diff --git a/.env b/.env deleted file mode 100644 index da7ccee60d..0000000000 --- a/.env +++ /dev/null @@ -1,5 +0,0 @@ -WANDB_API_KEY='' -ALLHANDS_API_KEY="" -SANDBOX_REMOTE_RUNTIME_API_URL="" -SGLANG_HOST_IP="127.0.0.1" -HOST_IP="127.0.0.1" \ No newline at end of file diff --git a/.gemini/config.yaml b/.gemini/config.yaml new file mode 100644 index 0000000000..231c59bccf --- /dev/null +++ b/.gemini/config.yaml @@ -0,0 +1,12 @@ +have_fun: false +code_review: + disable: false + comment_severity_threshold: MEDIUM + max_review_comments: -1 + pull_request_opened: + help: false + # disable PR summaries + summary: false + code_review: true + include_drafts: false +ignore_patterns: [] diff --git a/.github/workflows/cpu_ci.yaml b/.github/workflows/cpu_ci.yaml new file mode 100644 index 0000000000..9f262e69e5 --- /dev/null +++ b/.github/workflows/cpu_ci.yaml @@ -0,0 +1,89 @@ +name: SkyRL + +on: + push: + branches: + - main + - rc/** + pull_request: + +permissions: + checks: write # for status checks to appear + contents: read + +# Cancel runs for previous commits on the same branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check_code_quality: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + # This is the version of the action for setting up Python, not the Python version. + uses: actions/setup-python@v5 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.12' + cache: 'pip' + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + activate-environment: true + - name: Run pre-commit hooks + run: uv pip install pre-commit; pre-commit run --all-files --config .pre-commit-config.yaml + + skyrl_tests: + needs: check_code_quality + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./skyrl-train + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + # This is the version of the action for setting up Python, not the Python version. + uses: actions/setup-python@v5 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.12' + cache: 'pip' + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + activate-environment: true + - name: Install skyrl-train + run: uv sync --frozen --extra dev # installs from lock file + - name: Run cpu tests + run: uv run --frozen pytest tests/cpu/ + + skyrl_gym_tests: + needs: check_code_quality + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./skyrl-gym + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + # This is the version of the action for setting up Python, not the Python version. + uses: actions/setup-python@v5 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.12' + cache: 'pip' + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + activate-environment: true + - name: Install skyrl-gym + run: uv sync --frozen --extra dev # installs from lock file + - name: Run cpu tests + run: uv run --frozen pytest tests/ \ No newline at end of file diff --git a/.github/workflows/cpu_skyrl_tx.yaml b/.github/workflows/cpu_skyrl_tx.yaml new file mode 100644 index 0000000000..ca06fb3d54 --- /dev/null +++ b/.github/workflows/cpu_skyrl_tx.yaml @@ -0,0 +1,67 @@ +name: SkyRL-tx-CPU + +on: + push: + branches: [ main ] + paths: + - 'skyrl-tx/**' + - '.github/workflows/cpu_skyrl_tx.yaml' + pull_request: + paths: + - 'skyrl-tx/**' + - '.github/workflows/cpu_skyrl_tx.yaml' + workflow_dispatch: + +permissions: + checks: write # for status checks to appear + contents: read + +# Cancel runs for previous commits on the same branch +concurrency: + group: skyrl-tx-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + skyrl_tx_tests: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./skyrl-tx + steps: + - uses: actions/checkout@v4 + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + # - name: Check if reference docs are up to date + # run: | + # uv run --extra dev typer tx/run/main.py utils docs --name tx --output docs/reference.md && git diff --exit-code docs/reference.md + # - name: Test docs + # run: | + # uv run --extra dev mkdocs build --strict + - name: Run lint + run: | + uvx ruff check + # - name: Run type checking + # run: | + # uv run --extra tinker --extra dev ty check + - name: Run pytest + run: | + uv run --extra tinker --extra dev pytest --forked -s tests + - name: Run a single training step + run: | + uv run tx train --model pcmoritz/qwen3-tiny-test --dataset mahiatlinux/TinyStories-GPT4-V2-50K-SUBSET --output-dir /tmp --batch-size 2 --max-steps 1 --optimizer-args '{"learning_rate": 0.002, "weight_decay": 0.1}' + uv run tx train --model pcmoritz/qwen3-tiny-test --dataset mahiatlinux/TinyStories-GPT4-V2-50K-SUBSET --output-dir /tmp --batch-size 2 --max-steps 1 --optimizer-args '{"learning_rate": 0.002, "weight_decay": 0.1}' --tp-size 2 + - name: Run a single fine-tuning step on a chat dataset + run: | + uv run --with jinja2 tx train --model pcmoritz/qwen3-tiny-test --dataset smangrul/ultrachat-feedback-10k-chatml --output-dir /tmp --batch-size 2 --max-steps 1 --loader tx.loaders.chat --load-checkpoint-path /tmp + - name: Run a single fine-tuning step with Qwen3 MoE + run: | + uv run --with huggingface_hub hf download trl-internal-testing/tiny-Qwen3MoeForCausalLM --local-dir /tmp/qwen3_moe + uv run --with jinja2 tx train --model trl-internal-testing/tiny-Qwen3MoeForCausalLM --dataset smangrul/ultrachat-feedback-10k-chatml --output-dir /tmp --batch-size 2 --max-steps 1 --loader tx.loaders.chat --load-checkpoint-path /tmp/qwen3_moe + - name: Test experiment tracker integration + run: | + WANDB_MODE=offline WANDB_API_KEY=dummy uv run --with wandb tx train --model pcmoritz/qwen3-tiny-test --dataset mahiatlinux/TinyStories-GPT4-V2-50K-SUBSET --output-dir /tmp --batch-size 2 --max-steps 1 --tracker wandb --tracker-args '{"name": "Qwen3-8B", "project": "tx"}' + - name: Run engine benchmarks + run: | + uv run --extra tinker --extra dev python benchmarks/benchmark_engine.py --base-model trl-internal-testing/tiny-Qwen3ForCausalLM --backend-config '{"max_lora_adapters": 3, "max_lora_rank": 1}' --num-warmup-steps 1 --num-steps 1 --num-requests 1 --seq-len 8 --sample-max-tokens 16 diff --git a/.github/workflows/gpu_ci.yaml b/.github/workflows/gpu_ci.yaml new file mode 100644 index 0000000000..b6799b8e05 --- /dev/null +++ b/.github/workflows/gpu_ci.yaml @@ -0,0 +1,50 @@ +name: SkyRL-GPU + +on: + push: + branches: + - main + paths: + - 'skyrl-train/**' + - '!skyrl-train/docs/**' + - '!skyrl-train/examples/**' + - '.github/workflows/**' + workflow_dispatch: + + +permissions: + checks: write # for status checks to appear + contents: read + +jobs: + + skyrl_tests: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./skyrl-train + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + # This is the version of the action for setting up Python, not the Python version. + uses: actions/setup-python@v5 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.12' + cache: 'pip' + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + activate-environment: true + - name: Install basic dependencies + run: uv pip install anyscale==0.24.79 typer==0.9.0 + # Run tests + - name: GPU tests + env: + ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }} + ANYSCALE_HOST: https://console.anyscale.com + run: | + anyscale job submit -f ci/anyscale_gpu_ci.yaml --timeout 10000 + anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-ci --timeout 10000 diff --git a/.github/workflows/gpu_ci_megatron.yaml b/.github/workflows/gpu_ci_megatron.yaml new file mode 100644 index 0000000000..cab31b724d --- /dev/null +++ b/.github/workflows/gpu_ci_megatron.yaml @@ -0,0 +1,55 @@ +name: SkyRL-GPU-Megatron + +on: + push: + branches: + - main + paths: + - 'skyrl-train/skyrl_train/workers/**' + - 'skyrl-train/skyrl_train/distributed/megatron/**' + - 'skyrl-train/skyrl_train/config/**' + - 'skyrl-train/skyrl_train/trainer.py' + - 'skyrl-train/tests/gpu/gpu_ci/**' + - 'skyrl-train/pyproject.toml' + - '!skyrl-train/docs/**' + - '!skyrl-train/examples/**' + - '.github/workflows/**' + workflow_dispatch: + + +permissions: + checks: write # for status checks to appear + contents: read + +jobs: + + skyrl_tests_megatron: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./skyrl-train + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + # This is the version of the action for setting up Python, not the Python version. + uses: actions/setup-python@v5 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.12' + cache: 'pip' + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + activate-environment: true + - name: Install basic dependencies + run: uv pip install anyscale==0.24.79 typer==0.9.0 + # Run tests + - name: GPU tests + env: + ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }} + ANYSCALE_HOST: https://console.anyscale.com + run: | + anyscale job submit -f ci/anyscale_gpu_ci_megatron.yaml --timeout 5000 + anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-ci-megatron --timeout 5000 diff --git a/.github/workflows/gpu_e2e_ci.yaml b/.github/workflows/gpu_e2e_ci.yaml new file mode 100644 index 0000000000..ed89f5fecd --- /dev/null +++ b/.github/workflows/gpu_e2e_ci.yaml @@ -0,0 +1,47 @@ +name: SkyRL-GPU-E2E-CI + +on: + schedule: + - cron: '5 8 * * *' # Every day at 08:05 UTC (~00:05 PST / ~01:05 PDT) + workflow_dispatch: + +permissions: + checks: write # for status checks to appear + contents: read + +jobs: + + skyrl_gpu_e2e_test: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./skyrl-train + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + # This is the version of the action for setting up Python, not the Python version. + uses: actions/setup-python@v5 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.12' + cache: 'pip' + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + activate-environment: true + - name: Install basic dependencies + run: uv pip install anyscale==0.24.79 typer==0.9.0 + - name: Install envsubst + run: sudo apt-get update && sudo apt-get install -y gettext-base + - name: Basic convergence test + env: + ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }} + ANYSCALE_HOST: https://console.anyscale.com + WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} + run: | + envsubst < ci/anyscale_gpu_e2e_test.yaml > ci/anyscale_gpu_e2e_test_envsubst.yaml + anyscale job submit -f ci/anyscale_gpu_e2e_test_envsubst.yaml --timeout 4500 + anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test --timeout 4500 + rm -f ci/anyscale_gpu_e2e_test_envsubst.yaml \ No newline at end of file diff --git a/.github/workflows/gpu_e2e_ci_megatron.yaml b/.github/workflows/gpu_e2e_ci_megatron.yaml new file mode 100644 index 0000000000..6f1aa90314 --- /dev/null +++ b/.github/workflows/gpu_e2e_ci_megatron.yaml @@ -0,0 +1,47 @@ +name: SkyRL-GPU-E2E-CI-Megatron + +on: + schedule: + - cron: '5 8 * * *' # Every day at 08:05 UTC (~00:05 PST / ~01:05 PDT) + workflow_dispatch: + +permissions: + checks: write # for status checks to appear + contents: read + +jobs: + + skyrl_gpu_e2e_test_megatron: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./skyrl-train + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + # This is the version of the action for setting up Python, not the Python version. + uses: actions/setup-python@v5 + with: + # Semantic version range syntax or exact version of a Python version + python-version: '3.12' + cache: 'pip' + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + activate-environment: true + - name: Install basic dependencies + run: uv pip install anyscale==0.24.79 typer==0.9.0 + - name: Install envsubst + run: sudo apt-get update && sudo apt-get install -y gettext-base + - name: Basic convergence test + env: + ANYSCALE_CLI_TOKEN: ${{ secrets.ANYSCALE_CLI_TOKEN }} + ANYSCALE_HOST: https://console.anyscale.com + WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} + run: | + envsubst < ci/anyscale_gpu_e2e_test_megatron.yaml > ci/anyscale_gpu_e2e_test_megatron_envsubst.yaml + anyscale job submit -f ci/anyscale_gpu_e2e_test_megatron_envsubst.yaml --timeout 4500 + anyscale job wait --cloud sky-anyscale-aws-us-east-1 --name skyrl-train-gpu-e2e-test-megatron --timeout 4500 + rm -f ci/anyscale_gpu_e2e_test_megatron_envsubst.yaml \ No newline at end of file diff --git a/.github/workflows/openenv-fleet-train-tinker.yaml b/.github/workflows/openenv-fleet-train-tinker.yaml new file mode 100644 index 0000000000..65905e6ac3 --- /dev/null +++ b/.github/workflows/openenv-fleet-train-tinker.yaml @@ -0,0 +1,234 @@ +name: "Fleet Task Training (Tinker)" + +on: + workflow_dispatch: + inputs: + modality: + description: "Task modality" + type: choice + options: + - tool_use + - computer_use + default: tool_use + env_key: + description: "Fleet environment filter (empty=all)" + type: string + default: "" + max_tasks: + description: "Max tasks (empty=all, or number for testing)" + type: string + default: "" + model_name: + description: "Model to train" + type: choice + options: + - Qwen/Qwen3-8B + - Qwen/Qwen3-VL-30B-A3B-Instruct + default: Qwen/Qwen3-8B + max_steps: + description: "Maximum training steps" + type: string + default: "50" + batch_size: + description: "Batch size" + type: string + default: "8" + n_samples_per_prompt: + description: "Samples per prompt (for GRPO)" + type: string + default: "4" + data_version: + description: "Dataset version in S3 (e.g., v0.1, v0.2, v0.3)" + type: string + default: "v0.3" + +permissions: + contents: read + +# One training run at a time on the self-hosted runner +concurrency: + group: fleet-tinker-training + cancel-in-progress: false + +env: + TASKS_BUCKET: "fleet-internal-datasets" + TASKS_PREFIX: "${{ inputs.data_version }}/openenv" + +jobs: + train: + name: "Train ${{ inputs.model_name }} (by ${{ github.actor }})" + runs-on: [self-hosted, fleet-runner] + timeout-minutes: 4320 # 72 hours max + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh || true + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install dependencies + working-directory: skyrl-train + run: | + uv venv + source .venv/bin/activate + uv pip install -e . + uv pip install "tinker>=0.8.0" + uv pip install --no-cache-dir "openenv[fleet] @ git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" + + - name: Configure AWS credentials + run: | + mkdir -p ~/.aws + cat > ~/.aws/credentials << EOF + [default] + aws_access_key_id = ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key = ${{ secrets.AWS_SECRET_ACCESS_KEY }} + EOF + cat > ~/.aws/config << EOF + [default] + region = us-east-1 + EOF + + - name: Generate run name + id: run + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M") + RUN_ID=$(head -c 4 /dev/urandom | xxd -p) + MODALITY="${{ inputs.modality }}" + MODALITY_DASH=$(echo "$MODALITY" | tr '_' '-') + RUN_NAME="fleet-${MODALITY_DASH}-${RUN_ID}-${TIMESTAMP}" + echo "name=$RUN_NAME" >> $GITHUB_OUTPUT + echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT + + - name: Validate data version + run: | + DATA_VERSION="${{ inputs.data_version }}" + if ! aws s3 ls "s3://${{ env.TASKS_BUCKET }}/${DATA_VERSION}/openenv/" > /dev/null 2>&1; then + echo "ERROR: Data version '$DATA_VERSION' not found"; exit 1 + fi + + - name: Download tasks from S3 + run: | + MODALITY="${{ inputs.modality }}" + TASKS_FILE="all_${MODALITY}.json" + mkdir -p ./data + aws s3 cp "s3://${{ env.TASKS_BUCKET }}/${{ env.TASKS_PREFIX }}/${TASKS_FILE}" ./data/tasks.json + + - name: Prepare dataset + working-directory: skyrl-train + run: | + source .venv/bin/activate + PREPARE_CMD="python -m integrations.fleet.prepare_dataset --tasks-json ../data/tasks.json --output-dir ../data/fleet --modality ${{ inputs.modality }}" + [ -n "${{ inputs.env_key }}" ] && PREPARE_CMD="$PREPARE_CMD --env-key ${{ inputs.env_key }}" + [ -n "${{ inputs.max_tasks }}" ] && PREPARE_CMD="$PREPARE_CMD --max-tasks ${{ inputs.max_tasks }}" + eval "$PREPARE_CMD" + + - name: Notify Slack - Started + id: slack_start + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "text": "🚀 Tinker Training Started", + "blocks": [ + { "type": "header", "text": { "type": "plain_text", "text": "🚀 Tinker Training Started" } }, + { "type": "section", "fields": [ + { "type": "mrkdwn", "text": "*Run:*\n`${{ steps.run.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Model:*\n`${{ inputs.model_name }}`" }, + { "type": "mrkdwn", "text": "*Modality:*\n`${{ inputs.modality }}`" }, + { "type": "mrkdwn", "text": "*Max Steps:*\n`${{ inputs.max_steps }}`" } + ]}, + { "type": "context", "elements": [ + { "type": "mrkdwn", "text": "Triggered by: ${{ github.actor }} | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow> | " } + ]} + ] + } + + - name: Run training + id: training + working-directory: skyrl-train + env: + TINKER_API_URL: ${{ secrets.TINKER_API_URL }} + TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }} + FLEET_API_KEY: ${{ secrets.FLEET_API_KEY }} + LOGFIRE_TOKEN: ${{ secrets.LOGFIRE_TOKEN }} + WANDB_API_KEY: ${{ secrets.WANDB_API_KEY_TOOL_USE }} + run: | + source .venv/bin/activate + + python -m integrations.fleet.entrypoints.main_fleet_tinker \ + --model-name "${{ inputs.model_name }}" \ + --tasks-file ../data/tasks.json \ + --dataset-file ../data/fleet/train.parquet \ + --eval-dataset-file ../data/fleet/validation.parquet \ + --batch-size ${{ inputs.batch_size }} \ + --max-steps ${{ inputs.max_steps }} \ + --n-samples-per-prompt ${{ inputs.n_samples_per_prompt }} \ + --max-turns 50 \ + --eval-every 20 \ + --wandb-project fleet-tinker-grpo \ + --wandb-name "${{ steps.run.outputs.name }}" + + echo "status=COMPLETED" >> $GITHUB_OUTPUT + + - name: Notify Slack - Completed + if: steps.training.outputs.status == 'COMPLETED' + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "✅ Training Completed", + "blocks": [ + { "type": "header", "text": { "type": "plain_text", "text": "✅ Tinker Training Completed" } }, + { "type": "section", "fields": [ + { "type": "mrkdwn", "text": "*Run:*\n`${{ steps.run.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`COMPLETED`" } + ]}, + { "type": "section", "text": { "type": "mrkdwn", "text": "" } } + ] + } + + - name: Notify Slack - Failed + if: failure() + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "❌ Training Failed", + "blocks": [ + { "type": "header", "text": { "type": "plain_text", "text": "❌ Tinker Training Failed" } }, + { "type": "section", "fields": [ + { "type": "mrkdwn", "text": "*Run:*\n`${{ steps.run.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`FAILED`" } + ]}, + { "type": "section", "text": { "type": "mrkdwn", "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Logs>" } } + ] + } + + - name: Summary + if: always() + run: | + echo "## Tinker Training Summary" >> $GITHUB_STEP_SUMMARY + echo "| Field | Value |" >> $GITHUB_STEP_SUMMARY + echo "|-------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Run | \`${{ steps.run.outputs.name }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Model | \`${{ inputs.model_name }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Status | \`${{ steps.training.outputs.status || 'FAILED' }}\` |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/openenv-fleet-train.yaml b/.github/workflows/openenv-fleet-train.yaml new file mode 100644 index 0000000000..5110b8a1e3 --- /dev/null +++ b/.github/workflows/openenv-fleet-train.yaml @@ -0,0 +1,282 @@ +name: "Fleet Task Training (SkyPilot)" + +on: + workflow_dispatch: + inputs: + env_keys: + description: "Fleet environment filter (empty=all, or comma-separated: github,booking,reddit)" + type: string + default: "" + difficulty: + description: "Difficulty filter (1=easy, 2=medium, 3=hard)" + type: choice + options: + - all + - "1" + - "2" + - "3" + - "1,2" + - "2,3" + default: "all" + max_tasks: + description: "Max tasks (empty=all, or number for testing)" + type: string + default: "" + cloud: + description: "Preferred cloud (any = auto-select cheapest)" + type: choice + options: + - any + - lambda + - runpod + - vast + - nebius + - gcp + default: any + task_name: + description: "SkyPilot task name (YAML file in skyrl-train/tasks/)" + type: choice + options: + - openenv-fleet-grpo-qwen3_5-9b + - openenv-fleet-grpo-qwen3_5-35b + default: "openenv-fleet-grpo-qwen3_5-9b" + data_version: + description: "Dataset version in S3 (v3=20,557 tasks, v4=15,169 tasks, v52=latest curated)" + type: string + default: "v52" + resume_run_name: + description: "Resume from S3 checkpoint (W&B run name, e.g., fleet_tool_use_32b_d7167c1c)" + type: string + default: "" + disable_slack: + description: "Disable Slack notifications (for debugging)" + type: boolean + default: false + +permissions: + contents: read + +# Allow multiple concurrent training runs for experimentation +concurrency: + group: fleet-training-${{ github.run_id }} + cancel-in-progress: false + +jobs: + train: + name: "Train ${{ inputs.task_name }} (by ${{ github.actor }})" + runs-on: [self-hosted, fleet-runner] + timeout-minutes: 4320 # 72 hours max + + env: + WANDB_KEY_TOOL_USE: ${{ secrets.WANDB_API_KEY_TOOL_USE }} + WANDB_KEY_COMPUTER_USE: ${{ secrets.WANDB_API_KEY_COMPUTER_USE }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + steps: + - name: Clean runner state + run: | + rm -rf ~/actions-runner/_diag/pages/* 2>/dev/null || true + rm -rf ~/actions-runner/_work/_temp/* 2>/dev/null || true + + - name: Checkout + uses: actions/checkout@v4 + with: + clean: true + + - name: Configure Cloud Credentials + run: | + mkdir -p ~/.lambda_cloud && echo "api_key = ${{ secrets.LAMBDA_API_KEY }}" > ~/.lambda_cloud/lambda_keys + pip3 install runpod 2>/dev/null || true && runpod config "${{ secrets.RUNPOD_API_KEY }}" || true + mkdir -p ~/.config/vastai && echo "${{ secrets.VAST_API_KEY }}" > ~/.config/vastai/vast_api_key + pip3 install prime 2>/dev/null || true && prime config set-api-key "${{ secrets.PRIME_INTELLECT_API_KEY }}" || true + pip3 install "skypilot-nightly[primeintellect]" 2>/dev/null || true + mkdir -p ~/.nebius && echo '${{ secrets.NEBIUS_CREDENTIALS_JSON }}' > ~/.nebius/credentials.json + echo "${{ secrets.NEBIUS_TENANT_ID }}" > ~/.nebius/NEBIUS_TENANT_ID.txt + # GCP: install gcloud CLI, write service account key, and activate + if [ -n '${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}' ]; then + curl -sSL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz | tar -xz -C $HOME + echo "$HOME/google-cloud-sdk/bin" >> $GITHUB_PATH + export PATH="$HOME/google-cloud-sdk/bin:$PATH" + mkdir -p ~/.config/gcloud + echo '${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}' > ~/.config/gcloud/application_default_credentials.json + gcloud auth activate-service-account --key-file=$HOME/.config/gcloud/application_default_credentials.json + echo "GOOGLE_APPLICATION_CREDENTIALS=$HOME/.config/gcloud/application_default_credentials.json" >> $GITHUB_ENV + pip3 install google-api-python-client google-cloud-storage 2>/dev/null || true + # Set up kubeconfig for GKE RDMA cluster (enables cloud: kubernetes) + gcloud components install gke-gcloud-auth-plugin --quiet 2>/dev/null || true + gcloud container clusters get-credentials fleet-rdma --region=us-central1 --project=fleet-compute-489000 || true + else + echo "WARNING: GCP_SERVICE_ACCOUNT_KEY secret not set — GCP clouds will be unavailable" + fi + + - name: Verify Cloud Credentials + run: | + pip install "skypilot[kubernetes]" 2>/dev/null || true + sky api stop || true + sky api start || true + sky check lambda runpod vast primeintellect nebius gcp kubernetes || true + + - name: Validate data version + run: | + DATA_VERSION="${{ inputs.data_version }}" + if ! aws s3 ls "s3://fleet-internal-datasets/${DATA_VERSION}/openenv/" > /dev/null 2>&1; then + echo "ERROR: Data version '$DATA_VERSION' not found"; exit 1 + fi + + - name: Generate Cluster Name + id: cluster + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M") + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + MODALITY=$(grep -E '^\s+MODALITY:' skyrl-train/tasks/${{ inputs.task_name }}.yaml | awk -F'"' '{print $2}') + echo "modality=$MODALITY" >> $GITHUB_OUTPUT + NUM_NODES=$(grep -E '^\s*num_nodes:' skyrl-train/tasks/${{ inputs.task_name }}.yaml | awk '{print $2}') + if [ -n "$NUM_NODES" ] && [ "$NUM_NODES" != "1" ]; then + CLUSTER_NAME="fleet-${MODALITY}-${NUM_NODES}n-${SHORT_SHA}-${TIMESTAMP}" + else + CLUSTER_NAME="fleet-${MODALITY}-${SHORT_SHA}-${TIMESTAMP}" + fi + CLUSTER_NAME=$(echo "$CLUSTER_NAME" | tr '_' '-' | tr '[:upper:]' '[:lower:]') + echo "name=$CLUSTER_NAME" >> $GITHUB_OUTPUT + RUN_ID=$(head -c 4 /dev/urandom | xxd -p) + echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT + echo "wandb_run_name=fleet_${MODALITY}_${RUN_ID}" >> $GITHUB_OUTPUT + + - name: Notify Slack - Starting + if: ${{ inputs.disable_slack != true }} + id: slack_start + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "text": "🚀 Training Run Started", + "blocks": [ + { "type": "header", "text": { "type": "plain_text", "text": "🚀 Training Run Started" } }, + { "type": "section", "fields": [ + { "type": "mrkdwn", "text": "*Cluster:*\n`${{ steps.cluster.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Task:*\n`${{ inputs.task_name }}`" }, + { "type": "mrkdwn", "text": "*Modality:*\n`${{ steps.cluster.outputs.modality }}`" }, + { "type": "mrkdwn", "text": "*WandB:*\n`${{ steps.cluster.outputs.wandb_run_name }}`" } + ]}, + { "type": "context", "elements": [ + { "type": "mrkdwn", "text": "Triggered by: ${{ github.actor }} | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow> | " } + ]} + ] + } + + - name: Launch Training Cluster + id: launch + run: | + # Select WANDB key based on modality derived from task YAML + if [ "${{ steps.cluster.outputs.modality }}" = "computer_use" ]; then + WANDB_KEY="$WANDB_KEY_COMPUTER_USE" + else + WANDB_KEY="$WANDB_KEY_TOOL_USE" + fi + [ -z "$WANDB_KEY" ] && echo "ERROR: WANDB key not set for modality '${{ steps.cluster.outputs.modality }}'" && exit 1 + + LAUNCH_CMD="sky launch skyrl-train/tasks/${{ inputs.task_name }}.yaml" + LAUNCH_CMD="$LAUNCH_CMD --cluster ${{ steps.cluster.outputs.name }}" + LAUNCH_CMD="$LAUNCH_CMD --env WANDB_API_KEY=$WANDB_KEY" + LAUNCH_CMD="$LAUNCH_CMD --env FLEET_API_KEY=${{ secrets.FLEET_API_KEY }}" + LAUNCH_CMD="$LAUNCH_CMD --env DAYTONA_API_KEY=${{ secrets.DAYTONA_API_KEY }}" + LAUNCH_CMD="$LAUNCH_CMD --env LOGFIRE_TOKEN=${{ secrets.LOGFIRE_TOKEN }}" + LAUNCH_CMD="$LAUNCH_CMD --env DATA_VERSION=${{ inputs.data_version }}" + LAUNCH_CMD="$LAUNCH_CMD --env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" + LAUNCH_CMD="$LAUNCH_CMD --env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" + [ -n "${{ inputs.env_keys }}" ] && LAUNCH_CMD="$LAUNCH_CMD --env ENV_KEYS=${{ inputs.env_keys }}" + [ "${{ inputs.difficulty }}" != "all" ] && LAUNCH_CMD="$LAUNCH_CMD --env DIFFICULTY=${{ inputs.difficulty }}" + [ -n "${{ inputs.max_tasks }}" ] && LAUNCH_CMD="$LAUNCH_CMD --env MAX_TASKS=${{ inputs.max_tasks }}" + [ -n "${{ inputs.resume_run_name }}" ] && LAUNCH_CMD="$LAUNCH_CMD --env RESUME_RUN_NAME=${{ inputs.resume_run_name }}" + LAUNCH_CMD="$LAUNCH_CMD --env RUN_ID=${{ steps.cluster.outputs.run_id }}" + [ "${{ inputs.cloud }}" != "any" ] && LAUNCH_CMD="$LAUNCH_CMD --cloud ${{ inputs.cloud }}" + LAUNCH_CMD="$LAUNCH_CMD --retry-until-up -y" # Retry until GPUs available, run attached + + echo "Command: $LAUNCH_CMD" + eval "$LAUNCH_CMD" + echo "status=SUCCEEDED" >> $GITHUB_OUTPUT + + - name: Cleanup Cluster + if: always() + run: | + echo "Cleaning up cluster ${{ steps.cluster.outputs.name }}..." + sky down "${{ steps.cluster.outputs.name }}" -y || true + + - name: Notify Slack - Completed + if: steps.launch.outputs.status == 'SUCCEEDED' && inputs.disable_slack != true + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "✅ Training Completed", + "blocks": [ + { "type": "header", "text": { "type": "plain_text", "text": "✅ Training Completed" } }, + { "type": "section", "fields": [ + { "type": "mrkdwn", "text": "*Cluster:*\n`${{ steps.cluster.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`SUCCEEDED`" } + ]}, + { "type": "section", "text": { "type": "mrkdwn", "text": "" } } + ] + } + + - name: Notify Slack - Failed + if: failure() && inputs.disable_slack != true + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "❌ Training Failed", + "blocks": [ + { "type": "header", "text": { "type": "plain_text", "text": "❌ Training Failed" } }, + { "type": "section", "fields": [ + { "type": "mrkdwn", "text": "*Cluster:*\n`${{ steps.cluster.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`FAILED`" } + ]}, + { "type": "section", "text": { "type": "mrkdwn", "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Logs>" } } + ] + } + + - name: Notify Slack - Cancelled + if: cancelled() && inputs.disable_slack != true + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "⚠️ Training Cancelled", + "blocks": [ + { "type": "header", "text": { "type": "plain_text", "text": "⚠️ Training Cancelled" } }, + { "type": "section", "fields": [ + { "type": "mrkdwn", "text": "*Cluster:*\n`${{ steps.cluster.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`CANCELLED`" } + ]}, + { "type": "context", "elements": [ + { "type": "mrkdwn", "text": "Cancelled by user or superseded by new run" } + ]} + ] + } + + - name: Summary + if: always() + run: | + echo "## Training Summary" >> $GITHUB_STEP_SUMMARY + echo "| Field | Value |" >> $GITHUB_STEP_SUMMARY + echo "|-------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Cluster | \`${{ steps.cluster.outputs.name }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| WandB | \`${{ steps.cluster.outputs.wandb_run_name }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Status | \`${{ steps.launch.outputs.status || 'FAILED' }}\` |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/runner-health.yaml b/.github/workflows/runner-health.yaml new file mode 100644 index 0000000000..2ad7e7cd84 --- /dev/null +++ b/.github/workflows/runner-health.yaml @@ -0,0 +1,709 @@ +name: Runner Health Check + +on: + schedule: + # Run every 15 minutes + - cron: '*/15 * * * *' + workflow_dispatch: + +concurrency: + group: runner-health-check + cancel-in-progress: false + +permissions: + contents: read + actions: read + +jobs: + check-runners: + name: Check Runner Status + runs-on: ubuntu-latest + steps: + - name: Restore previous state + id: cache + uses: actions/cache/restore@v4 + with: + path: /tmp/runner-health-state + key: runner-health-state-${{ github.repository }} + restore-keys: runner-health-state- + + - name: Load previous state + id: prev + run: | + if [ -f /tmp/runner-health-state/state.json ]; then + cat /tmp/runner-health-state/state.json + PREV_COUNT=$(jq -r '.unhealthy_count // 0' /tmp/runner-health-state/state.json) + echo "unhealthy_count=$PREV_COUNT" >> $GITHUB_OUTPUT + echo "Previous unhealthy count: $PREV_COUNT" + elif [ -f /tmp/runner-health-state/last_unhealthy_count ]; then + # Migration from old format + PREV=$(cat /tmp/runner-health-state/last_unhealthy_count) + echo "unhealthy_count=$PREV" >> $GITHUB_OUTPUT + echo "Migrated from old state format, unhealthy count: $PREV" + else + echo "unhealthy_count=0" >> $GITHUB_OUTPUT + echo "No previous state found" + fi + + - name: Check EC2 instance health + id: ec2 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + run: | + set +e # Don't exit on errors + + # Runner instance IDs and names (from docs/gha-runner-setup.md) + RUNNERS="i-04a15158610df980f:fleet-runner-1 i-0f80c703294413a4c:fleet-runner-2 i-09321b67952c1a208:fleet-runner-3 i-0f2ccaa840ef29450:fleet-runner-4 i-05ecd98e74949dc87:fleet-runner-5" + + # Function to check instance with retries + check_instance() { + local id=$1 + local retries=3 + local delay=5 + for i in $(seq 1 $retries); do + RESULT=$(timeout 10 aws ec2 describe-instance-status --instance-ids "$id" --include-all-instances --query 'InstanceStatuses[0].[InstanceState.Name,InstanceStatus.Status]' --output text 2>/dev/null) + if [ -n "$RESULT" ] && [ "$RESULT" != "None" ]; then + echo "$RESULT" + return 0 + fi + [ $i -lt $retries ] && sleep $delay + done + echo "unknown unknown" + return 1 + } + + echo "## EC2 Runner Health" >> $GITHUB_STEP_SUMMARY + echo "| Runner | Instance | State | Health |" >> $GITHUB_STEP_SUMMARY + echo "|--------|----------|-------|--------|" >> $GITHUB_STEP_SUMMARY + + UNHEALTHY="" + UNHEALTHY_NAMES="" + TOTAL=0 + HEALTHY=0 + UNHEALTHY_DETAILS="[]" + + for entry in $RUNNERS; do + id=$(echo $entry | cut -d: -f1) + NAME=$(echo $entry | cut -d: -f2) + TOTAL=$((TOTAL + 1)) + + # Get instance state and status with retries + RESULT=$(check_instance "$id") + STATE=$(echo $RESULT | awk '{print $1}') + HEALTH=$(echo $RESULT | awk '{print $2}') + + # Check if healthy (running + ok) + if [ "$STATE" = "running" ] && [ "$HEALTH" = "ok" ]; then + HEALTHY=$((HEALTHY + 1)) + echo "| $NAME | $id | ✅ $STATE | $HEALTH |" >> $GITHUB_STEP_SUMMARY + else + UNHEALTHY="$UNHEALTHY $id" + UNHEALTHY_NAMES="$UNHEALTHY_NAMES $NAME" + UNHEALTHY_DETAILS=$(echo "$UNHEALTHY_DETAILS" | jq --arg id "$id" --arg name "$NAME" --arg state "$STATE" --arg health "$HEALTH" '. + [{"id": $id, "name": $name, "state": $state, "health": $health}]') + echo "| $NAME | $id | ❌ $STATE | $HEALTH |" >> $GITHUB_STEP_SUMMARY + fi + done + + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Summary:** $HEALTHY/$TOTAL healthy" >> $GITHUB_STEP_SUMMARY + + # If unhealthy, wait 30s and recheck to avoid transient false positives + UNHEALTHY_COUNT=$((TOTAL - HEALTHY)) + if [ "$UNHEALTHY_COUNT" -gt 0 ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Confirmation check in 30s...**" >> $GITHUB_STEP_SUMMARY + sleep 30 + + # Recheck only the unhealthy instances + CONFIRMED_UNHEALTHY="" + CONFIRMED_NAMES="" + CONFIRMED_DETAILS="[]" + for id in $UNHEALTHY; do + RESULT=$(check_instance "$id") + STATE=$(echo $RESULT | awk '{print $1}') + HEALTH=$(echo $RESULT | awk '{print $2}') + if [ "$STATE" != "running" ] || [ "$HEALTH" != "ok" ]; then + for entry in $RUNNERS; do + eid=$(echo $entry | cut -d: -f1) + ename=$(echo $entry | cut -d: -f2) + if [ "$eid" = "$id" ]; then + CONFIRMED_UNHEALTHY="$CONFIRMED_UNHEALTHY $id" + CONFIRMED_NAMES="$CONFIRMED_NAMES $ename" + CONFIRMED_DETAILS=$(echo "$CONFIRMED_DETAILS" | jq --arg id "$id" --arg name "$ename" --arg state "$STATE" --arg health "$HEALTH" '. + [{"id": $id, "name": $name, "state": $state, "health": $health}]') + break + fi + done + fi + done + + UNHEALTHY=$(echo $CONFIRMED_UNHEALTHY | xargs) + UNHEALTHY_NAMES=$(echo $CONFIRMED_NAMES | xargs) + UNHEALTHY_COUNT=$(echo $UNHEALTHY | wc -w | tr -d ' ') + UNHEALTHY_DETAILS="$CONFIRMED_DETAILS" + HEALTHY=$((TOTAL - UNHEALTHY_COUNT)) + + if [ "$UNHEALTHY_COUNT" -eq 0 ]; then + echo "✅ **Confirmation: All runners recovered (transient issue)**" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Confirmed unhealthy:** $UNHEALTHY_NAMES" >> $GITHUB_STEP_SUMMARY + fi + fi + + # Export for subsequent steps + echo "total=$TOTAL" >> $GITHUB_OUTPUT + echo "healthy=$HEALTHY" >> $GITHUB_OUTPUT + echo "unhealthy_count=$UNHEALTHY_COUNT" >> $GITHUB_OUTPUT + echo "unhealthy_ids=$UNHEALTHY" >> $GITHUB_OUTPUT + echo "unhealthy_names=$UNHEALTHY_NAMES" >> $GITHUB_OUTPUT + # Write details to file (too large for output variable) + echo "$UNHEALTHY_DETAILS" > /tmp/unhealthy_details.json + + - name: Fix runner process health (KillMode + zombie listeners) + if: steps.ec2.outputs.healthy != '0' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + GHA_RUNNER_SSH_KEY: ${{ secrets.GHA_RUNNER_SSH_KEY }} + run: | + set +e + if [ -z "$GHA_RUNNER_SSH_KEY" ]; then + echo "GHA_RUNNER_SSH_KEY secret not set, skipping process health check" + exit 0 + fi + + # Write SSH key to temp file + SSH_KEY=$(mktemp) + echo "$GHA_RUNNER_SSH_KEY" > "$SSH_KEY" + chmod 600 "$SSH_KEY" + SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes -i $SSH_KEY" + + RUNNERS="i-04a15158610df980f:fleet-runner-1 i-0f80c703294413a4c:fleet-runner-2 i-09321b67952c1a208:fleet-runner-3 i-0f2ccaa840ef29450:fleet-runner-4 i-05ecd98e74949dc87:fleet-runner-5" + + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Process Health" >> $GITHUB_STEP_SUMMARY + echo "| Runner | KillMode | Listeners | Action |" >> $GITHUB_STEP_SUMMARY + echo "|--------|----------|-----------|--------|" >> $GITHUB_STEP_SUMMARY + + for entry in $RUNNERS; do + INSTANCE_ID=$(echo $entry | cut -d: -f1) + NAME=$(echo $entry | cut -d: -f2) + + # Get public IP + IP=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null) + if [ -z "$IP" ] || [ "$IP" = "None" ]; then + echo "| $NAME | - | - | Skipped (no IP) |" >> $GITHUB_STEP_SUMMARY + continue + fi + + # SSH in and check/fix + RESULT=$(ssh $SSH_OPTS ubuntu@$IP bash <<'REMOTE_SCRIPT' + SVCFILE=$(ls /etc/systemd/system/actions.runner.fleet-ai-SkyRL.*.service 2>/dev/null | head -1) + if [ -z "$SVCFILE" ]; then echo "NO_SERVICE"; exit 0; fi + SVCNAME=$(basename "$SVCFILE" .service) + + # Check KillMode + KILLMODE=$(grep -o 'KillMode=.*' "$SVCFILE" | head -1) + FIXED_KILLMODE=false + if [ "$KILLMODE" != "KillMode=control-group" ]; then + sudo sed -i 's/KillMode=.*/KillMode=control-group/' "$SVCFILE" + sudo systemctl daemon-reload + FIXED_KILLMODE=true + fi + + # Count listeners (only from actions-runner/, not harbor) + LISTENER_COUNT=$(ps aux | grep '/home/ubuntu/actions-runner/bin/Runner.Listener' | grep -v grep | wc -l) + + # Kill orphans if more than 1 listener + KILLED_ORPHANS=false + if [ "$LISTENER_COUNT" -gt 1 ]; then + # Find the real runsvc.sh PID and its node child + REAL_RUNSVC=$(pgrep -f '/home/ubuntu/actions-runner/runsvc.sh' | head -1) + REAL_NODE="" + if [ -n "$REAL_RUNSVC" ]; then + REAL_NODE=$(pgrep -P "$REAL_RUNSVC" -f RunnerService 2>/dev/null) + fi + # Kill orphaned RunnerService.js (not the one under runsvc.sh) + for pid in $(pgrep -f './bin/RunnerService.js'); do + CWD=$(readlink /proc/$pid/cwd 2>/dev/null) + if [ "$CWD" = "/home/ubuntu/actions-runner" ] && [ "$pid" != "$REAL_NODE" ]; then + kill -9 "$pid" 2>/dev/null + fi + done + # Kill orphaned Runner.Listener not under the real node + REAL_LISTENER="" + if [ -n "$REAL_NODE" ]; then + REAL_LISTENER=$(pgrep -P "$REAL_NODE" -f Runner.Listener 2>/dev/null) + fi + for pid in $(pgrep -f '/home/ubuntu/actions-runner/bin/Runner.Listener'); do + if [ "$pid" != "$REAL_LISTENER" ]; then + kill -9 "$pid" 2>/dev/null + fi + done + sleep 2 + LISTENER_COUNT=$(ps aux | grep '/home/ubuntu/actions-runner/bin/Runner.Listener' | grep -v grep | wc -l) + KILLED_ORPHANS=true + fi + + # Clean stale diag pages + rm -rf ~/actions-runner/_diag/pages/* 2>/dev/null + + echo "KILLMODE=$KILLMODE FIXED=$FIXED_KILLMODE LISTENERS=$LISTENER_COUNT KILLED=$KILLED_ORPHANS" + REMOTE_SCRIPT + 2>&1) + + if echo "$RESULT" | grep -q "NO_SERVICE"; then + echo "| $NAME | - | - | No runner service |" >> $GITHUB_STEP_SUMMARY + elif echo "$RESULT" | grep -q "KILLMODE="; then + KILLMODE=$(echo "$RESULT" | grep -o 'KILLMODE=[^ ]*' | cut -d= -f2) + FIXED=$(echo "$RESULT" | grep -o 'FIXED=[^ ]*' | cut -d= -f2) + LISTENERS=$(echo "$RESULT" | grep -o 'LISTENERS=[^ ]*' | cut -d= -f2) + KILLED=$(echo "$RESULT" | grep -o 'KILLED=[^ ]*' | cut -d= -f2) + ACTION="OK" + if [ "$FIXED" = "true" ]; then ACTION="Fixed KillMode"; fi + if [ "$KILLED" = "true" ]; then ACTION="$ACTION, killed orphans"; fi + echo "| $NAME | $KILLMODE | $LISTENERS | $ACTION |" >> $GITHUB_STEP_SUMMARY + else + echo "| $NAME | ? | ? | SSH failed |" >> $GITHUB_STEP_SUMMARY + fi + done + + rm -f "$SSH_KEY" + + - name: Auto-recover unhealthy instances + id: recover + if: steps.ec2.outputs.unhealthy_count != '0' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + MAX_ATTEMPTS=3 + COOLDOWN_SECONDS=3600 # Reset attempt counter after 1 hour + + # Load per-instance recovery state from previous run + # Format: {"instance_id": {"count": N, "last_attempt": "ISO8601"}, ...} + # Also handles legacy format: {"instance_id": N} + if [ -f /tmp/runner-health-state/state.json ]; then + RAW_ATTEMPTS=$(jq -r '.recovery_attempts // {}' /tmp/runner-health-state/state.json) + else + RAW_ATTEMPTS="{}" + fi + + # Migrate legacy format (plain int) to new format (object with count + timestamp) + NOW_TS=$(date -u +%s) + NOW_ISO=$(date -u +%Y-%m-%dT%H:%M:%SZ) + PREV_ATTEMPTS=$(echo "$RAW_ATTEMPTS" | jq --arg now "$NOW_ISO" ' + to_entries | map( + if (.value | type) == "number" then + .value = {"count": .value, "last_attempt": $now} + else . end + ) | from_entries + ') + + # Get list of runners currently executing a job (busy runners) + # Note: requires admin:org or repo admin token; GITHUB_TOKEN may not have access + BUSY_RUNNERS_RAW=$(gh api repos/${{ github.repository }}/actions/runners --paginate --jq '.runners[] | select(.busy == true) | .name' 2>/dev/null) + if [ $? -eq 0 ] && echo "$BUSY_RUNNERS_RAW" | head -1 | grep -qv '"message"'; then + BUSY_RUNNERS="$BUSY_RUNNERS_RAW" + if [ -n "$BUSY_RUNNERS" ]; then + echo "Busy runners (will not stop/start):" + echo "$BUSY_RUNNERS" + fi + else + BUSY_RUNNERS="" + echo "Warning: Could not fetch runner status (token lacks admin permission), skipping busy check" + fi + + DETAILS=$(cat /tmp/unhealthy_details.json) + COUNT=$(echo "$DETAILS" | jq 'length') + + UPDATED_ATTEMPTS="$PREV_ATTEMPTS" + mkdir -p /tmp/recovery-results + + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Auto-Recovery" >> $GITHUB_STEP_SUMMARY + + # Function to recover a single instance (runs in background) + recover_instance() { + local INSTANCE_ID="$1" + local INSTANCE_NAME="$2" + local INSTANCE_STATE="$3" + local ATTEMPTS="$4" + local RESULT_FILE="/tmp/recovery-results/$INSTANCE_ID" + + echo "[$INSTANCE_NAME] Starting recovery (attempt $((ATTEMPTS + 1))/$MAX_ATTEMPTS)..." + + # Recovery strategy: stop/start (reboot doesn't fix impaired instances) + if [ "$INSTANCE_STATE" = "stopped" ]; then + echo "[$INSTANCE_NAME] Instance already stopped, starting..." + aws ec2 start-instances --instance-ids "$INSTANCE_ID" > /dev/null 2>&1 + elif [ "$INSTANCE_STATE" = "running" ]; then + echo "[$INSTANCE_NAME] Stopping instance..." + aws ec2 stop-instances --instance-ids "$INSTANCE_ID" > /dev/null 2>&1 + echo "[$INSTANCE_NAME] Waiting for instance to stop..." + aws ec2 wait instance-stopped --instance-ids "$INSTANCE_ID" 2>/dev/null + echo "[$INSTANCE_NAME] Starting instance..." + aws ec2 start-instances --instance-ids "$INSTANCE_ID" > /dev/null 2>&1 + elif [ "$INSTANCE_STATE" = "stopping" ]; then + echo "[$INSTANCE_NAME] Instance is stopping, waiting..." + aws ec2 wait instance-stopped --instance-ids "$INSTANCE_ID" 2>/dev/null + echo "[$INSTANCE_NAME] Starting instance..." + aws ec2 start-instances --instance-ids "$INSTANCE_ID" > /dev/null 2>&1 + else + echo "[$INSTANCE_NAME] Unexpected state '$INSTANCE_STATE', attempting start..." + aws ec2 start-instances --instance-ids "$INSTANCE_ID" > /dev/null 2>&1 + fi + + # Wait for instance to be running and pass status checks (up to 10 min) + echo "[$INSTANCE_NAME] Waiting for instance to pass status checks..." + if timeout 600 aws ec2 wait instance-status-ok --instance-ids "$INSTANCE_ID" 2>/dev/null; then + echo "recovered" > "$RESULT_FILE" + echo "[$INSTANCE_NAME] Recovered successfully" + else + echo "failed" > "$RESULT_FILE" + echo "[$INSTANCE_NAME] Recovery failed" + fi + } + + # Phase 1: Determine which instances to recover vs skip + RECOVER_PIDS="" + SKIPPED_NAMES="" + SKIPPED_COUNT=0 + + for i in $(seq 0 $((COUNT - 1))); do + INSTANCE_ID=$(echo "$DETAILS" | jq -r ".[$i].id") + INSTANCE_NAME=$(echo "$DETAILS" | jq -r ".[$i].name") + INSTANCE_STATE=$(echo "$DETAILS" | jq -r ".[$i].state") + INSTANCE_HEALTH=$(echo "$DETAILS" | jq -r ".[$i].health") + + # Get previous attempt count and last attempt time + ATTEMPTS=$(echo "$PREV_ATTEMPTS" | jq -r --arg id "$INSTANCE_ID" '.[$id].count // 0') + LAST_ATTEMPT=$(echo "$PREV_ATTEMPTS" | jq -r --arg id "$INSTANCE_ID" '.[$id].last_attempt // empty') + ELAPSED=0 + + # Reset attempts if cooldown has elapsed + if [ "$ATTEMPTS" -ge "$MAX_ATTEMPTS" ] && [ -n "$LAST_ATTEMPT" ]; then + LAST_TS=$(date -u -d "$LAST_ATTEMPT" +%s 2>/dev/null || date -u -j -f "%Y-%m-%dT%H:%M:%SZ" "$LAST_ATTEMPT" +%s 2>/dev/null || echo "0") + ELAPSED=$((NOW_TS - LAST_TS)) + if [ "$ELAPSED" -ge "$COOLDOWN_SECONDS" ]; then + echo "Cooldown elapsed for $INSTANCE_NAME ($ELAPSED >= $COOLDOWN_SECONDS sec), resetting attempts" + ATTEMPTS=0 + UPDATED_ATTEMPTS=$(echo "$UPDATED_ATTEMPTS" | jq --arg id "$INSTANCE_ID" 'del(.[$id])') + fi + fi + + echo "Processing $INSTANCE_NAME ($INSTANCE_ID): state=$INSTANCE_STATE health=$INSTANCE_HEALTH attempts=$ATTEMPTS/$MAX_ATTEMPTS" + + # Skip if runner is actively executing a job + if [ -n "$BUSY_RUNNERS" ] && echo "$BUSY_RUNNERS" | grep -qx "$INSTANCE_NAME"; then + echo "⏭️ **$INSTANCE_NAME**: Skipped — runner is busy (executing a job)" >> $GITHUB_STEP_SUMMARY + SKIPPED_NAMES="$SKIPPED_NAMES $INSTANCE_NAME" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Skip if max attempts reached (cooldown not yet elapsed) + if [ "$ATTEMPTS" -ge "$MAX_ATTEMPTS" ]; then + REMAINING=$((COOLDOWN_SECONDS - ELAPSED)) + REMAINING_MIN=$((REMAINING / 60)) + echo "⏭️ **$INSTANCE_NAME**: In cooldown — $ATTEMPTS attempts exhausted, retrying in ~${REMAINING_MIN}m" >> $GITHUB_STEP_SUMMARY + SKIPPED_NAMES="$SKIPPED_NAMES $INSTANCE_NAME" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Skip terminated/shutting-down instances — can't recover these + if [ "$INSTANCE_STATE" = "terminated" ] || [ "$INSTANCE_STATE" = "shutting-down" ]; then + echo "⏭️ **$INSTANCE_NAME**: Skipped — instance is $INSTANCE_STATE (needs replacement)" >> $GITHUB_STEP_SUMMARY + SKIPPED_NAMES="$SKIPPED_NAMES $INSTANCE_NAME" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + echo "🔄 **$INSTANCE_NAME**: Attempting recovery (attempt $((ATTEMPTS + 1))/$MAX_ATTEMPTS)..." >> $GITHUB_STEP_SUMMARY + + # Store attempt count for result collection + echo "$ATTEMPTS" > "/tmp/recovery-results/${INSTANCE_ID}.attempts" + echo "$INSTANCE_NAME" > "/tmp/recovery-results/${INSTANCE_ID}.name" + + # Launch recovery in parallel + recover_instance "$INSTANCE_ID" "$INSTANCE_NAME" "$INSTANCE_STATE" "$ATTEMPTS" & + RECOVER_PIDS="$RECOVER_PIDS $!:$INSTANCE_ID" + done + + # Phase 2: Wait for all parallel recoveries to complete + echo "Waiting for all recovery jobs to complete..." + wait + + # Phase 3: Collect results + RECOVERED_NAMES="" + RECOVERED_COUNT=0 + FAILED_NAMES="" + FAILED_COUNT=0 + + for entry in $RECOVER_PIDS; do + INSTANCE_ID=$(echo "$entry" | cut -d: -f2) + INSTANCE_NAME=$(cat "/tmp/recovery-results/${INSTANCE_ID}.name") + ATTEMPTS=$(cat "/tmp/recovery-results/${INSTANCE_ID}.attempts") + RESULT=$(cat "/tmp/recovery-results/$INSTANCE_ID" 2>/dev/null || echo "failed") + + if [ "$RESULT" = "recovered" ]; then + echo "✅ **$INSTANCE_NAME**: Recovered successfully" >> $GITHUB_STEP_SUMMARY + RECOVERED_NAMES="$RECOVERED_NAMES $INSTANCE_NAME" + RECOVERED_COUNT=$((RECOVERED_COUNT + 1)) + UPDATED_ATTEMPTS=$(echo "$UPDATED_ATTEMPTS" | jq --arg id "$INSTANCE_ID" 'del(.[$id])') + else + NEW_COUNT=$((ATTEMPTS + 1)) + echo "❌ **$INSTANCE_NAME**: Recovery failed (attempt $NEW_COUNT/$MAX_ATTEMPTS)" >> $GITHUB_STEP_SUMMARY + FAILED_NAMES="$FAILED_NAMES $INSTANCE_NAME" + FAILED_COUNT=$((FAILED_COUNT + 1)) + UPDATED_ATTEMPTS=$(echo "$UPDATED_ATTEMPTS" | jq --arg id "$INSTANCE_ID" --argjson n "$NEW_COUNT" --arg ts "$NOW_ISO" '.[$id] = {"count": $n, "last_attempt": $ts}') + fi + done + + RECOVERED_NAMES=$(echo $RECOVERED_NAMES | xargs) + FAILED_NAMES=$(echo $FAILED_NAMES | xargs) + SKIPPED_NAMES=$(echo $SKIPPED_NAMES | xargs) + STILL_UNHEALTHY=$((FAILED_COUNT + SKIPPED_COUNT)) + + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Recovery summary:** $RECOVERED_COUNT recovered, $FAILED_COUNT failed, $SKIPPED_COUNT skipped" >> $GITHUB_STEP_SUMMARY + + echo "recovered_count=$RECOVERED_COUNT" >> $GITHUB_OUTPUT + echo "recovered_names=$RECOVERED_NAMES" >> $GITHUB_OUTPUT + echo "failed_count=$FAILED_COUNT" >> $GITHUB_OUTPUT + echo "failed_names=$FAILED_NAMES" >> $GITHUB_OUTPUT + echo "skipped_count=$SKIPPED_COUNT" >> $GITHUB_OUTPUT + echo "skipped_names=$SKIPPED_NAMES" >> $GITHUB_OUTPUT + echo "still_unhealthy=$STILL_UNHEALTHY" >> $GITHUB_OUTPUT + + # Save updated attempts for state step + echo "$UPDATED_ATTEMPTS" > /tmp/updated_recovery_attempts.json + + - name: Check queued jobs + id: queue + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + QUEUED=$(gh run list --repo ${{ github.repository }} --status queued --json databaseId --jq 'length' 2>/dev/null || echo "0") + IN_PROGRESS=$(gh run list --repo ${{ github.repository }} --status in_progress --json databaseId --jq 'length' 2>/dev/null || echo "0") + + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Jobs:** $QUEUED queued, $IN_PROGRESS in progress" >> $GITHUB_STEP_SUMMARY + + echo "queued=$QUEUED" >> $GITHUB_OUTPUT + echo "in_progress=$IN_PROGRESS" >> $GITHUB_OUTPUT + + - name: Check if alert needed + id: alert_check + if: steps.recover.outputs.still_unhealthy != '' && steps.recover.outputs.still_unhealthy != '0' + run: | + # Only send Slack alert for NEW unhealthy runners not already alerted + if [ -f /tmp/runner-health-state/state.json ]; then + PREV_ALERTED=$(jq -r '.alerted_instances // [] | .[]' /tmp/runner-health-state/state.json) + else + PREV_ALERTED="" + fi + + FAILED="${{ steps.recover.outputs.failed_names }}" + SKIPPED="${{ steps.recover.outputs.skipped_names }}" + ALL_UNHEALTHY="$FAILED $SKIPPED" + + NEW_ALERTS="" + for name in $ALL_UNHEALTHY; do + if ! echo "$PREV_ALERTED" | grep -qx "$name"; then + NEW_ALERTS="$NEW_ALERTS $name" + fi + done + NEW_ALERTS=$(echo $NEW_ALERTS | xargs) + + if [ -n "$NEW_ALERTS" ]; then + echo "should_alert=true" >> $GITHUB_OUTPUT + echo "new_unhealthy=$NEW_ALERTS" >> $GITHUB_OUTPUT + echo "New unhealthy runners to alert on: $NEW_ALERTS" + else + echo "should_alert=false" >> $GITHUB_OUTPUT + echo "All unhealthy runners already alerted, skipping Slack" + fi + + # Update alerted list (will be persisted in save step) + ALL_ALERTED="$PREV_ALERTED" + for name in $ALL_UNHEALTHY; do + if ! echo "$ALL_ALERTED" | grep -qx "$name"; then + ALL_ALERTED="$ALL_ALERTED + $name" + fi + done + # Write as JSON array for save step + echo "$ALL_ALERTED" | grep -v '^$' | jq -R . | jq -s . > /tmp/updated_alerted.json + + - name: Save current state + if: always() + run: | + mkdir -p /tmp/runner-health-state + + # Load recovery attempts (from auto-recover step, or previous state) + if [ -f /tmp/updated_recovery_attempts.json ]; then + ATTEMPTS=$(cat /tmp/updated_recovery_attempts.json) + elif [ -f /tmp/runner-health-state/state.json ]; then + ATTEMPTS=$(jq -r '.recovery_attempts // {}' /tmp/runner-health-state/state.json) + else + ATTEMPTS="{}" + fi + + # Clean up recovery_attempts for instances that are now healthy + UNHEALTHY_IDS="${{ steps.ec2.outputs.unhealthy_ids }}" + ATTEMPTS=$(echo "$ATTEMPTS" | jq --arg ids "$UNHEALTHY_IDS" ' + ($ids | split(" ") | map(select(. != ""))) as $unhealthy | + to_entries | map(select(.key as $k | $unhealthy | index($k))) | from_entries + ') + + # Determine final unhealthy count (after recovery) + STILL_UNHEALTHY="${{ steps.recover.outputs.still_unhealthy }}" + if [ -z "$STILL_UNHEALTHY" ]; then + STILL_UNHEALTHY="${{ steps.ec2.outputs.unhealthy_count }}" + fi + + # Load alerted list (from alert_check step, or previous state, pruned to still-unhealthy) + if [ -f /tmp/updated_alerted.json ]; then + ALERTED=$(cat /tmp/updated_alerted.json) + elif [ -f /tmp/runner-health-state/state.json ]; then + ALERTED=$(jq -r '.alerted_instances // []' /tmp/runner-health-state/state.json) + else + ALERTED="[]" + fi + # Prune alerted list to only currently unhealthy runners + UNHEALTHY_NAMES="${{ steps.ec2.outputs.unhealthy_names }}" + ALERTED=$(echo "$ALERTED" | jq --arg names "$UNHEALTHY_NAMES" ' + ($names | split(" ") | map(select(. != ""))) as $current | + map(select(. as $a | $current | index($a))) | + unique + ') + + jq -n \ + --argjson attempts "$ATTEMPTS" \ + --argjson count "${STILL_UNHEALTHY:-0}" \ + --argjson alerted "$ALERTED" \ + '{unhealthy_count: $count, recovery_attempts: $attempts, alerted_instances: $alerted, updated_at: now | todate}' \ + > /tmp/runner-health-state/state.json + + # Remove old format file if it exists + rm -f /tmp/runner-health-state/last_unhealthy_count + + echo "Saved state:" + cat /tmp/runner-health-state/state.json + + - name: Update cache + if: always() + uses: actions/cache/save@v4 + with: + path: /tmp/runner-health-state + key: runner-health-state-${{ github.repository }}-${{ github.run_id }} + + - name: Alert - auto-recovery succeeded + if: steps.recover.outputs.recovered_count != '' && steps.recover.outputs.recovered_count != '0' && steps.recover.outputs.still_unhealthy == '0' + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "text": "🔧 Runners Auto-Recovered", + "blocks": [ + { + "type": "header", + "text": { "type": "plain_text", "text": "🔧 Runners Auto-Recovered" } + }, + { + "type": "section", + "fields": [ + { "type": "mrkdwn", "text": "*Recovered:*\n${{ steps.recover.outputs.recovered_names }}" }, + { "type": "mrkdwn", "text": "*Method:*\nEC2 stop/start" } + ] + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": "No action needed. | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" } + ] + } + ] + } + + - name: Alert - manual intervention needed + if: steps.alert_check.outputs.should_alert == 'true' + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "text": "🚨 Runner Recovery Failed — Manual Intervention Needed", + "blocks": [ + { + "type": "header", + "text": { "type": "plain_text", "text": "🚨 Manual Intervention Needed" } + }, + { + "type": "section", + "fields": [ + { "type": "mrkdwn", "text": "*Still unhealthy:*\n${{ steps.recover.outputs.failed_names }} ${{ steps.recover.outputs.skipped_names }}" }, + { "type": "mrkdwn", "text": "*Auto-recovered:*\n${{ steps.recover.outputs.recovered_names || 'none' }}" } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Auto-recovery failed. Will retry after 1h cooldown.\nManual fix: `aws ec2 stop-instances --instance-ids ` then `aws ec2 start-instances --instance-ids `\nOr launch a replacement runner per `docs/gha-runner-setup.md`." + } + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": " | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" } + ] + } + ] + } + + - name: Alert - all runners healthy (recovered from previous issue) + if: steps.ec2.outputs.unhealthy_count == '0' && steps.prev.outputs.unhealthy_count != '0' + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "text": "✅ All Runners Healthy", + "blocks": [ + { + "type": "header", + "text": { "type": "plain_text", "text": "✅ All Runners Healthy" } + }, + { + "type": "section", + "text": { "type": "mrkdwn", "text": "*Status:* ${{ steps.ec2.outputs.healthy }}/${{ steps.ec2.outputs.total }} runners online" } + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" } + ] + } + ] + } + + - name: Report status + if: steps.recover.outputs.still_unhealthy != '' && steps.recover.outputs.still_unhealthy != '0' + run: | + echo "::warning::${{ steps.recover.outputs.still_unhealthy }} runner(s) still unhealthy: ${{ steps.recover.outputs.failed_names }} ${{ steps.recover.outputs.skipped_names }} (will auto-retry after cooldown)" diff --git a/.github/workflows/test_metrics_small.yaml b/.github/workflows/test_metrics_small.yaml new file mode 100644 index 0000000000..e6007c3212 --- /dev/null +++ b/.github/workflows/test_metrics_small.yaml @@ -0,0 +1,352 @@ +name: "Test Metrics Small Config" + +on: + workflow_dispatch: + inputs: + modality: + description: "Task modality" + type: choice + options: + - tool_use + - computer_use + default: tool_use + env_key: + description: "Fleet environment filter (empty=all)" + type: string + default: "" + max_tasks: + description: "Max tasks (empty=all, or number for testing)" + type: string + default: "" + model_name: + description: "Model to train" + type: choice + options: + - Qwen/Qwen3-8B + - Qwen/Qwen3-VL-30B-A3B-Instruct + default: Qwen/Qwen3-8B + max_steps: + description: "Maximum training steps" + type: string + default: "50" + batch_size: + description: "Batch size" + type: string + default: "8" + n_samples_per_prompt: + description: "Samples per prompt (for GRPO)" + type: string + default: "4" + data_version: + description: "Dataset version in S3 (e.g., v0.1, v0.2)" + type: string + default: "v0.2" + +permissions: + contents: read + +env: + TASKS_BUCKET: "fleet-internal-datasets" + TASKS_PREFIX: "${{ inputs.data_version || 'v0.2' }}/openenv" + +jobs: + train: + runs-on: ubuntu-latest + timeout-minutes: 720 # 12 hours max + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Install dependencies + working-directory: skyrl-train + run: | + uv venv + source .venv/bin/activate + # Install base skyrl-train (no vLLM needed - Tinker handles inference) + uv pip install -e . + # Install tinker SDK + uv pip install "tinker>=0.8.0" + # Install OpenEnv with [fleet] extra for MCP support (includes mcp and fleet-python) + # Use --no-cache-dir to always fetch latest from git + uv pip install --no-cache-dir "openenv[fleet] @ git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" + + - name: Configure AWS credentials + run: | + mkdir -p ~/.aws + cat > ~/.aws/credentials << EOF + [default] + aws_access_key_id = ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key = ${{ secrets.AWS_SECRET_ACCESS_KEY }} + EOF + cat > ~/.aws/config << EOF + [default] + region = us-east-1 + EOF + + - name: Generate run name + id: run + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M") + RUN_ID=$(head -c 4 /dev/urandom | xxd -p) + MODALITY="${{ inputs.modality || 'tool_use' }}" + # Convert underscores to dashes (tool_use -> tool-use) + MODALITY_DASH=$(echo "$MODALITY" | tr '_' '-') + RUN_NAME="gmetrics-test-${MODALITY_DASH}-${RUN_ID}-${TIMESTAMP}" + echo "name=$RUN_NAME" >> $GITHUB_OUTPUT + echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT + echo "Generated run name: $RUN_NAME" + + - name: Validate data version + run: | + DATA_VERSION="${{ inputs.data_version || 'v0.2' }}" + echo "Checking if data version '$DATA_VERSION' exists in S3..." + if ! aws s3 ls "s3://${{ env.TASKS_BUCKET }}/${DATA_VERSION}/openenv/" > /dev/null 2>&1; then + echo "ERROR: Data version '$DATA_VERSION' not found in S3" + echo "Available versions:" + aws s3 ls "s3://${{ env.TASKS_BUCKET }}/" | grep PRE | awk '{print $2}' + exit 1 + fi + echo "Data version '$DATA_VERSION' validated" + + - name: Download tasks from S3 + run: | + MODALITY="${{ inputs.modality || 'tool_use' }}" + TASKS_FILE="all_${MODALITY}.json" + mkdir -p ./data + aws s3 cp "s3://${{ env.TASKS_BUCKET }}/${{ env.TASKS_PREFIX }}/${TASKS_FILE}" ./data/tasks.json + echo "Downloaded tasks to ./data/tasks.json" + TASK_COUNT=$(python -c "import json; d=json.load(open('./data/tasks.json')); print(len(d) if isinstance(d,list) else len(d.get('tasks',[])))") + echo "Task count: $TASK_COUNT" + + - name: Prepare dataset + working-directory: skyrl-train + run: | + source .venv/bin/activate + + MODALITY="${{ inputs.modality || 'tool_use' }}" + ENV_KEY="${{ inputs.env_key || '' }}" + MAX_TASKS="${{ inputs.max_tasks || '' }}" + + PREPARE_CMD="python -m integrations.fleet.prepare_dataset" + PREPARE_CMD="$PREPARE_CMD --tasks-json ../data/tasks.json" + PREPARE_CMD="$PREPARE_CMD --output-dir ../data/fleet" + PREPARE_CMD="$PREPARE_CMD --modality $MODALITY" + + if [ -n "$ENV_KEY" ]; then + PREPARE_CMD="$PREPARE_CMD --env-key $ENV_KEY" + fi + + if [ -n "$MAX_TASKS" ]; then + PREPARE_CMD="$PREPARE_CMD --max-tasks $MAX_TASKS" + fi + + echo "Running: $PREPARE_CMD" + eval "$PREPARE_CMD" + + echo "Dataset prepared:" + ls -la ../data/fleet/ + + - name: Notify Slack - Training Started + id: slack_start + if: success() + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "text": "🚀 Training Run Started", + "blocks": [ + { + "type": "header", + "text": { "type": "plain_text", "text": "🚀 Training Run Started" } + }, + { + "type": "section", + "fields": [ + { "type": "mrkdwn", "text": "*Run:*\n`${{ steps.run.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Backend:*\n`Tinker`" }, + { "type": "mrkdwn", "text": "*Model:*\n`${{ inputs.model_name || 'Qwen/Qwen3-8B' }}`" }, + { "type": "mrkdwn", "text": "*Modality:*\n`${{ inputs.modality || 'tool_use' }}`" } + ] + }, + { + "type": "section", + "fields": [ + { "type": "mrkdwn", "text": "*Env Filter:*\n`${{ inputs.env_key || 'all' }}`" }, + { "type": "mrkdwn", "text": "*Max Steps:*\n`${{ inputs.max_steps || '50' }}`" }, + { "type": "mrkdwn", "text": "*Batch Size:*\n`${{ inputs.batch_size || '8' }}`" } + ] + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": "Triggered by: ${{ github.actor }} | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow>" } + ] + } + ] + } + + - name: Run training + working-directory: skyrl-train + env: + # TINKER_API_URL is optional - SDK uses default if not set + TINKER_API_URL: ${{ secrets.TINKER_API_URL }} + TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }} + FLEET_API_KEY: ${{ secrets.FLEET_API_KEY }} + WANDB_API_KEY: ${{ secrets.WANDB_API_KEY_TOOL_USE }} + run: | + source .venv/bin/activate + + python -m integrations.fleet.entrypoints.main_fleet_tinker \ + --model-name "${{ inputs.model_name || 'Qwen/Qwen3-8B' }}" \ + --tasks-file ../data/tasks.json \ + --dataset-file ../data/fleet/train.parquet \ + --eval-dataset-file ../data/fleet/validation.parquet \ + --batch-size ${{ inputs.batch_size || '8' }} \ + --max-steps ${{ inputs.max_steps || '50' }} \ + --n-samples-per-prompt ${{ inputs.n_samples_per_prompt || '4' }} \ + --max-turns 50 \ + --eval-every 20 \ + --wandb-project fleet-tinker-grpo \ + --wandb-name "${{ steps.run.outputs.name }}" \ + --track-extra-gradient-metrics=true + + - name: Notify Slack - Training Completed + if: success() && steps.slack_start.outputs.ts + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "✅ Training Completed", + "blocks": [ + { + "type": "header", + "text": { "type": "plain_text", "text": "✅ Training Completed" } + }, + { + "type": "section", + "fields": [ + { "type": "mrkdwn", "text": "*Run:*\n`${{ steps.run.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Backend:*\n`Tinker`" }, + { "type": "mrkdwn", "text": "*Model:*\n`${{ inputs.model_name || 'Qwen/Qwen3-8B' }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`COMPLETED`" } + ] + }, + { + "type": "section", + "text": { "type": "mrkdwn", "text": "*WandB:* " } + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": "Triggered by: ${{ github.actor }} | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow>" } + ] + } + ] + } + + - name: Notify Slack - Training Failed + if: failure() && steps.slack_start.outputs.ts + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "❌ Training Failed", + "blocks": [ + { + "type": "header", + "text": { "type": "plain_text", "text": "❌ Training Failed" } + }, + { + "type": "section", + "fields": [ + { "type": "mrkdwn", "text": "*Run:*\n`${{ steps.run.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Backend:*\n`Tinker`" }, + { "type": "mrkdwn", "text": "*Model:*\n`${{ inputs.model_name || 'Qwen/Qwen3-8B' }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`FAILED`" } + ] + }, + { + "type": "section", + "text": { "type": "mrkdwn", "text": "*Check logs:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow Logs>" } + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": "Triggered by: ${{ github.actor }}" } + ] + } + ] + } + + - name: Notify Slack - Training Cancelled + if: cancelled() && steps.slack_start.outputs.ts + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + { + "channel": "#fleet-training-runs", + "thread_ts": "${{ steps.slack_start.outputs.ts }}", + "text": "⚠️ Training Cancelled", + "blocks": [ + { + "type": "header", + "text": { "type": "plain_text", "text": "⚠️ Training Cancelled" } + }, + { + "type": "section", + "fields": [ + { "type": "mrkdwn", "text": "*Run:*\n`${{ steps.run.outputs.name }}`" }, + { "type": "mrkdwn", "text": "*Backend:*\n`Tinker`" }, + { "type": "mrkdwn", "text": "*Model:*\n`${{ inputs.model_name || 'Qwen/Qwen3-8B' }}`" }, + { "type": "mrkdwn", "text": "*Status:*\n`CANCELLED`" } + ] + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": "Triggered by: ${{ github.actor }} | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Workflow>" } + ] + } + ] + } + + - name: Summary + if: always() + run: | + echo "## Tinker Fleet Training - ${{ job.status }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Field | Value |" >> $GITHUB_STEP_SUMMARY + echo "|-------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Run | \`${{ steps.run.outputs.name }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Model | \`${{ inputs.model_name || 'Qwen/Qwen3-8B' }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Backend | Tinker (hosted) |" >> $GITHUB_STEP_SUMMARY + echo "| Modality | \`${{ inputs.modality || 'tool_use' }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Env Filter | \`${{ inputs.env_key || 'all' }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Max Tasks | \`${{ inputs.max_tasks || 'all' }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Max Steps | \`${{ inputs.max_steps || '50' }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Batch Size | \`${{ inputs.batch_size || '8' }}\` |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Links" >> $GITHUB_STEP_SUMMARY + echo "- [WandB Dashboard](https://wandb.ai/thefleet/fleet-tinker-grpo)" >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..822c0e11b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,48 @@ +__pycache__/ +/wandb/ +/skyrl-train/wandb/ +**/*.egg-info/ +# hydra logs +/outputs/ +/skyrl-train/outputs/ +/data/lcb + +# Documentation build artifacts +/skyrl-train/docs/_build/ +docs/_build/ +docs/_static/ +docs/_templates/ +docs/_autosummary/ +docs/api/_autosummary/ +docs/generated/ +docs/_spelling/ + +# Documentation cache +.doctrees/ +.cache/ +.pytest_cache/ + +# NOTE (sumanthrh): Don't add .env to gitignore. .env file when passed to uv is used to set env vars for each ray worker process. +# If it's in .gitignore then it won't be a part of the working directory shipped by uv and your env vars will not be set. +# This will just appear as a warning (silent failure) and you're gonna have a bad time. +# .env + +# .env files inside directories can be ignored +/skyrl-train/.env +/skyrl-gym/.env + +/skyrl_train/.venv +/skyrl-gym/.venv +/.venv-skypilot + +# build +/skyrl-gym/build +/skyrl-gym/dist + +*.log + +# SQLite database files +*.db + +# uv lock files +uv.lock \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..1ca78edb12 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.9 + hooks: + - id: ruff + args: [ --fix, --exit-non-zero-on-fix ] + exclude: (^(skyrl-agent)/.*)$ + + # Black needs to be ran after ruff with --fix + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + + # Detect secrets and sensitive information + - repo: https://github.com/gitleaks/gitleaks + rev: v8.24.2 + hooks: + - id: gitleaks diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..a226c031bb --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version, and other tools you might need + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + jobs: + pre_create_environment: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + create_environment: + - cd skyrl-train && uv venv "${READTHEDOCS_VIRTUALENV_PATH}" + install: + - cd skyrl-train && UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --frozen --extra docs + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: skyrl-train/docs/conf.py + +# Optionally, but recommended, +# declare the Python requirements required to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..5bd68ca846 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,169 @@ +# Claude Code Instructions for SkyRL + +## Learnings + +The `learnings/` folder contains documented knowledge about SkyRL internals: +- `training-mechanics.md` - step_wise_trajectories, retokenize_chat_history, chat templates, loss reduction + +**Always check learnings/ before asking questions about SkyRL training mechanics.** + +## Project Documentation + +- `docs/experiments.md` - Main experiment log: dataset, method, changelog (v0.2→v0.2.4), analysis, future work + +## Critical Rules - DO NOT VIOLATE + +1. **NEVER push to main** - Always create a branch and open a PR for review. Do NOT merge PRs - the user will review and merge them. + +2. **Always push to origin (fleet-ai/SkyRL)** - Never push to upstream (NovaSky-AI/SkyRL). When creating PRs, use `--repo fleet-ai/SkyRL`. + +3. **Always run black before commits** - Format Python code before every commit: + ```bash + black skyrl-train/integrations/fleet/ + ``` + + Or run full pre-commit before creating a PR: + ```bash + uv pip install pre-commit + pre-commit run --all-files --config .pre-commit-config.yaml + ``` + + If pre-commit reformats files, stage them and commit again. + +4. **DO NOT use Fleet SDK directly** - Use OpenEnv as the abstraction layer. The integration should go through OpenEnv, not call Fleet SDK APIs directly. + - Fleet SDK repo: `/Users/deniz/repos/fleet-sdk` + - OpenEnv repo: `/Users/deniz/repos/OpenEnv` + - Use OpenEnv's `FleetTaskEnv` from `envs.fleet_env.task_env` for Fleet task integration + +5. **Always look at API documentation** - Do not guess flags or API parameters. Read the actual documentation first. + - Check the local fleet-sdk and OpenEnv repos for documentation + - Look at existing examples in those repos + +6. **Tinker integration must mimic SkyRL's skyrl_gym_generator** - Any development in Tinker code (`integrations/fleet/entrypoints/main_fleet_tinker.py`) must follow the same patterns as `skyrl_train/generators/skyrl_gym_generator.py`: + - Use same parameter names: `max_input_length`, `max_generate_length`, `max_sequence_length` + - Use same context length handling: check `if len(input_ids) > max_input_length` to end rollout + - Use DAPO overlong filtering: truncate sequences > `max_sequence_length` and zero out loss mask + - Match metrics naming: `pass_at_n`, per-environment metrics, etc. + +7. **Always run tests before creating PRs** - Ensure tests pass before pushing code: + ```bash + # Run all CPU tests + cd skyrl-train && uv run --isolated --extra dev pytest tests/cpu/ -v + + # Run specific test file + uv run --isolated --extra dev pytest tests/cpu/test_trainer_utils.py -v + ``` + + Test requirements: + - All existing tests must pass before creating or updating a PR + - New functionality should include test coverage + - Test edge cases and error conditions, not just happy paths + - When fixing bugs, add regression tests that would have caught the bug + - Use descriptive test names that explain what is being tested + +## Eval Trajectory Analysis (Required for Each Training Run) + +When documenting a training run, you MUST analyze eval trajectories from S3. + +### 1. Download Eval Trajectories + +```bash +aws s3 cp s3://skyrl-trajectories/evals/{run_name}/global_step_0/{env}.jsonl /tmp/{env}_step0.jsonl +``` + +### 2. Trajectory Format + +- `input_prompt`: Full prompt string +- `output_response`: Model's response string +- `score`: List of per-token scores (check if any > 0 for success) +- `stop_reason`: Why generation stopped +- `env_extras`: Contains `task_key`, `data_source` + +### 3. Compute These Metrics Per Environment + +| Metric | How to compute | +|--------|----------------| +| Avg Turns | Count `<\|im_start\|>assistant` in `output_response` | +| Avg Tool Calls | Count `"name":` patterns in `output_response` | +| Success Rate | From `aggregated_results.jsonl` pass@3 | + +### 4. Get Results from aggregated_results.jsonl + +```bash +aws s3 cp s3://skyrl-trajectories/evals/{run_name}/global_step_X/aggregated_results.jsonl - +``` + +Extract `eval/{env}/pass_at_3` for each environment. + +### 5. Document Error Patterns + +Categorize failures by pattern: +- Tool argument errors (validation failures) +- Authorization errors (permission denied) +- Resource not found errors +- False completions (model confident but wrong) +- Context overflow (hit max_input_length) + +### 6. Training Signal Analysis (variance_per_prompt) + +Extract `reward/{env}/variance_per_prompt` from WandB to measure GRPO learning signal: + +```python +import wandb +api = wandb.Api() +run = api.run("thefleet/fleet-task-grpo/{run_id}") +df = run.history(samples=1000) + +variance_cols = [c for c in df.columns if 'variance_per_prompt' in c and c.startswith('reward/')] +for col in variance_cols: + env = col.split('/')[1] + vals = df[col].dropna().tolist() + avg_var = sum(vals) / len(vals) if vals else 0 + pct_signal = sum(1 for v in vals if v > 0) / len(vals) * 100 if vals else 0 + print(f"{env}: avg={avg_var:.4f}, signal={pct_signal:.1f}%") +``` + +**Key metrics:** +- **Avg Variance**: Higher = more learning signal. 0.25 max (1 of 4 samples differs) +- **% Steps w/ Signal**: % of batches where variance > 0 + +**Interpretation:** +- Variance = 0: All 4 samples got same reward (no GRPO signal) +- Variance > 0: At least one sample differed (model can learn) +- < 25% signal = environment is too hard or too easy for current model + +### 7. Update experiments.md + +Document results in `fleet-research/threads/tool-use-training/experiments.md` following the template in Section 3.1. + +## Project Context + +- This is SkyRL, a reinforcement learning training framework +- Fleet integration should use OpenEnv's `FleetTaskEnv` class (`envs/fleet_env/task_env.py`) +- OpenEnv provides: `FleetEnvClient`, `FleetMCPTools`, `FleetTaskEnv`, `make_fleet_task_env` +- OpenEnv repo: https://github.com/fleet-ai/OpenEnv + +## Fleet Task Integration Pattern + +The correct way to integrate Fleet tasks with SkyRL: + +```python +# In SkyRL's Fleet integration env.py: +from envs.fleet_env import FleetTaskEnv, make_fleet_task_env + +# Create task environment from config +task_config = { + "task_key": "...", + "prompt": "...", + "env_key": "...", + "env_version": "...", + "verifier_code": "...", + "task_modality": "tool_use", +} +env = FleetTaskEnv(task_config, api_key=...) + +# Use Gymnasium-style interface +obs = env.reset() +obs, reward, done, info = env.step(action) +env.close() +``` diff --git a/LICENSE b/LICENSE index d645695673..261eeb9e9f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/Notice.txt b/Notice.txt deleted file mode 100644 index ade439da52..0000000000 --- a/Notice.txt +++ /dev/null @@ -1 +0,0 @@ -Copyright 2023-2024 Bytedance Ltd. and/or its affiliates \ No newline at end of file diff --git a/README.md b/README.md index c7fbe14331..f9ab737d5c 100644 --- a/README.md +++ b/README.md @@ -1,75 +1,90 @@
-# SkyRL-v0: Train Real-World Long-Horizon Agents via Reinforcement Learning +# SkyRL: A Modular Full-stack RL Library for LLMs -[![Github](https://img.shields.io/badge/SkyRL-000000?style=for-the-badge&logo=github&logoColor=000&logoColor=white)](https://github.com/NovaSky-AI/SkyRL) [![Twitter](https://img.shields.io/badge/NovaSky-white?style=for-the-badge&logo=X&logoColor=000&color=000&labelColor=white)](https://x.com/NovaSkyAI) [![Hugging Face Collection](https://img.shields.io/badge/NovaSky-fcd022?style=for-the-badge&logo=huggingface&logoColor=000&labelColor)](https://huggingface.co/NovaSky-AI) [![Discord](https://img.shields.io/badge/NovaSky-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/RBAjeWSA) - - -
-

- News • - Links • - Getting Started • - Evaluation • - Citation • - Acknowledgement -

-
+

+| Documentation | Twitter/X | Huggingface | Slack Workspace | +

+--- -# News -- **[2025/05/06]** 🎉 We released SkyRL-v0: the first open-source online RL training framework for multi-turn tool use LLMs, optimized for long-horizon, real-environment tasks like SWE-Bench! - - -# Links +# Overview -- 📜 [SkyRL-v0 Blog Post](https://novasky-ai.notion.site/skyrl-v0) +SkyRL is a full-stack RL library that provides the following components: +- [`skyrl-agent`](./skyrl-agent): Our agent layer for training long-horizon, real-world agents. For exact reproduction of [SkyRL-v0](https://novasky-ai.notion.site/skyrl-v0) results, please checkout to commit a0d50c482436af7fac8caffa4533616a78431d66. +- [`skyrl-train`](./skyrl-train): Our modular, performant training framework for RL. +- [`skyrl-gym`](./skyrl-gym): Our gymnasium of tool-use tasks, including a library of math, coding, search and SQL environments implemented in the Gymnasium API. +- [`skyrl-tx`](./skyrl-tx): A cross-platform library to enable users to expose a local [Tinker](https://thinkingmachines.ai/tinker/)-like REST API for model post-training. # Getting Started -This repository contains training code for the `SkyRL-v0` release. Our implementation is a fork of [VeRL](https://github.com/volcengine/verl). -## Installation +For a guide on developing with SkyRL, take at look at our [Development Guide](https://skyrl.readthedocs.io/en/latest/getting-started/development.html) docs. -The only pre-requisite is having `uv` [installed](https://docs.astral.sh/uv/getting-started/installation) on your system. We use the `uv` + `ray` integration to easily manage dependencies in multi-node training. +For model training, checkout [`skyrl-train`](./skyrl-train) to start using, modifying, or building on top of the SkyRL training stack. See our [quickstart docs](https://skyrl.readthedocs.io/en/latest/index.html) to ramp up! -### Clone SkyRL-OpenHands +For building environments, checkout [`skyrl-gym`](./skyrl-gym) to integrate your task in the simple gymnasium interface. -We use [SkyRL-OpenHands](https://github.com/NovaSky-AI/SkyRL-OpenHands) to be able to connect to our remote runtime server. Clone the repository and place it in the git root: +For agentic pipelines, check out [`skyrl-agent`](./skyrl-agent) for our work on optimizing and scaling pipelines for multi-turn tool use LLMs on long-horizon, real-environment tasks. -```bash -git clone https://github.com/NovaSky-AI/SkyRL-OpenHands -``` -### Installation dry run +# News +- **[2025/11/26]** 🎉 We released SkyRL-Agent: An agent layer for efficient, multi-turn, long-horizon agent training and evaluation. [[Paper](https://arxiv.org/pdf/2511.16108)] +- **[2025/10/06]** 🎉 We released SkyRL tx: An open implementation of a backend for the Tinker API to run a Tinker-like service on their own hardware. [[Blog](https://novasky-ai.notion.site/skyrl-tx)] +- **[2025/06/26]** 🎉 We released SkyRL-v0.1: A highly-modular, performant RL training framework. [[Blog](https://novasky-ai.notion.site/skyrl-v01)] +- **[2025/06/26]** 🎉 We released SkyRL-Gym: A library of RL environments for LLMs implemented with the Gymnasium API. [[Blog](https://novasky-ai.notion.site/skyrl-v01)] +- **[2025/05/20]** 🎉 We released SkyRL-SQL: a multi-turn RL training pipeline for Text-to-SQL, along with SkyRL-SQL-7B — a model trained on just 653 samples that outperforms both GPT-4o and o4-mini! +- **[2025/05/06]** 🎉 We released SkyRL-v0: our open RL training pipeline for multi-turn tool use LLMs, optimized for long-horizon, real-environment tasks like SWE-Bench! + +# Links +- 📜 [Fully Async RL with In-Flight Weight Updates in SkyRL](https://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html) +- 📜 [Open Recipes on SkyRL](https://skyrl.readthedocs.io/en/latest/recipes/overview.html) +- 📜 [SkyRL-Agent Paper](https://arxiv.org/pdf/2511.16108) +- 📜 [On-Policy Distillation on SkyRL Blog Post](https://novasky-ai.notion.site/on-policy-distillation) +- 📜 [Search-R1 on SkyRL Blog Post](https://novasky-ai.notion.site/skyrl-searchr1) +- 📜 [SkyRL-v0.1 Blog Post](https://novasky-ai.notion.site/skyrl-v01) +- 📜 [SkyRL-SQL Blog Post](https://novasky-ai.notion.site/skyrl-sql) +- 📜 [SkyRL-v0 Blog Post](https://novasky-ai.notion.site/skyrl-v0) -You can dry run your installation with the following command: +# Projects using SkyRL +- [Biomni-R0](https://biomni.stanford.edu/blog/biomni-r0-technical-report/): Using RL to Hill-Climb Biomedical Reasoning Agents to Expert-Level ![GitHub Repo stars](https://img.shields.io/github/stars/snap-stanford/Biomni) +- [How to Train Your Advisor](https://github.com/az1326/advisor-models): Steering Black-Box LLMs with Advisor Models ![GitHub Repo stars](https://img.shields.io/github/stars/az1326/advisor-models) +- [OpenThoughts-Agent](https://github.com/open-thoughts/OpenThoughts-Agent): Data recipes and robust infrastructure for training AI agents ![GitHub Repo stars](https://img.shields.io/github/stars/open-thoughts/OpenThoughts-Agent) -```bash -uv run --isolated --frozen pip show torch -``` +# Acknowledgement -NOTE: With a CPU head node, you might encounter installation issues with `torch-memory-saver`. To fix this, you need to install CUDA and make sure your CUDA libraries are linked in `/usr/lib`. For example, +This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/) in collaboration with [**Anyscale**](https://www.anyscale.com/), with generous compute support from [**Anyscale**](https://www.anyscale.com/), [**Databricks**](https://www.databricks.com/), [**NVIDIA**](https://developer.nvidia.com/brev), [**Lambda Labs**](https://lambdalabs.com/service/gpu-cloud?srsltid=AfmBOop5FnmEFTkavVtdZDsLWvHWNg6peXtat-OXJ9MW5GMNsk756PE5), [**AMD**](https://www.amd.com/en.html), [**AWS**](https://aws.amazon.com/), and [**Modal**](https://modal.com/). -```bash -sudo ln -s /usr/local/cuda-12.4/compat/libcuda.so /usr/lib/libcuda.so -sudo ln -s /usr/local/cuda-12.4/compat/libcuda.so.1 /usr/lib/libcuda.so.1 -``` +We adopt many lessons and code from several great projects such as [veRL](https://github.com/volcengine/verl), [OpenRLHF](https://github.com/OpenRLHF/OpenRLHF), [Search-R1](https://github.com/PeterGriffinJin/Search-R1), [OpenReasonerZero](https://github.com/Open-Reasoner-Zero/Open-Reasoner-Zero), and [NeMo-RL](https://github.com/NVIDIA-NeMo/RL). We appreciate each of these teams and their contributions to open-source research! -## Scripts for reproduction -For reproducing our results for SkyRL-Agent-14B-v0, SkyRL-Agent-8B-v0, and SkyRL-Agent-7B-v0 you can refer to [examples/sky](./examples/sky/README.md). +# Citation -# SWE-Bench-Verified Evaluation Results +If you find the work in this repository helpful, please consider citing: -| Model | Base | Base Performance | Performance | Training Time | -|--------------------|----------------------|------------------|-------------|---------------| -| SkyRL-Agent-7B-v0 | OpenHands-7B-Agent | 11% | 14.6% | 16hrs 8xH100 | -| SkyRL-Agent-8B-v0 | Qwen3-8B no thinking | 3.6% | 9.4% | 27hrs 8xH200 | -| SkyRL-Agent-14B-v0 | Qwen3-14B thinking | 18% | 21.6% | 20hrs 8xH200 | +```bibtex +@misc{cao2025skyrl, + title = {SkyRL-v0: Train Real-World Long-Horizon Agents via Reinforcement Learning}, + author = {Shiyi Cao and Sumanth Hegde and Dacheng Li and Tyler Griggs and Shu Liu and Eric Tang and Jiayi Pan and Xingyao Wang and Akshay Malik and Graham Neubig and Kourosh Hakhamaneshi and Richard Liaw and Philipp Moritz and Matei Zaharia and Joseph E. Gonzalez and Ion Stoica}, + year = {2025}, +} +``` +```bibtex +@misc{liu2025skyrlsql, + title={SkyRL-SQL: Matching GPT-4o and o4-mini on Text2SQL with Multi-Turn RL}, + author={Shu Liu and Sumanth Hegde and Shiyi Cao and Alan Zhu and Dacheng Li and Tyler Griggs and Eric Tang and Akshay Malik and Kourosh Hakhamaneshi and Richard Liaw and Philipp Moritz and Matei Zaharia and Joseph E. Gonzalez and Ion Stoica}, + year={2025}, +} +``` -# Acknowledgement -This work is done at [Berkeley Sky Computing Lab](https://sky.cs.berkeley.edu/), with the amazing compute support from [Lambda Labs](https://lambdalabs.com/service/gpu-cloud?srsltid=AfmBOop5FnmEFTkavVtdZDsLWvHWNg6peXtat-OXJ9MW5GMNsk756PE5), [Anyscale](https://www.anyscale.com/), and [Databricks](https://www.databricks.com/). \ No newline at end of file +```bibtex +@misc{griggs2025skrylv01, + title={Evolving SkyRL into a Highly-Modular RL Framework}, + author={Tyler Griggs and Sumanth Hegde and Eric Tang and Shu Liu and Shiyi Cao and Dacheng Li and Charlie Ruan and Philipp Moritz and Kourosh Hakhamaneshi and Richard Liaw and Akshay Malik and Matei Zaharia and Joseph E. Gonzalez and Ion Stoica}, + year={2025}, + note={Notion Blog} +} +``` diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..3716c8e83e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,19 @@ +FROM anyscale/ray:2.51.1-slim-py312-cu128 + +RUN sudo apt-get update -y && sudo apt-get install -y wget kmod libxml2 build-essential libnuma-dev + +# the cuda compiler here is needed for deepspeed +RUN wget https://developer.download.nvidia.com/compute/cuda/12.8.0/local_installers/cuda_12.8.0_570.86.10_linux.run \ + && sudo sh cuda_12.8.0_570.86.10_linux.run --silent --toolkit && rm -rf cuda_12.8.0_570.86.10_linux.run + +RUN curl -LsSf https://astral.sh/uv/0.9.4/install.sh | sh +RUN echo "export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook" >> /home/ray/.bashrc + +RUN sudo apt-get update \ + && sudo apt-get install -y openssh-server iputils-ping net-tools iproute2 traceroute netcat \ + libopenexr-dev libxi-dev libglfw3-dev libglew-dev libomp-dev libxinerama-dev libxcursor-dev tzdata \ + && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* + +RUN sudo apt update && sudo apt install --fix-broken && sudo apt install -y default-jre-headless openjdk-8-jdk \ + && sudo apt-get clean \ + && sudo rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.megatron b/docker/Dockerfile.megatron index dd355211dc..fe3cca931a 100644 --- a/docker/Dockerfile.megatron +++ b/docker/Dockerfile.megatron @@ -1,9 +1,50 @@ -FROM verlai/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3 +FROM anyscale/ray:2.51.1-slim-py312-cu128 -RUN pip install git+https://github.com/NVIDIA/TransformerEngine.git@stable +RUN sudo apt-get update -y && sudo apt-get install -y wget kmod libxml2 build-essential libnuma-dev -RUN cd /opt/nvidia && git clone --single-branch --branch core_r0.11.0 https://github.com/NVIDIA/Megatron-LM.git Megatron-LM +# the cuda compiler here is needed for deepspeed +RUN wget https://developer.download.nvidia.com/compute/cuda/12.8.0/local_installers/cuda_12.8.0_570.86.10_linux.run \ + && sudo sh cuda_12.8.0_570.86.10_linux.run --silent --toolkit && rm -rf cuda_12.8.0_570.86.10_linux.run -# only config pip index with https://pypi.tuna.tsinghua.edu.cn/simple if needed -# unset for now -RUN cd /opt/nvidia/Megatron-LM && pip3 install --no-deps -e . \ No newline at end of file +RUN curl -LsSf https://astral.sh/uv/0.9.4/install.sh | sh +RUN echo "export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook" >> /home/ray/.bashrc + + +RUN sudo apt-get update \ + && sudo apt-get install -y openssh-server iputils-ping net-tools iproute2 traceroute netcat \ + libopenexr-dev libxi-dev libglfw3-dev libglew-dev libomp-dev libxinerama-dev libxcursor-dev tzdata \ + && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* + +RUN sudo apt update && sudo apt install --fix-broken && sudo apt install -y default-jre-headless openjdk-8-jdk \ + && sudo apt-get clean \ + && sudo rm -rf /var/lib/apt/lists/* + +# build toolchain + nccl dev headers for transformer engine build +RUN sudo apt-get update && sudo apt-get install -y \ + cmake \ + ninja-build \ + libnccl-dev=2.25.1-1+cuda12.8 \ + libnccl2=2.25.1-1+cuda12.8 \ + && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* + +# ---------- PyTorch + cuDNN + Transformer Engine ---------- +# PyTorch + cuDNN + Transformer Engine +RUN pip install --no-cache-dir "torch==2.7.1" "nvidia-cudnn-cu12>=9.3" && \ + CUDNN_PATH="$(python -c 'import inspect, nvidia.cudnn as c, os; print(os.path.dirname(inspect.getfile(c)))')" && \ + sudo mkdir -p /opt && sudo ln -sfn "$CUDNN_PATH" /opt/cudnn && \ + echo "/opt/cudnn/lib" | sudo tee /etc/ld.so.conf.d/cudnn.conf >/dev/null && sudo ldconfig + +ENV CUDNN_PATH=/opt/cudnn +ENV CPATH=${CUDNN_PATH}/include:${CPATH} +ENV LD_LIBRARY_PATH=${CUDNN_PATH}/lib:${LD_LIBRARY_PATH} + +RUN pip install --no-cache-dir \ + nvidia-mathdx \ + pybind11 \ + setuptools \ + wheel + +ENV CMAKE_ARGS=-DPython_EXECUTABLE=/home/ray/anaconda3/bin/python3 + +RUN pip install --no-cache-dir --no-build-isolation "transformer_engine[pytorch]==2.9.0" +# -------------------- diff --git a/docker/Dockerfile.ngc.vllm b/docker/Dockerfile.ngc.vllm deleted file mode 100644 index 3a43a2504b..0000000000 --- a/docker/Dockerfile.ngc.vllm +++ /dev/null @@ -1,47 +0,0 @@ -# docker buildx build --platform linux/x86_64 -t "verlai/verl:ngc-th2.4.0-cu124-vllm0.6.3-ray2.4-te1.7-v0.0.6" -f docker/Dockerfile.ngc.vllm . --builder cloud-verlai-verl-builder --progress=plain --push -FROM nvcr.io/nvidia/pytorch:24.05-py3 - -# uninstall nv-pytorch fork -RUN pip3 uninstall pytorch-quantization \ - pytorch-triton \ - torch \ - torch-tensorrt \ - torchvision \ - xgboost transformer_engine flash_attn \ - apex megatron-core -y - -RUN pip3 install torch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 --index-url https://download.pytorch.org/whl/cu124 - -# =============== Megatron dependencies (optional) ================= -# install apex, set MAX_JOBS to avoid OOMs -RUN MAX_JOBS=4 pip3 install -v --disable-pip-version-check --no-cache-dir --no-build-isolation \ - --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" \ - git+https://github.com/NVIDIA/apex -# =============== End of Megatron dependencies (optional) ================= - -RUN pip3 install --no-cache-dir \ - accelerate \ - codetiming \ - datasets \ - dill \ - hydra-core \ - numpy \ - 'pandas' \ - 'peft' \ - 'pyarrow>=15.0.0' \ - 'pybind11' \ - 'pylatexenc' \ - 'ray>=2.10' \ - 'tensordict<0.6' \ - 'transformers' \ - 'vllm==0.6.3.post1' \ - 'wandb' - -# full dependencies -RUN pip3 install pytest yapf py-spy pyext liger-kernel - -# =============== Megatron dependencies (optional) ================= -# install Transformer Engine, which requires FA 2.5.8. Do it in a separate step for docker cache -RUN MAX_JOBS=4 NINJA_FLAGS="-j4" pip3 install flash-attn==2.5.8 --no-cache-dir --no-build-isolation -RUN MAX_JOBS=1 NINJA_FLAGS="-j1" TE_BUILD_WITH_NINJA=0 pip3 install git+https://github.com/eric-haibin-lin/TransformerEngine.git@v1.7.0 -# =============== End of Megatron dependencies (optional) ================= diff --git a/docker/Dockerfile.ngc.vllm0.8 b/docker/Dockerfile.ngc.vllm0.8 deleted file mode 100644 index c51df06c81..0000000000 --- a/docker/Dockerfile.ngc.vllm0.8 +++ /dev/null @@ -1,66 +0,0 @@ -# Start from the NVIDIA official image (ubuntu-22.04 + python-3.10) -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html -FROM nvcr.io/nvidia/pytorch:24.08-py3 - -# uninstall nv-pytorch fork -RUN pip3 uninstall -y pytorch-quantization \ - pytorch-triton torch torch-tensorrt torchvision \ - xgboost transformer_engine flash_attn apex megatron-core - -# Define environments -ENV MAX_JOBS=32 -ENV VLLM_WORKER_MULTIPROC_METHOD=spawn -ENV DEBIAN_FRONTEND=noninteractive -ENV NODE_OPTIONS="" -ENV HF_HUB_ENABLE_HF_TRANSFER="1" - -# Define installation arguments -ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ -ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple - -# Set apt source -RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ - { \ - echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ - echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ - echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ - echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ - } > /etc/apt/sources.list - -# Install systemctl -RUN apt-get update && \ - apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ - apt-get clean - -# Install tini -RUN apt-get update && \ - apt-get install -y tini && \ - apt-get clean - -# Change pip source -RUN pip config set global.index-url "${PIP_INDEX}" && \ - pip config set global.extra-index-url "${PIP_INDEX}" && \ - python -m pip install --upgrade pip - -# Install torch-2.6.0 + vllm-0.8.2 -RUN pip install --no-cache-dir vllm==0.8.2 torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 tensordict torchdata \ - transformers>=4.49.0 accelerate datasets peft hf-transfer \ - ray[default] codetiming hydra-core pandas pyarrow>=15.0.0 pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler \ - pytest yapf py-spy pyext pre-commit ruff - -# Install flash_attn-2.7.4.post1 -RUN pip uninstall -y transformer-engine flash-attn && \ - wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ - pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl - -# Fix cv2 -RUN pip uninstall -y pynvml nvidia-ml-py && \ - pip install --no-cache-dir nvidia-ml-py>=12.560.30 opencv-python-headless==4.8.0.74 fastapi==0.115.6 && \ - pip install --no-cache-dir --upgrade optree>=0.13.0 - -# Install verl -RUN pip install --no-cache-dir verl[vllm] -U - -# Reset pip config -RUN pip config unset global.index-url && \ - pip config unset global.extra-index-url diff --git a/docker/Dockerfile.ngc.vllm0.8.sagemaker b/docker/Dockerfile.ngc.vllm0.8.sagemaker deleted file mode 100644 index b9c458d0bf..0000000000 --- a/docker/Dockerfile.ngc.vllm0.8.sagemaker +++ /dev/null @@ -1,46 +0,0 @@ -# Using a pre-built image from AWS DLC which contains the current version of python (3.10) and supported cuda version (12.1) -FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.1.0-transformers4.36.0-gpu-py310-cu121-ubuntu20.04 - -# uninstall nv-pytorch fork -RUN pip3 uninstall -y pytorch-quantization \ - pytorch-triton torch torch-tensorrt torchvision \ - xgboost transformer_engine flash_attn apex megatron-core - -# Define environments -ENV MAX_JOBS=32 -ENV VLLM_WORKER_MULTIPROC_METHOD=spawn -ENV DEBIAN_FRONTEND=noninteractive -ENV NODE_OPTIONS="" -ENV HF_HUB_ENABLE_HF_TRANSFER="1" - -# Install systemctl -RUN apt-get update && \ - apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ - apt-get clean - -# Install tini -RUN apt-get update && \ - apt-get install -y tini && \ - apt-get clean - -# Install torch-2.6.0 + vllm-0.8.2 -RUN pip install --no-cache-dir vllm==0.8.2 torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 tensordict torchdata==0.11.0 \ - transformers>=4.49.0 accelerate datasets peft hf-transfer \ - ray[default] codetiming hydra-core pandas pyarrow>=15.0.0 pylatexenc qwen-vl-utils wandb dill pybind11 liger-kernel mathruler \ - pytest yapf py-spy pyext pre-commit ruff - -# Install flash_attn-2.7.4.post1 -RUN pip uninstall -y transformer-engine flash-attn && \ - pip install flash-attn==2.7.4.post1 --no-build-isolation - -# Fix cv2 -RUN pip uninstall -y pynvml nvidia-ml-py && \ - pip install --no-cache-dir nvidia-ml-py>=12.560.30 opencv-python-headless==4.8.0.74 fastapi==0.115.6 && \ - pip install --no-cache-dir --upgrade optree>=0.13.0 - -# Install verl -RUN pip install --no-cache-dir verl[vllm] -U - -# Reset pip config -RUN pip config unset global.index-url && \ - pip config unset global.extra-index-url diff --git a/docker/Dockerfile.ray244 b/docker/Dockerfile.ray244 new file mode 100644 index 0000000000..5a01951fc4 --- /dev/null +++ b/docker/Dockerfile.ray244 @@ -0,0 +1,22 @@ +FROM anyscale/ray:2.44.0-slim-py312-cu128 + +RUN sudo apt-get update -y && sudo apt-get install -y wget kmod libxml2 build-essential libnuma-dev + +# the cuda compiler here is needed for deepspeed +RUN wget https://developer.download.nvidia.com/compute/cuda/12.8.0/local_installers/cuda_12.8.0_570.86.10_linux.run \ + && sudo sh cuda_12.8.0_570.86.10_linux.run --silent --toolkit && rm -rf cuda_12.8.0_570.86.10_linux.run +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +RUN echo "export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook" >> /home/ray/.bashrc +RUN sudo apt-get update \ + && sudo apt-get install -y openssh-server iputils-ping net-tools iproute2 traceroute netcat \ + libopenexr-dev libxi-dev libglfw3-dev libglew-dev libomp-dev libxinerama-dev libxcursor-dev tzdata \ + && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* +RUN sudo apt update && sudo apt install --fix-broken && sudo apt install -y default-jre-headless openjdk-8-jdk \ + && sudo apt-get clean \ + && sudo rm -rf /var/lib/apt/lists/* + +# NOTE: vllm installation in base environment is needed for uv + vLLM to work +RUN pip install vllm==0.9.2 --extra-index-url https://download.pytorch.org/whl/cu128 \ + && pip install ray==2.44.0 omegaconf==2.3.0 loguru==0.7.3 \ + && rm -rf ~/.cache/pip +RUN pip install pyarrow==20.0.0 jaxtyping==0.3.2 && rm -rf ~/.cache/pip diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm deleted file mode 100644 index 6a0d20e41c..0000000000 --- a/docker/Dockerfile.rocm +++ /dev/null @@ -1,45 +0,0 @@ -# Build the docker in the repo dir: -# docker build -f docker/Dockerfile.rocm -t verl-rocm:03.04.2015 . -# docker images # you can find your built docker - - -FROM rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 - -# Set working directory -# WORKDIR $PWD/app - -# Set environment variables -ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" - -# Install vllm -RUN pip uninstall -y vllm && \ - rm -rf vllm && \ - git clone -b v0.6.3 https://github.com/vllm-project/vllm.git && \ - cd vllm && \ - MAX_JOBS=$(nproc) python3 setup.py install && \ - cd .. && \ - rm -rf vllm - -# Copy the entire project directory -COPY . . - -# Install dependencies -RUN pip install "tensordict<0.6" --no-deps && \ - pip install accelerate \ - codetiming \ - datasets \ - dill \ - hydra-core \ - liger-kernel \ - numpy \ - pandas \ - peft \ - "pyarrow>=15.0.0" \ - pylatexenc \ - "ray[data,train,tune,serve]" \ - torchdata \ - transformers \ - wandb \ - orjson \ - pybind11 && \ - pip install -e . --no-deps \ No newline at end of file diff --git a/docker/Dockerfile.vemlp.vllm.te b/docker/Dockerfile.vemlp.vllm.te deleted file mode 100644 index a03d7c102e..0000000000 --- a/docker/Dockerfile.vemlp.vllm.te +++ /dev/null @@ -1,41 +0,0 @@ -# docker buildx build --platform linux/x86_64 -t "verlai/verl:$TAG" -f docker/$FILE . - -# the one in docker.io is an alias for the one veturbo -# FROM vemlp-cn-beijing.cr.volces.com/veturbo/pytorch:2.4-cu124 -FROM docker.io/haibinlin/verl:v0.0.5-th2.4.0-cu124-base - -# only config pip index with https://pypi.tuna.tsinghua.edu.cn/simple if needed -# unset for now -RUN pip3 config unset global.index-url - -# transformers 4.47.0 contains the following bug: -# AttributeError: 'Gemma2Attention' object has no attribute '_flash_attn_uses_top_left_mask' -RUN pip3 install --no-cache-dir \ - torch==2.4.0 \ - accelerate \ - codetiming \ - dill \ - hydra-core \ - numpy \ - pybind11 \ - tensordict \ - "transformers <= 4.46.0" - -RUN pip3 install --no-cache-dir flash-attn==2.7.0.post2 --no-build-isolation - -# vllm depends on ray, and veRL does not support ray > 2.37 -RUN pip3 install --no-cache-dir vllm==0.6.3 ray==2.10 - -# install apex -RUN MAX_JOBS=4 pip3 install -v --disable-pip-version-check --no-cache-dir --no-build-isolation \ - --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" \ - git+https://github.com/NVIDIA/apex - -# install Transformer Engine -# - flash-attn pinned to 2.5.3 by TransformerEngine, switch to eric-haibin-lin/TransformerEngine.git@v1.7.0 to relax version req -# - install with: MAX_JOBS=1 NINJA_FLAGS="-j1" TE_BUILD_WITH_NINJA=0 to avoid OOM -# - cudnn is required by TransformerEngine -# RUN CUDNN_PATH=/opt/conda/lib/python3.11/site-packages/nvidia/cudnn \ -# pip3 install git+https://github.com/eric-haibin-lin/TransformerEngine.git@v1.7.0 -RUN MAX_JOBS=1 NINJA_FLAGS="-j1" pip3 install flash-attn==2.5.3 --no-cache-dir --no-build-isolation -RUN MAX_JOBS=1 NINJA_FLAGS="-j1" pip3 install git+https://github.com/NVIDIA/TransformerEngine.git@v1.7 diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 8bda904a9b..0000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = verl -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 2acaa9b12b..0000000000 --- a/docs/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# verl documents - -## Build the docs - -```bash -# Install dependencies. -pip install -r requirements-docs.txt - -# Build the docs. -make clean -make html -``` - -## Open the docs with your browser - -```bash -python -m http.server -d _build/html/ -``` -Launch your browser and navigate to http://localhost:8000 to view the documentation. \ No newline at end of file diff --git a/docs/README_vllm0.7.md b/docs/README_vllm0.7.md deleted file mode 100644 index 9a110ab440..0000000000 --- a/docs/README_vllm0.7.md +++ /dev/null @@ -1,71 +0,0 @@ -# Upgrading to vllm >= 0.7 - -## Installation - -Note: This version of veRL+vllm 0.7+ supports **FSDP** for training and **vLLM** for rollout. - -``` -# Create the conda environment -conda create -n verl python==3.10 -conda activate verl - -# Install verl -git clone https://github.com/volcengine/verl.git -cd verl -pip3 install -e . - -# Install the latest stable version of vLLM -pip3 install vllm==0.7.3 - -# Install flash-attn -pip3 install flash-attn --no-build-isolation - -``` - -Note that if you are installing lower versions of vLLM (0.7.0, 0.7.1, 0.7.2), you need to make some tiny patches manually on vllm (/path/to/site-packages/vllm after installation) after the above steps: - -- vllm/distributed/parallel_state.py: Remove the assertion below: - -``` -if (world_size - != tensor_model_parallel_size * pipeline_model_parallel_size): - raise RuntimeError( - f"world_size ({world_size}) is not equal to " - f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " - f"pipeline_model_parallel_size ({pipeline_model_parallel_size})") - -``` - -- vllm/executor/uniproc_executor.py: change `local_rank = rank` to `local_rank = int(os.environ["LOCAL_RANK"])` -- vllm/model_executor/model_loader/weight_utils.py: remove the `torch.cuda.empty_cache()` in `pt_weights_iterator` - -## Features - -### Use cuda graph - -After installation, examples using FSDP as training backends can be used. By default, the `enforce_eager` is set to True, which disables the cuda graph. To enjoy cuda graphs and the sleep mode of vLLM>=0.7, add the following lines to the bash script: - -``` -actor_rollout_ref.rollout.enforce_eager=False \ -actor_rollout_ref.rollout.free_cache_engine=False \ - -``` - -For a typical job like examples/ppo_trainer/run_qwen2-7b_seq_balance.sh, the rollout generation time is 115 seconds with vLLM0.6.3, while it is 85 seconds with vLLM0.7.0. By enabling the cudagraph, the generation duration is further reduced to 62 seconds. - -**Note:** Currently, if the `n` is greater than 1 in `SamplingParams` in vLLM>=0.7, there is a potential performance issue on the stability of rollout generation time (Some iterations would see generation time bursts) using vLLM's V0 Engine. - -### Use vLLM V1 Engine - -Using the vLLM V1 engine can avoid instability issues and achieve additional performance improvements. To use the V1 engine, you can first uninstall the previously installed vLLM and then follow the steps below to install the newer version. - -``` -git clone https://github.com/vllm-project/vllm.git -cd vllm -git checkout 2275784 -sed -i "903a\ data_parallel_size = world_size // pipeline_model_parallel_size // tensor_model_parallel_size" ./vllm/distributed/parallel_state.py -VLLM_USE_PRECOMPILED=1 pip install --editable . -``` - -Then you can enable the V1 engine by setting `export VLLM_USE_V1=1`. In some benchmark tests, the V1 engine demonstrates a 1.5x speed improvement over the vLLM V0 engine. -The stable support of the vLLM V1 engine will come soon. \ No newline at end of file diff --git a/docs/README_vllm0.8.md b/docs/README_vllm0.8.md deleted file mode 100644 index 9bc3d5613d..0000000000 --- a/docs/README_vllm0.8.md +++ /dev/null @@ -1,54 +0,0 @@ -# Upgrading to vLLM >= 0.8 - -## Installation - -Note: This version of veRL+vLLM 0.8+ supports **FSDP** for training and **vLLM** for rollout. - -```bash -# Create the conda environment -conda create -n verl python==3.10 -conda activate verl - -# Install verl -git clone https://github.com/volcengine/verl.git -cd verl -pip3 install -e . - -# Install the latest stable version of vLLM -pip3 install vllm==0.8.2 - -# Install flash-attn -pip3 install flash-attn --no-build-isolation - -``` - -We have a pre-built docker image for veRL+vLLM 0.8.2. You can direct import it with the following command: - -```bash -docker pull hiyouga/verl:ngc-th2.6.0-cu120-vllm0.8.2 -``` - -## Features - -vLLM 0.8+ supports cuda graph and V1 engine by default in veRL. To enable these features, remember to add the following lines to the bash script: - -```bash -actor_rollout_ref.rollout.enforce_eager=False \ -actor_rollout_ref.rollout.free_cache_engine=False \ -``` - -and also **remove** the environment variable if it exists: - -```bash -export VLLM_ATTENTION_BACKEND=XFORMERS -``` - -## Notes - -When you just directly upgrade vllm>=0.8, some dependency packages may undergo version changes. If you encounter the following problems: - -```bash -in from torch.multiprocessing.reductions import ForkingPickler ImportError: cannot import name 'ForkingPickler' from 'torch.multiprocessing.reductions' (/opt/conda/lib/python3.11/site-packages/torch/multiprocessing/reductions.py) -``` - -You need to upgrade `tensordict` to version 0.6.2 using the command `pip install tensordict==0.6.2`. diff --git a/docs/_static/logo.png b/docs/_static/logo.png deleted file mode 100644 index 6a3b61308d..0000000000 Binary files a/docs/_static/logo.png and /dev/null differ diff --git a/docs/advance/checkpoint.rst b/docs/advance/checkpoint.rst deleted file mode 100644 index f397e67d12..0000000000 --- a/docs/advance/checkpoint.rst +++ /dev/null @@ -1,122 +0,0 @@ -Using Checkpoints to Support Fault Tolerance Training -===================================================== - -There could be training errors or machine failure during the whole RLHF training process, -so it is recommended to enable checkpoints to minimize your loss. - -The API Interface has already been listed in :ref:`config-explain-page`, -and we will not repeat them. But there are still some technique details -we hope to clarify. - -.. note:: - - Notice that the ``checkpoint.contents`` field has no effect to FSDP checkpoint except ``hf_model``, - the other 3 fields are binded together to save and load. We recommend to include ``model``, ``optimizer`` and ``extra`` all. - -Checkpoint Saving Directory Structure -------------------------------------- - -Commonly, we use the ``default_local_dir`` declared in ``ppo_trainer.yaml`` or ``ppo_megatron_trainer.yml`` -to work as preffix when saving checkpoints, which is ``checkpoints/${trainer.project_name}/${trainer.experiment_name}``. - -So the inner checkpoint structure of **FSDP** is like: - -.. code:: - - checkpoints/${trainer.project_name}/${trainer.experiment_name} - ├── global_steps_${i} - │ ├── actor - │ │ ├── model_world_size_{self.world_size}_rank_{self.rank}.pt - │ │ ├── optim_world_size_{self.world_size}_rank_{self.rank}.pt - │ │ └── extra_state_world_size_{self.world_size}_rank_{self.rank}.pt - │ ├── actor_huggingface - │ ├── critic - │ │ ├── model_world_size_{self.world_size}_rank_{self.rank}.pt - │ │ ├── optim_world_size_{self.world_size}_rank_{self.rank}.pt - │ │ └── extra_state_world_size_{self.world_size}_rank_{self.rank}.pt - │ └── critic_huggingface - └── latest_checkpointed_iteration.txt - -All model shards, optimizers and extra states are stored togather, in a sharded and distributed way. - -While **Megatron** current checkpoint structure is: - -.. code:: - - checkpoints/${trainer.project_name}/${trainer.experiment_name} - ├── global_steps_${i} - │ ├── actor - │ │ ├── huggingface # default save tokenizer, save huggingface model if include ``hf_mode`` in checkpoint.contents - │ │ ├── model # save sharded model, naming the same as Megatron - │ │ │ ├── mp_rank_xx_yyy # xx is tp_rank in 2 digits, yyy is pp_rank in 3 digits - │ │ │ │ └── model_states.pt - │ │ │ └── mp_rank_xx_xxx - │ │ ├── optim - │ │ │ ├── distrib_optim_pp{x}_tp{y}.pt - │ │ │ └── distrib_optim_pp{x}_tp{y}.pt - │ │ └── rng_states - │ └── critic - │ │ ├── huggingface - │ │ ├── model - │ │ ├── optim - │ │ └── rng_states - └── latest_checkpointed_iteration.txt - -Convert FSDP and Megatron Checkpoints to HuggingFace Format Model ------------------------------------------------------------------ - -We provide a tool to convert the FSDP and Megatron checkpoints to HuggingFace format model. -The tool is located in ``scripts/model_merger.py``. - -The arguments are as follows: - -.. code:: bash - - usage: model_merger.py [-h] [--backend {fsdp,megatron}] - [--tie-word-embedding whether the model share embedding weights] - [--is-value-model whether the model is critic model] - [--hf_model_path $original_model_path, like {Qwen/Qwen2-7B}] - [--local_dir $local_directory saved fsdp or megatron models] - [--target_dir $target_dir to save converted models, default is tmp] - [--hf_upload_path $huggingface_repo to upload] - -So example use of Megatron model merger is: - -.. code:: bash - - python3 scripts/model_merger.py --backend megatron \ - --is-value-model \ - --hf_model_path Qwen/Qwen2-7B \ - --local_dir checkpoints/verl_megatron_gsm8k_examples/deepseek_megatron_checkpoint_saveload/global_step_1/actor/model - -Megatron Merger details ------------------------ - -Current implement of decoder layers uses ``nn.ModuleList`` to store the layers, -and thus the model layers on every PP rank and VPP rank starts their index from 0. - -There are 3 ways to correct this behavior: - -1. Modify the decoder layer's state_dict, add ``offset`` to each layer's index, thus rewrite ``nn.ModuleList`` implementation. -2. Modify the layer index when saving checkpoint and recover them when loading checkpoint. -3. The Checkpoint merger do this work, calculate the actual ``offset`` from ``state_dict`` only, a little complex. - -Current implementation use solution 2. - -Original Checkpoint Utils -------------------------- - -Original Checkpoint Utils refer to original checkpoint implementation in ``verl/models/[model]/megatron/checkpoint_utils``. - -We only need ``[model]_loader.py`` in original checkpoint utils now, since we get rid of storing ``hf_model`` every time (which is not recommended for large model training, try only saving sharded models if you can). - -.. note:: - - Note that ``[model]_loader`` only support environments where **storage clusters are able to connect with every calculation nodes**. - Because it utilizes **sharded load way to minimize the loading checkpoint overhead**. - Every rank loads its own data from ``state_dict`` which can be accessed by all of them. - While there is also no need to broadcast among DP ranks, since the saved state_dict is only produced by DP rank 0. - - For users who can **only place the huggingface model on one device**, we keep the original costly implementation in ``[model]_loader_deprecated``. In this implementation, rank 0 broadcast all weights to each tp and pp rank, and then dp rank 0 broadcast to all dp ranks. There may be at risks of OOM. - - To use deprecated loader, change the import package of ``load_state_dict_to_megatron_llama``. diff --git a/docs/advance/dpo_extension.rst b/docs/advance/dpo_extension.rst deleted file mode 100644 index 24833b69a2..0000000000 --- a/docs/advance/dpo_extension.rst +++ /dev/null @@ -1,271 +0,0 @@ -Extend to other RL(HF) algorithms -================================= - -We already implemented the complete training pipeline of the PPO -algorithms. To extend to other algorithms, we analyze the high-level -principle to use verl and provide a tutorial to implement the DPO -algorithm. Users can follow the similar paradigm to extend to other RL algorithms. - -.. note:: **Key ideas**: Single process drives multi-process computation and data communication. - -Overall Approach ----------------- - -Step 1: Consider what multi-machine multi-GPU computations are needed -for each model, such as ``generate_sequence`` , ``compute_log_prob`` and -``update_policy`` in the actor_rollout model. Implement distributed -single-process-multiple-data (SPMD) computation and encapsulate them -into APIs - -Step 2: Based on different distributed scenarios, including FSDP and 3D -parallelism in Megatron-LM, implement single-process control of data -interaction among multi-process computations. - -Step 3: Utilize the encapsulated APIs to implement the control flow - -Example: Online DPO -------------------- - -We use verl to implement a simple online DPO algorithm. The algorithm -flow of Online DPO is as follows: - -1. There is a prompt (rollout) generator which has the same weight as - the actor model. After a batch of prompts are fed into the generator, - it generates N responses for each prompt. -2. Send all the prompts + responses to a verifier for scoring, which can - be reward model or a rule-based function. Then sort them in pairs to - form a training batch. -3. Use this training batch to train the actor model using DPO. During - the process, a reference policy is needed. - -Step 1: What are the multi-machine multi-GPU computations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Sample Generator** - -Implementation details: - -.. code:: python - - from verl.single_controller.base import Worker - from verl.single_controller.ray import RayWorkerGroup, RayClassWithInitArgs, RayResourcePool - import ray - - @ray.remote - class SampleGenerator(Worker): - def __init__(self, config): - super().__init__() - self.config = config - - def generate_sequences(self, data): - pass - -Here, ``SampleGenerator`` can be viewed as a multi-process pulled up by -``torchrun``, with each process running the same code (SPMD). -``SampleGenerator`` needs to implement a ``generate_sequences`` API for -the control flow to call. The implementation details inside can use any -inference engine including vllm, sglang and huggingface. Users can -largely reuse the code in -verl/verl/workers/rollout/vllm_rollout/vllm_rollout.py and we won't -go into details here. - -**ReferencePolicy inference** - -API: compute reference log probability - -.. code:: python - - from verl.single_controller.base import Worker - import ray - - @ray.remote - class ReferencePolicy(Worker): - def __init__(self): - super().__init__() - self.model = Model() - - def infer(self, data): - return self.model(data) - -**Actor update** - -API: Update actor model parameters - -.. code:: python - - from verl.single_controller.base import Worker - import ray - - @ray.remote - class DPOActor(Worker): - def __init__(self): - super().__init__() - self.model = Model() - self.model = FSDP(self.model) # or other distributed strategy - self.optimizer = optim.Adam(self.model.parameters(), lr=1e-3) - self.loss_fn = xxx - - def update(self, data): - self.optimizer.zero_grad() - logits = self.model(data) - loss = self.loss_fn(logits) - loss.backward() - self.optimizer.step() - -**Notes: How to distinguish between control processes and distributed computation processes** -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- Control processes are generally functions directly decorated with - ``@ray.remote`` -- Computation processes are all wrapped into a ``RayWorkerGroup``. - -Users can reuse most of the distribtued computation logics implemented -in PPO algorithm, including FSDP and Megatron-LM backend in -verl/verl/trainer/ppo. - -Step 2: Based on different distributed scenarios, implement single-process control of multi-process data interaction -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**The core problem to solve here is how a single process sends data to -multiple processes, drives multi-process computation, and how the -control process obtains the results of multi-process computation.** -First, we initialize the multi-process ``WorkerGroup`` in the control -process. - -.. code:: python - - @ray.remote(num_cpus=1) - def main_task(config): - # construct SampleGenerator - resource_pool = RayResourcePool(process_on_nodes=[8] * 2) # 16 GPUs - ray_cls = RayClassWithInitArgs(SampleGenerator, config=config) - # put SampleGenerator onto resource pool - worker_group = RayWorkerGroup(resource_pool, ray_cls) - - # construct reference policy - -As we can see, in the control process, multiple processes are wrapped -into a ``RayWorkerGroup``. Inside this ``WorkerGroup``, there is a -``self._workers`` member, where each worker is a RayActor -(https://docs.ray.io/en/latest/ray-core/actors.html) of SampleGenerator. -ray_trainer.md also provide an implementation of -``MegatronRayWorkerGroup``. - -Assuming the model is distributed using FSDP, and there is a batch of -data on the control process, for data parallelism, the underlying -calling process is: - -.. code:: python - - data = xxx - data_list = data.chunk(dp_size) - - output = [] - for d in data_list: - # worker_group._workers[i] is a SampleGenerator - output.append(worker_group._workers[i].generate_sequences.remote(d)) - - output = ray.get(output) - output = torch.cat(output) - -Single process calling multiple processes involves the following 3 -steps: - -1. Split the data into DP parts on the control process. -2. Send the data to remote, call the remote computation through RPC, and - utilize multi-process computation. -3. Obtain the computation results of each worker on the control process - and merge them. - -Frequently calling these 3 steps on the controller process greatly hurts -code readability. **In verl, we have abstracted and encapsulated these 3 -steps, so that the worker's method + dispatch + collect can be -registered into the worker_group** - -.. code:: python - - from verl.single_controller.base.decorator import register - - def dispatch_data(worker_group, data): - return data.chunk(worker_group.world_size) - - def collect_data(worker_group, data): - return torch.cat(data) - - dispatch_mode = { - 'dispatch_fn': dispatch_data, - 'collect_fn': collect_data - } - - @register(dispatch_mode=dispatch_mode) - def generate_sequences(self, data): - pass - -In this way, we can directly call the method inside the worker through -the ``worker_group`` on the control (driver) process (which is a single -process): - -.. code:: python - - output = worker_group.generate_sequences(data) - -This single line includes data splitting, data distribution and -computation, and data collection. - -Furthermore, the model parallelism size of each model is usually fixed, -including dp, tp, pp. So for these common distributed scenarios, we have -pre-implemented specific dispatch and collect methods,in `decorator.py `_, which can be directly used to wrap the computations. - -.. code:: python - - from verl.single_controller.base.decorator import register, Dispatch - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def generate_sequences(self, data: DataProto) -> DataProto: - pass - -Here it requires the data interface to be ``DataProto``. Definition of -``DataProto`` is in `protocol.py `_. - -Step 3: Main training loop -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -With the above training flows, we can implement the algorithm's control -flow. It is recommended that ``main_task`` is also a ray remote process. - -.. code:: python - - @ray.remote(num_cpus=1) - def main_task(config): - # construct SampleGenerator - resource_pool = RayResourcePool(process_on_nodes=[8] * 2) # 16 GPUs - ray_cls = RayClassWithInitArgs(SampleGenerator, config=config) - # put SampleGenerator onto resource pool - sample_gen = RayWorkerGroup(resource_pool, ray_cls) - - # construct reference policy - ray_cls = RayClassWithInitArgs(ReferencePolicy) - ref_policy = RayWorkerGroup(resource_pool, ray_cls) - - # construct actor - ray_cls = RayClassWithInitArgs(DPOActor) - dpo_policy = RayWorkerGroup(resource_pool, ray_cls) - - dataloader = DataLoader() - - for data in dataloader: - # generate data - data = sample_gen.generate_sequences(data) - # generate scores for each data - data = generate_scores(data) - # generate pairwise data using scores - data = generate_pairwise_data(data) - # generate ref_log_prob - data.batch['ref_log_prob'] = ref_policy.infer(data) - # update using dpo - dpo_policy.update(data) - # logging - -Here, different ``WorkerGroups`` can be placed in the same resource pool or -in different resource pools using ``create_colocated_worker_cls`` -similar as in `ray_trainer.py `_. diff --git a/docs/advance/fsdp_extension.rst b/docs/advance/fsdp_extension.rst deleted file mode 100644 index c708bf2501..0000000000 --- a/docs/advance/fsdp_extension.rst +++ /dev/null @@ -1,95 +0,0 @@ - -Add models with the FSDP backend -================================== - -Model --------------------------- - -In principle, our FSDP backend can support any HF model and we can -sychronoize the actor model weight with vLLM using `hf_weight_loader.py `_. -However, ``hf_weight_loader`` is will gather the full state_dict of a -model during synchronization, which may cause OOM. We suggest using -``dtensor_weight_loader`` which gather the full model parameter layer by -layer to reduce the peak memory usage. We already support dtensor weight -loader for the models below in `dtensor_weight_loader.py `_.: - -- ``GPT2LMHeadModel`` -- ``LlamaForCausalLM`` -- ``LLaMAForCausalLM`` -- ``MistralForCausalLM`` -- ``InternLMForCausalLM`` -- ``AquilaModel`` -- ``AquilaForCausalLM`` -- ``Phi3ForCausalLM`` -- ``GemmaForCausalLM`` -- ``Gemma2ForCausalLM`` -- ``GPTBigCodeForCausalLM`` -- ``Starcoder2ForCausalLM`` -- ``Qwen2ForCausalLM`` -- ``DeepseekV2ForCausalLM`` - -To implement ``dtensor_weight_loader`` of a model that's supported in -vLLM, follow the guide of gemma model below: - -1. Copy the - ``load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]])`` from the vllm model class - to ``dtensor_weight_loaders.py`` -2. Modify the arguments to - ``(actor_weights: Dict, vllm_model: nn.Module)`` -3. Replace the ``self`` to ``vllm_model`` -4. Add the - ``local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight)`` - before each ``param = params_dict[name]`` and modify the following - weight loading using ``local_loaded_weight``. -5. Register the implemented dtensor weight loader to ``__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__``. - -.. code-block:: diff - - - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): - + def gemma_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - + params_dict = dict(vllm_model.named_parameters()) - loaded_params = set() - - for name, loaded_weight in weights: - + for name, loaded_weight in actor_weights.items(): - for (param_name, shard_name, shard_id) in stacked_params_mapping: - if shard_name not in name: - continue - name = name.replace(shard_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - + local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - - weight_loader(param, loaded_weight, shard_id) - + weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # lm_head is not used in vllm as it is tied with embed_token. - # To prevent errors, skip loading lm_head.weight. - if "lm_head.weight" in name: - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - + local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", - default_weight_loader) - - weight_loader(param, loaded_weight) - + weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - loaded_params.add(name) - unloaded_params = params_dict.keys() - loaded_params - if unloaded_params: - raise RuntimeError( - "Some weights are not initialized from checkpoints: " - f"{unloaded_params}") \ No newline at end of file diff --git a/docs/advance/megatron_extension.rst b/docs/advance/megatron_extension.rst deleted file mode 100644 index 91d744a8a9..0000000000 --- a/docs/advance/megatron_extension.rst +++ /dev/null @@ -1,26 +0,0 @@ -Add models with the Megatron-LM backend -========================================= - -Model ------------ - -The most challenging aspect to use the Megatron-LM backend is implementing -the models for training. Currently, we implement Llama model that -support data parallelism, tensor parallelism, pipeline parallelism (also -vPP) and sequence parallelism. We also implement remove padding (sequence packing) on Llama -model, which can be found in `modeling_llama_megatron.py `_. - -To support other model, users are required to implement: - -1. Implemnt a model similar to ``modeling_llama_megatron.py`` that satisfy the - parallelism requirements of Megatron-LM. Then register your model in - the `registry.py `_. -2. Checkpoint utils that can load full checkpoint (e.g. huggingface - checkpoint) to partitioned models during the runtime. Then register - your loader to ``weight_loader_registry`` in `weight_loader_registry.py `_. -3. Weight loader that synchronize the weight from Megatron to rollout - (vLLM) model. Note that both the actor model and rollout model are - partitioned during runtime. So, it's advisable to map the model name - in actor model implementation. Otherwise, you may need an additional - name mapping and even weight transformation. The weight loader implementation - is in `megatron_weight_loaders.py `_. \ No newline at end of file diff --git a/docs/advance/placement.rst b/docs/advance/placement.rst deleted file mode 100644 index a98caa116e..0000000000 --- a/docs/advance/placement.rst +++ /dev/null @@ -1,11 +0,0 @@ -Ray API Design Tutorial -======================================= - -We provide a tutorial for our Ray API design, including: - -- Ray basic concepts -- Resource Pool and RayWorkerGroup -- Data Dispatch, Execution and Collection -- Initialize the RayWorkerGroup and execute the distributed computation in the given Resource Pool - -See details in `tutorial.ipynb `_. \ No newline at end of file diff --git a/docs/amd_tutorial/amd_build_dockerfile_page.rst b/docs/amd_tutorial/amd_build_dockerfile_page.rst deleted file mode 100644 index 50ac1985a6..0000000000 --- a/docs/amd_tutorial/amd_build_dockerfile_page.rst +++ /dev/null @@ -1,512 +0,0 @@ -Getting started with AMD (ROCM Kernel) -===================================================== - -Author: `Yusheng Su `_ - -Setup ------ - -If you run on AMD GPUs (MI300) with ROCM platform, you cannot use the previous quickstart to run VeRL. You should follow the following steps to build a docker and assign ``HIP_VISIBLE_DEVICES`` and ``ROCR_VISIBLE_DEVICES`` when starting RLHF training. - - - -docker/Dockerfile.rocm -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - # Build the docker in the repo dir: - # docker build -f docker/Dockerfile.rocm -t verl-rocm:03.04.2015 . - # docker images # you can find your built docker - FROM rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 - - # Set working directory - # WORKDIR $PWD/app - - # Set environment variables - ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" - - # Install vllm - RUN pip uninstall -y vllm && \ - rm -rf vllm && \ - git clone -b v0.6.3 https://github.com/vllm-project/vllm.git && \ - cd vllm && \ - MAX_JOBS=$(nproc) python3 setup.py install && \ - cd .. && \ - rm -rf vllm - - # Copy the entire project directory - COPY . . - - # Install dependencies - RUN pip install "tensordict<0.6" --no-deps && \ - pip install accelerate \ - codetiming \ - datasets \ - dill \ - hydra-core \ - liger-kernel \ - numpy \ - pandas \ - datasets \ - peft \ - "pyarrow>=15.0.0" \ - pylatexenc \ - "ray[data,train,tune,serve]" \ - torchdata \ - transformers \ - wandb \ - orjson \ - pybind11 && \ - pip install -e . --no-deps - -Build the image: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - docker build -t verl-rocm . - -Run the container -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -Optional: Running without root and with user permissions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: bash - - docker run --rm -it \ - --device /dev/dri \ - --device /dev/kfd \ - -p 8265:8265 \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v $HOME/.ssh:/root/.ssh \ - -v $HOME:$HOME \ - --shm-size 128G \ - -w $PWD \ - verl-rocm \ - /bin/bash - -(Optional): If you do not want to root mode and require assign yuorself as the user -Please add ``-e HOST_UID=$(id -u)`` and ``-e HOST_GID=$(id -g)`` into the above docker launch script. - -Example -------- - -Due to to special setting in AMD (ROCM) torch, you need to assign ``HIP_VISIBLE_DEVICES`` and ``ROCR_VISIBLE_DEVICES`` when starting Ray in VeRL's RLHF training. - -PPO -~~~ - -.. code-block:: bash - - YOUR_PROJECT_NAME=r1-verl-ppo-upstream - YOUR_RUN_NAME=r1-training_ppo-upstream - # export HYDRA_FULL_ERROR=1 - export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - GPUS_PER_NODE=8 - MODEL_PATH=Qwen/Qwen2.5-0.5B-Instruct - python3 examples/data_preprocess/gsm8k.py --local_dir data/gsm8k - python3 -c "import transformers; transformers.pipeline('text-generation', model='$MODEL_PATH')" - PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ - data.train_files=data/gsm8k/train.parquet \ - data.val_files=data/gsm8k/test.parquet \ - data.train_batch_size=256 \ - data.val_batch_size=1312 \ - data.max_prompt_length=512 \ - data.max_response_length=256 \ - actor_rollout_ref.model.path=$MODEL_PATH \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=64 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - critic.optim.lr=1e-5 \ - critic.model.path=$MODEL_PATH \ - critic.ppo_micro_batch_size_per_gpu=4 \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.logger=['console'] \ - trainer.project_name=$YOUR_PROJECT_NAME \ - trainer.experiment_name=$YOUR_RUN_NAME \ - trainer.val_before_train=False \ - trainer.default_hdfs_dir=null \ - trainer.n_gpus_per_node=$GPUS_PER_NODE \ - trainer.nnodes=1 \ - trainer.save_freq=10 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 #2>&1 | tee verl_demo.log - -GRPO -~~~~ - -.. code-block:: bash - - YOUR_PROJECT_NAME=r1-verl-grpo-upstream - YOUR_RUN_NAME=r1-training_grpo-upstream - # export HYDRA_FULL_ERROR=1 - # export FSDP_VERBOSE=1 - export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - GPUS_PER_NODE=8 - MODEL_PATH=Qwen/Qwen2.5-0.5B-Instruct - # MODEL_PATH=Qwen/Qwen2-7B-Instruct - python3 examples/data_preprocess/gsm8k.py --local_dir data/gsm8k - python3 -c "import transformers; transformers.pipeline('text-generation', model='$MODEL_PATH')" - python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=data/gsm8k/train.parquet \ - data.val_files=data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.val_batch_size=1312 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - actor_rollout_ref.model.path=$MODEL_PATH \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.model.enable_gradient_checkpointing=Flase \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.fsdp_config.param_offload=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name=$YOUR_PROJECT_NAME \ - trainer.experiment_name=$YOUR_RUN_NAME \ - trainer.n_gpus_per_node=$GPUS_PER_NODE \ - trainer.val_before_train=False \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 - - - -Multi-node training: slurm with Docker/Podman container ---------------------------------------------------------------------------------------- - -If you want to run multi-node training with slurm, you can use the following script. - -.. note:: - 1. You need to use ``podman`` or ``docker`` in the following script. We will release the apptainer script later. - 2. If you want to use ``podman``, you just replace ``docker`` with ``podman`` in the following script. - -The script includes the following steps: - -1. SLURM Configuration -2. Environment Setup -3. Docker/Podman Container Setup -4. Ray Cluster Initialization -5. Data Preprocessing -6. Model Setup -7. Training Launch - - -slurm_script.sh -~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - #!/bin/bash - - #SBATCH --job-name=verl-ray-on-slurm - #SBATCH --nodes=2 - #SBATCH --ntasks-per-node=2 - #SBATCH --mem=200G - #SBATCH --time=30-00:00:00 - #SBATCH --gpus-per-node=8 - #SBATCH --cpus-per-task=28 - #SBATCH --output=../verl_log/slurm-%j.out - #SBATCH --error=../verl_log/slurm-%j.err - #SBATCH --nodelist=gpu-[0,1] - - - # load necessary modules - ### Run this setup - # [Cluster]: Use docker - # docker pull docker.io/rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 - - - ########################################################################## - ###The following setting should be set in different project and cluster### - ########################################################################## - - ### Project - CONTAINER_NAME="multinode_verl_training" - IMG="verl.rocm" - DOCKERFILE="docker/Dockerfile.rocm" - # echo $PWD - verl_workdir="${HOME}/projects/verl_upstream" - export TRANSFORMERS_CACHE="${HOME}/.cache/huggingface" - export HF_HOME=$TRANSFORMERS_CACHE - - ### Cluster Network Setting - export NCCL_DEBUG=TRACE - export GPU_MAX_HW_QUEUES=2 - export TORCH_NCCL_HIGH_PRIORITY=1 - export NCCL_CHECKS_DISABLE=1 - # export NCCL_IB_HCA=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 - export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9 - export NCCL_IB_GID_INDEX=3 - export NCCL_CROSS_NIC=0 - export CUDA_DEVICE_MAX_CONNECTIONS=1 - export NCCL_PROTO=Simple - export RCCL_MSCCL_ENABLE=0 - export TOKENIZERS_PARALLELISM=false - export HSA_NO_SCRATCH_RECLAIM=1 - ########################################################################## - - ### For rocm and training script - export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - export CUDA_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - - - # Build and launch the Docker container - srun bash -c " - # Exit on any error - set -e - - # Clean up dangling images (images with tag) - docker image prune -f - - # Need to pull the docker first - docker pull docker.io/rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 - - if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMG}"; then - echo \"Building ${IMG} image...\" - docker build -f \"${DOCKERFILE}\" -t \"${IMG}\" . - else - echo \"${IMG} image already exists, skipping build\" - fi - - # Removing old container if exists - docker rm \"${CONTAINER_NAME}\" 2>/dev/null || true - - # Checking network devices - ibdev2netdev - - # Launch the docker - docker run --rm -d \ - -e HYDRA_FULL_ERROR=1 \ - -e HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES} \ - -e ROCR_VISIBLE_DEVICES=${ROCR_VISIBLE_DEVICES} \ - -e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \ - -e NCCL_DEBUG=${NCCL_DEBUG} \ - -e GPU_MAX_HW_QUEUES=${GPU_MAX_HW_QUEUES} \ - -e TORCH_NCCL_HIGH_PRIORITY=${TORCH_NCCL_HIGH_PRIORITY} \ - -e NCCL_CHECKS_DISABLE=${NCCL_CHECKS_DISABLE} \ - -e NCCL_IB_HCA=${NCCL_IB_HCA} \ - -e NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} \ - -e NCCL_CROSS_NIC=${NCCL_CROSS_NIC} \ - -e CUDA_DEVICE_MAX_CONNECTIONS=${CUDA_DEVICE_MAX_CONNECTIONS} \ - -e NCCL_PROTO=${NCCL_PROTO} \ - -e RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE} \ - -e TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM} \ - -e HSA_NO_SCRATCH_RECLAIM=${HSA_NO_SCRATCH_RECLAIM} \ - -e TRANSFORMERS_CACHE=${TRANSFORMERS_CACHE} \ - -e HF_HOME=${HF_HOME} \ - --network host \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v \${HOME}:\${HOME} \ - -v \${HOME}/.ssh:/root/.ssh \ - -w "${verl_workdir}" \ - --shm-size 128G \ - --name \"${CONTAINER_NAME}\" \ - \"${IMG}\" \ - tail -f /dev/null - - echo \"Container setup completed\" - " - # (Optional): If you do not want to root mode and require assign yuorself as the user - # Please add `-e HOST_UID=$(id -u)` and `-e HOST_GID=$(id -g)` into the above docker launch script. - - - - - - ### Ray launch the nodes before training - - # Getting the node names - nodes_array=($(scontrol show hostnames "$SLURM_JOB_NODELIST" | tr '\n' ' ')) - - head_node=${nodes_array[0]} - head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) - - # if we detect a space character in the head node IP, we'll - # convert it to an ipv4 address. This step is optional. - if [[ "$head_node_ip" == *" "* ]]; then - IFS=' ' read -ra ADDR <<<"$head_node_ip" - if [[ ${#ADDR[0]} -gt 16 ]]; then - head_node_ip=${ADDR[1]} - else - head_node_ip=${ADDR[0]} - fi - echo "IPV6 address detected. We split the IPV4 address as $head_node_ip" - fi - - port=6379 - ip_head=$head_node_ip:$port - export ip_head - echo "IP Head: $ip_head" - - # make sure we set environment variables before Ray initialization - export VLLM_ATTENTION_BACKEND=XFORMERS - - # Print out all env variables - printenv - - echo "Starting HEAD at $head_node" - srun --nodes=1 --ntasks=1 -w "$head_node" \ - docker exec "${CONTAINER_NAME}" \ - ray start --head --node-ip-address="$head_node_ip" --port=$port \ - --dashboard-port=8266 \ - --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & - # optional, though may be useful in certain versions of Ray < 1.0. - sleep 10 - - # number of nodes other than the head node - worker_num=$((SLURM_JOB_NUM_NODES - 1)) - - for ((i = 1; i <= worker_num; i++)); do - node_i=${nodes_array[$i]} - echo "Debug: Starting worker on node_i = ${node_i}" - if [ -z "$node_i" ]; then - echo "Error: Empty node name for worker $i" - continue - fi - echo "Starting WORKER $i at $node_i" - srun --nodes=1 --ntasks=1 -w "$node_i" \ - docker exec "${CONTAINER_NAME}" \ - ray start --address "$ip_head" --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & - sleep 5 - done - - - - - # Ray initlization test (See whether any error in the above excution) - echo "Testing Ray initialization in the slurm nodes..." - docker exec "${CONTAINER_NAME}" python3 -c ' - import ray - try: - ray.init(address="auto") - print("\n=== Ray Cluster Status ===") - print(f"Number of nodes: {len(ray.nodes())}") - for node in ray.nodes(): - print("Node: {}, Status: {}".format(node["NodeManagerHostname"], node["Alive"])) - # print(f"Node: {node}") - ray.shutdown() - print("Ray initialization successful!") - except Exception as e: - print(f"Ray initialization failed: {str(e)}") - ' - echo "=== Ray test completed ===" - ###### - - - - # Run data preprocessing - - echo "Starting data preprocessing..." - docker exec "${CONTAINER_NAME}" \ - python3 "examples/data_preprocess/gsm8k.py" "--local_dir" "../data/gsm8k" - - echo "Starting data preprocessing..." - docker exec "${CONTAINER_NAME}" \ - python3 "examples/data_preprocess/math_dataset.py" "--local_dir" "../data/math" - - train_files="../data/gsm8k/train.parquet" - val_files="../data/gsm8k/test.parquet" - - # Download and test model - echo "Loading model..." - docker exec "${CONTAINER_NAME}" \ - python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" - MODEL_PATH="Qwen/Qwen2-7B-Instruct" - - # Set model path after pipeline test - MODEL_PATH="Qwen/Qwen2.5-0.5B-Instruct" - - echo "== Data and model loading Done ==" - - echo "Start to train..." - - docker exec "${CONTAINER_NAME}" \ - python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" - MODEL_PATH="Qwen/Qwen2-7B-Instruct" - - - PYTHONUNBUFFERED=1 srun --overlap --nodes=${SLURM_NNODES} --ntasks=1 -w "$head_node" \ - docker exec "${CONTAINER_NAME}" \ - python3 -m verl.trainer.main_ppo \ - data.train_files=$train_files \ - data.val_files=$val_files \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - actor_rollout_ref.model.path=$MODEL_PATH \ - actor_rollout_ref.model.enable_gradient_checkpointing=False \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=$MODEL_PATH \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=8 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.kl_ctrl.kl_coef=0.0001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-32B-Instruct_function_rm' \ - trainer.n_gpus_per_node=${SLURM_GPUS_PER_NODE} \ - trainer.val_before_train=False \ - trainer.nnodes=${SLURM_NNODES} \ - trainer.save_freq=-1 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 - - -Run slurm_script.sh -~~~~~~~~~~~~~~~~~~~~ -Just sbatch your slurm_script.sh - -.. code-block:: bash - - sbatch slurm_script.sh - diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 24a488d36a..0000000000 --- a/docs/conf.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - -# -- Project information ----------------------------------------------------- - -project = u'verl' -# pylint: disable=W0622 -copyright = u'2024 ByteDance Seed Foundation MLSys Team' -author = u'Guangming Sheng, Chi Zhang, Yanghua Peng, Haibin Lin' - - -# -- General configuration --------------------------------------------------- -# The master toctree document. -master_doc = 'index' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ['recommonmark', - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.autosectionlabel', -] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -source_suffix = ['.rst', 'rest', '.md'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = u'en' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/data.rst b/docs/data.rst deleted file mode 100644 index e7efca203d..0000000000 --- a/docs/data.rst +++ /dev/null @@ -1,59 +0,0 @@ -Data interface -========================= - -DataProto is the interface for data exchange. - -The :class:`verl.DataProto` class contains two key members: - -- batch: a :class:`tensordict.TensorDict` object for the actual data -- meta_info: a :class:`Dict` with additional meta information - -TensorDict -~~~~~~~~~~~~ - -:attr:`DataProto.batch` is built on top of :class:`tensordict`, a project in the PyTorch ecosystem. -A TensorDict is a dict-like container for tensors. To instantiate a TensorDict, you must specify key-value pairs as well as the batch size. - -.. code-block:: python - - >>> import torch - >>> from tensordict import TensorDict - >>> tensordict = TensorDict({"zeros": torch.zeros(2, 3, 4), "ones": torch.ones(2, 3, 5)}, batch_size=[2,]) - >>> tensordict["twos"] = 2 * torch.ones(2, 5, 6) - >>> zeros = tensordict["zeros"] - >>> tensordict - TensorDict( - fields={ - ones: Tensor(shape=torch.Size([2, 3, 5]), device=cpu, dtype=torch.float32, is_shared=False), - twos: Tensor(shape=torch.Size([2, 5, 6]), device=cpu, dtype=torch.float32, is_shared=False), - zeros: Tensor(shape=torch.Size([2, 3, 4]), device=cpu, dtype=torch.float32, is_shared=False)}, - batch_size=torch.Size([2]), - device=None, - is_shared=False) - -One can also index a tensordict along its batch_size. The contents of the TensorDict can be manipulated collectively as well. - -.. code-block:: python - - >>> tensordict[..., :1] - TensorDict( - fields={ - ones: Tensor(shape=torch.Size([1, 3, 5]), device=cpu, dtype=torch.float32, is_shared=False), - twos: Tensor(shape=torch.Size([1, 5, 6]), device=cpu, dtype=torch.float32, is_shared=False), - zeros: Tensor(shape=torch.Size([1, 3, 4]), device=cpu, dtype=torch.float32, is_shared=False)}, - batch_size=torch.Size([1]), - device=None, - is_shared=False) - >>> tensordict = tensordict.to("cuda:0") - >>> tensordict = tensordict.reshape(6) - -For more about :class:`tensordict.TensorDict` usage, see the official tensordict_ documentation. - -.. _tensordict: https://pytorch.org/tensordict/overview.html - - -Core APIs -~~~~~~~~~~~~~~~~~ - -.. autoclass:: verl.DataProto - :members: to, select, union, make_iterator, concat diff --git a/docs/examples/config.rst b/docs/examples/config.rst deleted file mode 100644 index d6c1ddcc12..0000000000 --- a/docs/examples/config.rst +++ /dev/null @@ -1,481 +0,0 @@ -.. _config-explain-page: - -Config Explanation -=================== - -ppo_trainer.yaml for FSDP Backend ---------------------------------- - -Data -~~~~ - -.. code:: yaml - - data: - tokenizer: null - train_files: ~/data/rlhf/gsm8k/train.parquet - val_files: ~/data/rlhf/gsm8k/test.parquet - prompt_key: prompt - max_prompt_length: 512 - max_response_length: 512 - train_batch_size: 1024 - return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs - return_raw_chat: False - shuffle: True - filter_overlong_prompts: False # for large-scale dataset, filtering overlong prompts could be timeconsuming. You should disable this and set `truncation='left' - truncation: error - image_key: images - custom_cls: - path: null - name: null - -- ``data.train_files``: Training set parquet. Can be a list or a single - file. The program will read all files into memory, so it can't be too - large (< 100GB). The path can be either local path or HDFS path. For - HDFS path, we provide utils to download it to DRAM and convert the - HDFS path to local path. -- ``data.val_files``: Validation parquet. Can be a list or a single - file. -- ``data.prompt_key``: The field in the dataset where the prompt is - located. Default is 'prompt'. -- ``data.max_prompt_length``: Maximum prompt length. All prompts will be - left-padded to this length. An error will be reported if the length is - too long -- ``data.max_response_length``: Maximum response length. Rollout in RL - algorithms (e.g. PPO) generates up to this length -- ``data.train_batch_size``: Batch size sampled for one training - iteration of different RL algorithms. -- ``data.return_raw_input_ids``: Whether to return the original - input_ids without adding chat template. This is mainly used to - accommodate situations where the reward model's chat template differs - from the policy. It needs to be decoded first, then apply the RM's - chat template. If using a model-based RM, and the policy and RM - chat_templates are different, this flag needs to be set -- ``data.return_raw_chat``: -- ``data.shuffle``: Whether to shuffle the data in the dataloader. -- ``data.filter_overlong_prompts``: Default don't filter. You can filter for small-scale dataset. - For large-scale dataset, filtering overlong prompts could be timeconsuming. - You should disable this and set ``truncation='left`` -- ``data.truncation``: Truncate the input_ids or prompt length if they - exceed max_prompt_length. Default is 'error', not allow exceed the - max_prompt_length. The users should increase the max_prompt_length if - throwing the error. You can also set ``left`` and ``right``. -- ``data.image_key``: The field in the multi-modal dataset where the image is - located. Default is 'images'. - -Customized Dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Customized dataset extension is implemented for the SFT trainer and can be extended to other trainers with similar changes. - -.. code:: yaml - - custom_cls: - path: null - name: null - -- ``data.custom_cls.path``: The path to the file containing your customized dataset class. If not specified, pre-implemented dataset will be used. -- ``data.custom_cls.name``: The name of the dataset class within the specified file. - -Actor/Rollout/Reference Policy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: yaml - - actor_rollout_ref: - hybrid_engine: True - model: - path: ~/models/deepseek-llm-7b-chat - external_lib: null - override_config: { } - enable_gradient_checkpointing: False - use_remove_padding: False - actor: - strategy: fsdp # This is for backward-compatibility - ppo_mini_batch_size: 256 - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: 8 - use_dynamic_bsz: False - ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length} - grad_clip: 1.0 - clip_ratio: 0.2 - entropy_coeff: 0.001 - use_kl_loss: False # True for GRPO - use_torch_compile: True # False to disable torch compile - kl_loss_coef: 0.001 # for grpo - kl_loss_type: low_var_kl # for grpo - ppo_epochs: 1 - shuffle: False - ulysses_sequence_parallel_size: 1 # sp size - optim: - lr: 1e-6 - lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - fsdp_config: - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - param_offload: False - optimizer_offload: False - fsdp_size: -1 - checkpoint: - contents: ['model', 'optimizer', 'extra'] - ref: - fsdp_config: - param_offload: False - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: 16 - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} # sp size - rollout: - name: vllm - temperature: 1.0 - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1 - prompt_length: ${data.max_prompt_length} # not use for opensource - response_length: ${data.max_response_length} - # for vllm rollout - dtype: bfloat16 # should align with FSDP - gpu_memory_utilization: 0.5 - ignore_eos: False - enforce_eager: True - free_cache_engine: True - load_format: dummy_dtensor - tensor_model_parallel_size: 2 - max_num_batched_tokens: 8192 - max_num_seqs: 1024 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: 16 - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - # for hf rollout - do_sample: True - # number of responses (i.e. num sample times) - n: 1 # > 1 for grpo, rloo - -**Common config for actor, rollout and reference model** - -- ``actor_rollout_ref.hybrid_engine``: Whether it's a hybrid engine, - currently only supports hybrid engine -- ``actor_rollout_ref.model.path``: Huggingface model path. This can be - either local path or HDFS path. For HDFS path, we provide utils to - download it to DRAM and convert the HDFS path to local path. -- ``actor_rollout_ref.model.external_libs``: Additional Python packages - that need to be imported. Used to register models or tokenizers into - the Huggingface system. -- ``actor_rollout_ref.model.override_config``: Used to override some of - the model's original configurations, mainly dropout -- ``actor_rollout_ref.model.enable_gradient_checkpointing``: Whether to - enable gradient checkpointing for the actor - -**Actor model** - -- ``actor_rollout_ref.actor.strategy``: fsdp or megatron. In this - example, we use fsdp backend. - -- ``actor_rollout_ref.actor.ppo_mini_batch_size``: One sample is split - into multiple sub-batches with batch_size=ppo_mini_batch_size for PPO - updates. The ppo_mini_batch_size is a global num across all workers/gpus - -- ``actor_rollout_ref.actor.ppo_micro_batch_size``: [Will be deprecated, use ppo_micro_batch_size_per_gpu] - Similar to gradient accumulation, the micro_batch_size_per_gpu for one forward pass, - trading speed for GPU memory. The value represent the global view. - -- ``actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu``: Similar to gradient - accumulation, the micro_batch_size_per_gpu for one forward pass, trading speed - for GPU memory. The value represent the local num per gpu. - -- ``actor_rollout_ref.actor.grad_clip``: Gradient clipping for actor - updates -- ``actor_rollout_ref.actor.use_kl_loss``: to use kl loss in actor. When used, we are not applying KL in the reward function. - -- ``actor_rollout_ref.actor.clip_ratio``: PPO clip ratio - -- ``actor_rollout_ref.actor.use_torch_compile``: Whether to use torch compile in actor - -- ``actor_rollout_ref.actor.entropy_coeff``: The weight of entropy when - calculating PPO loss - -- ``actor_rollout_ref.actor.ppo_epochs``: Number of epochs for PPO - updates on one set of sampled data - -- ``actor_rollout_ref.actor.shuffle``: Whether to shuffle data when - there are multiple epochs - -- ``actor_rollout_ref.actor.optim``: Actor's optimizer parameters - -- ``actor_rollout_ref.actor.fsdp_config``: FSDP config for actor - training - - - ``wrap_policy``: FSDP wrap policy. By default, it uses Huggingface's - wrap policy, i.e., wrapping by DecoderLayer - - - No need to set transformer_layer_cls_to_wrap, so we comment it. - - - ``*_offload``: Whether to enable parameter, gradient and optimizer - offload - - - Trading speed for GPU memory. - -- ``actor_rollout_ref.actor.use_kl_loss``: Whether to enable kl loss. Default is False. - -- ``actor_rollout_ref.actor.kl_loss_coef``: The coefficient of kl loss. Default is 0.001. - -- ``actor_rollout_ref.actor.kl_loss_type``: Support ``kl``, ``abs``, ``mse``, ``low_var_kl`` and ``full``. How to calculate the kl divergence between actor and reference policy. For - specific options, refer to `kl_penalty()` in `core_algos.py `_ . - -- ``actor_rollout_ref.actor.checkpoint``: The configurations of checkpoint function in actor - - - ``contents``: The contents to save in the checkpoint. By default, we save model, optimizer and extra information in the checkpoint. - The extra information includes Rng states currently, FSDP supported lr_scheduler, and Megatron opt_param_scheduler will coming soon. - We do not store hf_model in checkpoint by default, but we provide a tool in `scripts/model_merge.py` to convert checkpoint format to hf format. - -**Reference Model** - -Reference model will be enabled when ``actor.use_kl_loss`` or/and ``algorithm.use_kl_in_reward`` is/are True. - -- ``actor_rollout_ref.ref``: FSDP config same as actor. **For models - larger than 7B, it's recommended to turn on offload for ref by - default** - -- ``actor_rollout_ref.ref.log_prob_micro_batch_size``: [Will be deprecate, use log_prob_micro_batch_size_per_gpu] - The batch size for one forward pass in the computation of ``ref_log_prob``. The value represent the global num. - -- ``actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu``: The batch size - for one forward pass in the computation of ``ref_log_prob``. The value represent the local num per gpu. - -**Rollout Model** - -- ``actor_rollout_ref.rollout.name``: hf/vllm/sglang. - -- Rollout (Auto-regressive) parameters. The key should be equal to the - property name in vLLM's ``SamplingParams``. - - - ``temperature``, ``top_k``, ``top_p`` and others: Sampling - parameters in ``SamplingParams``. - -- ``dtype``: Rollout model parameters type. This should be align with - the actor model parameter type in FSDP/Megatron backend. - -- ``gpu_memory_utilization``: The proportion of the remaining GPU memory - allocated for kv cache after other models have initialized when using - vllm. - -- ``tensor_model_parallel_size``: TP size for rollout. Only effective - for vllm. - -- ``actor_rollout_ref.ref.log_prob_micro_batch_size``: [Will be deprecate, use log_prob_micro_batch_size_per_gpu] - The batch size for one forward pass in the computation of ``log_prob``. The value represent the global num. - -- ``log_prob_micro_batch_size_per_gpu``: Micro batch size per gpu (The batch size for - one forward pass) for recalculating ``log_prob``. The value represent the local num per gpu. - -- ``do_sample``: Whether to sample. If set to False, the rollout model - will perform greedy sampling. We disable ``do_sample`` during - validation. - -- ``actor_rollout_ref.rollout.ignore_eos``: Whether to ignore the EOS - token and continue generating tokens after the EOS token is generated. - -- ``actor_rollout_ref.rollout.free_cache_engine``: Offload the KVCache - after rollout generation stage. Default is True. When set to True, we - need to disable the usage of CUDAGraph (set ``enforce_eager`` to - True.) - -- ``actor_rollout_ref.rollout.enforce_eager``: Whether to use CUDAGraph - in vLLM generation. Default set to True to disable CUDAGraph. - -- ``actor_rollout_ref.rollout.load_format``: Which weight loader to use - to load the actor model weights to the rollout model. - - - ``auto``: Use Megatron weight loader. - - ``megatron``: Use Megatron weight loader. Deployed with Megatron - backend. The input model ``state_dict()`` is already partitioned - along TP dimension and already gathered along PP dimension. This - weight loader requires that the Rollout model and Actor model's - parameters shape and name should be identical. - - ``dtensor``: Default solution when using Huggingface weight loader. - Deployed with FSDP backend and the state_dict_type is - ``StateDictType.SHARDED_STATE_DICT``. Recommend to use this weight - loader - - ``hf``: Use Huggingface weight loader. Deployed with FSDP backend - and the state_dict_type is ``StateDictType.FULL_STATE_DICT``. This - solution doesn't need to rewrite the weight loader for each model - implemented in vLLM but it results in larger peak memory usage. - - ``dummy_hf``, ``dummy_megatron``, ``dummy_dtensor``: Random - initialization. - -.. note:: **NOTED**: In this config field, users only need to select from ``dummy_megatron``, ``dummy_dtensor``, ``dummy_hf`` for rollout initialization and our hybrid engine will select the corresponding weight loader (i.e., ``megatron``, ``dtensor``, ``hf``) during actor/rollout weight synchronization. - -Critic Model -~~~~~~~~~~~~ - -Most parameters for Critic are similar to Actor Model. - -Reward Model -~~~~~~~~~~~~ - -.. code:: yaml - - reward_model: - enable: False - model: - input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical - path: ~/models/Anomy-RM-v0.1 - external_lib: ${actor_rollout_ref.model.external_lib} - fsdp_config: - min_num_params: 0 - param_offload: False - micro_batch_size_per_gpu: 16 - max_length: null - reward_manager: naive - -- ``reward_model.enable``: Whether to enable reward model. If False, we - compute the reward only with the user-defined reward functions. In - GSM8K and Math examples, we disable reward model. For RLHF alignment - example using full_hh_rlhf, we utilize reward model to assess the - responses. If False, the following parameters are not effective. -- ``reward_model.model`` - - - ``input_tokenizer``: Input tokenizer. If the reward model's chat - template is inconsistent with the policy, we need to first decode to - plaintext, then apply the rm's chat_template. Then score with RM. If - chat_templates are consistent, it can be set to null. - - ``path``: RM's HDFS path or local path. Note that RM only supports - AutoModelForSequenceClassification. Other model types need to define - their own RewardModelWorker and pass it from the code. -- ``reward_model.reward_manager``: Reward Manager. This defines the mechanism - of computing rule-based reward and handling different reward sources. Default - is ``naive``. If all verification functions are multiprocessing-safe, the reward - manager can be set to ``prime`` for parallel verification. - -Customized Reward Function -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: yaml - - custom_reward_function: - path: null - name: compute_score - -- ``custom_reward_function.path``: The path to the file containing your customized reward function. If not specified, pre-implemented reward functions will be used. -- ``custom_reward_function.name`` (Optional) : The name of the reward function within the specified file. Default is 'compute_score'. - -Algorithm -~~~~~~~~~ - -.. code:: yaml - - algorithm: - gamma: 1.0 - lam: 1.0 - adv_estimator: gae - use_kl_in_reward: False - kl_penalty: kl # how to estimate kl divergence - kl_ctrl: - type: fixed - kl_coef: 0.005 - horizon: 10000 - target_kl: 0.1 - -- ``gemma``: discount factor -- ``lam``: Trade-off between bias and variance in the GAE estimator -- ``adv_estimator``: Support ``gae``, ``grpo``, ``reinforce_plus_plus``, ``rloo`` -- ``use_kl_in_reward``: Whether to enable in-reward kl penalty. Default is False. -- ``kl_penalty``: Support ``kl``, ``abs``, ``mse``, ``low_var_kl`` and ``full``. How to - calculate the kl divergence between actor and reference policy. For - specific options, refer to `kl_penalty()` in `core_algos.py `_ . -- ``kl_ctrl``: Config for in-reward kl_penalty controller - - ``kl_coef``: The (initial) coefficient of in-reward kl_penalty. Default is 0.001. - - ``type``: 'fixed' for FixedKLController and 'adaptive' for AdaptiveKLController. - - ``horizon`` and ``target_kl``: See source code of AdaptiveKLController for details. - -Trainer -~~~~~~~ - -.. code:: yaml - - trainer: - total_epochs: 30 - project_name: verl_examples - experiment_name: gsm8k - logger: ['console', 'wandb'] - log_val_generations: 0 - nnodes: 1 - n_gpus_per_node: 8 - save_freq: -1 - val_before_train: True - test_freq: 2 - critic_warmup: 0 - default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name} # hdfs checkpoint path - default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} # local checkpoint path - resume_mode: auto # or disable or resume_path if resume_from_path is set - resume_from_path: null - remove_previous_ckpt_in_save: False - del_local_ckpt_after_load: False - -- ``trainer.total_epochs``: Number of epochs in training. -- ``trainer.project_name``: For wandb, swanlab, mlflow -- ``trainer.experiment_name``: For wandb, swanlab, mlflow -- ``trainer.logger``: Support console and wandb, swanlab, mlflow, tensorboard -- ``trainer.log_val_generations``: The number of logged generation during validation (default ``0``) -- ``trainer.nnodes``: Number of nodes used in the training. -- ``trainer.n_gpus_per_node``: Number of GPUs per node. -- ``trainer.save_freq``: The frequency (by iteration) to save checkpoint - of the actor and critic model. -- ``trainer.val_before_train``: Whether to run validation before training. -- ``trainer.test_freq``: The validation frequency (by iteration). -- ``trainer.critic_warmup``: The number of iteration to train the critic - model before actual policy learning. -- ``trainer.resume_mode``: The mode of resuming training. Support - ``disable``, ``auto`` and ``resume_path``. If set to ``auto`` as default, the - program will automatically resume from the latest checkpoint in the - default_hdfs_dir. If set to ``resume_path``, the program will resume - from the path specified in ``resume_from_path``. -- ``trainer.resume_from_path``: The path to resume training from. Only - effective when ``resume_mode`` is set to ``resume_path``. -- ``trainer.remove_previous_ckpt_in_save``: Whether to remove previous - checkpoints in the save directory. Default is False. -- ``trainer.del_local_ckpt_after_load``: Whether to delete local - checkpoints after loading them. Default is False. - - -evaluation.yaml ---------------- - -Data -~~~~ - -.. code:: yaml - - data: - path: /tmp/math_Qwen2-7B-Instruct.parquet - prompt_key: prompt - response_key: responses - data_source_key: data_source - reward_model_key: reward_model - -- ``data.path``: Path to the dataset file (Parquet format). -- ``data.prompt_key``: The field in the dataset where the prompt is located. Default is 'prompt'. -- ``data.response_key``: The key holds the generated responses. This should be a list of strings representing the responses. Default is 'responses'. -- ``data.data_source_key``: This is used to separate metric calculations for different data sources, ensuring that metrics are calculated independently for each source. -- ``data.reward_model_key``: The key holds the reference answers. These reference answers typically serve as the ground truth or test cases for the task. - -Customized Reward Function -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: yaml - - custom_reward_function: - path: null - name: compute_score - -- ``custom_reward_function.path``: The path to the file containing your customized reward function. If not specified, pre-implemented reward functions will be used. -- ``custom_reward_function.name`` (Optional) : The name of the reward function within the specified file. Default is 'compute_score'. diff --git a/docs/examples/gsm8k_example.rst b/docs/examples/gsm8k_example.rst deleted file mode 100644 index 17eb76aca7..0000000000 --- a/docs/examples/gsm8k_example.rst +++ /dev/null @@ -1,189 +0,0 @@ -GSM8K Example -============= - -Introduction ------------- - -In this example, we train an LLM to tackle the GSM8k task. - -Paper: https://arxiv.org/pdf/2110.14168 - -Dataset: https://huggingface.co/datasets/gsm8k - -Note that the original paper mainly focuses on training a verifier (a -reward model) to solve math problems via Best-of-N sampling. In this -example, we train an RLHF agent using a rule-based reward model. - -Dataset Introduction --------------------- - -GSM8k is a math problem dataset. The prompt is an elementary school -problem. The LLM model is required to answer the math problem. - -The training set contains 7473 samples and the test set contains 1319 -samples. - -**An example** - -Prompt - - Katy makes coffee using teaspoons of sugar and cups of water in the - ratio of 7:13. If she used a total of 120 teaspoons of sugar and cups - of water, calculate the number of teaspoonfuls of sugar she used. - -Solution - - The total ratio representing the ingredients she used to make the - coffee is 7+13 = <<7+13=20>>20 Since the fraction representing the - number of teaspoons she used is 7/20, she used 7/20\ *120 = - <<7/20*\ 120=42>>42 #### 42 - -Step 1: Prepare dataset ------------------------ - -.. code:: bash - - cd examples/data_preprocess - python3 gsm8k.py --local_dir ~/data/gsm8k - -Step 2: Download Model ----------------------- - -There're three ways to prepare the model checkpoints for post-training: - -- Download the required models from huggingface or modelscope - -.. code:: bash - - huggingface-cli download deepseek-ai/deepseek-math-7b-instruct --local-dir ~/models/deepseek-math-7b-instruct --local-dir-use-symlinks False - # or - modelscope download --model deepseek-ai/deepseek-math-7b-instruct --local_dir ~/models/deepseek-math-7b-instruct - -- Already store your store model in the local directory or HDFS path. -- Also, you can directly use the model name in huggingface (e.g., - deepseek-ai/deepseek-math-7b-instruct) in - ``actor_rollout_ref.model.path`` and ``critic.model.path`` field in - the run script. You can also download models from modelscope by setting environmental variable ``VERL_USE_MODELSCOPE=True``. - See examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh for example. - -Noted that users should prepare checkpoints for actor, critic and reward -model. - -[Optional] Step 3: SFT your Model ---------------------------------- - -We provide a SFT Trainer using PyTorch FSDP in -`fsdp_sft_trainer.py `_. -Users can customize their own SFT -script using our FSDP SFT Trainer. - -We also provide various training scripts for SFT on GSM8K dataset in `gsm8k sft directory `_. - -.. code:: shell - - set -x - - torchrun -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=question \ - data.response_key=answer \ - data.micro_batch_size_per_gpu=8 \ - model.partial_pretrain=deepseek-ai/deepseek-coder-6.7b-instruct \ - trainer.default_hdfs_dir=hdfs://user/verl/experiments/gsm8k/deepseek-coder-6.7b-instruct/ \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-deepseek-coder-6.7b-instruct \ - trainer.total_epochs=4 \ - trainer.logger=['console','wandb'] - - -If you use AMD GPUs (ROCm kernel), you need to add the following environment variables into the run script: - - .. code-block:: bash - - export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - export CUDA_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - - -Step 4: Perform PPO training with your model on GSM8K Dataset -------------------------------------------------------------- - -- Prepare your own run.sh script. Here's an example for GSM8k dataset - and deepseek-llm-7b-chat model. -- Users could replace the ``data.train_files`` ,\ ``data.val_files``, - ``actor_rollout_ref.model.path`` and ``critic.model.path`` based on - their environment. -- See :doc:`config` for detailed explanation of each config field. - -**Reward Model/Function** - -We use a rule-based reward model. We force the model to produce a final -answer following 4 “#” as shown in the solution. We extract the final -answer from both the solution and model's output using regular -expression matching. We compare them and assign a reward of 1 to correct -answer, 0.1 to incorrect answer and 0 to no answer. - -**Training Script** - -The training script example for FSDP and Megatron-LM backend are stored in examples/ppo_trainer directory. - -.. code:: bash - - cd ../ppo_trainer - bash run_deepseek7b_llm.sh - -The script of run_deepseek7b_llm.sh - -.. code:: bash - - set -x - - python3 -m verl.trainer.main_ppo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_micro_batch_size_per_gpu=32 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 $@ - - -If you use AMD GPUs (ROCm kernel), you need to add the following environment variables into the run script: - - .. code-block:: bash - - export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - export CUDA_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - -If you encounter any issues in using AMD GPUs running VeRL, feel free to contact me - `Yusheng Su `_. \ No newline at end of file diff --git a/docs/examples/ppo_code_architecture.rst b/docs/examples/ppo_code_architecture.rst deleted file mode 100644 index 1ad7c9658d..0000000000 --- a/docs/examples/ppo_code_architecture.rst +++ /dev/null @@ -1,207 +0,0 @@ -PPO Example Architecture -======================== - -Let's start with the Proximal Policy Optimization algorithm, which is -most widely used algorithm in LLM post-training. - -The main entry point of the PPO algorithm example is: -`main_ppo.py `_. -In this tutorial, we will go through the code architecture in `main_ppo.py `_. - -Define the data ---------------- - -Users need to preprocess and store the dataset in parquet files. -And we implement `RLHFDataset` to load and tokenize the parquet files. - -For ``RLHFDataset`` (Default), at least 1 fields are required: - -- ``prompt``: Contains the string prompt - -We already provide some examples of processing the datasets to parquet -files in `data_preprocess directory `_. Currently, we support -preprocess of GSM8k, MATH, Hellasage, Full_hh_rlhf datasets. See :doc:`../preparation/prepare_data` for -more information. - -Define the reward functions for different datasets --------------------------------------------------- - -In this main entry point, the users only need to define their own reward -function based on the datasets (or applications) utilized in PPO -training. - -For example, we already provide reward functions for `GSM8k `_ -and `MATH `_ -datasets in the ``_select_rm_score_fn``. In the ``RewardManager``, we -will compute the reward score based on the data_source to select -corresponding reward functions. For some RLHF datasets (e.g., -full_hh_rlhf), the reward model is utilized to assess the responses -without any reward functions. In this case, the ``RewardManager`` will -return the ``rm_score`` computed by the reward model directly. - -See `reward functions `_ for detailed implementation. - -Define worker classes ---------------------- - -.. code:: python - - if config.actor_rollout_ref.actor.strategy == 'fsdp': # for FSDP backend - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray import RayWorkerGroup - ray_worker_group_cls = RayWorkerGroup - - elif config.actor_rollout_ref.actor.strategy == 'megatron': # for Megatron backend - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup - ray_worker_group_cls = NVMegatronRayWorkerGroup # Ray worker class for Megatron-LM - - else: - raise NotImplementedError - - from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role - - role_worker_mapping = { - Role.ActorRollout: ActorRolloutRefWorker, - Role.Critic: CriticWorker, - Role.RefPolicy: ActorRolloutRefWorker - } - - global_pool_id = 'global_pool' - resource_pool_spec = { - global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, - } - mapping = { - Role.ActorRollout: global_pool_id, - Role.Critic: global_pool_id, - Role.RefPolicy: global_pool_id, - } - -Step 1: Construct the mapping between roles and workers -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A role represents a group of workers in the same process. We have -pre-defined several roles in `ray_trainer.py `_. - -.. code:: python - - class Role(Enum): - """ - To create more roles dynamically, you can subclass Role and add new members - """ - Actor = 0 # This worker only has Actor - Rollout = 1 # This worker only has Rollout - ActorRollout = 2 # This worker has both actor and rollout, it's a HybridEngine - Critic = 3 # This worker only has critic - RefPolicy = 4 # This worker only has reference policy - RewardModel = 5 # This worker only has reward model - ActorRolloutRef = 6 # This worker contains actor, rollout and reference policy simultaneously - -Step 2: Define the worker class corresponding to this role -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- We have pre-implemented the ``ActorRolloutRefWorker``. Through - different configs, it can be a standalone actor, a standalone rollout, - an ActorRollout HybridEngine, or an ActorRolloutRef HybridEngine -- We also pre-implemented workers for ``Actor``, ``Rollout``, - ``Critic``, ``Reward Model`` and ``Reference model`` on two different - backend: PyTorch FSDP - and Megatron-LM. - See `FSDP Workers `_ - and `Megatron-LM Workers `_ - for more information. - -Step 3: Define resource pool id and resource pool spec -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- Resource pool is a division of global GPU resources, - ``resource_pool_spec`` is a dict, mapping from id to # of GPUs - - - In the above example, we defined a global resource pool: - global_pool_id, and then put all roles on this one resource pool - with all the GPUs in this post-training task. This refers to - *co-locate* placement where all the models share the same set of - GPUs. - -- See resource pool and placement for advance usage. - -Defining reward model/function ------------------------------- - -.. code:: python - - # we should adopt a multi-source reward function here - # - for rule-based rm, we directly call a reward score - # - for model-based rm, we call a model - # - for code related prompt, we send to a sandbox if there are test cases - # - finally, we combine all the rewards together - # - The reward type depends on the tag of the data - if config.reward_model.enable: - from verl.workers.fsdp_workers import RewardModelWorker - role_worker_mapping[Role.RewardModel] = RewardModelWorker - mapping[Role.RewardModel] = global_pool_id - - reward_fn = RewardManager(tokenizer=tokenizer, num_examine=0) - - # Note that we always use function-based RM for validation - val_reward_fn = RewardManager(tokenizer=tokenizer, num_examine=1) - - resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) - -Since not all tasks use model-based RM, users need to define here -whether it's a model-based RM or a function-based RM - -- If it's a model-based RM, directly add the ``RewardModel`` role in the - resource mapping and add it to the resource pool mapping. - - - Note that the pre-defined ``RewardModelWorker`` only supports models - with the structure of huggingface - ``AutoModelForSequenceClassification``. If it's not this model, you - need to define your own RewardModelWorker in `FSDP Workers `_ - and `Megatron-LM Workers `_. - -- If it's a function-based RM, the users are required to classified the - reward function for each datasets. - -.. code:: python - - def _select_rm_score_fn(data_source): - if data_source == 'openai/gsm8k': - return gsm8k.compute_score - elif data_source == 'lighteval/MATH': - return math.compute_score - else: - raise NotImplementedError - -See reward functions implemented in `directory `_ -for more information. - -Define, init and run the PPO Trainer ------------------------------------- - -.. code:: python - - trainer = RayPPOTrainer(config=config, - tokenizer=tokenizer, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - ray_worker_group_cls=ray_worker_group_cls, - reward_fn=reward_fn, - val_reward_fn=val_reward_fn) - trainer.init_workers() - trainer.fit() - -- We first initialize the ``RayPPOTrainer`` with user config, tokenizer - and all the above worker mapping, resource pool, worker group and - reward functions -- We first call the ``trainer.init_workers()`` to initialize the models - on the allocated GPUs (in the resource pool) -- The actual PPO training will be executed in ``trainer.fit()`` - -verl can be easily extended to other RL algorithms by reusing the Ray -model workers, resource pool and reward functions. See :doc:`extension<../advance/dpo_extension>` for -more information. - -Details of the ``RayPPOTrainer`` is discussed in :doc:`Ray Trainer<../workers/ray_trainer>`. diff --git a/docs/experiment/ppo.rst b/docs/experiment/ppo.rst deleted file mode 100644 index 679c663e03..0000000000 --- a/docs/experiment/ppo.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. _algo-baseline-page: - -Algorithm Baselines -=================== - -Datasets ------------------- - -Assuming GSM8k/math dataset is preprocess via ``python3 examples/data_preprocess/*.py`` - -Refer to the table below to reproduce RL training from different pre-trained models. - -NVIDIA GPUs --------------------------------- - -.. _Huggingface: https://huggingface.co/google/gemma-2-2b-it#benchmark-results -.. _SFT Command and Logs: https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/gemma-2-2b-it-sft-0.411.log -.. _SFT+PPO Command and Logs: https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/gemma-2-2b-it-ppo-bsz512_4-prompt1024-resp-512-0.640.log -.. _wandb: https://api.wandb.ai/links/verl-team/h7ux8602 -.. _Qwen Blog: https://qwenlm.github.io/blog/qwen2.5-llm/ -.. _PPO Command and Logs: https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/Qwen2.5-0.5B-bsz256_2-prompt1024-resp512-0.567.log -.. _Megatron PPO Command and Logs: https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/deepseek-llm-7b-chat-megatron-bsz256_4-prompt512-resp512-0.695.log -.. _Qwen7b GRPO Script: https://github.com/volcengine/verl/blob/a65c9157bc0b85b64cd753de19f94e80a11bd871/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh -.. _Megatron wandb: https://wandb.ai/verl-team/verl_megatron_gsm8k_examples/runs/10fetyr3 -.. _Qwen7b ReMax Script: https://github.com/eric-haibin-lin/verl/blob/main/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh -.. _Qwen7b ReMax Wandb: https://wandb.ai/liziniu1997/verl_remax_example_gsm8k/runs/vxl10pln -.. _Qwen0.5b PRIME Script: https://github.com/volcengine/verl/blob/main/recipe/prime/run_prime_qwen.sh -.. _Qwen0.5b PRIME Wandb: https://api.wandb.ai/links/zefan-wang-thu-tsinghua-university/rxd1btvb -.. _Megatron Qwen2 7b GRPO Script with Math and GSM8k: https://github.com/eric-haibin-lin/verl-data/blob/experiments/gsm8k/qwen2-7b_math_megatron.log - -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Model | Method | Test score | Details | -+==================================+========================+============+=====================+=========================================================================+ -| google/gemma-2-2b-it | pretrained checkpoint | 23.9 | `Huggingface`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| google/gemma-2-2b-it | SFT | 52.06 | `SFT Command and Logs`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| google/gemma-2-2b-it | SFT + PPO | 64.02 | `SFT+PPO Command and Logs`_, `wandb`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Qwen/Qwen2.5-0.5B-Instruct | pretrained checkpoint | 36.4 | `Qwen Blog`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Qwen/Qwen2.5-0.5B-Instruct | PPO | 56.7 | `PPO Command and Logs`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Qwen/Qwen2.5-0.5B-Instruct | PRIME | 58.7 | `Qwen0.5b PRIME Script`_, `Qwen0.5b PRIME Wandb`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| deepseek-ai/deepseek-llm-7b-chat | PPO (Megatron) | 69.5 [1]_ | `Megatron PPO Command and Logs`_, `Megatron wandb`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Qwen/Qwen2-7B-Instruct | GRPO | 89 | `Qwen7b GRPO Script`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Qwen/Qwen2-7B-Instruct | GRPO (Megatron) | 89.6 | `Megatron Qwen2 7b GRPO Script with Math and GSM8k`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Qwen/Qwen2.5-7B-Instruct | ReMax | 97 | `Qwen7b ReMax Script`_, `Qwen7b ReMax Wandb`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ - - -AMD GPUs (MI300) --------------------------------- - -.. _ppo_run_deepseek7b_llm.sh: https://github.com/yushengsu-thu/verl_training_log/blob/main/gsm8k/ppo_run_deepseek7b_llm.log -.. _grpo_run_deepseek7b_llm.sh: https://github.com/yushengsu-thu/verl_training_log/blob/main/gsm8k/grpo_run_deepseek7b_llm.log - -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| Model | Method | Test score | Details | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| deepseek-ai/deepseek-llm-7b-chat | PPO | 70.5 [1]_ | `ppo_run_deepseek7b_llm.sh`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ -| deepseek-ai/deepseek-llm-7b-chat | GRPO | 71.4 [1]_ | `grpo_run_deepseek7b_llm.sh`_ | -+----------------------------------+------------------------+------------+-----------------------------------------------------------------------------------------------+ - -.. [1] During the evaluation, we have only extracted answers following the format "####". A more flexible answer exaction, longer response length and better prompt engineering may lead to higher score. diff --git a/docs/faq/faq.rst b/docs/faq/faq.rst deleted file mode 100644 index 6fe4d066d3..0000000000 --- a/docs/faq/faq.rst +++ /dev/null @@ -1,64 +0,0 @@ -Frequently Asked Questions -==================================== - -Ray related ------------- - -How to add breakpoint for debugging with distributed Ray? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Please checkout the official debugging guide from Ray: https://docs.ray.io/en/latest/ray-observability/ray-distributed-debugger.html - - -Distributed training ------------------------- - -How to run multi-node post-training with Ray? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can start a ray cluster and submit a ray job, following the official guide from Ray: https://docs.ray.io/en/latest/ray-core/starting-ray.html - -Then in the configuration, set the ``trainer.nnode`` config to the number of machines for your job. - -How to use verl on a Slurm-managed cluster? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Ray provides users with `this `_ official -tutorial to start a Ray cluster on top of Slurm. We have verified the :doc:`GSM8K example<../examples/gsm8k_example>` -on a Slurm cluster under a multi-node setting with the following steps. - -1. [Optional] If your cluster support `Apptainer or Singularity `_ and you wish -to use it, convert verl's Docker image to an Apptainer image. Alternatively, set up the environment with the package -manager available on your cluster or use other container runtimes (e.g. through `Slurm's OCI support `_) available to you. - -.. code:: bash - - apptainer pull /your/dest/dir/vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3.sif docker://verlai/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3 - -2. Follow :doc:`GSM8K example<../examples/gsm8k_example>` to prepare the dataset and model checkpoints. - -3. Modify `examples/slurm/ray_on_slurm.slurm `_ with your cluster's own information. - -4. Submit the job script to the Slurm cluster with `sbatch`. - -Please note that Slurm cluster setup may vary. If you encounter any issues, please refer to Ray's -`Slurm user guide `_ for common caveats. - -If you changed Slurm resource specifications, please make sure to update the environment variables in the job script if necessary. - -Illegal memory access ---------------------------------- - -If you encounter the error message like ``CUDA error: an illegal memory access was encountered`` during rollout, most likely it is due to a known issue from vllm. -Please set the following environment variable. The env var must be set before the ``ray start`` command if any. - -.. code:: bash - - export VLLM_ATTENTION_BACKEND=XFORMERS - -If in doubt, print this env var in each rank to make sure it is properly set. - -Checkpoints ------------------------- - -If you want to convert the model checkpoint into huggingface safetensor format, please refer to ``scripts/model_merger.py``. diff --git a/docs/fleet-integration-architecture.md b/docs/fleet-integration-architecture.md new file mode 100644 index 0000000000..125f2db79b --- /dev/null +++ b/docs/fleet-integration-architecture.md @@ -0,0 +1,198 @@ +# Fleet Integration Architecture + +This document describes how Fleet environments integrate with SkyRL and Tinker training loops. + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Training Loops │ +├───────────────────────────────────┬─────────────────────────────────────────┤ +│ SkyRL Training Loop │ Tinker Training Loop │ +│ (grpo_trainer.py + │ (main_fleet_tinker.py) │ +│ skyrl_gym_generator.py) │ │ +│ │ │ +│ • generate_batched() collects │ • collect_batch_rollouts() collects │ +│ rollouts via vLLM (GPU) │ rollouts via Tinker inference API │ +│ • Computes GRPO advantages │ • Computes GRPO advantages │ +│ • Trainer runs forward/backward │ • prepare_training_data() builds batch │ +│ on local GPU │ • Tinker forward_backward() (hosted) │ +│ │ │ +└───────────────────────────────────┴─────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ SkyRL FleetTaskEnv Wrapper │ +│ (integrations/fleet/env.py) │ +│ │ +│ FleetTaskEnv (extends BaseTextEnv) │ +│ ───────────────────────────────────────────────────────────────────────── │ +│ │ +│ __init__(env_config, extras) │ +│ • Loads task config from JSON file (tasks_file) │ +│ • Validates task_key exists in config │ +│ • Sets up API key and TTL for Fleet environments │ +│ │ +│ init() / init_async() │ +│ • Creates OpenEnv FleetTaskEnv instance │ +│ • Calls openenv.reset_async() to create Fleet env and fetch tools │ +│ • Builds system prompt with tool definitions and date context │ +│ • Returns initial chat_history: [system_msg, user_task_prompt] │ +│ │ +│ step() / step_async(action) │ +│ • Parses tool call from LLM response (...) │ +│ • Executes tool via openenv.step_async() → Fleet platform │ +│ • Tracks turns, tool_calls for metrics │ +│ • Returns BaseTextEnvStepOutput with observation, reward, done │ +│ │ +│ close() │ +│ • Cleanup Fleet environment resources │ +│ │ +│ State: chat_history, tools, turns, tool_calls, openenv_task_env │ +│ Note: sync methods wrap async methods with asyncio.run() │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ OpenEnv FleetTaskEnv │ +│ (envs/fleet_env/task_env.py) │ +│ │ +│ reset_async() │ +│ • Creates Fleet environment via fleet.make() │ +│ • Fetches available tools via MCP list_tools │ +│ • Returns observation with tools list │ +│ │ +│ step_async(action) │ +│ • Executes tool call on Fleet environment │ +│ • Runs verifier if episode done (reward computation) │ +│ • Returns (observation, reward, done, info) │ +│ │ +│ close() │ +│ • Destroys Fleet environment instance │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + │ HTTP/MCP + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Fleet Platform │ +│ │ +│ Hosted task environments with MCP tool interfaces: │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Outlook │ │ GitHub │ │ Booking │ │ Reddit │ │ +│ │ (email, │ │ (repos, │ │ (search, │ │ (posts, │ │ +│ │ calendar) │ │ issues, PR) │ │ reserve) │ │ comments) │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +## Key Abstractions + +| Layer | Component | Purpose | +|-------|-----------|---------| +| Training | `main_fleet_tinker.py` | Tinker-based training: hosted inference + training | +| Training | `grpo_trainer.py` + `skyrl_gym_generator.py` | SkyRL training: vLLM inference (GPU) + local training | +| Wrapper | `FleetTaskEnv` (SkyRL) | Chat history management, tool call parsing, prompt construction | +| Env | `FleetTaskEnv` (OpenEnv) | Low-level Fleet API: env creation, tool execution, verifier | +| Platform | Fleet | Hosted task environments (Outlook, GitHub, etc.) | + +## Dataset Configuration + +Both training loops use the same dataset preparation process: + +### Data Versioning + +Tasks are stored in S3 at `s3://fleet-internal-datasets/{data_version}/openenv/`: +- `all_tool_use.json` - Tool use tasks +- `all_computer_use.json` - Computer use tasks + +The `data_version` parameter (e.g., `v0.1`, `v0.2`) is configurable in both workflows and validated against S3 before training starts. + +### Train/Test Split (`prepare_dataset.py`) + +| Parameter | Value | Description | +|-----------|-------|-------------| +| `eval_ratio` | 10% | Fraction for evaluation | +| `MAX_EVAL_SAMPLES` | 30 | Cap per environment | +| `MIN_EVAL_SAMPLES` | 1 | Minimum to create eval split | + +Split strategy: +- **Stratified by environment**: Each env maintains the train/eval ratio +- **Hash-based assignment**: Deterministic (same task → same split) +- **Held-out envs**: `instacart` (computer_use only) goes to eval only + +## Communication Flow + +### Tinker Training Loop +```python +# 1. Create environment +env = FleetTaskEnv(env_config, extras={"task_key": task_key, ...}) + +# 2. Initialize - creates Fleet env, fetches tools, builds system prompt +chat_history, metadata = await env.init_async([]) + +# 3. Rollout loop +while not done: + input_ids = tokenize(env.chat_history) + output = tinker_sample(input_ids) # Tinker hosted inference + step_output = await env.step_async(output) # Parse tool call, execute on Fleet + done = step_output.done + +# 4. Cleanup +env.close() +``` + +### SkyRL Training Loop +```python +# 1. Create environment +env = skyrl_gym.make("fleet_task", env_config, extras) + +# 2. Initialize (uses asyncio.run internally) +chat_history, metadata = env.init(prompt) + +# 3. Rollout loop +while not done: + output = inference_engine.generate(chat_history) # Local vLLM inference + step_output = env.step(output) # Uses asyncio.run internally + done = step_output.done + +# 4. Cleanup +env.close() +``` + +## Data Flow + +``` +┌────────────┐ prompt ┌────────────┐ action ┌────────────┐ +│ Training │ ───────────▶ │ FleetTask │ ───────────▶ │ Fleet │ +│ Loop │ │ Env │ │ Platform │ +│ │ ◀─────────── │ (SkyRL) │ ◀─────────── │ │ +└────────────┘ chat_history └────────────┘ observation └────────────┘ + step_output +``` + +## Key Data Structures + +**init() / init_async() returns:** +- `chat_history`: List of messages `[{role, content}, ...]` + - System message with tools JSON and instructions + - User message with task prompt +- `metadata`: `{task_key, env_key, tools, modality}` + +**step() / step_async() returns:** +- `BaseTextEnvStepOutput`: + - `observations`: New messages to append (tool result or error) + - `reward`: Float (0.0 during rollout, verifier reward on done) + - `done`: Boolean (agent said `` or max_turns reached) + - `metadata`: `{task_key, turn, tool_call, tool_result, error}` + +## Training Data Preparation + +Both training loops follow the same pattern for preparing training data: + +1. **DAPO Overlong Filtering**: Zero out loss mask for responses not ending with EOS token +2. **Sequence Truncation**: Truncate to `max_sequence_length`, preserving prompt +3. **Advantage Computation**: GRPO advantages from rewards grouped by prompt +4. **Build Training Batch**: Combine into format expected by trainer diff --git a/docs/gha-runner-setup.md b/docs/gha-runner-setup.md new file mode 100644 index 0000000000..3ec35c9fa8 --- /dev/null +++ b/docs/gha-runner-setup.md @@ -0,0 +1,118 @@ +# GitHub Actions Self-Hosted Runners + +Self-hosted runners for long training jobs (24+ hours) that exceed GitHub's 6-hour limit. + +## Runner Pool + +| Runner | Instance ID | Type | RAM | Disk | Swap | +|--------|-------------|------|-----|------|------| +| fleet-runner-1 | i-04a15158610df980f | t3.medium | 4GB | 50GB | 4GB | +| fleet-runner-2 | i-0f80c703294413a4c | t3.xlarge | 16GB | 50GB | 4GB | +| fleet-runner-3 | i-09321b67952c1a208 | t3.xlarge | 16GB | 50GB | 4GB | +| fleet-runner-4 | i-0f2ccaa840ef29450 | t3.medium | 4GB | 50GB | 4GB | +| fleet-runner-5 | i-05ecd98e74949dc87 | t3.xlarge | 16GB | 50GB | 4GB | + +> **Note:** Runners 1 and 4 are still t3.medium. They should be upgraded to t3.xlarge +> when idle (stop instance, change type, start instance). + +**Cost:** ~$120/month per t3.xlarge runner, ~$30/month per t3.medium runner + +[View runner status](https://github.com/fleet-ai/SkyRL/settings/actions/runners) + +## Auto-Recovery + +- **Watchdog:** Auto-restarts crashed/hung services (5-min timeout) +- **Health check:** Cron every 5 min detects GitHub disconnection +- **Central monitoring:** `runner-health.yaml` checks all runners every 15 min, auto-recovers unhealthy instances via EC2 stop/start, and alerts Slack + +The central health check (`runner-health.yaml`) will automatically attempt to recover unhealthy runners by stopping and restarting the EC2 instance. It tracks recovery attempts per instance and escalates to a manual intervention Slack alert after 2 failed attempts. + +## Installed Cloud Providers + +The setup script installs SkyPilot with cloud plugins and additional CLIs: + +| Provider | SkyPilot Plugin | Separate CLI | Config Location | +|----------|-----------------|--------------|-----------------| +| Lambda | ✓ | - | `~/.lambda_cloud/lambda_keys` | +| RunPod | ✓ | `runpod` | `~/.runpod/config.toml` | +| Vast | ✓ | - | `~/.config/vastai/vast_api_key` | +| Nebius | ✓ | - | `~/.nebius/credentials.json` | +| AWS | ✓ | `aws` | `~/.aws/credentials` | +| Prime Intellect | - | `prime` | via `prime config` | + +Credentials are configured during workflow runs from GitHub secrets. + +## Add a Runner + +```bash +# 1. Launch EC2 (copy config from existing runner) +aws ec2 run-instances --image-id ami-0c7217cdde317cfec --instance-type t3.xlarge --key-name gha-runner-key --security-group-ids sg-00fefd8181d51909d --subnet-id subnet-03879810067f57f85 --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":50,"VolumeType":"gp3"}}]' --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=fleet-runner-N}]' + +# 2. Get token +gh api -X POST repos/fleet-ai/SkyRL/actions/runners/registration-token --jq '.token' + +# 3. SSH and setup +curl -O https://raw.githubusercontent.com/fleet-ai/SkyRL/main/scripts/setup-gha-runner.sh +chmod +x setup-gha-runner.sh && ./setup-gha-runner.sh fleet-runner-N +``` + +## Troubleshooting + +**Runner offline:** + +The health check workflow will auto-recover most cases. If manual intervention is needed: + +```bash +# Stop/start the instance (reboot does NOT fix impaired instances): +aws ec2 stop-instances --instance-ids +aws ec2 wait instance-stopped --instance-ids +aws ec2 start-instances --instance-ids +``` + +> **Note:** `reboot-instances` does not work for instances with `impaired` status. Always use stop/start instead. + +**Check instance health:** +```bash +aws ec2 describe-instance-status --instance-ids i-04a15158610df980f i-0f80c703294413a4c i-09321b67952c1a208 i-0f2ccaa840ef29450 i-05ecd98e74949dc87 --query 'InstanceStatuses[].[InstanceId,InstanceStatus.Status]' --output table +``` + +**Manual service restart (via SSH):** +```bash +sudo systemctl restart actions.runner.fleet-ai-SkyRL.* +``` + +**Zombie Runner.Listener processes (checkout failures with "Missing file at path" or "file already exists"):** + +The default `KillMode=process` in the runner's systemd service only kills the parent `runsvc.sh` on restart, leaving `RunnerService.js` and `Runner.Listener` children orphaned. Multiple listeners fight over `_work/_temp/_runner_file_commands/`, causing checkout to fail. + +Fix: ensure `KillMode=control-group` is set in the override (the setup script does this automatically for new runners): +```bash +# Check current KillMode +grep KillMode /etc/systemd/system/actions.runner.fleet-ai-SkyRL.*.service + +# Fix if it says KillMode=process +sudo sed -i 's/KillMode=process/KillMode=control-group/' /etc/systemd/system/actions.runner.fleet-ai-SkyRL.*.service + +# Kill orphaned processes, clean state, restart +sudo pkill -9 -f 'actions-runner/bin/Runner.Listener' +sudo pkill -9 -f 'actions-runner/externals/node.*RunnerService' +rm -rf ~/actions-runner/_diag/pages/* ~/actions-runner/_work/_temp/* +sudo systemctl daemon-reload +sudo systemctl restart actions.runner.fleet-ai-SkyRL.* + +# Verify: should show exactly 1 listener +ps aux | grep Runner.Listener | grep -v grep | wc -l +``` + +**SkyPilot or RunPod not found (`sky: command not found` or `runpod: command not found`):** + +The setup script installs tools to `~/.local/bin/` and creates symlinks at `/usr/local/bin/`. If symlinks are missing, the GHA service can't find them: + +```bash +# Check if symlinks exist +ls -la /usr/local/bin/sky /usr/local/bin/runpod + +# If missing, create them (via EC2 Instance Connect): +aws ec2-instance-connect send-ssh-public-key --instance-id --instance-os-user ubuntu --ssh-public-key file://~/.ssh/.pub +ssh -i ~/.ssh/ ubuntu@ "sudo ln -sf /home/ubuntu/.local/bin/sky /usr/local/bin/sky && sudo ln -sf /home/ubuntu/.local/bin/runpod /usr/local/bin/runpod && sky --version" +``` diff --git a/docs/hybrid_flow.rst b/docs/hybrid_flow.rst deleted file mode 100644 index 1802cbadaf..0000000000 --- a/docs/hybrid_flow.rst +++ /dev/null @@ -1,269 +0,0 @@ -========================================================= -HybridFlow Programming Guide -========================================================= - -.. _vermouth: https://github.com/vermouth1992 - -Author: `Chi Zhang `_ - -verl is an open source implementation of the paper `HybridFlow `_ [1]_. In this section, we will introduce the basic concepts of HybridFlow, the motivation and how to program with verl APIs. - -Motivation and Design ------------------------- -We use dataflow to represent RL systems. [4]_. - -DataFlow -~~~~~~~~~~~~~~~~~~~~ - -Dataflow is an abstraction of computations. Neural Network training is a typical dataflow. It can be represented by computational graph. - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/dataflow.jpeg?raw=true - :alt: The dataflow graph from CS231n 2024 lecture 4 - -This figure [2]_ represents the computation graph of a polynomial function followed by a sigmoid function. In the data flow of neural network computation, each node represents an operator, and each edge represents the direction of forward/backward propagation. The computation graph determines the architecture of the neural network. - -RL as a dataflow problem -++++++++++++++++++++++++++++++++++++++++++++++ - -Reinforcement learning (RL) training can also be represented as a dataflow. Below is the dataflow graph that represents the PPO algorithm used in RLHF [3]_: - -.. image:: https://picx.zhimg.com/70/v2-cb8ab5ee946a105aab6a563e92682ffa_1440w.avis?source=172ae18b&biz_tag=Post - :alt: PPO dataflow graph, credit to Zhihu 低级炼丹师 - -However, the dataflow of RL has fundamental differences compared with dataflow of neural network training as follows: - -+--------------------------+--------------------------------------------------+---------------------+ -| Workload | Node | Edge | -+--------------------------+--------------------------------------------------+---------------------+ -| Neural Network Training | Operator (+/-/matmul/softmax) | Tensor movement | -+--------------------------+--------------------------------------------------+---------------------+ -| Reinforcement Learning | High-level operators (rollout/model forward) | Data Movement | -+--------------------------+--------------------------------------------------+---------------------+ - -In the case of tabular reinforcement learning, each operator is a simple scalar math operation (e.g., bellman update). In deep reinforcement learning(DRL), each operator is a high-level neural network computation such as model inference/update. This makes RL a two-level dataflow problem: - -- Control flow: defines how the high-level operators are executed (e.g., In PPO, we first perform rollout. Then, we perform advantage computation. Finally, we perform training). It expresses the **core logics of RL algorithms**. -- Computation flow: defines the dataflow of **neural network computation** (e.g., model forward/backward/optimizer). - - -Design Choices -~~~~~~~~~~~~~~~~~~~~ -The model size used in DRL before the LLM era is typically small. Thus, the high-level neural network computation can be done in a single process. This enables embedding the computation flow inside the control flow as a single process. - -However, in the LLM era, the computation flow (e.g., training neural network) becomes a multi-process program. This naturally leads to two design choices: - -1. Convert the control flow into a multi-process program as well. Then colocate with computation flow (unified multi-controller) - -- Advantages: - - - Achieves the **optimal performance** under fixed computation flow and control flow as the communication overhead in both training and data transfer is minimized. - -- Disadvantages: - - - The computation and/or control flow is **hard to reuse** from software perspective as computation code is coupled with specific controller code. For example, the training loop of PPO is generic. Say we have an PPO training flow implemented with a specific computation flow such as FSDP. Neither the control flow or computation flow can be reused if we want to switch the computation flow from FSDP to Megatron, due to the coupling of control and computation flows. - - Requires more efforts from the user under flexible and dynamic control flows, due to the multi-process nature of the program. - -2. Separate the flows: single process for the control flow and multi-process for computation flow - -- Advantages: - - - The computation flow defined elsewhere can be **easily reused** after the decoupling. - - The controller runs on a single process. Implementing a new RL algorithm with a **different control flow is simple and easy**. - -- Disadvantages: - - - Additional **data communication overhead** each time the controller process and computatation processes interact. The data has to be sent back and forth. - -In verl, the latter strategy with separate control flow and computation flow is adopted. verl is designed to decouple the control flow of RL algorithms, and the implementation of computation engines. - -Overall Execution Diagram -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is a simplified diagram denoting the execution of a reinforcement learning job. In the diagram, the controller runs on a single process, while the generator/actor workers, critic workers run on multiple processes, placed with specific resource groups. For rollout, the controller passes the data to the generator to perform sample generation. When the rollout is done, the data is passed back to controller for the next step of the algorithm. Similar execution is done for other workers. With the hybrid controller design, the data flow and computation is decoupled to provide both efficiency in computation and flexiblity in defining algorithm training loops. - -.. figure:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/driver_worker.png?raw=true - :alt: The execution diagram - -Codebase walkthrough (PPO) ------------------------------------------------- - -Entry function -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Code: https://github.com/volcengine/verl/blob/main/verl/trainer/main_ppo.py - -In this file, we define a remote function `main_task` that serves as the controller (driver) process as shown in the above figure. We also define a ``RewardManager``, where users can customize their reward function based on the data source in the dataset. Note that `RewardManager` should return the final token-level reward that is optimized by RL algorithms. Note that users can combine model-based rewards and rule-based rewards. -The ``main_task`` constructs a RayPPOTrainer instance and launch the fit. Note that ``main_task`` **runs as a single process**. - -We highly recommend that the ``main_task`` is NOT scheduled on the head of the ray cluster because ``main_task`` will consume a lot of memory but the head usually contains very few resources. - -Ray trainer -~~~~~~~~~~~~~~~~~~~~ -Code: https://github.com/volcengine/verl/blob/main/verl/trainer/ppo/ray_trainer.py - -The RayPPOTrainer manages - -- Worker and WorkerGroup construction -- Runs the main loop of PPO algorithm - -Note that, the fit function of RayPPOTrainer **runs as a single process**. - -Worker and WorkerGroup construction -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Each workerGroup manages a list of workers that runs remotely. Note that the worker group runs in the process of its construtor. -Each worker inside the WorkerGroup runs on a GPU. The worker group serves as a proxy for the controller process to interact with a list of workers, in order to perform certain computations. **In order to do so, we have to bind the methods of the worker into the method of the WorkerGroup and define the data dispatch and data collection**. This is done via simple decoration that will be introduced in the Worker definition section. - -For example, in PPO, we define 3 worker groups: - -- ActorRolloutRef: manages actor, rollout and reference policy. ActorRolloutRefWorker can be instantiated as a single actor, a single rollout, a single reference policy, a combined actor/rollout or a combined actor/rollout/ref. This design is aimed for the maximum code reuse in various scenarios. The reason for colocating actor and rollout is for fast weight transfer using nccl. The reason for coloating actor and reference is to implement an efficient lora PPO as the reference policy is simply the base model of PPO in lora. -- Critic: manages the critic model -- Reward: manages the reward model - -The worker group will be constructed on the resource pool it designates. The resource pool is a set of GPUs in the ray cluster. - -Worker definition -~~~~~~~~~~~~~~~~~~~~ - -.. _ActorRolloutRefWorker: https://github.com/volcengine/verl/blob/main/verl/workers/fsdp_workers.py - -We take `ActorRolloutRefWorker `_ for an example. -The APIs it should expose to the controller process are: - -- init_model: build the underlying model -- generate_sequences: given prompts, generate responses -- compute_log_prob: compute the log-probability of a generated sequence using actor -- compute_ref_log_prob: compute the log-probability of a generated sequence using reference policy -- save_checkpoint: save the checkpoint - -Note that these methods are defined in the worker that can only be invoked via remote calls. For example, if the controller process wants to initialize the model, it has to call - -.. code-block:: python - - for worker in actor_rollout_ref_wg: - worker.init_model.remote() - -If the controller process wants to generate sequences, it has to call - -.. code-block:: python - - data = xxx - # split the data into dp chunks - data_dp_lst = data.split(dp_size) - output_dp_lst = [] - for i, worker in enumerate(actor_rollout_ref_wg): - output_future = worker.generate_sequences.remote(data_dp_lst[i]) - output_dp_lst.append(output_future) - output = torch.cat(ray.get(output_dp_lst), dim=0) - -We observe that controll process calling worker group methods in general can be divided into 3 parts: - -- Split the data into data parallel sizes -- Dispatch the corresponding data into each worker -- Collect and concatenate the data when the computation finishes - -In verl, we design a syntax sugar to encapsulate the 3 processes into a single call from the controller process. - -.. code-block:: python - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def generate_sequences(data): - ... - - # on the driver - output = actor_rollout_ref_wg.generate_sequences(data) - -We decorate the method of the worker with a ``register`` that explicitly defines how the input data should be splitted and dispatch to each worker, and how the output data should be collected and concatenated by the controller. For example, ``Dispatch.DP_COMPUTE_PROTO`` splits the input data into dp chunks, dispatch each data to each worker, collect the output and concatenate the results. Note that this function requires the input and output to be a DataProto defined here (https://github.com/volcengine/verl/blob/main/verl/protocol.py). - - -PPO main loop -~~~~~~~~~~~~~~~~~~~~ -With the aforementioned APIs, we can implement the main loop of PPO as if it is a single process program - -.. code-block:: python - - for prompt in dataloader: - output = actor_rollout_ref_wg.generate_sequences(prompt) - old_log_prob = actor_rollout_ref_wg.compute_log_prob(output) - ref_log_prob = actor_rollout_ref_wg.compute_ref_log_prob(output) - values = critic_wg.compute_values(output) - rewards = reward_wg.compute_scores(output) - # compute_advantages is running directly on the control process - advantages = compute_advantages(values, rewards) - output = output.union(old_log_prob) - output = output.union(ref_log_prob) - output = output.union(values) - output = output.union(rewards) - output = output.union(advantages) - # update actor - actor_rollout_ref_wg.update_actor(output) - critic.update_critic(output) - -Takeaways -~~~~~~~~~~~~~~~~~~~~ -- This programming paradigm enables users to use different computation backend without modification of the control process. -- This programming paradigm enables flexible placement (by changing the mapping of WorkerGroup and ResourcePool) without modification of the control process. - -Repository organization ------------------------------------------------- - -Important code files in the repository are organized as below: - -.. code-block:: bash - - verl # the verl package - trainer - main_ppo.py # the entrypoint for RL training - ppo - ray_trainer.py # the training loop for RL algorithms such as PPO - fsdp_sft_trainer.py # the SFT trainer with FSDP backend - config - generation.yaml # configuration template for rollout - ppo_trainer.yaml # configuration template for the RL trainer - workers - protocol.py # the interface of DataProto - fsdp_workers.py # the FSDP worker interfaces: ActorRolloutRefWorker, CriticWorker, RewardModelWorker - megatron_workers.py # the Megatron worker interfaces: ActorRolloutRefWorker, CriticWorker, RewardModelWorker - actor - dp_actor.py # data parallel actor with FSDP backend - megatron_actor.py # nD parallel actor with Megatron backend - critic - dp_critic.py # data parallel critic with FSDP backend - megatron_critic.py # nD parallel critic with FSDP backend - reward_model - megatron - reward_model.py # reward model with Megatron backend - rollout - vllm - vllm_rollout.py # rollout with vllm backend - hf_rollout.py # rollout with huggingface TGI backend - sharding_manager - fsdp_ulysses.py # data and model resharding when using FSDP + ulysses - fsdp_vllm.py # data and model resharding when using FSDP + ulysses + vllm - megatron_vllm.py # data and model resharding when using Megatron + vllm - utils - dataset # datasets for SFT/RM/RL - reward_score # function based reward - gsm8k.py # reward function for gsm8k dataset - math.py # reward function for math dataset - seqlen_balancing.py # the sequence balance optimization - models - llama # Megatron implementation for llama, deepseek, mistral, etc - transformers # ulysses integration with transformer models such as llama, qwen, etc - weight_loader_registery.py # registry of weight loaders for loading hf ckpt into Megatron - third_party - vllm # adaptor for vllm's usage in RL - vllm_v_0_6_3 # vllm v0.6.3 adaptor - llm.py # entrypoints for generate, sync_model_weight, offload_model_weights - parallel_state.py # vllm related device mesh and process groups - dtensor_weight_loaders.py # weight loader for huggingface models with FSDP - megatron_weight_loaders.py # weight loader for Megatron models - vllm_spmd # vllm >= v0.7 adaptor (coming soon) - examples # example scripts - tests # integration and unit tests - .github # the configuration of continuous integration tests - - -.. [1] HybridFlow: A Flexible and Efficient RLHF Framework: https://arxiv.org/abs/2409.19256v2 -.. [2] Data flow graph credit to CS231n 2024 lecture 4: https://cs231n.stanford.edu/slides/2024/lecture_4.pdf -.. [3] PPO dataflow graph credit to 低级炼丹师 from Zhihu​: https://zhuanlan.zhihu.com/p/635757674 -.. [4] RLFlow \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 9562e13548..0000000000 --- a/docs/index.rst +++ /dev/null @@ -1,121 +0,0 @@ -Welcome to verl's documentation! -================================================ - -.. _hf_arxiv: https://arxiv.org/pdf/2409.19256 - -verl is a flexible, efficient and production-ready RL training framework designed for large language models (LLMs) post-training. It is an open source implementation of the `HybridFlow `_ paper. - -verl is flexible and easy to use with: - -- **Easy extension of diverse RL algorithms**: The Hybrid programming model combines the strengths of single-controller and multi-controller paradigms to enable flexible representation and efficient execution of complex Post-Training dataflows. Allowing users to build RL dataflows in a few lines of code. - -- **Seamless integration of existing LLM infra with modular APIs**: Decouples computation and data dependencies, enabling seamless integration with existing LLM frameworks, such as PyTorch FSDP, Megatron-LM and vLLM. Moreover, users can easily extend to other LLM training and inference frameworks. - -- **Flexible device mapping and parallelism**: Supports various placement of models onto different sets of GPUs for efficient resource utilization and scalability across different cluster sizes. - -- Ready integration with popular HuggingFace models - - -verl is fast with: - -- **State-of-the-art throughput**: By seamlessly integrating existing SOTA LLM training and inference frameworks, verl achieves high generation and training throughput. - -- **Efficient actor model resharding with 3D-HybridEngine**: Eliminates memory redundancy and significantly reduces communication overhead during transitions between training and generation phases. - --------------------------------------------- - -.. _Contents: - -.. toctree:: - :maxdepth: 5 - :caption: Quickstart - - start/install - start/quickstart - start/multinode - -.. toctree:: - :maxdepth: 4 - :caption: Programming guide - - hybrid_flow - -.. toctree:: - :maxdepth: 5 - :caption: Data Preparation - - preparation/prepare_data - preparation/reward_function - -.. toctree:: - :maxdepth: 5 - :caption: Configurations - - examples/config - -.. toctree:: - :maxdepth: 2 - :caption: PPO Example - - examples/ppo_code_architecture - examples/gsm8k_example - -.. toctree:: - :maxdepth: 1 - :caption: PPO Trainer and Workers - - workers/ray_trainer - workers/fsdp_workers - workers/megatron_workers - -.. toctree:: - :maxdepth: 1 - :caption: Performance Tuning Guide - - perf/perf_tuning - README_vllm0.8.md - -.. toctree:: - :maxdepth: 1 - :caption: Experimental Results - - experiment/ppo - -.. toctree:: - :maxdepth: 1 - :caption: Advance Usage and Extension - - advance/placement - advance/dpo_extension - advance/fsdp_extension - advance/megatron_extension - advance/checkpoint - -.. toctree:: - :maxdepth: 1 - :caption: API References - - data.rst - - -.. toctree:: - :maxdepth: 1 - :caption: FAQ - - faq/faq - -Contribution -------------- - -verl is free software; you can redistribute it and/or modify it under the terms -of the Apache License 2.0. We welcome contributions. -Join us on `GitHub `_, `Slack `_ and `Wechat `_ for discussions. - -Code formatting -^^^^^^^^^^^^^^^^^^^^^^^^ -We use yapf (Google style) to enforce strict code formatting when reviewing MRs. Run yapf at the top level of verl repo: - -.. code-block:: bash - - pip3 install yapf - yapf -ir -vv --style ./.style.yapf verl examples tests diff --git a/docs/perf/perf_tuning.rst b/docs/perf/perf_tuning.rst deleted file mode 100644 index fa2d416d69..0000000000 --- a/docs/perf/perf_tuning.rst +++ /dev/null @@ -1,164 +0,0 @@ -Performance Tuning Guide -============================== - -Author: `Guangming Sheng `_ - -In this section, we will discuss how to tune the performance of all the stages in verl, including: - -1. Rollout generation throughput. - -2. Enable ``use_remove_padding=True`` for sequence packing (i.e., data packing and remove padding). - -3. Batch size tuning for forward and backward computation - -4. Enable ``use_dynamic_bsz=True`` for higher throughput. - -5. Utilize Ulysses Sequence Parallel for Long Context Training - -6. LigerKernel for SFT performance optimization - -Rollout Generation Tuning --------------------------- - -verl currently supports two rollout backends: vLLM and TGI (with SGLang support coming soon). - -Below are key factors for tuning vLLM-based rollout. Before tuning, we recommend setting ``actor_rollout_ref.rollout.disable_log_stats=False`` so that rollout statistics are logged. - -- Increase ``gpu_memory_utilization``. The vLLM pre-allocates GPU KVCache by using gpu_memory_utilization% of the remaining memory. - However, if model parameters and optimizer states are not offloaded, using too high a fraction can lead to OOM. - A value between 0.5 and 0.7 often strikes a good balance between high throughput and avoiding OOM. - -- Adjust ``max_num_seqs`` or ``max_num_batched_tokens``. - If the GPU cache utilization is relatively low in the log, increase ``max_num_seqs`` or ``max_num_batched_tokens`` - can enlarge the effective batch size in the decoding stage, allowing more concurrent requests per batch. - We recommend setting ``max_num_batched_tokens > 2048`` for higher throughput. - -- Use a smaller ``tensor_parallel_size``. - When GPU resources allow, a smaller tensor parallel size spawns more vLLM replicas. - Data parallelism (DP) can yield higher throughput than tensor parallelism (TP), but also increases KVCache consumption. - Carefully balance the trade-off between more replicas and higher memory usage. - Our experient in Sec. 8.4 of `HybridFlow paper `_ evaluate this trade-off. - -More tuning details such as dealing with Preemption and Chunked-prefill -can be found in `vLLM official tuning guide `_ - -The performance of vllm can be further increased if upgrading from v0.6.3 to v0.7. See https://github.com/volcengine/verl/blob/main/docs/README_vllm0.7.md for details on how to upgrade. - -Enable remove padding (sequence packing) ------------------------------------------ - -Currently, for llama, mistral, gemma1 and qwen based models, users can enable `use_remove_padding=True` to utilize the -sequence packing implementation provided by transformers library. - -For other models, transformers library may also support it but we haven't tested it yet. -Users can add the desired model config to the `test_transformer.py `_ file. -And test its functionaility by running the following command: - -.. code-block:: bash - - pytest -s tests/model/test_transformer.py - -If the test passes, you can add your desired model into the model `registry.py `_ file. -Then, you can enjoy the performance boost of sequence packing -and welcome to PR your tested model to verl! - - -Batch Size Tuning ------------------ - -To achieve higher throughput in experience preparation (i.e., model fwd) and model update (i.e., actor/critic fwd/bwd), -users may need to tune the ``*micro_batch_size_per_gpu`` for different computation. - -In verl, the core principle for setting batch sizes is: - -- **Algorithmic metrics** (train batch size, PPO mini-batch size) are *global* (from a single-controller perspective), - normalized in each worker. See the `normalization code `_. - -- **Performance-related parameters** (micro batch size, max token length for dynamic batch size) are *local* parameters that define the per-GPU data allocations. - See the `normalization code `_. - -.. note:: In your training script, please use ``*micro_batch_size_per_gpu`` instead of ``*micro_batch_size``. - So that you don't need to consider the normalization of the ``micro_batch_size`` and ``micro_batch_size`` will be deprecated. - -Batch Size Tuning tips -"""""""""""""""""""""" - -Therefore, users may need to tune the ``*micro_batch_size_per_gpu`` to accelerate training. Here're some tips: - -1. **Enable gradient checkpointing**: - Set ``actor_rollout_ref.model.enable_gradient_checkpointing=True`` and ``critic.model.enable_gradient_checkpointing=True``. - This often allows for larger micro-batch sizes and will be beneficial for large mini-batch training. - -2. Increase the ``*micro_batch_size_per_gpu`` as much as possible till equals to normalized ``mini_batch_size``. - -3. **Use larger forward-only parameters**: - Forward only parameter, such as ``actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu``, - ``actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu``, ``critic.forward_micro_batch_size_per_gpu`` could be larger (e.g., 2x) than training related micro batch sizes, - such as ``actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu``, ``critic.ppo_micro_batch_size_per_gpu``. - -4. **Allow larger micro-batch sizes for Critic and Reward models**: - micro batch size of Critic and Reward model could be larger than Actor model. This is because the actor model has much larger vocab size in the final layer. - - -Tuning for Dynamic Batch Size ------------------------------ - -Dynamic batch size is a technique that allows the model to process similar number of tokens in a single forward pass (with different actual batch sizes). -This can significantly improve the training efficiency and reduce the memory usage. - -To utilize this technique, users can set ``use_dynamic_bsz=True`` in actor, ref, critic and reward models. -With ``use_dynamic_bsz=True``, users don't need to tune ``*micro_batch_size_per_gpu``. -Instead, users should tune the following parameters: - -- ``actor_rollout_ref.actor.ppo_max_token_len_per_gpu``, ``critic.ppo_max_token_len_per_gpu``: - The maximum number of tokens to be processed in fwd and bwd of ``update_policy`` and ``update_critic``. - -- ``actor_rollout_ref.ref.log_prob_max_token_len_per_gpu`` and ``actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu``: - The maximum number of tokens to be processed in a the fwd computation of ``compute_log_prob`` and ``comptue_ref_log_prob``. - -- ``critic.forward_micro_batch_size_per_gpu``, ``reward_model.forward_micro_batch_size_per_gpu``: - The maximum number of tokens to be processed in a the fwd computation of ``compute_values``, ``compute_rm_score``. - -Dynamic Batch Size Tuning tips -"""""""""""""""""""""""""""""" - -Here're some tips to tune the above parameters: - -1. **Increase** ``actor_rollout_ref.actor.ppo_max_token_len_per_gpu`` - Make it at least 2 x (max_prompt_length + max_response_length). We set it to 3x in `run_qwen2-7b_rm_seq_balance.sh `_. - Try to increase it to get higher throughput. - -2. **Forward-only parameters can be larger**: - Similar to the non-dynamic-batch scenario, forward-only token limits can exceed those used in forward/backward operations. - -3. **Use larger limits for Critic and Reward models**: - Critic and Reward parameters can be set at least 2× the Actor’s limits. For instance, we set them to 4× here: - `run_qwen2-7b_rm_seq_balance.sh `_ - -.. :math:`\text{critic.ppo_max_token_len_per_gpu} = 2 \times \text{actor.ppo_max_token_len_per_gpu})`. - -Ulysses Sequence Parallel for Long Context Training ----------------------------------------------------- - -To utilize this technique, users can set ``ulysses_sequence_parallel_size>1`` in actor, ref, critic and reward models. - -We support different model utilize different ulysses_sequence_parallel_size sizes. - -To train log sequence (>32k), users may need to decrease the ``*micro_batch_size_per_gpu`` and ``*max_token_len_per_gpu`` to avoid OOM. - -LigerKernel for SFT ----------------------- - -LigerKernel is a high-performance kernel for Supervised Fine-Tuning (SFT) that can improve training efficiency. To enable LigerKernel in your SFT training: - -1. Install liger-kernel via ``pip3 install liger-kernel``. In your SFT configuration file (e.g., ``verl/trainer/config/sft_trainer.yaml``), set the ``use_liger`` parameter: - - .. code-block:: yaml - - model: - use_liger: True # Enable LigerKernel for SFT - -2. The default value is ``False``. Enable it only when you want to use LigerKernel's optimizations. - -3. LigerKernel is particularly useful for improving training performance in SFT scenarios. - diff --git a/docs/preparation/prepare_data.rst b/docs/preparation/prepare_data.rst deleted file mode 100644 index de88d8d91a..0000000000 --- a/docs/preparation/prepare_data.rst +++ /dev/null @@ -1,126 +0,0 @@ -Prepare Data for Post-Training -======================================== - -Before starting the post-training job, we need to prepare the data for -the policy training. The data should be stored in the parquet format. - -We provide several data preprocess scripts for different datasets, -including GSM8K, MATH, HelloSwag, Full_hh_rlhf. To prepare other datasets, we need -to follow the following steps: The data preprocess script can be divided -into two parts: - -1. The first part is the common part, which loads the dataset from - huggingface's ``datasets`` package. Then preprocess the datasets with - the ``make_map_fn`` and then store in the parquet format. - -.. code:: python - - import re - import os - import datasets - - from verl.utils.hdfs_io import copy, makedirs - import argparse - - # To extract the solution for each prompts in the dataset - # def extract_solution(solution_str): - # ... - - - if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--local_dir', default='/opt/tiger/gsm8k') - parser.add_argument('--hdfs_dir', default=None) - - args = parser.parse_args() - - num_few_shot = 5 - data_source = 'openai/gsm8k' - - dataset = datasets.load_dataset(data_source, 'main') - - train_dataset = dataset['train'] - test_dataset = dataset['test'] - - # Construct a `def make_map_fn(split)` for the corresponding datasets. - # ... - - train_dataset = train_dataset.map(function=make_map_fn('train'), with_indices=True) - test_dataset = test_dataset.map(function=make_map_fn('test'), with_indices=True) - - local_dir = args.local_dir - hdfs_dir = args.hdfs_dir - - train_dataset.to_parquet(os.path.join(local_dir, 'train.parquet')) - test_dataset.to_parquet(os.path.join(local_dir, 'test.parquet')) - - makedirs(hdfs_dir) - - copy(src=local_dir, dst=hdfs_dir) - -2. The users are required to implement the ``make_map_fn()`` function - (as well as the ``extract_solution``) on their own to support - different datasets or tasks. - -We already implemented the data preprocess of GSM8k, MATH, Hellaswag and Full_hh_rlhf -datasets. And we take the GSM8k dataset as an example: - -**GSM8K** - -In the ``make_map_fn``, each data field should consist of the following -5 fields: - -1. ``data_source``: The name of the dataset. To index the corresponding - reward function in the ``RewardModule`` -2. ``prompt``: This field should be constructed in the format of - huggingface chat_template. The tokenizer in ``RLHFDataset`` will - apply chat template and tokenize the prompt. -3. ``ability``: Define the task category. -4. ``reward_model``: Currently, we only utilize the ``ground_truth`` - field during evaluation. The ``ground_truth`` is computed by the - ``extract_solution`` function. **NOTED** that the implementation of - the corresponding reward function should align with this extracted - ``ground_truth``. -5. ``extra_info``: Record some information of the current prompt. Not - use for now. - -.. code:: python - - def extract_solution(solution_str): - solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) # extract the solution after #### - assert solution is not None - final_solution = solution.group(0) - final_solution = final_solution.split('#### ')[1].replace(',', '') - return final_solution - - instruction_following = "Let's think step by step and output the final answer after \"####\"." - - # add a row to each data item that represents a unique id - def make_map_fn(split): - - def process_fn(example, idx): - question = example.pop('question') - - question = question + ' ' + instruction_following - - answer = example.pop('answer') - solution = extract_solution(answer) - data = { - "data_source": data_source, - "prompt": [{ - "role": "user", - "content": question - }], - "ability": "math", - "reward_model": { - "style": "rule", - "ground_truth": solution - }, - "extra_info": { - 'split': split, - 'index': idx - } - } - return data - - return process_fn diff --git a/docs/preparation/reward_function.rst b/docs/preparation/reward_function.rst deleted file mode 100644 index 431ea7a8dd..0000000000 --- a/docs/preparation/reward_function.rst +++ /dev/null @@ -1,69 +0,0 @@ -Implement Reward Function for Dataset -====================================== - -For each dataset, we need to implement a reward function or utilize a reward model to compute the rewards for the generated responses. -We already pre-implemented some reward functions in `reward_score directory `_. -You can also use customized reward functions. - -Currently, we support reward functions for GSM8k and MATH datasets. For RLHF datasets (e.g., -full_hh_rlhf) and Code Generation (e.g., APPS), we utilize reward model -and SandBox (will opensource soon) for evaluation respectively. - -RewardManager -------------- - -In the entrypoint of the PPO Post-Training script `main_ppo.py `_, -we implement a ``RewardManager`` that utilze pre-implemented reward functions to compute the scores for each response. - -In the ``RewardManager``, we implemented a ``__call__`` function to -compute the score for each response. -All the reward functions are executed by ``compute_score_fn``. -The input is a ``DataProto``, which includes: - -- ``input_ids``, ``attention_mask``: ``input_ids`` and ``attention_mask`` after applying - chat_template, including prompt and response -- ``responses``: response tokens -- ``ground_truth``: The ground truth string of the current prompt. - Stored in ``non_tensor_batch`` in the ``DataProto``, which should be - preprocessed in the parquet files. -- ``data_source``: The dataset name of the current prompt. Stored in - ``non_tensor_batch`` in the ``DataProto``, which should be - preprocessed in the parquet files. - -After detokenize the responses, the responses string and the ground -truth string will be input to the ``compute_score_fn`` to compute the -score for each response. - -Reward Functions ----------------- - -Pre-implemented -~~~~~~~~~~~~~~~ - -We already pre-implemented some reward functions in `reward_score directory `_. - -- In the `GSM8k example `_, we - force the response to output the final answer after four ####, then - use string matching to compare with the ground truth. If completely - correct, score 1 point; if the format is correct, score 0.1 points; if - the format is incorrect, score 0 points. -- In the `MATH example `_, we follow - the implementation in `lm-evaluation-harness repository `_. - -Customized -~~~~~~~~~~ - -You can implement customized reward functions in a separate file and specify them using ``custom_reward_function.path`` and ``custom_reward_function.name``. For the set of them, please refer to :ref:`config-explain-page`. - -The parameters of your reward function should be ``data_source``, ``solution_str``, ``ground_truth``, and ``extra_info``. -For example: - -.. code:: python - - def my_reward_fn(data_source, solution_str, ground_truth, extra_info=None): - return len(solution_str)/100 - -If you are testing only a single customized reward function, you can simply name it 'compute_score' and leave ``custom_reward_function.name`` unset. - -To run multiple tests with different customized reward functions, you can modify both ``custom_reward_function.path`` and ``custom_reward_function.name`` for each trial. -For instance, you might create a single `my_reward.py` file and implement multiple reward functions within it. This way, for different trials, you only need to adjust ``custom_reward_function.name``, making it more convenient to conduct multiple tests within scripts. \ No newline at end of file diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt deleted file mode 100644 index 49ecc0af40..0000000000 --- a/docs/requirements-docs.txt +++ /dev/null @@ -1,12 +0,0 @@ -# markdown suport -recommonmark -# markdown table suport -sphinx-markdown-tables - -# theme default rtd - -# crate-docs-theme -sphinx-rtd-theme - -# pin tokenizers version to avoid env_logger version req -tokenizers==0.19.1 diff --git a/docs/start/install.rst b/docs/start/install.rst deleted file mode 100644 index d288a02793..0000000000 --- a/docs/start/install.rst +++ /dev/null @@ -1,217 +0,0 @@ -Installation -============ - -Requirements ------------- - -- **Python**: Version >= 3.9 -- **CUDA**: Version >= 12.1 - -verl supports various backends. Currently, the following configurations are available: - -- **FSDP** and **Megatron-LM** (optional) for training. -- **SGLang** (preview), **vLLM** and **TGI** for rollout generation. - -Choices of Backend Engines ----------------------------- - -1. Training: - -We recommend using **FSDP** backend to investigate, research and prototype different models, datasets and RL algorithms. The guide for using FSDP backend can be found in :doc:`FSDP Workers<../workers/fsdp_workers>`. - -For users who pursue better scalability, we recommend using **Megatron-LM** backend. Currently, we support Megatron-LM v0.11 [1]_. The guide for using Megatron-LM backend can be found in :doc:`Megatron-LM Workers<../workers/megatron_workers>`. - -.. note:: - - We are announcing the direct support of GPTModel, without need to implement your own model any more. Also it's easy to use TransformerEngine's support for even higher performance. - The main branch of verl has enabled this feature, but may not stable enough. If you encounter issues, please feel free to report and try `0.3.x branch `_ instead. - -2. Inference: - -For inference, the integration of both vllm v0.6.3 and v0.8.2 is stable. For huggingface TGI integration, it is usually used for debugging and single GPU exploration. Regarding sglang integration, it is blazing fast and under rapid development - we release it as a preview feature and please give us feedbacks. - - -Install from docker image -------------------------- - -We provide pre-built Docker images for quick setup. For SGLang usage, please follow the later sections in this doc. - -Image and tag: ``whatcanyousee/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te2.0-megatron0.11.0-v0.0.6``. See files under ``docker/`` for NGC-based image or if you want to build your own. - -1. Launch the desired Docker image: - -.. code:: bash - - docker run --runtime=nvidia -it --rm --shm-size="10g" --cap-add=SYS_ADMIN -v - - -2. Inside the container, install verl: - -.. code:: bash - - # install the nightly version (recommended) - git clone https://github.com/volcengine/verl && cd verl && pip3 install -e . - # or install from pypi via `pip3 install verl` - -.. note:: - - The Docker image is built with the following configurations: - - - **PyTorch**: 2.4.0+cu124 - - **CUDA**: 12.4 - - **Megatron-LM**: core_r0.11.0 - - **vLLM**: 0.6.3 - - **Ray**: 2.10.0 - - **TransformerEngine**: 2.0.0+754d2a0 - - Now verl has been **compatible to Megatron-LM core_r0.11.0**, and there is **no need to apply patches** to Megatron-LM. Also, the image has integrated **Megatron-LM core_r0.11.0**, located at ``/opt/nvidia/Meagtron-LM``. One more thing, because verl only use ``megatron.core`` module for now, there is **no need to modify** ``PATH`` if you have installed Megatron-LM with this docker image. - - -Install verl-SGLang from scratch ---------------------------------------------- - -If you want to use SGLang instead of vllm for inference, please follow the instruction here. SGLang has largely support the rearch and inference workload at xAI. For verl-sglang installation, ignore the version conflicts reported by pip with vllm. And, SGLang support native API for RLHF, do not need to patch a single line of code. - -The following steps are quick installation guide for verl-SGLang. - -.. code:: bash - - # Create a virtual environment and use uv for quick installation - python3 -m venv ~/.python/verl-sglang && source ~/.python/verl-sglang/bin/activate - python3 -m pip install --upgrade pip && python3 -m pip install --upgrade uv - - # Install verl-SGLang - git clone https://github.com/volcengine/verl verl-sglang && cd verl-sglang - python3 -m uv pip install . - - # Install the latest stable version of sglang with verl support, currently, the latest version is 0.4.4.post3 - # For SGLang installation, you can also refer to https://docs.sglang.ai/start/install.html - python3 -m uv pip install "sglang[all]==0.4.4.post3" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python - - -Install from custom environment ---------------------------------------------- - -If you do not want to use the official docker image, here is how to start from your own environment. To manage environment, we recommend using conda: - -.. code:: bash - - conda create -n verl python==3.10 - conda activate verl - -For installing the latest version of verl, the best way is to clone and -install it from source. Then you can modify our code to customize your -own post-training jobs. - -.. code:: bash - - # install verl together with some lightweight dependencies in setup.py - pip3 install torch==2.4.0 --index-url https://download.pytorch.org/whl/cu124 - pip3 install flash-attn --no-build-isolation - git clone https://github.com/volcengine/verl.git - cd verl - pip3 install -e . - - -Megatron is optional. It's dependencies can be setup as below: - -.. code:: bash - - # apex - pip3 install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" \ - git+https://github.com/NVIDIA/apex - - # transformer engine - pip3 install git+https://github.com/NVIDIA/TransformerEngine.git@stable - # megatron core - pip3 install megatron-core==0.11.0 - - -Install with AMD GPUs - ROCM kernel support ------------------------------------------------------------------- - -When you run on AMD GPUs (MI300) with ROCM platform, you cannot use the previous quickstart to run verl. You should follow the following steps to build a docker and run it. - -If you encounter any issues in using AMD GPUs running verl, feel free to contact me - `Yusheng Su `_. - -Find the docker for AMD ROCm: `docker/Dockerfile.rocm `_ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - # Build the docker in the repo dir: - # docker build -f docker/Dockerfile.rocm -t verl-rocm:03.04.2015 . - # docker images # you can find your built docker - FROM rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 - - # Set working directory - # WORKDIR $PWD/app - - # Set environment variables - ENV PYTORCH_ROCM_ARCH="gfx90a;gfx942" - - # Install vllm - RUN pip uninstall -y vllm && \ - rm -rf vllm && \ - git clone -b v0.6.3 https://github.com/vllm-project/vllm.git && \ - cd vllm && \ - MAX_JOBS=$(nproc) python3 setup.py install && \ - cd .. && \ - rm -rf vllm - - # Copy the entire project directory - COPY . . - - # Install dependencies - RUN pip install "tensordict<0.6" --no-deps && \ - pip install accelerate \ - codetiming \ - datasets \ - dill \ - hydra-core \ - liger-kernel \ - numpy \ - pandas \ - datasets \ - peft \ - "pyarrow>=15.0.0" \ - pylatexenc \ - "ray[data,train,tune,serve]" \ - torchdata \ - transformers \ - wandb \ - orjson \ - pybind11 && \ - pip install -e . --no-deps - -Build the image: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - docker build -t verl-rocm . - -Launch the container -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - docker run --rm -it \ - --device /dev/dri \ - --device /dev/kfd \ - -p 8265:8265 \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v $HOME/.ssh:/root/.ssh \ - -v $HOME:$HOME \ - --shm-size 128G \ - -w $PWD \ - verl-rocm \ - /bin/bash - -(Optional): If you do not want to root mode and require assign yuorself as the user -Please add ``-e HOST_UID=$(id -u)`` and ``-e HOST_GID=$(id -g)`` into the above docker launch script. - -(Currently Support): Training Engine: FSDP; Inference Engine: vLLM - We will support Megatron and SGLang in the future. diff --git a/docs/start/multinode.rst b/docs/start/multinode.rst deleted file mode 100644 index cd4742c2c9..0000000000 --- a/docs/start/multinode.rst +++ /dev/null @@ -1,422 +0,0 @@ -Multinode Training -================== - -.. _wuxibin89: https://github.com/wuxibin89 - -Author: `Xibin Wu `_, `Yusheng Su `_. - -Manual ------- - -Set up multinode ray cluster -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -1. Start head node with ``ray start --head --dashboard-host=0.0.0.0``, there're 2 address you should care about: - -- GCS address: ``ray start --address=
``, where worker node should connect to. -- Dashboard address: ``
:8265``, where you should submit job to the cluster. - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/head.png?raw=true - -2. Start worker node with ``ray start --address=
`` you get above. - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/worker.png?raw=true - -3. Now you should see the cluster have 2 nodes with ``ray status``. - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/status.png?raw=true - -4. Additionally, you can access dashboard in the browser with the address you get above. - -*Firewall rules maybe need configure to access the dashboard, if there's any trouble, please contact your network administrator.* - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/overview.png?raw=true - -Submit job to ray cluster -~~~~~~~~~~~~~~~~~~~~~~~~~ -1. Submit ray job to cluster with the dashboard address you get above. - -.. code-block:: bash - - ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env=verl/trainer/runtime_env.yaml \ - --no-wait \ - -- \ - python3 -m verl.trainer.main_ppo \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=2 \ - ... - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/submit.png?raw=true - -2. Then you can check the job status with the following commands: - -- ray job list: list all jobs submitted to the cluster. -- ray job logs : query the logs of the job. -- ray job status : query the status of the job. -- ray job stop : request the job to be stopped. - -3. You can also access driver/task/actor logs in ``/tmp/ray/session_latest/logs/``, driver log is ``job-driver-raysubmit_.log``. - -4. We strongly recommend you to view job detail from dashboard in multinode training, because it provide more structure way to view the job information. - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/job.png?raw=true -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/job_detail.png?raw=true - - -Slurm ------ -TBD - -How to debug? ---------------------- - -Legacy Ray Debugger -~~~~~~~~~~~~~~~~~~~ -1. Ray has a builtin legacy `debugger `_ that allows you to debug your distributed applications. To enable debugger, start ray cluster with ``RAY_DEBUG=legacy`` and ``--ray-debugger-external``. - -.. code-block:: bash - - # start head node - RAY_DEBUG=legacy ray start --head --dashboard-host=0.0.0.0 --ray-debugger-external - # start worker node - RAY_DEBUG=legacy ray start --address='10.124.46.192:6379' --ray-debugger-external - -2. Set up breakpoint in your code, and submit job to cluster. Then run ``ray debug`` to wait breakpoint: - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/legacy.png?raw=true - -Ray Distributed Debugger VSCode Extension -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. Starting with Ray 2.39, Anyscale introduce a new `Ray Distributed Debugger `_ VSCode extension. Please follow the instruction to install the extension, and then add cluster with the dashboard address you get above. - -*NOTE: Don't forget remove RAY_DEBUG=legacy and --ray-debugger-external in ray start* - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/debugger.png?raw=true - -2. Set up breakpoint in your code, and submit job to cluster. Then the extension will show the breakpoint information. - -.. image:: https://github.com/eric-haibin-lin/verl-community/blob/main/docs/ray/breakpoint.png?raw=true - - - - - - - - - -Multi-node training on AMD clusters ---------------------------------------------------------------------------------------- - -If you want to run multi-node training with slurm with Docker/Podman container on AMD Cluster, you can use the following script. - -If you encounter any issues in using AMD GPUs running verl, please contact `Yusheng Su `_. - -.. note:: - 1. You need to use ``podman`` or ``docker`` in the following script. We will release the apptainer script later. - 2. If you want to use ``podman``, you just replace ``docker`` with ``podman`` in the following script. - -The script includes the following steps: - -1. SLURM Configuration -2. Environment Setup -3. Docker/Podman Container Setup -4. Ray Cluster Initialization -5. Data Preprocessing -6. Model Setup -7. Training Launch - - -slurm_script.sh -~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - #!/bin/bash - - #SBATCH --job-name=verl-ray-on-slurm - #SBATCH --nodes=2 - #SBATCH --ntasks-per-node=2 - #SBATCH --mem=200G - #SBATCH --time=30-00:00:00 - #SBATCH --gpus-per-node=8 - #SBATCH --cpus-per-task=28 - #SBATCH --output=../verl_log/slurm-%j.out - #SBATCH --error=../verl_log/slurm-%j.err - #SBATCH --nodelist=gpu-[0,1] - - - # load necessary modules - ### Run this setup - # [Cluster]: Use docker - # docker pull docker.io/rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 - - - ########################################################################## - ###The following setting should be set in different project and cluster### - ########################################################################## - - ### Project - CONTAINER_NAME="multinode_verl_training" - IMG="verl.rocm" - DOCKERFILE="docker/Dockerfile.rocm" - # echo $PWD - verl_workdir="${HOME}/projects/verl_upstream" - export TRANSFORMERS_CACHE="${HOME}/.cache/huggingface" - export HF_HOME=$TRANSFORMERS_CACHE - - ### Cluster Network Setting - export NCCL_DEBUG=TRACE - export GPU_MAX_HW_QUEUES=2 - export TORCH_NCCL_HIGH_PRIORITY=1 - export NCCL_CHECKS_DISABLE=1 - # export NCCL_IB_HCA=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 - export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_8,mlx5_9 - export NCCL_IB_GID_INDEX=3 - export NCCL_CROSS_NIC=0 - export CUDA_DEVICE_MAX_CONNECTIONS=1 - export NCCL_PROTO=Simple - export RCCL_MSCCL_ENABLE=0 - export TOKENIZERS_PARALLELISM=false - export HSA_NO_SCRATCH_RECLAIM=1 - ########################################################################## - - ### For rocm and training script - export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - export ROCR_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - export CUDA_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES - - - # Build and launch the Docker container - srun bash -c " - # Exit on any error - set -e - - # Clean up dangling images (images with tag) - docker image prune -f - - # Need to pull the docker first - docker pull docker.io/rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 - - if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMG}"; then - echo \"Building ${IMG} image...\" - docker build -f \"${DOCKERFILE}\" -t \"${IMG}\" . - else - echo \"${IMG} image already exists, skipping build\" - fi - - # Removing old container if exists - docker rm \"${CONTAINER_NAME}\" 2>/dev/null || true - - # Checking network devices - ibdev2netdev - - # Launch the docker - docker run --rm -d \ - -e HYDRA_FULL_ERROR=1 \ - -e HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES} \ - -e ROCR_VISIBLE_DEVICES=${ROCR_VISIBLE_DEVICES} \ - -e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \ - -e NCCL_DEBUG=${NCCL_DEBUG} \ - -e GPU_MAX_HW_QUEUES=${GPU_MAX_HW_QUEUES} \ - -e TORCH_NCCL_HIGH_PRIORITY=${TORCH_NCCL_HIGH_PRIORITY} \ - -e NCCL_CHECKS_DISABLE=${NCCL_CHECKS_DISABLE} \ - -e NCCL_IB_HCA=${NCCL_IB_HCA} \ - -e NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} \ - -e NCCL_CROSS_NIC=${NCCL_CROSS_NIC} \ - -e CUDA_DEVICE_MAX_CONNECTIONS=${CUDA_DEVICE_MAX_CONNECTIONS} \ - -e NCCL_PROTO=${NCCL_PROTO} \ - -e RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE} \ - -e TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM} \ - -e HSA_NO_SCRATCH_RECLAIM=${HSA_NO_SCRATCH_RECLAIM} \ - -e TRANSFORMERS_CACHE=${TRANSFORMERS_CACHE} \ - -e HF_HOME=${HF_HOME} \ - --network host \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v \${HOME}:\${HOME} \ - -v \${HOME}/.ssh:/root/.ssh \ - -w "${verl_workdir}" \ - --shm-size 128G \ - --name \"${CONTAINER_NAME}\" \ - \"${IMG}\" \ - tail -f /dev/null - - echo \"Container setup completed\" - " - # (Optional): If you do not want to root mode and require assign yuorself as the user - # Please add `-e HOST_UID=$(id -u)` and `-e HOST_GID=$(id -g)` into the above docker launch script. - - - - - - ### Ray launch the nodes before training - - # Getting the node names - nodes_array=($(scontrol show hostnames "$SLURM_JOB_NODELIST" | tr '\n' ' ')) - - head_node=${nodes_array[0]} - head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) - - # if we detect a space character in the head node IP, we'll - # convert it to an ipv4 address. This step is optional. - if [[ "$head_node_ip" == *" "* ]]; then - IFS=' ' read -ra ADDR <<<"$head_node_ip" - if [[ ${#ADDR[0]} -gt 16 ]]; then - head_node_ip=${ADDR[1]} - else - head_node_ip=${ADDR[0]} - fi - echo "IPV6 address detected. We split the IPV4 address as $head_node_ip" - fi - - port=6379 - ip_head=$head_node_ip:$port - export ip_head - echo "IP Head: $ip_head" - - # make sure we set environment variables before Ray initialization - export VLLM_ATTENTION_BACKEND=XFORMERS - - # Print out all env variables - printenv - - echo "Starting HEAD at $head_node" - srun --nodes=1 --ntasks=1 -w "$head_node" \ - docker exec "${CONTAINER_NAME}" \ - ray start --head --node-ip-address="$head_node_ip" --port=$port \ - --dashboard-port=8266 \ - --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & - # optional, though may be useful in certain versions of Ray < 1.0. - sleep 10 - - # number of nodes other than the head node - worker_num=$((SLURM_JOB_NUM_NODES - 1)) - - for ((i = 1; i <= worker_num; i++)); do - node_i=${nodes_array[$i]} - echo "Debug: Starting worker on node_i = ${node_i}" - if [ -z "$node_i" ]; then - echo "Error: Empty node name for worker $i" - continue - fi - echo "Starting WORKER $i at $node_i" - srun --nodes=1 --ntasks=1 -w "$node_i" \ - docker exec "${CONTAINER_NAME}" \ - ray start --address "$ip_head" --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & - sleep 5 - done - - - - - # Ray initlization test (See whether any error in the above excution) - echo "Testing Ray initialization in the slurm nodes..." - docker exec "${CONTAINER_NAME}" python3 -c ' - import ray - try: - ray.init(address="auto") - print("\n=== Ray Cluster Status ===") - print(f"Number of nodes: {len(ray.nodes())}") - for node in ray.nodes(): - print("Node: {}, Status: {}".format(node["NodeManagerHostname"], node["Alive"])) - # print(f"Node: {node}") - ray.shutdown() - print("Ray initialization successful!") - except Exception as e: - print(f"Ray initialization failed: {str(e)}") - ' - echo "=== Ray test completed ===" - ###### - - - - # Run data preprocessing - - echo "Starting data preprocessing..." - docker exec "${CONTAINER_NAME}" \ - python3 "examples/data_preprocess/gsm8k.py" "--local_dir" "../data/gsm8k" - - echo "Starting data preprocessing..." - docker exec "${CONTAINER_NAME}" \ - python3 "examples/data_preprocess/math_dataset.py" "--local_dir" "../data/math" - - train_files="../data/gsm8k/train.parquet" - val_files="../data/gsm8k/test.parquet" - - # Download and test model - echo "Loading model..." - docker exec "${CONTAINER_NAME}" \ - python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" - MODEL_PATH="Qwen/Qwen2-7B-Instruct" - - # Set model path after pipeline test - MODEL_PATH="Qwen/Qwen2.5-0.5B-Instruct" - - echo "== Data and model loading Done ==" - - echo "Start to train..." - - docker exec "${CONTAINER_NAME}" \ - python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2-7B-Instruct')" - MODEL_PATH="Qwen/Qwen2-7B-Instruct" - - - PYTHONUNBUFFERED=1 srun --overlap --nodes=${SLURM_NNODES} --ntasks=1 -w "$head_node" \ - docker exec "${CONTAINER_NAME}" \ - python3 -m verl.trainer.main_ppo \ - data.train_files=$train_files \ - data.val_files=$val_files \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - actor_rollout_ref.model.path=$MODEL_PATH \ - actor_rollout_ref.model.enable_gradient_checkpointing=False \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=$MODEL_PATH \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=8 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.kl_ctrl.kl_coef=0.0001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-32B-Instruct_function_rm' \ - trainer.n_gpus_per_node=${SLURM_GPUS_PER_NODE} \ - trainer.val_before_train=False \ - trainer.nnodes=${SLURM_NNODES} \ - trainer.save_freq=-1 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 - - -Run multi-node training with above slurm_script.sh -~~~~~~~~~~~~~~~~~~~~ -Just sbatch your slurm_script.sh - -.. code-block:: bash - - sbatch slurm_script.sh - diff --git a/docs/start/quickstart.rst b/docs/start/quickstart.rst deleted file mode 100644 index ac8c75d451..0000000000 --- a/docs/start/quickstart.rst +++ /dev/null @@ -1,142 +0,0 @@ -.. _quickstart: - -========================================================= -Quickstart: PPO training on GSM8K dataset -========================================================= - -Post-train a LLM using GSM8K dataset. - -Introduction ------------- - -.. _hf_dataset_gsm8k: https://huggingface.co/datasets/gsm8k - -In this example, we train an LLM to tackle the `GSM8k `_ task with function-based rewards. [1]_ - -Prerequisite: - -- the latest version of ``verl`` and its dependencies installed following the installation guide. Using the docker image is recommended. - -- a GPU with at least 24 GB HBM - - -Dataset Introduction --------------------- - -GSM8k is a math problem dataset. The prompt is an elementary school -problem. The LLM model is asked to solve the math problem. Below is an example: - -Prompt - - Katy makes coffee using teaspoons of sugar and cups of water in the - ratio of 7:13. If she used a total of 120 teaspoons of sugar and cups - of water, calculate the number of teaspoonfuls of sugar she used. - -Solution - - The total ratio representing the ingredients she used to make the - coffee is 7+13 = <<7+13=20>>20 Since the fraction representing the - number of teaspoons she used is 7/20, she used 7/20\ *120 = - <<7/20*\ 120=42>>42 #### 42 - -Step 1: Prepare the dataset ----------------------------- - -We preprocess the dataset in parquet format so that (1) it contains necessary fields for computing RL rewards and (2) is faster to read. - -.. code-block:: bash - - python3 examples/data_preprocess/gsm8k.py --local_dir ~/data/gsm8k - -Step 2: Download a model for post-training -------------------------------------------- - -In this example, we start with the ``Qwen2.5-0.5B-Instruct`` model. - -If you want to perform SFT before RL, refer to the :doc:`Complete GSM8K Example<../examples/gsm8k_example>`, the `sft directory `_ and `SFT Trainer `_ for further details. - -.. code-block:: bash - - python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2.5-0.5B-Instruct')" - -Step 3: Perform PPO training with the instruct model ----------------------------------------------------------------------- - -**Reward Model/Function** - -We use a pre-defined rule-based reward model. We force the model to produce a final -answer following 4 “#” as shown in the solution. We extract the final -answer from both the solution and model's output using regular -expression matching. We assign a reward of 1 to correct -answer, 0.1 to incorrect answer and 0 to no answer. - -For more details, please refer to `verl/utils/reward_score/gsm8k.py `_. - -**Training Script** - -Now let's run PPO training with the dataset and model above. [2]_ - - -Set the ``data.train_files`` ,\ ``data.val_files``, ``actor_rollout_ref.model.path`` and ``critic.model.path`` based on your dataset and model names or paths. -You may set ``VERL_USE_MODELSCOPE=True`` to download models from modelscope instead of huggingface. - -.. code-block:: bash - - PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=256 \ - data.max_prompt_length=512 \ - data.max_response_length=256 \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=64 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - critic.optim.lr=1e-5 \ - critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ - critic.ppo_micro_batch_size_per_gpu=4 \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.logger=['console'] \ - trainer.val_before_train=False \ - trainer.default_hdfs_dir=null \ - trainer.n_gpus_per_node=1 \ - trainer.nnodes=1 \ - trainer.save_freq=10 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 2>&1 | tee verl_demo.log - -You are expected to see the following logs, indicating training in progress. The key metric ``val/test_score/openai/gsm8k`` is computed every ``trainer.test_freq`` steps: - -.. code-block:: bash - - step:0 - timing/gen:21.470 - timing/ref:4.360 - timing/values:5.800 - actor/reward_kl_penalty:0.000 - actor/reward_kl_penalty_coeff:0.001 - timing/adv:0.109 - timing/update_critic:15.664 - critic/vf_loss:14.947 - critic/vf_clipfrac:0.000 - critic/vpred_mean:-2.056 - critic/grad_norm:1023.278 - critic/lr(1e-4):0.100 - timing/update_actor:20.314 - actor/entropy_loss:0.433 - actor/pg_loss:-0.005 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/grad_norm:1.992 - actor/lr(1e-4):0.010 - critic/score/mean:0.004 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.004 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:-0.000 - critic/advantages/max:2.360 - critic/advantages/min:-2.280 - critic/returns/mean:0.003 - critic/returns/max:0.000 - critic/returns/min:0.000 - critic/values/mean:-2.045 - critic/values/max:9.500 - critic/values/min:-14.000 - response_length/mean:239.133 - response_length/max:256.000 - response_length/min:77.000 - prompt_length/mean:104.883 - prompt_length/max:175.000 - prompt_length/min:68.000 - step:1 - timing/gen:23.020 - timing/ref:4.322 - timing/values:5.953 - actor/reward_kl_penalty:0.000 - actor/reward_kl_penalty:0.001 - timing/adv:0.118 - timing/update_critic:15.646 - critic/vf_loss:18.472 - critic/vf_clipfrac:0.384 - critic/vpred_mean:1.038 - critic/grad_norm:942.924 - critic/lr(1e-4):0.100 - timing/update_actor:20.526 - actor/entropy_loss:0.440 - actor/pg_loss:0.000 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.060 - actor/lr(1e-4):0.010 - critic/score/mean:0.000 - critic/score/max:0.000 - critic/score/min:0.000 - critic/rewards/mean:0.000 - critic/rewards/max:0.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.000 - critic/advantages/max:2.702 - critic/advantages/min:-2.616 - critic/returns/mean:0.000 - critic/returns/max:0.000 - critic/returns/min:0.000 - critic/values/mean:-2.280 - critic/values/max:11.000 - critic/values/min:-16.000 - response_length/mean:232.242 - response_length/max:256.000 - response_length/min:91.000 - prompt_length/mean:102.398 - prompt_length/max:185.000 - prompt_length/min:70.000 - -Checkout :ref:`algo-baseline-page` for full training and validation logs for reference. - -The checkpoint is saved at the following dir by default: ``checkpoints/${trainer.project_name}/${trainer.experiment_name}`` - -To enable ``wandb`` for experiment tracking, set the following configs: - -.. code-block:: bash - - trainer.logger=['console','wandb'] \ - trainer.project_name=$YOUR_PROJECT_NAME \ - trainer.experiment_name=$YOUR_RUN_NAME \ - -If you encounter out of memory issues with HBM less than 32GB, enable the following configs would help: - -.. code-block:: bash - - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - critic.ppo_micro_batch_size_per_gpu=1 \ - -For the full set of configs, please refer to :ref:`config-explain-page` for detailed explanation and performance tuning. - - -.. [1] The original paper (https://arxiv.org/pdf/2110.14168) mainly focuses on training a verifier (a reward model) to solve math problems via Best-of-N sampling. In this example, we train an RL agent using a rule-based reward model. -.. [2] More training script examples for FSDP and Megatron-LM backend are stored in `examples/ppo_trainer `_ directory. diff --git a/docs/training-runbook.md b/docs/training-runbook.md new file mode 100644 index 0000000000..2a2509333b --- /dev/null +++ b/docs/training-runbook.md @@ -0,0 +1,150 @@ +# Fleet Training Runbook + +Quick reference for launching training runs on Fleet tasks. + +## Backends + +| Backend | Infrastructure | Best For | +|---------|---------------|----------| +| **SkyRL (SkyPilot)** | Self-managed GPU (Lambda/RunPod/Vast) | Full control, longer runs | +| **Tinker** | Hosted GPU| Quick iteration, no GPU setup | + +## Launching a Run + +### Via GitHub Actions (Recommended) + +1. Go to **Actions** → Select workflow: + - `Fleet Task Training (SkyPilot)` - SkyRL backend + - `Fleet Task Training (Tinker)` - Tinker backend + +2. Click **Run workflow** and configure: + +#### Common Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `modality` | `tool_use` or `computer_use` | `tool_use` | +| `env_key` | Filter by environment (empty=all) | `""` | +| `max_tasks` | Limit tasks for testing | `""` (all) | +| `data_version` | Dataset version in S3 (e.g., `v0.3`) | `v0.3` | +| `enable_context_tools` | Enable context management tools (`check_context`, `manage_context`, etc.) | `false` | + +#### SkyRL-Specific + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `cloud` | `any`, `lambda`, `runpod`, `vast` | `any` | +| `task_name` | SkyPilot YAML in `skyrl-train/tasks/` | `openenv-fleet-grpo` | + +#### Tinker-Specific + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `model_name` | Model to train | `Qwen/Qwen3-8B` | +| `max_steps` | Training steps | `50` | +| `batch_size` | Batch size | `8` | +| `n_samples_per_prompt` | GRPO samples per prompt | `4` | + +## Monitoring + +### Slack Notifications + +Channel: `#fleet-training-runs` + +- Started: Run name, backend, modality, config +- Completed: Final status, WandB link +- Failed: Error logs link +- Cancelled: Cleanup status + +### WandB Dashboards + +| Backend | Project | +|---------|---------| +| SkyRL | [fleet-task-grpo](https://wandb.ai/thefleet/fleet-task-grpo) | +| Tinker | [fleet-tinker-grpo](https://wandb.ai/thefleet/fleet-tinker-grpo) | + +### Key Metrics + +- `reward/avg_pass_at_1` - Primary success metric +- `reward/avg_raw_reward` - Average reward per episode +- `reward/{env_key}/pass_at_1` - Per-environment breakdown + +## Testing a Small Run + +Quick validation before full training: + +``` +modality: tool_use +env_key: github # Single environment +max_tasks: 4 # Few tasks +max_steps: 10 # Tinker only +``` + +## Context Management Tools + +When `enable_context_tools=true`, the model gets 4 additional tools for managing long contexts: + +| Tool | Description | +|------|-------------| +| `check_context` | Check current token count and context budget | +| `manage_context` | Drop old turns to free up context space | +| `search_tool_output` | Search within truncated tool outputs | +| `view_tool_output` | View specific lines of truncated tool outputs | + +**Requirements for context tools:** +- `step_wise_trajectories=true` (set automatically in YAML) +- `inject_context_status=true` (adds context status to observations) +- YaRN rope scaling extends context to 65K tokens + +**When to use:** Enable for environments with long tool schemas (e.g., GitHub with 100+ tools) where the model needs to proactively manage context before hitting the 48K limit. + +## Troubleshooting + +### MCP Connection Timeouts + +``` +httpx.ConnectTimeout +ERROR mcp.client.streamable_http: Error in post_writer +``` + +**Cause**: Too many concurrent Fleet environment connections. + +**Fix**: Tinker uses `max_concurrent=2` semaphore to limit parallel connections. + +### "No tools found" Errors + +``` +RuntimeError: Task X: no tools found in observation +``` + +**Cause**: MCP client failed to fetch tools from Fleet. + +**Fix**: OpenEnv retries with exponential backoff (3 attempts). + +### SkyPilot Cluster Issues + +```bash +# Check cluster status +sky status + +# View logs +sky logs + +# Force cleanup +sky down -y +``` + +## Required Secrets + +| Secret | Used By | +|--------|---------| +| `FLEET_API_KEY` | Both | +| `WANDB_API_KEY_TOOL_USE` | Both (tool_use modality) | +| `WANDB_API_KEY_COMPUTER_USE` | Both (computer_use modality) | +| `TINKER_API_KEY` | Tinker | +| `LAMBDA_API_KEY` | SkyRL | +| `RUNPOD_API_KEY` | SkyRL | +| `VAST_API_KEY` | SkyRL | +| `AWS_ACCESS_KEY_ID` | Both (S3 datasets) | +| `AWS_SECRET_ACCESS_KEY` | Both (S3 datasets) | +| `SLACK_BOT_TOKEN` | Notifications | diff --git a/docs/workers/fsdp_workers.rst b/docs/workers/fsdp_workers.rst deleted file mode 100644 index b1c78e7c83..0000000000 --- a/docs/workers/fsdp_workers.rst +++ /dev/null @@ -1,142 +0,0 @@ -PyTorch FSDP Backend -====================== - -We support PyTorch FSDP Backend by implementing various workers for -actor, critic, reference, rollout and reward models. We also implement -the ``FSDPVLLMShardingManager`` that reshard weight between FSDP and -vLLM in `fsdp_vllm.py `_. - -**Pros** - -- Readily support various models. - - - Users only need to implement the corresponding - ``dtensor_weight_loader`` for weight synchronization between FSDP - and vLLM. While for ``hf_weight_loader``, users can directly apply - any models supported both in HF and vLLM without any code change. - -- Easy to organize the forward and backward computation for each model. - -**Cons** - -- Poor scalability when it comes to large-scale models (e.g. Llama 70B - and 405B) -- The resharding overhead between actor and rollout could be larger than - Megatron-LM backend. - -Due to the simplicity, we recommend using FSDP backend for algorithm -research and prototyping. - -FSDP Workers --------------- - -ActorRolloutRefWorker -^^^^^^^^^^^^^^^^^^^^^ - -Actor/Rollout HybridEngine -'''''''''''''''''''''''''' - -1. HybridEngine, Actor and Rollout initialization API. - -.. code:: python - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - -``ONE_TO_ALL``: when calling the ``init_model`` function from the driver -process, each worker (on a GPU) will execute the following model -initialization process. - -The initialization details of HybridEngine, Actor and Rollout are -highlighted below: - -1. ``DataParallelPPOActor`` implements the simple PPO computation logics - when the model is built with FSDP, including compute log prob, model - update. -2. ``vLLMRollout`` support generation with vLLM. We modify the vLLM - Engine and make it executed under SPMD to fit into our - ``WorkerGroup`` design. -3. ``FSDPVLLMShardingManager`` a context manager to perform actual - resharding between actor and rollout. - -See `source code `_. for more information. - -1. Generate sequence and recompute log prob - -.. code:: python - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def generate_sequences(self, prompts: DataProto): - -- ``Dispatch.DP_COMPUTE_PROTO``: The data will be dispatched and - collected along the DP dimension - -- In this function, the rollout model will perform auto-regressive - generation and the actor model will recompute the old log prob for the - generated response. - -3. Update actor model - -.. code:: python - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def update_actor(self, data: DataProto): - -- Update the actor model weight using PPO & entropy loss. - -ReferenceModel -'''''''''''''' - -1. Reference model initialization - -The reference model is initialized using the same function as the actor -model without initializing the HybridEngine and Optimizer. Then the -actor model is also wrapped by the ``DataParallelPPOActor``. - -2. Compute reference log prob - -.. code:: python - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_ref_log_prob(self, data: DataProto): - -- In this function, the reference model will call the compute log prob - function in ``DataParallelPPOActor`` to compute the reference log - prob. - -CriticWorker and RewardWorker -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. Model initialization - -Quite similar to reference model. The CriticWorker will perform -additional initialization for the Optimizer. - -2. Compute Values for CriticWorker - -.. code:: python - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_values(self, data: DataProto): - -3. Update Critic - -.. code:: python - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def update_critic(self, data: DataProto): - -4. Compute Reward - -.. code:: python - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_rm_score(self, data: DataProto): - - -HybridShard ------------- - -We didn't support FSDP `HybridShard`. To support this, we may need to -construct a 2D device mesh and test the corresponding -``dtensor_weight_loader`` and ``hf_weight_loader`` for each model. diff --git a/docs/workers/megatron_workers.rst b/docs/workers/megatron_workers.rst deleted file mode 100644 index d967c3f514..0000000000 --- a/docs/workers/megatron_workers.rst +++ /dev/null @@ -1,203 +0,0 @@ -Megatron-LM Backend -===================== - -We support Megatron Backend by implementing various workers for actor, -critic, reference, rollout and reward models. We also implement the -``3DHybridEngine`` using Megatron-LM and vLLM in `megatron_vllm.py `_. - -**Pros** - -- Support 3D parallelism and sequence parallelism for best scalablility - and throughput. -- 3D HybridEngine can significantly reduce peak memory usage and reduce - weight synchronize overhead between actor and rollout. - -**Cons** - -- Users should implement their own models for Megatron-LM -- Users should implement the corresponding weight_loader to - - - synchronize the model weight between actor (in Megatron) and rollout - (in vLLM). - - load weights from checkpoints to corresponding model in Megatron-LM - -Megatron Workers ----------------- - -MegatronWorker -^^^^^^^^^^^^^^ - -``MegatronWorker`` is the base class of different megatron worker -classes. In this class, ``get_megatron_global_info`` and -``get_megatron_rank_info`` function to retrive the 3D parallel world -size and rank of each ``Worker`` running on specific GPU. These information -will be used in transfer protocol for Megatron Backend. - -The following ``Worker`` class for different models will be utilized to -construct the ``WorkerGroup`` . - -We implement various of APIs for each ``Worker`` class decorated by the -``@register(dispatch_mode=)`` . These APIs can be called by the ray -driver process. The data can be correctly collect and dispatch following -the ``dispatch_mode`` on each function. The supported dispatch_model -(i.e., transfer protocols) can be found in `decorator.py `_. - -ActorRolloutRefWorker -^^^^^^^^^^^^^^^^^^^^^ - -This class is implemented for Actor/Rollout HybridEngine or for the -reference model to initialize their model and perform computation. - -Actor/Rollout HybridEngine -'''''''''''''''''''''''''' - -1. HybridEngine, Actor and Rollout initialization API. - -.. code:: python - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - -``ONE_TO_ALL``: when calling the ``init_model`` function from the driver -process, each worker (on a GPU) will execute the following model -initialization process. - -The initialization details of HybridEngine, Actor and Rollout are -highlighted below: - -1. ``AllGatherPPModel`` holds memory buffer for both Actor and Rollout - and support weight resharding between actor and rollout. -2. ``MegatronPPOActor`` implements the simple PPO computation logics - when the model is built with Megatron, including compute log prob, - model update. -3. ``vLLMRollout`` support generation with vLLM. We modify the vLLM - Engine and make it executed under SPMD to fit into our - ``WorkerGroup`` design. -4. ``MegatronVLLMShardingManager`` a context manager to perform actual - resharding between actor and rollout. - -See `source code `_ for more information. - -.. code:: python - - # Initialize the 3D HybridEngine - hybrid_engine = AllGatherPPModel(model_provider=megatron_actor_model_provider) - # Fetch the model at current rank - actor_module = hybrid_engine.this_rank_models - ... - - # build actor model - self.actor = MegatronPPOActor(config=self.config.actor, - model_config=self.actor_model_config, - megatron_config=megatron_config, - actor_module=self.actor_module, - actor_optimizer=self.actor_optimizer, - actor_optimizer_config=self.actor_optim_config) - - # build rollout - # rollout initialization - rollout = vLLMRollout(actor_module=params, - config=self.config.rollout, - tokenizer=self.tokenizer, - model_hf_config=self.actor_model_config, - train_tp=mpu.get_tensor_model_parallel_world_size()) - # perform weight resharding between actor and rollout - sharding_manager = MegatronVLLMShardingManager(module=self.hybrid_engine, - inference_engine=rollout.inference_engine, - model_config=self.actor_model_config, - layer_name_mapping=layer_name_mapping) - ... - -2. Generate sequence and recompute log prob - -.. code:: python - - @register(dispatch_mode=Dispatch.MEGATRON_PP_AS_DP_PROTO) - def generate_sequences(self, prompts: DataProto): - -- ``Dispatch.MEGATRON_PP_AS_DP_PROTO``: The PP dimension of the actor - model will be regarded as DP dimension. Then the driver process will - dispatch and collect the data according to this reorganization. This - is because, in HybridEngine, the actor weight, which usually applied - larger 3D parallel sizes, will be gathered along the PP dimension and - TP dimension. Therefore, the corresponding data should be dispatched - and collected through the 3D parallel group of the rollout model, - rather than the actor model. However, the world_size and rank - information can only be retrived from ``get_megatron_global_info`` and - ``get_megatron_rank_info``, which records the 3D information for the - actor model. Moreover, the data resharding inside TP dimension will be - processed within the HybridEngine. - -- In this function, the rollout model will perform auto-regressive - generation and the actor model will recompute the old log prob for the - generated response. - -3. Update actor model - -.. code:: python - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def update_actor(self, data: DataProto): - -- ``Dispatch.MEGATRON_COMPUTE_PROTO``: User passes the data partitioned - by DP dimension. The data is dispatched to all tp/pp ranks within the - same dp group, and ultimately only collects output data from tp=0 and - the last pp. -- Update the actor model weight using PPO & entropy loss. - -ReferenceModel -'''''''''''''' - -1. Reference model initialization - -The reference model is initialized using the same function as the actor -model without initializing the HybridEngine and Optimizer. Then the -actor model is also wrapped by the ``MegatronPPOActor``. - -2. Compute reference log prob - -.. code:: python - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def compute_ref_log_prob(self, data: DataProto): - -- In this function, the reference model will call the compute log prob - function in ``MegatronPPOActor`` to compute the reference log prob. - -CriticWorker and RewardWorker -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. Model initialization - -Quite similar to reference model. The CriticWorker will perform -additional initialization for the Optimizer. - -2. Compute Values for CriticWorker - -.. code:: python - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def compute_values(self, data: DataProto): - -3. Update Critic - -.. code:: python - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def update_critic(self, data: DataProto): - -4. Compute Reward - -.. code:: python - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def compute_rm_score(self, data: DataProto): - -Context Parallel ----------------- - -Currently we can only use LLaMa and Qwen models implemented in verl, and context parallel is not supported by far. - -We are working in progress to support Megatron implementation of GPTModel, with TransformerEngine support. So if the itegration goes well, we can support Ulysses, Ring and AllGather context parallel in the future. - -Now we support Megatron checkpointing save/load function with original models. Please check the :ref:`config-explain-page` page to see how to use the APIs. \ No newline at end of file diff --git a/docs/workers/ray_trainer.rst b/docs/workers/ray_trainer.rst deleted file mode 100644 index a6e0b08497..0000000000 --- a/docs/workers/ray_trainer.rst +++ /dev/null @@ -1,243 +0,0 @@ -PPO Ray Trainer -=============== - -We implement the RayPPOTrainer, which is a trainer runs on the driver -process on a single CPU/GPU node (default is CPU). - -The PPORayTrainer include 3 core functions for data preparation, -WorkerGroup initialization and PPO training loop. - -Data Preparation ----------------- - -The ``PPORayTrainer``, as a single process, is responsible for loading a -complete batch of samples (prompts) from the dataset and then dispatch -to different worker_groups running on different GPUs. - -To generalize the data loading, we implement the ``RLHFDataset`` class -to load the preprocessed parquet files, apply chat templates to the -prompts, add padding, truncate prompts that exceed max prompt length and -then tokenize. - -.. code:: python - - self.train_dataset = RLHFDataset(parquet_files=self.config.data.train_files, - tokenizer=self.tokenizer, - prompt_key=self.config.data.prompt_key, - max_prompt_length=self.config.data.max_prompt_length, - filter_prompts=True, - return_raw_chat=self.config.data.get('return_raw_chat', False), - truncation='error') - -Then, the dataloader will iterate the dataset under PPO mini batch size. - -WorkerGroup Initialization --------------------------- - -We first introduce a basic implementation of initializing the -``WorkerGroup`` of the actor model on a given set of GPUs. - -.. code:: python - - # max_colocate_count means the number of WorkerGroups (i.e. processes) in each RayResourcePool - # For FSDP backend, we recommend using max_colocate_count=1 that merge all WorkerGroups into one. - # For Megatron backend, we recommend using max_colocate_count>1 that can utilize different WorkerGroup for differnt models - resource_pool = RayResourcePool(process_on_nodes=[config.trainer.n_gpus_per_node] * config.trainer.nnodes, - use_gpu=True, - max_colocate_count=1) - # define actor rollout cls to be init on remote - actor_rollout_cls = RayClassWithInitArgs(cls=ActorRolloutWorker) - # define actor_rollout worker group - actor_rollout_worker_group = MegatronRayWorkerGroup(resource_pool=resource_pool, - ray_cls_with_init=actor_rollout_cls, - default_megatron_kwargs=config.actor_rollout.megatron) - -Different WorkerGroups, like ``actor_rollout_worker_group`` , -``critic_worker_group`` and ``ref_worker_group`` lies on a separate -process in the above implementation. - -The driver process can then call the distributed compute function within -the ``actor_rollout_worker_group`` and other roles to construct the RL -training loop. - -For models colocated in the same set of GPUs, we further provide a -fine-grain optimization, which merge the ``worker_group`` of different roles -in the same process. This optimization can save the redundant -CUDA/distributed context in different processes. - -.. code:: python - - # initialize WorkerGroup - # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, - # you should not use `create_colocated_worker_cls`. Instead, directly pass different resource pool to different worker groups. - # See TODO(url) for more information. - all_wg = {} - for resource_pool, class_dict in self.resource_pool_to_cls.items(): - worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) - wg_dict = self.ray_worker_group_cls(resource_pool=resource_pool, ray_cls_with_init=worker_dict_cls) - spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) - all_wg.update(spawn_wg) - - if self.use_critic: - self.critic_wg = all_wg['critic'] - self.critic_wg.init_model() - - if self.use_reference_policy: - self.ref_policy_wg = all_wg['ref'] - self.ref_policy_wg.init_model() - - if self.use_rm: - self.rm_wg = all_wg['rm'] - self.rm_wg.init_model() - - # we should create rollout at the end so that vllm can have a better estimation of kv cache memory - self.actor_rollout_wg = all_wg['actor_rollout'] - self.actor_rollout_wg.init_model() - -.. note:: For megatron backend, if we merge the ``worker_groups`` into the same processes, all the roles will utilize the same 3D parallel size. To optimize this, we may need to maintain several 3D process groups for each role in the same distributed context. If you want to use different 3D parallel size for different roles, please follow the similar architecture of the first code block to initialize each role's ``worker_group`` - - -PPO Training Loop ------------------ - -We implement the PPO training loop by calling the functions in -worker_group of each role. The input and output data of each function is -a ``DataProto`` object implemented in `protocol.py `_. In the training -loop, trainer will dispatch/collect the data to/from different GPUs -following the transfer protocols wrapped in the workers' functions. The -computation of PPO micro batches is processed in ``update_actor`` and -``update_critic`` functions. - -To extend to other RLHF algorithms, such as DPO, GRPO, please refer to -:doc:`../advance/dpo_extension`. - -.. code:: python - - def fit(self): - """ - The training loop of PPO. - The driver process only need to call the compute functions of the worker group through RPC to construct the PPO dataflow. - The light-weight advantage computation is done on the driver process. - """ - from verl.utils.tracking import Tracking - from omegaconf import OmegaConf - - logger = Tracking(project_name=self.config.trainer.project_name, - experiment_name=self.config.trainer.experiment_name, - default_backend=self.config.trainer.logger, - config=OmegaConf.to_container(self.config, resolve=True)) - - global_steps = 0 - - # perform validation before training - # currently, we only support validation using the reward_function. - if self.val_reward_fn is not None: - val_metrics = self._validate() - pprint(f'Initial validation metrics: {val_metrics}') - - for epoch in range(self.config.trainer.total_epochs): - for batch_dict in self.train_dataloader: - metrics = {} - - batch: DataProto = DataProto.from_single_dict(batch_dict) - # batch = batch.to('cuda') - - # pop those keys for generation - gen_batch = batch.pop(batch_keys=['input_ids', 'attention_mask', 'position_ids']) - - # generate a batch - with Timer(name='gen', logger=None) as timer: - gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) - metrics['timing/gen'] = timer.last - - batch = batch.union(gen_batch_output) - - if self.use_reference_policy: - # compute reference log_prob - with Timer(name='ref', logger=None) as timer: - ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) - batch = batch.union(ref_log_prob) - metrics['timing/ref'] = timer.last - - # compute values - with Timer(name='values', logger=None) as timer: - values = self.critic_wg.compute_values(batch) - batch = batch.union(values) - metrics['timing/values'] = timer.last - - with Timer(name='adv', logger=None) as timer: - # compute scores. Support both model and function-based. - # We first compute the scores using reward model. Then, we call reward_fn to combine - # the results from reward model and rule-based results. - if self.use_rm: - # we first compute reward model score - reward_tensor = self.rm_wg.compute_rm_score(batch) - batch = batch.union(reward_tensor) - - # we combine with rule-based rm - reward_tensor = self.reward_fn(batch) - batch.batch['token_level_scores'] = reward_tensor - - # compute rewards. apply_kl_penalty if available - batch, kl_metrics = apply_kl_penalty(batch, - kl_ctrl=self.kl_ctrl_in_reward, - kl_penalty=self.config.algorithm.kl_penalty) - metrics.update(kl_metrics) - - # compute advantages, executed on the driver process - batch = compute_advantage(batch, - self.config.algorithm.gamma, - self.config.algorithm.lam, - adv_estimator=self.config.algorithm.adv_estimator) - metrics['timing/adv'] = timer.last - - # update critic - if self.use_critic: - with Timer(name='update_critic', logger=None) as timer: - critic_output = self.critic_wg.update_critic(batch) - metrics['timing/update_critic'] = timer.last - critic_output_metrics = reduce_metrics(critic_output.meta_info['metrics']) - metrics.update(critic_output_metrics) - - # implement critic warmup - if self.config.trainer.critic_warmup <= global_steps: - # update actor - with Timer(name='update_actor', logger=None) as timer: - actor_output = self.actor_rollout_wg.update_actor(batch) - metrics['timing/update_actor'] = timer.last - actor_output_metrics = reduce_metrics(actor_output.meta_info['metrics']) - metrics.update(actor_output_metrics) - - # validate - if self.val_reward_fn is not None and (global_steps + 1) % self.config.trainer.test_freq == 0: - with Timer(name='testing', logger=None) as timer: - val_metrics: dict = self._validate() - val_metrics = {f'val/{key}': val for key, val in val_metrics.items()} - metrics['timing/testing'] = timer.last - metrics.update(val_metrics) - - # collect metrics - data_metrics = compute_data_metrics(batch=batch) - metrics.update(data_metrics) - - # TODO: make a canonical logger that supports various backend - logger.log(data=metrics, step=global_steps) - - if self.config.trainer.save_freq > 0 and (global_steps + 1) % self.config.trainer.save_freq == 0: - actor_local_path = os.path.join(self.config.trainer.default_local_dir, 'actor', - f'global_step_{global_steps}') - actor_remote_path = os.path.join(self.config.trainer.default_hdfs_dir, 'actor') - self.actor_rollout_wg.save_checkpoint(actor_local_path, actor_remote_path) - - if self.use_critic: - critic_local_path = os.path.join(self.config.trainer.default_local_dir, 'critic', - f'global_step_{global_steps}') - critic_remote_path = os.path.join(self.config.trainer.default_hdfs_dir, 'critic') - self.critic_wg.save_checkpoint(critic_local_path, critic_remote_path) - - global_steps += 1 - - # perform validation after training - if self.val_reward_fn is not None: - val_metrics = self._validate() - pprint(f'Final validation metrics: {val_metrics}') diff --git a/examples/checkpoint/run_deepseek_megatron_ckpt.sh b/examples/checkpoint/run_deepseek_megatron_ckpt.sh deleted file mode 100644 index 8706e68054..0000000000 --- a/examples/checkpoint/run_deepseek_megatron_ckpt.sh +++ /dev/null @@ -1,91 +0,0 @@ -set -x - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -huggingface-cli download deepseek-ai/deepseek-llm-7b-chat - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \ - critic.optim.lr=2e-5 \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=4 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_checkpoint' \ - trainer.experiment_name='deepseek_megatron_checkpoint_saveload' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=50 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=50 $@ - - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \ - critic.optim.lr=2e-5 \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=4 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_checkpoint' \ - trainer.experiment_name='deepseek_megatron_checkpoint_saveload' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.resume_mode=auto \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=150 $@ \ No newline at end of file diff --git a/examples/checkpoint/run_qwen_megatron_ckpt.sh b/examples/checkpoint/run_qwen_megatron_ckpt.sh deleted file mode 100644 index 6ceeda11c7..0000000000 --- a/examples/checkpoint/run_qwen_megatron_ckpt.sh +++ /dev/null @@ -1,97 +0,0 @@ -set -x - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -huggingface-cli download Qwen/Qwen2-7B-Instruct - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \ - critic.optim.lr=2e-5 \ - critic.model.path=Qwen/Qwen2-7B-Instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=4 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_checkpoint' \ - trainer.experiment_name='qwen2_7b_megatron_saveload' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=100 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=100 $@ - - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \ - critic.optim.lr=2e-5 \ - critic.model.path=Qwen/Qwen2-7B-Instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=4 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_checkpoint' \ - trainer.experiment_name='qwen2_7b_megatron_saveload' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.resume_mode=auto \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=150 $@ \ No newline at end of file diff --git a/examples/data_preprocess/full_hh_rlhf.py b/examples/data_preprocess/full_hh_rlhf.py deleted file mode 100644 index 07e0884cb9..0000000000 --- a/examples/data_preprocess/full_hh_rlhf.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -- Preprocess data and split the training set into 75% for training RM and 25% for validting RM. -- All the training data is used to train SFT and RL. -- Both chosen and rejected is used to train SFT -""" -import argparse -import os - -import pandas as pd -from datasets import load_dataset - -from tqdm.auto import tqdm - -from verl.utils.fs import copy, makedirs - - -def generate_sft_dataset(target_hdfs_path_dir, local_dir='~/data/full_hh_rlh/sft'): - dataset = load_dataset('Dahoas/full-hh-rlhf') - output = {'prompt': [], 'response': []} - for data in tqdm(dataset['train']): - # add chosen - output['prompt'].append(data['prompt']) - output['response'].append(data['chosen']) - - # add rejection - output['prompt'].append(data['prompt']) - output['response'].append(data['rejected']) - - df = pd.DataFrame(output) - - local_dir = os.path.expanduser(local_dir) - os.makedirs(local_dir, exist_ok=True) - - local_path = os.path.join(local_dir, 'train.parquet') - - df.to_parquet(path=local_path) - - if target_hdfs_path_dir is not None: - hdfs_dir = target_hdfs_path_dir + '/' + 'train.parquet' - makedirs(hdfs_dir) - - copy(local_path, hdfs_dir) - - -def generate_rm_dataset(target_hdfs_path_dir, local_dir='~/data/full_hh_rlh/rm'): - train_dataset = load_dataset('Dahoas/full-hh-rlhf', split='train[:75%]') - test_dataset = load_dataset('Dahoas/full-hh-rlhf', split='train[-25%:]') - - local_dir = os.path.expanduser(local_dir) - os.makedirs(local_dir, exist_ok=True) - - for dataset, name in zip([train_dataset, test_dataset], ['train', 'test']): - output = {'prompt': [], 'chosen': [], 'rejected': []} - for data in tqdm(dataset): - # add chosen - output['prompt'].append(data['prompt']) - output['chosen'].append(data['chosen']) - output['rejected'].append(data['rejected']) - - df = pd.DataFrame(output) - - local_path = os.path.join(local_dir, name + '.parquet') - - df.to_parquet(path=local_path) - - if target_hdfs_path_dir is not None: - hdfs_dir = target_hdfs_path_dir + '/' + name + '.parquet' - makedirs(hdfs_dir) - - copy(local_path, hdfs_dir) - - -def generate_rl_dataset(target_hdfs_path_dir, local_dir='~/data/full_hh_rlhf/rl'): - dataset = load_dataset('Dahoas/full-hh-rlhf') - train_dataset = dataset['train'] - - data_source = 'Dahoas/full-hh-rlhf' - - # add a row to each data item that represents a unique id - def make_map_fn(split): - - def process_fn(example, idx): - prompt = example.pop('prompt') - response = example.pop('response') - - data = { - "data_source": data_source, - "prompt": [{ - "role": "user", - "content": prompt - }], - "ability": "alignment", - "reward_model": { - "style": "model", - "ground_truth": response # should not be used - }, - "extra_info": { - 'split': split, - 'index': idx - } - } - return data - - return process_fn - - train_dataset = train_dataset.map(function=make_map_fn('train'), with_indices=True) - local_dir = os.path.expanduser(local_dir) - local_path = os.path.join(local_dir, 'train.parquet') - train_dataset.to_parquet(local_path) - - if target_hdfs_path_dir is not None: - hdfs_dir = target_hdfs_path_dir + '/' + 'train.parquet' - makedirs(hdfs_dir) - - copy(local_path, hdfs_dir) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--split', type=str, choices=['sft', 'rm', 'rl'], required=True) - parser.add_argument('--local_dir', type=str, default='~/data/full_hh_rlhf') - parser.add_argument('--hdfs_dir', type=str, required=False, default=None) - - args = parser.parse_args() - - if args.split == 'sft': - generate_sft_dataset(args.hdfs_dir, os.path.join(args.local_dir, args.split)) - elif args.split == 'rm': - generate_rm_dataset(args.hdfs_dir, os.path.join(args.local_dir, args.split)) - elif args.split == 'rl': - generate_rl_dataset(args.hdfs_dir, os.path.join(args.local_dir, args.split)) - else: - raise NotImplementedError diff --git a/examples/data_preprocess/geo3k.py b/examples/data_preprocess/geo3k.py deleted file mode 100644 index 3a0c776738..0000000000 --- a/examples/data_preprocess/geo3k.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Preprocess the Geometry3k dataset to parquet format -""" - -import os -import datasets - -from verl.utils.hdfs_io import copy, makedirs -import argparse - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--local_dir', default='~/data/geo3k') - parser.add_argument('--hdfs_dir', default=None) - - args = parser.parse_args() - - data_source = 'hiyouga/geometry3k' - - dataset = datasets.load_dataset(data_source) - - train_dataset = dataset['train'] - test_dataset = dataset['test'] - - instruction_following = ( - r'You FIRST think about the reasoning process as an internal monologue and then provide the final answer. ' - r'The reasoning process MUST BE enclosed within tags. The final answer MUST BE put in \boxed{}.' - ) - - # add a row to each data item that represents a unique id - def make_map_fn(split): - - def process_fn(example, idx): - problem = example.pop('problem') - prompt = problem + ' ' + instruction_following - answer = example.pop('answer') - images = example.pop('images') - - data = { - "data_source": data_source, - "prompt": [{ - "role": "user", - "content": prompt, - }], - "images": images, - "ability": "math", - "reward_model": { - "style": "rule", - "ground_truth": answer - }, - "extra_info": { - 'split': split, - 'index': idx, - 'answer': answer, - "question": problem, - } - } - return data - - return process_fn - - train_dataset = train_dataset.map(function=make_map_fn('train'), with_indices=True, num_proc=8) - test_dataset = test_dataset.map(function=make_map_fn('test'), with_indices=True, num_proc=8) - - local_dir = args.local_dir - hdfs_dir = args.hdfs_dir - - train_dataset.to_parquet(os.path.join(local_dir, 'train.parquet')) - test_dataset.to_parquet(os.path.join(local_dir, 'test.parquet')) - - if hdfs_dir is not None: - makedirs(hdfs_dir) - copy(src=local_dir, dst=hdfs_dir) diff --git a/examples/data_preprocess/gsm8k.py b/examples/data_preprocess/gsm8k.py deleted file mode 100644 index b82d7d71ad..0000000000 --- a/examples/data_preprocess/gsm8k.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Preprocess the GSM8k dataset to parquet format -""" - -import re -import os -import datasets - -from verl.utils.hdfs_io import copy, makedirs -import argparse - - -def extract_solution(solution_str): - solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) - assert solution is not None - final_solution = solution.group(0) - final_solution = final_solution.split('#### ')[1].replace(',', '') - return final_solution - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--local_dir', default='~/data/gsm8k') - parser.add_argument('--hdfs_dir', default=None) - - args = parser.parse_args() - - data_source = 'openai/gsm8k' - - dataset = datasets.load_dataset(data_source, 'main') - - train_dataset = dataset['train'] - test_dataset = dataset['test'] - - instruction_following = "Let's think step by step and output the final answer after \"####\"." - - # add a row to each data item that represents a unique id - def make_map_fn(split): - - def process_fn(example, idx): - question_raw = example.pop('question') - - question = question_raw + ' ' + instruction_following - - answer_raw = example.pop('answer') - solution = extract_solution(answer_raw) - data = { - "data_source": data_source, - "prompt": [{ - "role": "user", - "content": question, - }], - "ability": "math", - "reward_model": { - "style": "rule", - "ground_truth": solution - }, - "extra_info": { - 'split': split, - 'index': idx, - 'answer': answer_raw, - "question": question_raw, - } - } - return data - - return process_fn - - train_dataset = train_dataset.map(function=make_map_fn('train'), with_indices=True) - test_dataset = test_dataset.map(function=make_map_fn('test'), with_indices=True) - - local_dir = args.local_dir - hdfs_dir = args.hdfs_dir - - train_dataset.to_parquet(os.path.join(local_dir, 'train.parquet')) - test_dataset.to_parquet(os.path.join(local_dir, 'test.parquet')) - - if hdfs_dir is not None: - makedirs(hdfs_dir) - - copy(src=local_dir, dst=hdfs_dir) diff --git a/examples/data_preprocess/hellaswag.py b/examples/data_preprocess/hellaswag.py deleted file mode 100644 index 39c8e8f559..0000000000 --- a/examples/data_preprocess/hellaswag.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Preprocess Hellaswag dataset. - -""" - -import re -import os -import datasets - -from verl.utils.hdfs_io import copy, makedirs -import argparse - - -def preprocess(text): - text = text.strip() - # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag. - text = text.replace(" [title]", ". ") - text = re.sub("\\[.*?\\]", "", text) - text = text.replace(" ", " ") - return text - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--local_dir', default='/opt/tiger/hellaswag') - parser.add_argument('--hdfs_dir', default=None) - - args = parser.parse_args() - - data_source = 'Rowan/hellaswag' - - dataset = datasets.load_dataset(data_source, trust_remote_code=True) - - train_dataset = dataset['train'] - val_dataset = dataset['validation'] - test_dataset = dataset['test'] - - instruction = 'Please complete the following sentence.\n' - - def make_map_fn(split): - - def process_fn(doc, idx): - ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize() - query = preprocess(doc["activity_label"] + ": " + ctx) - choices = [preprocess(ending) for ending in doc["endings"]] - gold = int(doc["label"]) - - data = { - "data_source": data_source, - "prompt": [{ - "role": "user", - "content": query - }], - "ability": "nlp", - "reward_model": { - "style": "model", - "eval": "multiple_choice", # using loglikelihood - "ground_truth": gold, - "choices": choices - }, - "extra_info": { - 'split': split, - 'index': idx - } - } - return data - - return process_fn - - # filter data that doesn't have a label - train_dataset = train_dataset.filter(lambda x: len(x['label']) > 0) - val_dataset = val_dataset.filter(lambda x: len(x['label']) > 0) - test_dataset = test_dataset.filter(lambda x: len(x['label']) > 0) - - train_dataset = train_dataset.map(function=make_map_fn('train'), with_indices=True) - val_dataset = val_dataset.map(function=make_map_fn('validation'), with_indices=True) - test_dataset = test_dataset.map(function=make_map_fn('test'), with_indices=True) - - local_dir = args.local_dir - hdfs_dir = args.hdfs_dir - - train_dataset.to_parquet(os.path.join(local_dir, 'train.parquet')) - val_dataset.to_parquet(os.path.join(local_dir, 'validation.parquet')) - test_dataset.to_parquet(os.path.join(local_dir, 'test.parquet')) - - if hdfs_dir is not None: - makedirs(hdfs_dir) - - copy(src=local_dir, dst=hdfs_dir) diff --git a/examples/data_preprocess/math_dataset.py b/examples/data_preprocess/math_dataset.py deleted file mode 100644 index 632f28801a..0000000000 --- a/examples/data_preprocess/math_dataset.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Preprocess the MATH-lighteval dataset to parquet format -""" - -import os -import datasets - -from verl.utils.hdfs_io import copy, makedirs -import argparse - -from verl.utils.reward_score.math import remove_boxed, last_boxed_only_string - - -def extract_solution(solution_str): - return remove_boxed(last_boxed_only_string(solution_str)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--local_dir', default='~/data/math') - parser.add_argument('--hdfs_dir', default=None) - - args = parser.parse_args() - - # 'lighteval/MATH' is no longer available on huggingface. - # Use mirror repo: DigitalLearningGmbH/MATH-lighteval - data_source = 'DigitalLearningGmbH/MATH-lighteval' - print(f"Loading the {data_source} dataset from huggingface...", flush=True) - dataset = datasets.load_dataset(data_source, trust_remote_code=True) - - train_dataset = dataset['train'] - test_dataset = dataset['test'] - - instruction_following = "Let's think step by step and output the final answer within \\boxed{}." - - # add a row to each data item that represents a unique id - def make_map_fn(split): - - def process_fn(example, idx): - question = example.pop('problem') - - question = question + ' ' + instruction_following - - answer = example.pop('solution') - solution = extract_solution(answer) - data = { - "data_source": data_source, - "prompt": [{ - "role": "user", - "content": question - }], - "ability": "math", - "reward_model": { - "style": "rule", - "ground_truth": solution - }, - "extra_info": { - 'split': split, - 'index': idx - } - } - return data - - return process_fn - - train_dataset = train_dataset.map(function=make_map_fn('train'), with_indices=True) - test_dataset = test_dataset.map(function=make_map_fn('test'), with_indices=True) - - local_dir = args.local_dir - hdfs_dir = args.hdfs_dir - - train_dataset.to_parquet(os.path.join(local_dir, 'train.parquet')) - test_dataset.to_parquet(os.path.join(local_dir, 'test.parquet')) - - if hdfs_dir is not None: - makedirs(hdfs_dir) - - copy(src=local_dir, dst=hdfs_dir) diff --git a/examples/generation/run_deepseek7b_mutli_node.sh b/examples/generation/run_deepseek7b_mutli_node.sh deleted file mode 100644 index e939268ff8..0000000000 --- a/examples/generation/run_deepseek7b_mutli_node.sh +++ /dev/null @@ -1,22 +0,0 @@ -set -x - -data_path=$HOME/data/rlhf/gsm8k/test.parquet -save_path=$HOME/data/rlhf/math/deepseek_v2_lite_gen_test.parquet -model_path=deepseek-ai/deepseek-llm-7b-chat - -python3 -m verl.trainer.main_generation \ - trainer.nnodes=2 \ - trainer.n_gpus_per_node=8 \ - data.path=$data_path \ - data.prompt_key=prompt \ - data.n_samples=1 \ - data.output_path=$save_path \ - model.path=$model_path\ - +model.trust_remote_code=True \ - rollout.temperature=1.0 \ - rollout.top_k=50 \ - rollout.top_p=0.7 \ - rollout.prompt_length=2048 \ - rollout.response_length=1024 \ - rollout.tensor_model_parallel_size=16 \ - rollout.gpu_memory_utilization=0.8 diff --git a/examples/generation/run_deepseek_v2_lite_math.sh b/examples/generation/run_deepseek_v2_lite_math.sh deleted file mode 100644 index 6f13fef53e..0000000000 --- a/examples/generation/run_deepseek_v2_lite_math.sh +++ /dev/null @@ -1,22 +0,0 @@ -set -x - -data_path=$HOME/data/rlhf/gsm8k/test.parquet -save_path=$HOME/data/rlhf/math/deepseek_v2_lite_gen_test.parquet -model_path=deepseek-ai/deepseek-llm-7b-chat - -python3 -m verl.trainer.main_generation \ - trainer.nnodes=1 \ - trainer.n_gpus_per_node=8 \ - data.path=$data_path \ - data.prompt_key=prompt \ - data.n_samples=1 \ - data.output_path=$save_path \ - model.path=$model_path \ - +model.trust_remote_code=True \ - rollout.temperature=1.0 \ - rollout.top_k=50 \ - rollout.top_p=0.7 \ - rollout.prompt_length=2048 \ - rollout.response_length=1024 \ - rollout.tensor_model_parallel_size=2 \ - rollout.gpu_memory_utilization=0.8 diff --git a/examples/generation/run_sglang.sh b/examples/generation/run_sglang.sh deleted file mode 100644 index 8b1ba7a2ae..0000000000 --- a/examples/generation/run_sglang.sh +++ /dev/null @@ -1,35 +0,0 @@ -set -x -source /root/miniconda3/etc/profile.d/conda.sh -conda activate sglang-verl -data_path=/verl/data/gsm8k/test.parquet -save_path=/verl/data/math/deepseek_v2_lite_gen_test.parquet -model_path=Qwen/Qwen2.5-Coder-7B-Instruct -export HOST_IP="127.0.0.1" -export SGLANG_HOST_IP="127.0.0.1" - -# port=6379 -# ray start --head \ -# --port=$port \ -# --num-gpus=8 \ -# --include-dashboard=false \ -# --block & - -python3 -m verl.trainer.main_generation \ - trainer.nnodes=1 \ - trainer.n_gpus_per_node=1 \ - data.path=$data_path \ - data.prompt_key=prompt \ - data.n_samples=1 \ - data.output_path=$save_path \ - model.path=$model_path \ - +model.trust_remote_code=True \ - rollout.name=async \ - rollout.temperature=1.0 \ - rollout.top_k=50 \ - rollout.top_p=0.7 \ - rollout.prompt_length=2048 \ - rollout.response_length=1024 \ - rollout.tensor_model_parallel_size=1 \ - rollout.gpu_memory_utilization=0.8 \ - +rollout.enable_memory_saver=True \ - +trainer.hybrid_engine=True diff --git a/examples/grpo_trainer/run_deepseek7b_llm.sh b/examples/grpo_trainer/run_deepseek7b_llm.sh deleted file mode 100644 index 4f7f79ea87..0000000000 --- a/examples/grpo_trainer/run_deepseek7b_llm.sh +++ /dev/null @@ -1,40 +0,0 @@ -set -x - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=80 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=160 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=160 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_deepseek7b_llm_math.sh b/examples/grpo_trainer/run_deepseek7b_llm_math.sh deleted file mode 100644 index 1666167fb6..0000000000 --- a/examples/grpo_trainer/run_deepseek7b_llm_math.sh +++ /dev/null @@ -1,50 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=40 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm_math' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_deepseek7b_llm_math_megatron.sh b/examples/grpo_trainer/run_deepseek7b_llm_math_megatron.sh deleted file mode 100644 index 89a13d9b88..0000000000 --- a/examples/grpo_trainer/run_deepseek7b_llm_math_megatron.sh +++ /dev/null @@ -1,49 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=grpo \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm_math_megatron' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_deepseek7b_llm_megatron.sh b/examples/grpo_trainer/run_deepseek7b_llm_megatron.sh deleted file mode 100644 index bdf4fc9709..0000000000 --- a/examples/grpo_trainer/run_deepseek7b_llm_megatron.sh +++ /dev/null @@ -1,40 +0,0 @@ -set -x - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm_megatron' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_deepseek7b_llm_seq_balance.sh b/examples/grpo_trainer/run_deepseek7b_llm_seq_balance.sh deleted file mode 100644 index 1e3a3a0691..0000000000 --- a/examples/grpo_trainer/run_deepseek7b_llm_seq_balance.sh +++ /dev/null @@ -1,39 +0,0 @@ -set -x - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm_seq_packing' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_qwen2-7b.sh b/examples/grpo_trainer/run_qwen2-7b.sh deleted file mode 100644 index db0947fa5b..0000000000 --- a/examples/grpo_trainer/run_qwen2-7b.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=40 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=40 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=40 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_qwen2-7b_math.sh b/examples/grpo_trainer/run_qwen2-7b_math.sh deleted file mode 100644 index e990bf04f7..0000000000 --- a/examples/grpo_trainer/run_qwen2-7b_math.sh +++ /dev/null @@ -1,50 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_qwen2-7b_math_megatron.sh b/examples/grpo_trainer/run_qwen2-7b_math_megatron.sh deleted file mode 100644 index 5f3f989661..0000000000 --- a/examples/grpo_trainer/run_qwen2-7b_math_megatron.sh +++ /dev/null @@ -1,50 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=grpo \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm_megatron' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_qwen2-7b_megatron.sh b/examples/grpo_trainer/run_qwen2-7b_megatron.sh deleted file mode 100644 index e8fca48781..0000000000 --- a/examples/grpo_trainer/run_qwen2-7b_megatron.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm_megatron' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh b/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh deleted file mode 100644 index 211bfcb91c..0000000000 --- a/examples/grpo_trainer/run_qwen2-7b_seq_balance.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm_kl1e-3' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_qwen2_5_vl-7b.sh b/examples/grpo_trainer/run_qwen2_5_vl-7b.sh deleted file mode 100644 index 19ec6c5e7e..0000000000 --- a/examples/grpo_trainer/run_qwen2_5_vl-7b.sh +++ /dev/null @@ -1,46 +0,0 @@ -set -x -ENGINE=${1:-vllm} -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/geo3k/train.parquet \ - data.val_files=$HOME/data/geo3k/test.parquet \ - data.train_batch_size=512 \ - data.max_prompt_length=1024 \ - data.max_response_length=2048 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.image_key=images \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=128 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=10 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.01 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=20 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=$ENGINE \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.enable_chunked_prefill=False \ - actor_rollout_ref.rollout.enforce_eager=False \ - actor_rollout_ref.rollout.free_cache_engine=False \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=20 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_geo3k' \ - trainer.experiment_name='qwen2_5_vl_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/grpo_trainer/run_sglang.sh b/examples/grpo_trainer/run_sglang.sh deleted file mode 100644 index 4ed1e9fc9e..0000000000 --- a/examples/grpo_trainer/run_sglang.sh +++ /dev/null @@ -1,56 +0,0 @@ -set -x -source /root/miniconda3/etc/profile.d/conda.sh -conda activate sglang-verl - -export VLLM_ATTENTION_BACKEND=XFORMERS - -gsm8k_train_path=/verl/data/gsm8k/train.parquet -gsm8k_test_path=/verl/data/gsm8k/test.parquet -# math_train_path=$HOME/data/math/train.parquet -# math_test_path=$HOME/data/math/test.parquet -export HOST_IP="127.0.0.1" -export SGLANG_HOST_IP="127.0.0.1" - -train_files="['$gsm8k_train_path']" -test_files="['$gsm8k_test_path']" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=256 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - +data.task_type='gsm8k' \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-Coder-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=async \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ - actor_rollout_ref.rollout.n=1 \ - actor_rollout_ref.rollout.enable_memory_saver=True \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=-1 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/ppo_trainer/run_deepseek7b_llm.sh b/examples/ppo_trainer/run_deepseek7b_llm.sh deleted file mode 100644 index 0805ba87a1..0000000000 --- a/examples/ppo_trainer/run_deepseek7b_llm.sh +++ /dev/null @@ -1,41 +0,0 @@ -set -x - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_micro_batch_size_per_gpu=32 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh b/examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh deleted file mode 100644 index 46c351d2b5..0000000000 --- a/examples/ppo_trainer/run_deepseek7b_llm_modelscope.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -x - -VERL_USE_MODELSCOPE=True \ -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_micro_batch_size_per_gpu=32 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_deepseek7b_llm_sp2.sh b/examples/ppo_trainer/run_deepseek7b_llm_sp2.sh deleted file mode 100644 index 5f9c9a31a4..0000000000 --- a/examples/ppo_trainer/run_deepseek7b_llm_sp2.sh +++ /dev/null @@ -1,43 +0,0 @@ -set -x - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=64 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - critic.optim.lr=1e-5 \ - critic.ulysses_sequence_parallel_size=2 \ - critic.model.use_remove_padding=True \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_micro_batch_size_per_gpu=64 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm_sp2' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_deepseek_full_hh_rlhf.sh b/examples/ppo_trainer/run_deepseek_full_hh_rlhf.sh deleted file mode 100644 index c0c1c2b2e9..0000000000 --- a/examples/ppo_trainer/run_deepseek_full_hh_rlhf.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -x - -train_files=$HOME/data/full_hh_rlhf/rl/train.parquet -test_files=$HOME/data/full_hh_rlhf/rl/train.parquet # no use - -python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=512 \ - data.max_prompt_length=128 \ - data.max_response_length=128 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=128 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - reward_model.enable=True \ - reward_model.megatron.tensor_model_parallel_size=4 \ - reward_model.model.path=deepseek-ai/deepseek-llm-7b-chat \ - reward_model.micro_batch_size_per_gpu=4 \ - reward_model.param_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_full_hh_rlhf_examples' \ - trainer.experiment_name='deepseek_llm_7b_model_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=100 $@ diff --git a/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron.sh b/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron.sh deleted file mode 100644 index 80f137ab51..0000000000 --- a/examples/ppo_trainer/run_deepseek_math_gsm8k_megatron.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -x - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-coder-6.7b-instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.path=deepseek-ai/deepseek-coder-6.7b-instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_math_gsm8k_examples' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=100 $@ diff --git a/examples/ppo_trainer/run_deepseek_megatron.sh b/examples/ppo_trainer/run_deepseek_megatron.sh deleted file mode 100644 index 66247a38a1..0000000000 --- a/examples/ppo_trainer/run_deepseek_megatron.sh +++ /dev/null @@ -1,56 +0,0 @@ -set -x - -# prepare pre-trained model ckpt -# deepseek-llm-7b-chat has 30 layers, which is not good to use with PP=2 and VPP=2, try using a 6.7b model instead -# huggingface-cli download deepseek-ai/deepseek-llm-7b-chat --local-dir $HOME/models/deepseek-llm-7b-chat -huggingface-cli download deepseek-ai/deepseek-coder-6.7b-instruct - -# ``actor_rollout_ref.rollout.tensor_model_parallel_size`` in theory could be different from -# ``**.megatron.tensor_model_parallel_size`` - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml -# tested on L20-16 GPUs per nodes, for other machines please adjust the n_gpus_per_node config accordingly -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-coder-6.7b-instruct \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \ - critic.optim.lr=2e-5 \ - critic.model.path=deepseek-ai/deepseek-coder-6.7b-instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=4 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_epochs=15 \ - trainer.val_before_train=False $@ diff --git a/examples/ppo_trainer/run_gemma.sh b/examples/ppo_trainer/run_gemma.sh deleted file mode 100644 index 504f320e2d..0000000000 --- a/examples/ppo_trainer/run_gemma.sh +++ /dev/null @@ -1,40 +0,0 @@ -set -x - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=512 \ - data.max_prompt_length=1024 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=google/gemma-2-2b-it \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=False \ - actor_rollout_ref.actor.ppo_mini_batch_size=128 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=False \ - critic.model.path=google/gemma-2-2b-it \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example' \ - trainer.experiment_name='gemma2b_function_rm' \ - trainer.n_gpus_per_node=2 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_qwen2-7b_math_gsm8k_megatron.sh b/examples/ppo_trainer/run_qwen2-7b_math_gsm8k_megatron.sh deleted file mode 100644 index cf1ae4739c..0000000000 --- a/examples/ppo_trainer/run_qwen2-7b_math_gsm8k_megatron.sh +++ /dev/null @@ -1,44 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megatron_trainer'\ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.path=Qwen/Qwen2-7B-Instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_math_gsm8k_examples' \ - trainer.experiment_name='qwen2_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=100 $@ diff --git a/examples/ppo_trainer/run_qwen2-7b_megatron.sh b/examples/ppo_trainer/run_qwen2-7b_megatron.sh deleted file mode 100644 index a4ddb38604..0000000000 --- a/examples/ppo_trainer/run_qwen2-7b_megatron.sh +++ /dev/null @@ -1,57 +0,0 @@ -set -x - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -# tested on L20-16 GPUs per nodes, for other machines please adjust the n_gpus_per_node config accordingly -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=4 \ - critic.optim.lr=1e-5 \ - critic.model.path=Qwen/Qwen2-7B-Instruct \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=4 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='Qwen2-7B-Instruct_function_rm_megatron' \ - trainer.n_gpus_per_node=16 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_qwen2-7b_rm.sh b/examples/ppo_trainer/run_qwen2-7b_rm.sh deleted file mode 100644 index 914a3db7f6..0000000000 --- a/examples/ppo_trainer/run_qwen2-7b_rm.sh +++ /dev/null @@ -1,72 +0,0 @@ -# Discliamer: the model used in the script is only for academic purpose. -set -x - -# Data preparation scripts are available in ``examples/data_preprocess``. -# Example usage: -# -# python3 examples/data_preprocess/math_dataset.py --local_dir ~/data/math -# python3 examples/data_preprocess/gsm8k.py --local_dir ~/data/gsm8k - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -export VLLM_ATTENTION_BACKEND=XFORMERS # vllm + qwen2-7b with flash_attn has some issues - -# prepare model ckpt -huggingface-cli download Qwen/Qwen2-7B-Instruct --local-dir $HOME/models/Qwen2-7B-Instruct & -huggingface-cli download sfairXC/FsfairX-LLaMA3-RM-v0.1 --local-dir $HOME/models/FsfairX-LLaMA3-RM-v0.1 & -wait - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path="$HOME/models/Qwen2-7B-Instruct" \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.optim.lr_warmup_steps_ratio=0.05 \ - critic.model.path="$HOME/models/Qwen2-7B-Instruct" \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_micro_batch_size_per_gpu=32 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - reward_model.enable=True \ - reward_model.model.path="$HOME/models/FsfairX-LLaMA3-RM-v0.1" \ - reward_model.model.use_remove_padding=True \ - reward_model.model.fsdp_config.param_offload=True \ - reward_model.micro_batch_size_per_gpu=32 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example' \ - trainer.val_before_train=False \ - trainer.experiment_name='Qwen2-7B-Instruct_hybrid_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance.sh b/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance.sh deleted file mode 100644 index b029b39f66..0000000000 --- a/examples/ppo_trainer/run_qwen2-7b_rm_seq_balance.sh +++ /dev/null @@ -1,60 +0,0 @@ -set -x - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=4096 \ - data.max_prompt_length=4096 \ - data.max_response_length=4096 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=512 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=Qwen/Qwen2-7B-Instruct \ - critic.model.enable_gradient_checkpointing=True \ - critic.use_dynamic_bsz=True \ - critic.ppo_max_token_len_per_gpu=98304 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - reward_model.enable=True \ - reward_model.model.path=sfairXC/FsfairX-LLaMA3-RM-v0.1\ - reward_model.model.use_remove_padding=True \ - reward_model.model.fsdp_config.param_offload=True \ - reward_model.micro_batch_size_per_gpu=32 \ - reward_model.use_dynamic_bsz=True \ - reward_model.forward_max_token_len_per_gpu=98304 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen2-7b_hybrid_rm_bsz8k_p4k_r4k_seq_packing' \ - trainer.n_gpus_per_node=8 \ - trainer.val_before_train=False \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_qwen2-7b_seq_balance.sh b/examples/ppo_trainer/run_qwen2-7b_seq_balance.sh deleted file mode 100644 index 0182aa7ca4..0000000000 --- a/examples/ppo_trainer/run_qwen2-7b_seq_balance.sh +++ /dev/null @@ -1,51 +0,0 @@ -set -x - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=4096 \ - data.max_prompt_length=4096 \ - data.max_response_length=4096 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=512 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=Qwen/Qwen2-7B-Instruct \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_max_token_len_per_gpu=98304 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen2-7b_function_rm_bsz8k_p4k_r4k_seq_packing' \ - trainer.n_gpus_per_node=8 \ - trainer.val_before_train=False \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/run_qwen2.5-32b.sh b/examples/ppo_trainer/run_qwen2.5-32b.sh deleted file mode 100644 index da2bab9e30..0000000000 --- a/examples/ppo_trainer/run_qwen2.5-32b.sh +++ /dev/null @@ -1,50 +0,0 @@ -set -x - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-32B-Instruct \ - actor_rollout_ref.model.enable_gradient_checkpointing=False \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=Qwen/Qwen2.5-32B-Instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=8 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-32B-Instruct_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=4 \ - trainer.save_freq=-1 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 $@ diff --git a/examples/ppo_trainer/verl_getting_started.ipynb b/examples/ppo_trainer/verl_getting_started.ipynb deleted file mode 100644 index 73df079ea4..0000000000 --- a/examples/ppo_trainer/verl_getting_started.ipynb +++ /dev/null @@ -1,3596 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "eXkR4NjYhezg" - }, - "source": [ - "# Run Qwen PPO with [verl](https://github.com/volcengine/verl)\n", - "\n", - "This tutorial provides a step-by-step guide to using verl for executing your RLHF pipeline. You can find our [github repo](https://github.com/volcengine/verl/) and [documentation](https://verl.readthedocs.io/en/latest/index.html) for mode details.\n", - "\n", - "This notebook is also published on the [Lightning Studio](https://lightning.ai/hlin-verl/studios/verl-getting-started) platform, which provides free GPU quota every month. Checkout the published notebook with pre-installed dependencies using a free L4 GPU [here](https://lightning.ai/hlin-verl/studios/verl-getting-started) (no credit card required).\n", - "\n", - "### You will learn:\n", - "\n", - "- How to install verl from scratch.\n", - "- How to use existing scripts to run an RLHF pipeline with your own models and data." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XSDNzNuQkJJh" - }, - "source": [ - "# Dependency Installation\n", - "\n", - "If you are running on Lightning Studio using the published notebook, the dependencies are **already installed** and you can proceed to step \"**Load Pretrained Language Model**\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "gnfZyMm-3BNC", - "outputId": "9e8e5116-5344-4c9b-bc34-e9e9be752ff5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pip in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (24.3.1)\n", - "Requirement already satisfied: setuptools in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (75.8.0)\n", - "Requirement already satisfied: wheel in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (0.45.1)\n", - "Requirement already satisfied: torch==2.4.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (2.4.0)\n", - "Requirement already satisfied: torchvision==0.19.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (0.19.0)\n", - "Requirement already satisfied: filelock in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (3.16.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (4.12.2)\n", - "Requirement already satisfied: sympy in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (1.13.3)\n", - "Requirement already satisfied: networkx in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (3.4.2)\n", - "Requirement already satisfied: jinja2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (3.1.4)\n", - "Requirement already satisfied: fsspec in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (2024.9.0)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (9.1.0.70)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (2.20.5)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (12.1.105)\n", - "Requirement already satisfied: triton==3.0.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0) (3.0.0)\n", - "Requirement already satisfied: numpy in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torchvision==0.19.0) (1.26.4)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torchvision==0.19.0) (10.4.0)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.4.0) (12.6.77)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from jinja2->torch==2.4.0) (3.0.2)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from sympy->torch==2.4.0) (1.3.0)\n", - "pytorch-lightning 2.4.0\n", - "torch 2.4.0\n", - "torchmetrics 1.3.1\n", - "torchvision 0.19.0\n", - "Requirement already satisfied: flash-attn in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (2.7.2.post1)\n", - "Requirement already satisfied: torch in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from flash-attn) (2.4.0)\n", - "Requirement already satisfied: einops in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages/einops-0.8.0-py3.10.egg (from flash-attn) (0.8.0)\n", - "Requirement already satisfied: filelock in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (3.16.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (4.12.2)\n", - "Requirement already satisfied: sympy in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (1.13.3)\n", - "Requirement already satisfied: networkx in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (3.4.2)\n", - "Requirement already satisfied: jinja2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (3.1.4)\n", - "Requirement already satisfied: fsspec in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (2024.9.0)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (9.1.0.70)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (2.20.5)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n", - "Requirement already satisfied: triton==3.0.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch->flash-attn) (3.0.0)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->flash-attn) (12.6.77)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from jinja2->torch->flash-attn) (3.0.2)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from sympy->torch->flash-attn) (1.3.0)\n" - ] - } - ], - "source": [ - "!pip3 install --upgrade pip setuptools wheel\n", - "!pip3 install torch==2.4.0 torchvision==0.19.0\n", - "!pip3 list | grep torch\n", - "!pip3 install flash-attn --no-build-isolation" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HzV28CwOmruV" - }, - "source": [ - "## Install and verify verl\n", - "Now we're ready to install verl!" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0mtIn1VOk2E7", - "outputId": "8a83156e-c3aa-4921-97e2-a9472f22ed9d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Obtaining file:///teamspace/studios/this_studio/verl_repo\n", - " Installing build dependencies ... \u001b[?25ldone\n", - "\u001b[?25h Checking if build backend supports build_editable ... \u001b[?25ldone\n", - "\u001b[?25h Getting requirements to build editable ... \u001b[?25ldone\n", - "\u001b[?25h Preparing editable metadata (pyproject.toml) ... \u001b[?25ldone\n", - "\u001b[?25hRequirement already satisfied: accelerate in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (1.1.1)\n", - "Requirement already satisfied: codetiming in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (1.4.0)\n", - "Requirement already satisfied: datasets in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (3.1.0)\n", - "Requirement already satisfied: dill in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (0.3.8)\n", - "Requirement already satisfied: hydra-core in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (1.3.2)\n", - "Requirement already satisfied: numpy in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (1.26.4)\n", - "Requirement already satisfied: pybind11 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (2.13.6)\n", - "Requirement already satisfied: ray in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (2.10.0)\n", - "Requirement already satisfied: tensordict in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (0.5.0)\n", - "Requirement already satisfied: transformers in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (4.46.3)\n", - "Requirement already satisfied: vllm<=0.6.3 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from verl==0.1) (0.5.4)\n", - "Requirement already satisfied: cmake>=3.21 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (3.31.1)\n", - "Requirement already satisfied: ninja in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (1.11.1.2)\n", - "Requirement already satisfied: psutil in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (6.1.0)\n", - "Requirement already satisfied: sentencepiece in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.2.0)\n", - "Requirement already satisfied: requests in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (2.32.3)\n", - "Requirement already satisfied: tqdm in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (4.67.1)\n", - "Requirement already satisfied: py-cpuinfo in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (9.0.0)\n", - "Requirement already satisfied: tokenizers>=0.19.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.20.3)\n", - "Requirement already satisfied: fastapi in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.115.4)\n", - "Requirement already satisfied: aiohttp in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (3.10.10)\n", - "Requirement already satisfied: openai in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (1.55.3)\n", - "Requirement already satisfied: uvicorn[standard] in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.32.0)\n", - "Requirement already satisfied: pydantic>=2.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (2.9.2)\n", - "Requirement already satisfied: pillow in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (10.4.0)\n", - "Requirement already satisfied: prometheus-client>=0.18.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.21.0)\n", - "Requirement already satisfied: prometheus-fastapi-instrumentator>=7.0.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (7.0.0)\n", - "Requirement already satisfied: tiktoken>=0.6.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.7.0)\n", - "Requirement already satisfied: lm-format-enforcer==0.10.3 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.10.3)\n", - "Requirement already satisfied: outlines<0.1,>=0.0.43 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.0.46)\n", - "Requirement already satisfied: typing-extensions in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (4.12.2)\n", - "Requirement already satisfied: filelock>=3.10.4 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (3.16.1)\n", - "Requirement already satisfied: pyzmq in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (26.2.0)\n", - "Requirement already satisfied: nvidia-ml-py in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (12.560.30)\n", - "Requirement already satisfied: torch==2.4.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (2.4.0)\n", - "Requirement already satisfied: torchvision==0.19 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.19.0)\n", - "Requirement already satisfied: xformers==0.0.27.post2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (0.0.27.post2)\n", - "Requirement already satisfied: vllm-flash-attn==2.6.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from vllm<=0.6.3->verl==0.1) (2.6.1)\n", - "Requirement already satisfied: interegular>=0.3.2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from lm-format-enforcer==0.10.3->vllm<=0.6.3->verl==0.1) (0.3.3)\n", - "Requirement already satisfied: packaging in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from lm-format-enforcer==0.10.3->vllm<=0.6.3->verl==0.1) (24.1)\n", - "Requirement already satisfied: pyyaml in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from lm-format-enforcer==0.10.3->vllm<=0.6.3->verl==0.1) (6.0.2)\n", - "Requirement already satisfied: sympy in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (1.13.3)\n", - "Requirement already satisfied: networkx in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (3.4.2)\n", - "Requirement already satisfied: jinja2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (3.1.4)\n", - "Requirement already satisfied: fsspec in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (2024.9.0)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (9.1.0.70)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (2.20.5)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (12.1.105)\n", - "Requirement already satisfied: triton==3.0.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from torch==2.4.0->vllm<=0.6.3->verl==0.1) (3.0.0)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.4.0->vllm<=0.6.3->verl==0.1) (12.6.77)\n", - "Requirement already satisfied: click>=7.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from ray->verl==0.1) (8.1.7)\n", - "Requirement already satisfied: jsonschema in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from ray->verl==0.1) (4.23.0)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from ray->verl==0.1) (1.1.0)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from ray->verl==0.1) (4.23.4)\n", - "Requirement already satisfied: aiosignal in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from ray->verl==0.1) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from ray->verl==0.1) (1.5.0)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from transformers->verl==0.1) (0.26.3)\n", - "Requirement already satisfied: regex!=2019.12.17 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from transformers->verl==0.1) (2023.10.3)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from transformers->verl==0.1) (0.4.5)\n", - "Requirement already satisfied: pyarrow>=15.0.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from datasets->verl==0.1) (18.1.0)\n", - "Requirement already satisfied: pandas in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from datasets->verl==0.1) (2.1.4)\n", - "Requirement already satisfied: xxhash in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from datasets->verl==0.1) (3.5.0)\n", - "Requirement already satisfied: multiprocess<0.70.17 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from datasets->verl==0.1) (0.70.16)\n", - "Requirement already satisfied: omegaconf<2.4,>=2.2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from hydra-core->verl==0.1) (2.3.0)\n", - "Requirement already satisfied: antlr4-python3-runtime==4.9.* in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from hydra-core->verl==0.1) (4.9.3)\n", - "Requirement already satisfied: cloudpickle in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from tensordict->verl==0.1) (3.1.0)\n", - "Requirement already satisfied: orjson in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from tensordict->verl==0.1) (3.10.12)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from aiohttp->vllm<=0.6.3->verl==0.1) (2.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from aiohttp->vllm<=0.6.3->verl==0.1) (24.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from aiohttp->vllm<=0.6.3->verl==0.1) (6.1.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.12.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from aiohttp->vllm<=0.6.3->verl==0.1) (1.17.1)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from aiohttp->vllm<=0.6.3->verl==0.1) (4.0.3)\n", - "Requirement already satisfied: lark in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (1.2.2)\n", - "Requirement already satisfied: nest-asyncio in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (1.6.0)\n", - "Requirement already satisfied: diskcache in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (5.6.3)\n", - "Requirement already satisfied: numba in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (0.60.0)\n", - "Requirement already satisfied: referencing in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (0.35.1)\n", - "Requirement already satisfied: pycountry in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (24.6.1)\n", - "Requirement already satisfied: pyairports in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (2.1.1)\n", - "Requirement already satisfied: starlette<1.0.0,>=0.30.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from prometheus-fastapi-instrumentator>=7.0.0->vllm<=0.6.3->verl==0.1) (0.41.2)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from pydantic>=2.0->vllm<=0.6.3->verl==0.1) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.23.4 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from pydantic>=2.0->vllm<=0.6.3->verl==0.1) (2.23.4)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from requests->vllm<=0.6.3->verl==0.1) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from requests->vllm<=0.6.3->verl==0.1) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from requests->vllm<=0.6.3->verl==0.1) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from requests->vllm<=0.6.3->verl==0.1) (2024.8.30)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from jsonschema->ray->verl==0.1) (2024.10.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from jsonschema->ray->verl==0.1) (0.20.1)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from openai->vllm<=0.6.3->verl==0.1) (4.6.2.post1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from openai->vllm<=0.6.3->verl==0.1) (1.9.0)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from openai->vllm<=0.6.3->verl==0.1) (0.27.2)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from openai->vllm<=0.6.3->verl==0.1) (0.8.0)\n", - "Requirement already satisfied: sniffio in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from openai->vllm<=0.6.3->verl==0.1) (1.3.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from pandas->datasets->verl==0.1) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from pandas->datasets->verl==0.1) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from pandas->datasets->verl==0.1) (2024.2)\n", - "Requirement already satisfied: h11>=0.8 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from uvicorn[standard]->vllm<=0.6.3->verl==0.1) (0.14.0)\n", - "Requirement already satisfied: httptools>=0.5.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from uvicorn[standard]->vllm<=0.6.3->verl==0.1) (0.6.4)\n", - "Requirement already satisfied: python-dotenv>=0.13 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from uvicorn[standard]->vllm<=0.6.3->verl==0.1) (1.0.1)\n", - "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from uvicorn[standard]->vllm<=0.6.3->verl==0.1) (0.21.0)\n", - "Requirement already satisfied: watchfiles>=0.13 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from uvicorn[standard]->vllm<=0.6.3->verl==0.1) (0.24.0)\n", - "Requirement already satisfied: websockets>=10.4 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from uvicorn[standard]->vllm<=0.6.3->verl==0.1) (13.1)\n", - "Requirement already satisfied: exceptiongroup>=1.0.2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai->vllm<=0.6.3->verl==0.1) (1.2.2)\n", - "Requirement already satisfied: httpcore==1.* in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai->vllm<=0.6.3->verl==0.1) (1.0.6)\n", - "Requirement already satisfied: six>=1.5 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets->verl==0.1) (1.16.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from yarl<2.0,>=1.12.0->aiohttp->vllm<=0.6.3->verl==0.1) (0.2.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from jinja2->torch==2.4.0->vllm<=0.6.3->verl==0.1) (3.0.2)\n", - "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from numba->outlines<0.1,>=0.0.43->vllm<=0.6.3->verl==0.1) (0.43.0)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from sympy->torch==2.4.0->vllm<=0.6.3->verl==0.1) (1.3.0)\n", - "Building wheels for collected packages: verl\n", - " Building editable for verl (pyproject.toml) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for verl: filename=verl-0.1-0.editable-py3-none-any.whl size=13000 sha256=8fd1f1241dfe89d7f8384fe884f50ec4e070d18029c37472e5584300f5a326de\n", - " Stored in directory: /tmp/pip-ephem-wheel-cache-pz36kou4/wheels/f4/30/ea/7a2d2086bd780aba22048a0b415dc5e5a9e50b2c87e39e9717\n", - "Successfully built verl\n", - "Installing collected packages: verl\n", - "Successfully installed verl-0.1\n" - ] - } - ], - "source": [ - "# In case you run this notebook and have not cloned verl yet:\n", - "# !git clone https://github.com/volcengine/verl $HOME/verl_repo\n", - "\n", - "!cd $HOME/verl_repo && pip3 install -e . -U" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Restart the python kernel" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'status': 'ok', 'restart': True}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import IPython\n", - "\n", - "# Restart the kernel to pickup the latest python packages\n", - "IPython.get_ipython().kernel.do_shutdown(restart=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "mOBX8Jqc-ZBe" - }, - "outputs": [], - "source": [ - "import torch\n", - "try:\n", - " assert torch.cuda.is_available() is True\n", - " torch.ones(1, dtype=torch.bfloat16).cuda()\n", - "except AssertionError:\n", - " print(\"Please switch to an env with GPUs supporting bfloat16 (L4 RTX 5000, A5000, A100, H100, A10, etc)\")\n", - "\n", - "try:\n", - " import verl\n", - "except Exception as e:\n", - " print(\"Please install verl via pip and restart the kernel\")\n", - " raise e\n", - "\n", - "import flash_attn" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9mawNxDfo3Uu" - }, - "source": [ - "# Load Pretrained Language Model\n", - "\n", - "verl supports models available in Huggingface transformers (as well as custom Megatron models).\n", - "\n", - "Let's download the model first." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 404, - "referenced_widgets": [ - "763679906f9248a7a5f4c8de952d98ae", - "b319db13c64b43a38250342c81708f70", - "5110a596739443a8a640cfd50030644b", - "e93bf508749940909c3233904e898497", - "b45a42483d64410ba245feda17ae3e16", - "538e82daa19140098a4053da6e23de45", - "6b14f827b15f4e34be6590a5d2085b64", - "07f455e5e6dd45b7ba52f78bfc7ec7d6", - "fdf06125a50249b8878dbf01993306f4", - "0ab915aba5e14e5bba7ba1c22a682b89", - "2749b87567ea4b6cbc4cf825e2282615", - "645fee7bcccd42a794e4aa889c1fe145", - "aa19071cede44a089d7f3b19227d51e0", - "412349b6e00b4994bc3f63f8405b3ec2", - "a921f9b0d3c74381b75aa60f4d1cac1c", - "b707bf4c56744f05ac9245b07f6d1788", - "252e651f687e47f3bd20518f2ac5fb9f", - "835a6a0a56554d158ee40ccc5ccdffc5", - "be14bccf9f114d9f839c805afef08f61", - "52268a2d657b4e19badd66f781f68d93", - "06873240926949d98e13872546c5231d", - "936061cb57ad445195efc0aa24dd8d66", - "144df34a87334a6d8eb13055e7a9b9e4", - "1e9ee1c383074f638a688b029d72bc79", - "5cfeadb8ff394f38ac2e23f1a66beeb3", - "0eeef594fb564491ad8d80f86a8fbfdc", - "771c5ca9460f4539b30f452dd3f36b12", - "fab6aab315214fcb884529a4dbf84fe5", - "06e1b9b5d49d4ee3ab8d1a523659bcbf", - "e3848f0a11f8472fba3ecb624bc86dd9", - "c7b67dd574ad4c15b36930047553e9d3", - "9fbafd9fc26748b7889b5c52600f80a8", - "889e01d618544f7c9a9d748730255007", - "69e57962129241a689cfd2933b64127c", - "4bdbe0a8bb434bfc8e2172ecb5189705", - "b0bbbf7f9f264dfda2c0d6775567e446", - "6c9485ecc56f4027ad8f3824554e3968", - "3447ed64518746cabb0176348fc88d96", - "35e124a16d2945ddbb3ade95ef2b5519", - "7de86c10755f4e0da7974bdf1815a85d", - "4957b3690466495997721afab68ad93a", - "9e2c1dcd2cd643bbb941d6697fcc75a0", - "b10402691cc3480693dcf49d19336c72", - "f0350562775a4c4ca83772a78d05122b", - "1a382959fdeb4554827397823284d2fa", - "f52d7af1a82249a3aa7785476e10c2ad", - "afcc65785fef4b71b03ac83a4b14d97f", - "c0b19ca098a443598c662921832e8799", - "ca24445f8af44c8397f12d15d66eebf5", - "6cd310d2188d424eb20c3bf83ac34f56", - "ddecda628c6a4a5680b4241633153ebd", - "e49f1b46d8ae4c3e8f894b1f411922b9", - "0c9b8ffe4b8c4b5ca72a21cc54a1feb9", - "c3651669cb084d86b9b8c427c665d185", - "35bacfb8aa4c4a25bf8ce2d13a00f2b8", - "c1020ed4d8a44747838ed59287d284ed", - "a726ef24d10c42bf859e4c76cebde672", - "40259328dd5e4256939d7b1a3f038d98", - "ee0b85738cbf4376a6427fadbdecfad7", - "1491cbb53c6e4bfb9d17cf123dea83dd", - "5c8c3c4d700540f089f671d4f5d0dd9f", - "7c45a87d87f44b2384a4fd316ae36663", - "866c770e39b64dfd9764de755f6a9ec5", - "2babfcd555104f9d8ecf98e164ec40fc", - "7920655156a44e629514673dde2b9663", - "c24df93c305e42cdbaed3d6111d72010", - "c0e97dba53284330b0fb8cefc852d552", - "4d1a260957214732940766c874d3a02b", - "89a180c90767474b8e699e264620666e", - "7363ebea3a3a4f55b69b2d813c3b2fa5", - "d49791321218419d8b7af314dd904777", - "e6b66ca90c9c4b0ead5153e4a07cdc86", - "3e1dd2fd3bb049ab83aa987d748f5b9e", - "a1255e85757e495a86ae366857fb64f1", - "6f3742161c4f4bcc891c82aff7ece69f", - "e9f9be6fa1744f3380d21c451bc81555", - "c5024f35870446a0ae8fd747101ab719" - ] - }, - "id": "k8FsgBYnpR-R", - "outputId": "57e0e9ae-2c9e-498d-849f-7eafe59c4c03" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching 10 files: 0%| | 0/10 [00:00>20 Since the fraction representing the number of teaspoons she used is 7/20, she used 7/20120 = <<7/20120=42>>42 #### 42" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "AgRCvb6V6B3A", - "outputId": "f45c7f69-2f81-4b19-98de-8dc4b869736d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Map: 100%|████████████████████████| 7473/7473 [00:00<00:00, 24893.94 examples/s]\n", - "Map: 100%|████████████████████████| 1319/1319 [00:00<00:00, 25219.21 examples/s]\n", - "Creating parquet from Arrow format: 100%|████████| 8/8 [00:00<00:00, 257.77ba/s]\n", - "Creating parquet from Arrow format: 100%|████████| 2/2 [00:00<00:00, 370.05ba/s]\n" - ] - } - ], - "source": [ - "!mkdir -p $HOME/data/gsm8k\n", - "!python3 $HOME/verl_repo/examples/data_preprocess/gsm8k.py --local_dir $HOME/data/gsm8k" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JPZZKBxunAoj" - }, - "source": [ - "# the reward\n", - "\n", - "We use a rule-based reward model. We force the model to produce a final answer following 4 `#` as shown in the solution. We extract the final answer from both the solution and model's output using regular expression matching. We compare them and assign a reward of 1 to correct answer, 0.1 to incorrect answer and 0 to no answer." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "SjjLVuO60WD1", - "outputId": "affb562c-7f6c-41f7-ef47-4dfea0020e90" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "def compute_score(solution_str, ground_truth, method='strict', format_score=0., score=1.):\n", - " \"\"\"The scoring function for GSM8k.\n", - "\n", - " Reference: Trung, Luong, et al. \"Reft: Reasoning with reinforced fine-tuning.\" Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). 2024.\n", - "\n", - " Args:\n", - " solution_str: the solution text\n", - " ground_truth: the ground truth\n", - " method: the method to extract the solution, choices are 'strict' and 'flexible'\n", - " format_score: the score for the format\n", - " score: the score for the correct answer\n", - " \"\"\"\n", - " answer = extract_solution(solution_str=solution_str, method=method)\n", - " if answer is None:\n", - " return 0\n", - " else:\n", - " if answer == ground_truth:\n", - " return score\n", - " else:\n", - " return format_score\n", - "\n" - ] - } - ], - "source": [ - "import inspect\n", - "from verl.utils.reward_score.gsm8k import compute_score as gsm8k_reward\n", - "\n", - "print(inspect.getsource(gsm8k_reward))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NPBGPdSD0sCF" - }, - "source": [ - "# Run the RL Pipeline\n", - "Let's start with the Proximal Policy Optimization (PPO) algorithm, one of the most widely used methods for post-training large language models.\n", - "\n", - "The main entry point of the PPO algorithm example is: `main_ppo.py`. A detailed guide to understanding the code architecture of `main_ppo.py` is available [here](https://verl.readthedocs.io/en/latest/examples/ppo_code_architecture.html).\n", - "\n", - "In this tutorial, we will demonstrate how to run the PPO algorithm with **Qwen 2.5-0.5B** by setting:\n", - "- `trainer.n_gpus_per_node`: Number of GPUs per node.\n", - "\n", - "- `actor_rollout_ref.rollout.tensor_model_parallel_size`: TP size for rollout. Only effective for vllm.\n", - "\n", - "- `actor_rollout_ref/critic.model.path`: Huggingface model path. This can be either local path or HDFS path. For HDFS path, we provide utils to download it to DRAM and convert the HDFS path to local path.\n", - "\n", - "- `data.train_batch_size`: Batch size sampled for one training iteration of different RL algorithms.\n", - "\n", - "- `data.max_prompt_length`: Maximum prompt length. All prompts will be left-padded to this length. An error will be reported if the length is too long.\n", - "\n", - "- `data.max_response_length`: Maximum response length. Rollout in RL algorithms (e.g. PPO) generates up to this length.\n", - "\n", - "- `actor_rollout_ref.actor.ppo_mini_batch_size`: One sample is split into multiple sub-batches with batch_size=ppo_mini_batch_size for PPO updates.\n", - "\n", - "- `actor_rollout_ref/critic.actor.ppo_micro_batch_size`: Similar to gradient accumulation, the micro_batch_size for one forward pass, trading speed for GPU memory.\n", - "\n", - "The full configuration explanation is available [here](https://verl.readthedocs.io/en/latest/examples/config.html).\n", - "\n", - "The training may take a few hours to finish but you can observe how the model performance increases. It will progressively output:\n", - "\n", - "- generated sentences.\n", - "\n", - "- step information with RL metrics, such as entropy loss, kl, and ``val/test_score/openai/gsm8k`` (validated every ``trainer.test_freq`` steps)\n", - "\n", - "If you come across GPU out of memory issues, set smaller values for the micro batch size used for gradient accumulation:\n", - "\n", - "- actor_rollout_ref.actor.ppo_micro_batch_size=1\n", - "- critic.ppo_micro_batch_size=1" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "GvyEebBB4eCA", - "outputId": "a0bb8f75-6f79-456c-c71f-254c84503763" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-01-10 21:40:29,298\tINFO worker.py:1752 -- Started a local Ray instance.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m {'actor_rollout_ref': {'actor': {'clip_ratio': 0.2,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'entropy_coeff': 0.001,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'fsdp_config': {'grad_offload': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'optimizer_offload': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'param_offload': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'wrap_policy': {'min_num_params': 0}},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'grad_clip': 1.0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'optim': {'lr': 1e-06,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'lr_warmup_steps_ratio': 0.0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'min_lr_ratio': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'total_training_steps': -1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'warmup_style': 'constant'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ppo_epochs': 1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ppo_micro_batch_size': 4,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ppo_mini_batch_size': 64,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'shuffle': True,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'strategy': 'fsdp'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'hybrid_engine': True,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'model': {'enable_gradient_checkpointing': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'external_lib': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'override_config': {},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'path': '/teamspace/studios/this_studio/models/Qwen2.5-0.5B-Instruct'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ref': {'fsdp_config': {'param_offload': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'wrap_policy': {'min_num_params': 0}},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'log_prob_micro_batch_size_per_gpu': 4},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'rollout': {'do_sample': True,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'dtype': 'bfloat16',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'enforce_eager': True,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'free_cache_engine': True,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'gpu_memory_utilization': 0.4,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ignore_eos': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'load_format': 'dummy_dtensor',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'log_prob_micro_batch_size_per_gpu': 1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'max_num_batched_tokens': 8192,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'max_num_seqs': 1024,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'n': 1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'name': 'vllm',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'prompt_length': 512,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'response_length': 256,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'temperature': 1.0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'tensor_model_parallel_size': 1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'top_k': -1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'top_p': 1}},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'algorithm': {'adv_estimator': 'gae',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'gamma': 1.0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'kl_ctrl': {'kl_coef': 0.001, 'type': 'fixed'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'kl_penalty': 'kl',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'lam': 1.0},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'critic': {'cliprange_value': 0.5,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'forward_micro_batch_size_per_gpu': 4,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'grad_clip': 1.0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'model': {'enable_gradient_checkpointing': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'external_lib': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'fsdp_config': {'grad_offload': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'optimizer_offload': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'param_offload': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'wrap_policy': {'min_num_params': 0}},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'override_config': {},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'path': '/teamspace/studios/this_studio/models/Qwen2.5-0.5B-Instruct',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'tokenizer_path': '/teamspace/studios/this_studio/models/Qwen2.5-0.5B-Instruct'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'optim': {'lr': 1e-05,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'lr_warmup_steps_ratio': 0.0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'min_lr_ratio': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'total_training_steps': -1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'warmup_style': 'constant'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ppo_epochs': 1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ppo_micro_batch_size': 4,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'ppo_mini_batch_size': 64,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'shuffle': True,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'strategy': 'fsdp'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'data': {'max_prompt_length': 512,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'max_response_length': 256,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'prompt_key': 'prompt',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'return_raw_chat': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'return_raw_input_ids': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'tokenizer': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'train_batch_size': 256,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'train_files': '/teamspace/studios/this_studio/data/gsm8k/train.parquet',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'val_batch_size': 1312,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'val_files': '/teamspace/studios/this_studio/data/gsm8k/test.parquet'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'reward_model': {'enable': False,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'max_length': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'micro_batch_size': 64,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'model': {'external_lib': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'fsdp_config': {'min_num_params': 0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'param_offload': False},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'input_tokenizer': '/teamspace/studios/this_studio/models/Qwen2.5-0.5B-Instruct',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'path': '~/models/FsfairX-LLaMA3-RM-v0.1'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'strategy': 'fsdp'},\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'trainer': {'critic_warmup': 0,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'default_hdfs_dir': None,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'default_local_dir': 'checkpoints/verl_examples/gsm8k',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'experiment_name': 'gsm8k',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'logger': ['console'],\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'n_gpus_per_node': 1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'nnodes': 1,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'project_name': 'verl_examples',\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'save_freq': 10,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'test_freq': 10,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'total_epochs': 15,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 'val_before_train': False}}\n", - "\u001b[36m(main_task pid=28294)\u001b[0m original dataset len: 7473\n", - "\u001b[36m(main_task pid=28294)\u001b[0m filter dataset len: 7473\n", - "\u001b[36m(main_task pid=28294)\u001b[0m original dataset len: 1319\n", - "\u001b[36m(main_task pid=28294)\u001b[0m filter dataset len: 1319\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Size of train dataloader: 29\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Size of val dataloader: 1\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Critic overriding config {'bos_token_id': None, 'eos_token_id': 151645, 'pad_token_id': 151643}\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen2ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained(\"openai/whisper-tiny\", attn_implementation=\"flash_attention_2\", torch_dtype=torch.float16)`\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Qwen2ForCausalLM contains 494.03M parameters\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Before critic FSDP, memory allocated (GB): 0.0, memory reserved (GB): 0.0\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_init_utils.py:440: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m NCCL version 2.20.5+cuda12.4\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m ip-10-192-12-228:28545:28765 [0] nccl_net_ofi_init:1472 NCCL WARN NET/OFI aws-ofi-nccl initialization failed\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m After critic FSDP, memory allocated (GB): 1.8410954475402832, memory reserved (GB): 2.95703125\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Total steps: 435, num_warmup_steps: 0\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Model config after override: Qwen2Config {\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"_name_or_path\": \"/teamspace/studios/this_studio/models/Qwen2.5-0.5B-Instruct\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"architectures\": [\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"Qwen2ForCausalLM\"\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m ],\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"attention_dropout\": 0.0,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"eos_token_id\": 151645,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"hidden_act\": \"silu\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"hidden_size\": 896,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"initializer_range\": 0.02,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"intermediate_size\": 4864,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"max_position_embeddings\": 32768,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"max_window_layers\": 21,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"model_type\": \"qwen2\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"num_attention_heads\": 14,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"num_hidden_layers\": 24,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"num_key_value_heads\": 2,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"pad_token_id\": 151643,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"rms_norm_eps\": 1e-06,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"rope_scaling\": null,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"rope_theta\": 1000000.0,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"sliding_window\": null,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"tie_word_embeddings\": true,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"torch_dtype\": \"bfloat16\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"transformers_version\": \"4.46.3\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"use_cache\": true,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"use_sliding_window\": false,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"vocab_size\": 151936\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m }\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Qwen2ForCausalLM contains 494.03M parameters\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m wrap_policy: functools.partial(, transformer_layer_cls={})\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_init_utils.py:440: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Model config after override: Qwen2Config {\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"_name_or_path\": \"/teamspace/studios/this_studio/models/Qwen2.5-0.5B-Instruct\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"architectures\": [\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"Qwen2ForCausalLM\"\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m ],\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"attention_dropout\": 0.0,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"eos_token_id\": 151645,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"hidden_act\": \"silu\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"hidden_size\": 896,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"initializer_range\": 0.02,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"intermediate_size\": 4864,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"max_position_embeddings\": 32768,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"max_window_layers\": 21,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"model_type\": \"qwen2\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"num_attention_heads\": 14,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"num_hidden_layers\": 24,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"num_key_value_heads\": 2,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"pad_token_id\": 151643,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"rms_norm_eps\": 1e-06,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"rope_scaling\": null,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"rope_theta\": 1000000.0,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"sliding_window\": null,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"tie_word_embeddings\": true,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"torch_dtype\": \"bfloat16\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"transformers_version\": \"4.46.3\",\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"use_cache\": true,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"use_sliding_window\": false,\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \"vocab_size\": 151936\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m }\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m \n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Qwen2ForCausalLM contains 494.03M parameters\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m wrap_policy: functools.partial(, transformer_layer_cls={})\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_init_utils.py:440: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Total steps: 435, num_warmup_steps: 0\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Before building vllm rollout, memory allocated (GB): 4.602750778198242, memory reserved (GB): 5.78125\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m local rank 0\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m no hf weight loader need to be updated\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m before init cache memory allocated: 5.944623104GB, reserved: 6.067060736GB\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m after init cache memory allocated: 21.606612992GB, reserved: 21.770534912GB\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Using LocalLogger is deprecated. The constructor API will change \n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m kwargs: {'n': 1, 'logprobs': 1, 'max_tokens': 256, 'detokenize': False, 'temperature': 1.0, 'top_k': -1, 'top_p': 1, 'ignore_eos': False}\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m After building vllm rollout, memory allocated (GB): 19.201326370239258, memory reserved (GB): 20.275390625\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m After building sharding manager, memory allocated (GB): 19.201326370239258, memory reserved (GB): 20.275390625\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:0 - critic/kl:0.000 - critic/kl_coeff:0.001 - critic/vf_loss:10.488 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.533 - critic/grad_norm:377.446 - critic/lr:0.000 - actor/entropy_loss:0.461 - actor/pg_loss:0.001 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/grad_norm:2.304 - actor/lr:0.000 - critic/score/mean:0.012 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.012 - critic/rewards/max:1.001 - critic/rewards/min:-0.002 - critic/advantages/mean:0.000 - critic/advantages/max:3.686 - critic/advantages/min:-4.200 - critic/returns/mean:0.010 - critic/returns/max:1.001 - critic/returns/min:-0.002 - critic/values/mean:0.527 - critic/values/max:19.625 - critic/values/min:-16.250 - response_length/mean:235.977 - response_length/max:256.000 - response_length/min:83.000 - prompt_length/mean:104.766 - prompt_length/max:183.000 - prompt_length/min:66.000 - timing/gen:24.928 - timing/ref:3.332 - timing/values:3.153 - timing/adv:0.064 - timing/update_critic:9.192 - timing/update_actor:10.928 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:1 - critic/kl:0.000 - critic/kl_coeff:0.001 - critic/vf_loss:10.038 - critic/vf_clipfrac:0.441 - critic/vpred_mean:-0.168 - critic/grad_norm:263.810 - critic/lr:0.000 - actor/entropy_loss:0.463 - actor/pg_loss:-0.002 - actor/pg_clipfrac:0.001 - actor/ppo_kl:-0.000 - actor/grad_norm:2.290 - actor/lr:0.000 - critic/score/mean:0.000 - critic/score/max:0.000 - critic/score/min:0.000 - critic/rewards/mean:-0.000 - critic/rewards/max:0.001 - critic/rewards/min:-0.001 - critic/advantages/mean:0.000 - critic/advantages/max:3.767 - critic/advantages/min:-3.851 - critic/returns/mean:0.000 - critic/returns/max:0.002 - critic/returns/min:-0.001 - critic/values/mean:0.494 - critic/values/max:18.125 - critic/values/min:-16.750 - response_length/mean:233.168 - response_length/max:256.000 - response_length/min:103.000 - prompt_length/mean:101.836 - prompt_length/max:206.000 - prompt_length/min:72.000 - timing/gen:24.277 - timing/ref:3.390 - timing/values:3.204 - timing/adv:0.070 - timing/update_critic:9.131 - timing/update_actor:10.871 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:2 - critic/kl:-0.000 - critic/kl_coeff:0.001 - critic/vf_loss:3.138 - critic/vf_clipfrac:0.420 - critic/vpred_mean:1.064 - critic/grad_norm:237.830 - critic/lr:0.000 - actor/entropy_loss:0.442 - actor/pg_loss:0.005 - actor/pg_clipfrac:0.001 - actor/ppo_kl:-0.000 - actor/grad_norm:2.197 - actor/lr:0.000 - critic/score/mean:0.008 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.008 - critic/rewards/max:1.001 - critic/rewards/min:-0.002 - critic/advantages/mean:0.000 - critic/advantages/max:5.567 - critic/advantages/min:-5.070 - critic/returns/mean:0.006 - critic/returns/max:1.001 - critic/returns/min:-0.002 - critic/values/mean:1.789 - critic/values/max:11.938 - critic/values/min:-9.375 - response_length/mean:234.031 - response_length/max:256.000 - response_length/min:84.000 - prompt_length/mean:104.551 - prompt_length/max:184.000 - prompt_length/min:67.000 - timing/gen:24.146 - timing/ref:3.414 - timing/values:3.167 - timing/adv:0.063 - timing/update_critic:9.121 - timing/update_actor:10.832 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:3 - critic/kl:0.000 - critic/kl_coeff:0.001 - critic/vf_loss:1.683 - critic/vf_clipfrac:0.380 - critic/vpred_mean:0.302 - critic/grad_norm:239.985 - critic/lr:0.000 - actor/entropy_loss:0.436 - actor/pg_loss:-0.001 - actor/pg_clipfrac:0.001 - actor/ppo_kl:0.000 - actor/grad_norm:2.097 - actor/lr:0.000 - critic/score/mean:0.008 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.008 - critic/rewards/max:1.000 - critic/rewards/min:-0.003 - critic/advantages/mean:-0.000 - critic/advantages/max:6.460 - critic/advantages/min:-7.391 - critic/returns/mean:0.006 - critic/returns/max:1.000 - critic/returns/min:-0.003 - critic/values/mean:1.414 - critic/values/max:11.312 - critic/values/min:-7.250 - response_length/mean:237.445 - response_length/max:256.000 - response_length/min:72.000 - prompt_length/mean:104.500 - prompt_length/max:195.000 - prompt_length/min:70.000 - timing/gen:24.064 - timing/ref:3.368 - timing/values:3.173 - timing/adv:0.066 - timing/update_critic:9.179 - timing/update_actor:11.105 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:4 - critic/kl:0.000 - critic/kl_coeff:0.001 - critic/vf_loss:0.736 - critic/vf_clipfrac:0.250 - critic/vpred_mean:0.188 - critic/grad_norm:155.038 - critic/lr:0.000 - actor/entropy_loss:0.442 - actor/pg_loss:-0.001 - actor/pg_clipfrac:0.001 - actor/ppo_kl:0.000 - actor/grad_norm:2.056 - actor/lr:0.000 - critic/score/mean:0.004 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.004 - critic/rewards/max:1.000 - critic/rewards/min:-0.002 - critic/advantages/mean:-0.000 - critic/advantages/max:6.189 - critic/advantages/min:-7.601 - critic/returns/mean:0.002 - critic/returns/max:1.000 - critic/returns/min:-0.002 - critic/values/mean:0.777 - critic/values/max:8.938 - critic/values/min:-5.875 - response_length/mean:233.352 - response_length/max:256.000 - response_length/min:46.000 - prompt_length/mean:102.660 - prompt_length/max:217.000 - prompt_length/min:66.000 - timing/gen:24.128 - timing/ref:3.418 - timing/values:3.275 - timing/adv:0.065 - timing/update_critic:9.343 - timing/update_actor:10.813 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:5 - critic/kl:0.000 - critic/kl_coeff:0.001 - critic/vf_loss:0.284 - critic/vf_clipfrac:0.156 - critic/vpred_mean:0.284 - critic/grad_norm:93.840 - critic/lr:0.000 - actor/entropy_loss:0.484 - actor/pg_loss:-0.004 - actor/pg_clipfrac:0.001 - actor/ppo_kl:-0.000 - actor/grad_norm:2.098 - actor/lr:0.000 - critic/score/mean:0.008 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.008 - critic/rewards/max:1.000 - critic/rewards/min:-0.002 - critic/advantages/mean:0.000 - critic/advantages/max:12.584 - critic/advantages/min:-9.160 - critic/returns/mean:0.007 - critic/returns/max:1.000 - critic/returns/min:-0.002 - critic/values/mean:0.527 - critic/values/max:6.812 - critic/values/min:-8.125 - response_length/mean:235.293 - response_length/max:256.000 - response_length/min:51.000 - prompt_length/mean:105.004 - prompt_length/max:201.000 - prompt_length/min:67.000 - timing/gen:24.027 - timing/ref:3.427 - timing/values:3.193 - timing/adv:0.060 - timing/update_critic:9.194 - timing/update_actor:10.883 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:6 - critic/kl:0.001 - critic/kl_coeff:0.001 - critic/vf_loss:0.210 - critic/vf_clipfrac:0.181 - critic/vpred_mean:0.276 - critic/grad_norm:98.484 - critic/lr:0.000 - actor/entropy_loss:0.458 - actor/pg_loss:0.001 - actor/pg_clipfrac:0.001 - actor/ppo_kl:-0.000 - actor/grad_norm:2.095 - actor/lr:0.000 - critic/score/mean:0.012 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.012 - critic/rewards/max:0.999 - critic/rewards/min:-0.002 - critic/advantages/mean:0.000 - critic/advantages/max:8.644 - critic/advantages/min:-14.228 - critic/returns/mean:0.011 - critic/returns/max:1.000 - critic/returns/min:-0.002 - critic/values/mean:0.594 - critic/values/max:7.656 - critic/values/min:-3.719 - response_length/mean:234.320 - response_length/max:256.000 - response_length/min:88.000 - prompt_length/mean:102.762 - prompt_length/max:177.000 - prompt_length/min:68.000 - timing/gen:24.469 - timing/ref:3.435 - timing/values:3.224 - timing/adv:0.062 - timing/update_critic:9.298 - timing/update_actor:10.804 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:7 - critic/kl:0.001 - critic/kl_coeff:0.001 - critic/vf_loss:0.122 - critic/vf_clipfrac:0.129 - critic/vpred_mean:0.235 - critic/grad_norm:76.795 - critic/lr:0.000 - actor/entropy_loss:0.475 - actor/pg_loss:-0.004 - actor/pg_clipfrac:0.001 - actor/ppo_kl:0.000 - actor/grad_norm:2.153 - actor/lr:0.000 - critic/score/mean:0.012 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.012 - critic/rewards/max:1.000 - critic/rewards/min:-0.002 - critic/advantages/mean:-0.000 - critic/advantages/max:7.246 - critic/advantages/min:-10.160 - critic/returns/mean:0.010 - critic/returns/max:1.001 - critic/returns/min:-0.003 - critic/values/mean:0.508 - critic/values/max:4.094 - critic/values/min:-1.828 - response_length/mean:230.289 - response_length/max:256.000 - response_length/min:66.000 - prompt_length/mean:100.691 - prompt_length/max:186.000 - prompt_length/min:56.000 - timing/gen:23.792 - timing/ref:3.449 - timing/values:3.217 - timing/adv:0.064 - timing/update_critic:9.305 - timing/update_actor:10.897 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:8 - critic/kl:0.001 - critic/kl_coeff:0.001 - critic/vf_loss:0.078 - critic/vf_clipfrac:0.038 - critic/vpred_mean:-0.035 - critic/grad_norm:73.372 - critic/lr:0.000 - actor/entropy_loss:0.459 - actor/pg_loss:-0.003 - actor/pg_clipfrac:0.001 - actor/ppo_kl:0.000 - actor/grad_norm:2.053 - actor/lr:0.000 - critic/score/mean:0.012 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.011 - critic/rewards/max:1.000 - critic/rewards/min:-0.002 - critic/advantages/mean:0.000 - critic/advantages/max:9.498 - critic/advantages/min:-8.295 - critic/returns/mean:0.009 - critic/returns/max:1.001 - critic/returns/min:-0.003 - critic/values/mean:0.297 - critic/values/max:2.438 - critic/values/min:-1.625 - response_length/mean:235.316 - response_length/max:256.000 - response_length/min:93.000 - prompt_length/mean:103.734 - prompt_length/max:174.000 - prompt_length/min:66.000 - timing/gen:24.379 - timing/ref:3.439 - timing/values:3.213 - timing/adv:0.063 - timing/update_critic:9.164 - timing/update_actor:10.861 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m validation generation end\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>system\n", - "\u001b[36m(main_task pid=28294)\u001b[0m You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>user\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Frances sells 20 cupcakes for $2 for each cupcake and 40 cookies at $1 each. She buys five trays at $4 for each tray. How much money does Frances have left? Let's think step by step and output the final answer after \"####\".<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>assistant\n", - "\u001b[36m(main_task pid=28294)\u001b[0m To determine how much money Frances has left after selling and buying items, we need to follow these steps:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 1. Calculate the total revenue from selling cupcakes.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 2. Calculate the total revenue from selling cookies.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 3. Calculate the total revenue from buying the trays.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 4. Sum up all the revenues to get the total revenue.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 5. Subtract the total revenue from the initial amount of money Frances had to find out how much money she has left.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m Let's start with the first step:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 1. **Revenue from selling cupcakes:**\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Frances sells 20 cupcakes at $2 each.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Total revenue from cupcakes = 20 cupcakes * $2/cupcake = $40.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 2. **Revenue from selling cookies:**\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Frances sells 40 cookies at $1 each.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Total revenue from cookies = 40 cookies * $1/cookie = $40.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 3. **Revenue from buying the trays:**\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Frances buys 5 trays at $4 each.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Total revenue from buying trays = 5 trays * $4/tray = $20.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 4. **Total revenue:**\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Total revenue = Revenue from cupcakes + Revenue from cookies + Revenue from buying trays\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Total revenue\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:9 - critic/kl:0.002 - critic/kl_coeff:0.001 - critic/vf_loss:0.041 - critic/vf_clipfrac:0.008 - critic/vpred_mean:-0.073 - critic/grad_norm:38.623 - critic/lr:0.000 - actor/entropy_loss:0.480 - actor/pg_loss:0.001 - actor/pg_clipfrac:0.001 - actor/ppo_kl:-0.000 - actor/grad_norm:2.148 - actor/lr:0.000 - val/test_score/openai/gsm8k:0.004 - critic/score/mean:0.012 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.011 - critic/rewards/max:1.000 - critic/rewards/min:-0.004 - critic/advantages/mean:0.000 - critic/advantages/max:7.622 - critic/advantages/min:-10.036 - critic/returns/mean:0.009 - critic/returns/max:1.000 - critic/returns/min:-0.004 - critic/values/mean:0.080 - critic/values/max:2.609 - critic/values/min:-1.609 - response_length/mean:228.992 - response_length/max:256.000 - response_length/min:60.000 - prompt_length/mean:100.484 - prompt_length/max:183.000 - prompt_length/min:65.000 - timing/gen:23.970 - timing/ref:3.402 - timing/values:3.268 - timing/adv:0.063 - timing/update_critic:9.110 - timing/update_actor:10.833 - timing/testing:59.966 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving actor checkpoint to checkpoints/verl_examples/gsm8k/actor/global_step_9\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving critic checkpoint to checkpoints/verl_examples/gsm8k/critic/global_step_9\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:10 - critic/kl:0.003 - critic/kl_coeff:0.001 - critic/vf_loss:0.041 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.098 - critic/grad_norm:37.952 - critic/lr:0.000 - actor/entropy_loss:0.498 - actor/pg_loss:-0.000 - actor/pg_clipfrac:0.001 - actor/ppo_kl:0.000 - actor/grad_norm:2.188 - actor/lr:0.000 - critic/score/mean:0.020 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.019 - critic/rewards/max:1.000 - critic/rewards/min:-0.004 - critic/advantages/mean:0.000 - critic/advantages/max:8.178 - critic/advantages/min:-9.838 - critic/returns/mean:0.018 - critic/returns/max:1.001 - critic/returns/min:-0.004 - critic/values/mean:0.226 - critic/values/max:2.391 - critic/values/min:-1.391 - response_length/mean:224.570 - response_length/max:256.000 - response_length/min:36.000 - prompt_length/mean:100.480 - prompt_length/max:167.000 - prompt_length/min:68.000 - timing/gen:25.188 - timing/ref:3.398 - timing/values:3.243 - timing/adv:0.059 - timing/update_critic:9.055 - timing/update_actor:10.881 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:11 - critic/kl:0.003 - critic/kl_coeff:0.001 - critic/vf_loss:0.041 - critic/vf_clipfrac:0.005 - critic/vpred_mean:0.097 - critic/grad_norm:43.225 - critic/lr:0.000 - actor/entropy_loss:0.479 - actor/pg_loss:-0.008 - actor/pg_clipfrac:0.001 - actor/ppo_kl:-0.000 - actor/grad_norm:2.087 - actor/lr:0.000 - critic/score/mean:0.023 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.023 - critic/rewards/max:1.001 - critic/rewards/min:-0.005 - critic/advantages/mean:-0.000 - critic/advantages/max:9.081 - critic/advantages/min:-6.945 - critic/returns/mean:0.023 - critic/returns/max:1.001 - critic/returns/min:-0.005 - critic/values/mean:0.291 - critic/values/max:1.672 - critic/values/min:-1.562 - response_length/mean:231.227 - response_length/max:256.000 - response_length/min:69.000 - prompt_length/mean:103.645 - prompt_length/max:184.000 - prompt_length/min:70.000 - timing/gen:24.139 - timing/ref:3.439 - timing/values:3.204 - timing/adv:0.060 - timing/update_critic:9.110 - timing/update_actor:10.813 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:12 - critic/kl:0.003 - critic/kl_coeff:0.001 - critic/vf_loss:0.031 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.093 - critic/grad_norm:31.382 - critic/lr:0.000 - actor/entropy_loss:0.466 - actor/pg_loss:-0.002 - actor/pg_clipfrac:0.001 - actor/ppo_kl:-0.000 - actor/grad_norm:2.228 - actor/lr:0.000 - critic/score/mean:0.023 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.023 - critic/rewards/max:1.001 - critic/rewards/min:-0.005 - critic/advantages/mean:0.000 - critic/advantages/max:6.806 - critic/advantages/min:-6.111 - critic/returns/mean:0.018 - critic/returns/max:1.002 - critic/returns/min:-0.006 - critic/values/mean:0.242 - critic/values/max:1.484 - critic/values/min:-0.688 - response_length/mean:229.203 - response_length/max:256.000 - response_length/min:86.000 - prompt_length/mean:104.930 - prompt_length/max:256.000 - prompt_length/min:62.000 - timing/gen:24.259 - timing/ref:3.418 - timing/values:3.193 - timing/adv:0.060 - timing/update_critic:9.209 - timing/update_actor:10.883 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:13 - critic/kl:0.004 - critic/kl_coeff:0.001 - critic/vf_loss:0.030 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.150 - critic/grad_norm:34.615 - critic/lr:0.000 - actor/entropy_loss:0.484 - actor/pg_loss:0.006 - actor/pg_clipfrac:0.001 - actor/ppo_kl:0.000 - actor/grad_norm:2.046 - actor/lr:0.000 - critic/score/mean:0.008 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.007 - critic/rewards/max:0.998 - critic/rewards/min:-0.006 - critic/advantages/mean:0.000 - critic/advantages/max:10.923 - critic/advantages/min:-5.137 - critic/returns/mean:0.008 - critic/returns/max:1.000 - critic/returns/min:-0.007 - critic/values/mean:0.283 - critic/values/max:0.961 - critic/values/min:-0.660 - response_length/mean:232.605 - response_length/max:256.000 - response_length/min:94.000 - prompt_length/mean:101.812 - prompt_length/max:171.000 - prompt_length/min:69.000 - timing/gen:24.266 - timing/ref:3.374 - timing/values:3.190 - timing/adv:0.059 - timing/update_critic:9.099 - timing/update_actor:10.888 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:14 - critic/kl:0.005 - critic/kl_coeff:0.001 - critic/vf_loss:0.039 - critic/vf_clipfrac:0.005 - critic/vpred_mean:-0.000 - critic/grad_norm:43.469 - critic/lr:0.000 - actor/entropy_loss:0.496 - actor/pg_loss:0.002 - actor/pg_clipfrac:0.002 - actor/ppo_kl:-0.000 - actor/grad_norm:2.275 - actor/lr:0.000 - critic/score/mean:0.031 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.030 - critic/rewards/max:0.999 - critic/rewards/min:-0.006 - critic/advantages/mean:0.000 - critic/advantages/max:6.506 - critic/advantages/min:-3.311 - critic/returns/mean:0.028 - critic/returns/max:1.001 - critic/returns/min:-0.007 - critic/values/mean:-0.228 - critic/values/max:0.363 - critic/values/min:-1.023 - response_length/mean:226.652 - response_length/max:256.000 - response_length/min:16.000 - prompt_length/mean:102.953 - prompt_length/max:170.000 - prompt_length/min:69.000 - timing/gen:24.303 - timing/ref:3.398 - timing/values:3.207 - timing/adv:0.060 - timing/update_critic:9.210 - timing/update_actor:10.870 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:15 - critic/kl:0.007 - critic/kl_coeff:0.001 - critic/vf_loss:0.040 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.122 - critic/grad_norm:29.667 - critic/lr:0.000 - actor/entropy_loss:0.509 - actor/pg_loss:-0.003 - actor/pg_clipfrac:0.002 - actor/ppo_kl:-0.000 - actor/grad_norm:2.313 - actor/lr:0.000 - critic/score/mean:0.051 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.049 - critic/rewards/max:1.000 - critic/rewards/min:-0.008 - critic/advantages/mean:0.000 - critic/advantages/max:5.265 - critic/advantages/min:-3.315 - critic/returns/mean:0.043 - critic/returns/max:1.001 - critic/returns/min:-0.008 - critic/values/mean:-0.033 - critic/values/max:0.773 - critic/values/min:-0.602 - response_length/mean:222.965 - response_length/max:256.000 - response_length/min:45.000 - prompt_length/mean:107.332 - prompt_length/max:222.000 - prompt_length/min:68.000 - timing/gen:23.736 - timing/ref:3.414 - timing/values:3.171 - timing/adv:0.058 - timing/update_critic:9.015 - timing/update_actor:10.800 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:16 - critic/kl:0.012 - critic/kl_coeff:0.001 - critic/vf_loss:0.033 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.039 - critic/grad_norm:30.166 - critic/lr:0.000 - actor/entropy_loss:0.519 - actor/pg_loss:-0.000 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.363 - actor/lr:0.000 - critic/score/mean:0.039 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.037 - critic/rewards/max:1.000 - critic/rewards/min:-0.012 - critic/advantages/mean:-0.000 - critic/advantages/max:6.580 - critic/advantages/min:-3.047 - critic/returns/mean:0.030 - critic/returns/max:1.001 - critic/returns/min:-0.012 - critic/values/mean:-0.096 - critic/values/max:0.475 - critic/values/min:-0.582 - response_length/mean:224.695 - response_length/max:256.000 - response_length/min:17.000 - prompt_length/mean:103.379 - prompt_length/max:190.000 - prompt_length/min:68.000 - timing/gen:23.597 - timing/ref:3.361 - timing/values:3.187 - timing/adv:0.059 - timing/update_critic:8.919 - timing/update_actor:10.995 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:17 - critic/kl:0.016 - critic/kl_coeff:0.001 - critic/vf_loss:0.064 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.040 - critic/grad_norm:33.541 - critic/lr:0.000 - actor/entropy_loss:0.476 - actor/pg_loss:-0.009 - actor/pg_clipfrac:0.002 - actor/ppo_kl:-0.000 - actor/grad_norm:2.292 - actor/lr:0.000 - critic/score/mean:0.094 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.091 - critic/rewards/max:0.999 - critic/rewards/min:-0.011 - critic/advantages/mean:-0.000 - critic/advantages/max:4.398 - critic/advantages/min:-1.815 - critic/returns/mean:0.081 - critic/returns/max:1.002 - critic/returns/min:-0.011 - critic/values/mean:-0.122 - critic/values/max:0.375 - critic/values/min:-0.832 - response_length/mean:222.750 - response_length/max:256.000 - response_length/min:14.000 - prompt_length/mean:104.809 - prompt_length/max:186.000 - prompt_length/min:69.000 - timing/gen:23.406 - timing/ref:3.361 - timing/values:3.109 - timing/adv:0.060 - timing/update_critic:9.127 - timing/update_actor:10.768 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:18 - critic/kl:0.020 - critic/kl_coeff:0.001 - critic/vf_loss:0.057 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.080 - critic/grad_norm:32.595 - critic/lr:0.000 - actor/entropy_loss:0.496 - actor/pg_loss:-0.011 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.423 - actor/lr:0.000 - critic/score/mean:0.102 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.098 - critic/rewards/max:0.999 - critic/rewards/min:-0.015 - critic/advantages/mean:0.000 - critic/advantages/max:4.333 - critic/advantages/min:-2.007 - critic/returns/mean:0.084 - critic/returns/max:1.001 - critic/returns/min:-0.015 - critic/values/mean:-0.081 - critic/values/max:0.438 - critic/values/min:-0.605 - response_length/mean:220.078 - response_length/max:256.000 - response_length/min:29.000 - prompt_length/mean:105.336 - prompt_length/max:176.000 - prompt_length/min:69.000 - timing/gen:23.512 - timing/ref:3.375 - timing/values:3.197 - timing/adv:0.059 - timing/update_critic:9.130 - timing/update_actor:10.796 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m validation generation end\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>system\n", - "\u001b[36m(main_task pid=28294)\u001b[0m You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>user\n", - "\u001b[36m(main_task pid=28294)\u001b[0m A loaf of bread at the bakery costs $2. Bagels cost $1 each. How much more do 3 loaves of bread cost than 2 bagels? Let's think step by step and output the final answer after \"####\".<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>assistant\n", - "\u001b[36m(main_task pid=28294)\u001b[0m To solve this problem, we need to calculate the total cost of 3 loaves of bread and 2 bagels, and then find the difference between these two costs.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 1. **Calculate the cost of 3 loaves of bread:**\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Each loaf of bread costs $2.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Therefore, 3 loaves of bread cost \\( 3 \\times 2 = \\$6 \\).\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 2. **Calculate the cost of 2 bagels:**\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Each bagel costs $1.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - Therefore, 2 bagels cost \\( 2 \\times 1 = \\$2 \\).\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 3. **Find the difference between the cost of 3 loaves of bread and 2 bagels:**\n", - "\u001b[36m(main_task pid=28294)\u001b[0m - The difference is \\( 6 - 2 = \\$4 \\).\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m #### Final Answer: #### $4<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:19 - critic/kl:0.027 - critic/kl_coeff:0.001 - critic/vf_loss:0.055 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.112 - critic/grad_norm:26.858 - critic/lr:0.000 - actor/entropy_loss:0.449 - actor/pg_loss:-0.008 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.423 - actor/lr:0.000 - val/test_score/openai/gsm8k:0.187 - critic/score/mean:0.113 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.109 - critic/rewards/max:0.999 - critic/rewards/min:-0.016 - critic/advantages/mean:0.000 - critic/advantages/max:4.134 - critic/advantages/min:-1.849 - critic/returns/mean:0.101 - critic/returns/max:1.003 - critic/returns/min:-0.016 - critic/values/mean:-0.035 - critic/values/max:0.436 - critic/values/min:-0.504 - response_length/mean:221.965 - response_length/max:256.000 - response_length/min:6.000 - prompt_length/mean:103.352 - prompt_length/max:189.000 - prompt_length/min:72.000 - timing/gen:23.618 - timing/ref:3.393 - timing/values:3.151 - timing/adv:0.060 - timing/update_critic:9.072 - timing/update_actor:10.790 - timing/testing:57.314 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving actor checkpoint to checkpoints/verl_examples/gsm8k/actor/global_step_19\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving critic checkpoint to checkpoints/verl_examples/gsm8k/critic/global_step_19\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:20 - critic/kl:0.024 - critic/kl_coeff:0.001 - critic/vf_loss:0.067 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.128 - critic/grad_norm:30.260 - critic/lr:0.000 - actor/entropy_loss:0.471 - actor/pg_loss:-0.015 - actor/pg_clipfrac:0.001 - actor/ppo_kl:0.000 - actor/grad_norm:3.157 - actor/lr:0.000 - critic/score/mean:0.152 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.148 - critic/rewards/max:0.999 - critic/rewards/min:-0.016 - critic/advantages/mean:-0.000 - critic/advantages/max:3.422 - critic/advantages/min:-4.571 - critic/returns/mean:0.129 - critic/returns/max:1.006 - critic/returns/min:-0.017 - critic/values/mean:-0.009 - critic/values/max:1.398 - critic/values/min:-0.656 - response_length/mean:219.305 - response_length/max:256.000 - response_length/min:21.000 - prompt_length/mean:104.742 - prompt_length/max:199.000 - prompt_length/min:65.000 - timing/gen:24.937 - timing/ref:3.418 - timing/values:3.153 - timing/adv:0.059 - timing/update_critic:9.206 - timing/update_actor:10.821 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:21 - critic/kl:0.028 - critic/kl_coeff:0.001 - critic/vf_loss:0.067 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.147 - critic/grad_norm:23.865 - critic/lr:0.000 - actor/entropy_loss:0.431 - actor/pg_loss:-0.014 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.468 - actor/lr:0.000 - critic/score/mean:0.164 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.159 - critic/rewards/max:0.999 - critic/rewards/min:-0.017 - critic/advantages/mean:-0.000 - critic/advantages/max:3.344 - critic/advantages/min:-1.174 - critic/returns/mean:0.139 - critic/returns/max:1.002 - critic/returns/min:-0.017 - critic/values/mean:0.021 - critic/values/max:0.305 - critic/values/min:-0.371 - response_length/mean:218.129 - response_length/max:256.000 - response_length/min:61.000 - prompt_length/mean:103.434 - prompt_length/max:216.000 - prompt_length/min:68.000 - timing/gen:23.752 - timing/ref:3.414 - timing/values:3.145 - timing/adv:0.060 - timing/update_critic:9.022 - timing/update_actor:10.779 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:22 - critic/kl:0.029 - critic/kl_coeff:0.001 - critic/vf_loss:0.077 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.160 - critic/grad_norm:31.984 - critic/lr:0.000 - actor/entropy_loss:0.427 - actor/pg_loss:-0.011 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.264 - actor/lr:0.000 - critic/score/mean:0.199 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.193 - critic/rewards/max:0.999 - critic/rewards/min:-0.023 - critic/advantages/mean:-0.000 - critic/advantages/max:2.826 - critic/advantages/min:-1.320 - critic/returns/mean:0.161 - critic/returns/max:1.001 - critic/returns/min:-0.023 - critic/values/mean:0.038 - critic/values/max:0.354 - critic/values/min:-0.379 - response_length/mean:221.094 - response_length/max:256.000 - response_length/min:62.000 - prompt_length/mean:105.645 - prompt_length/max:215.000 - prompt_length/min:65.000 - timing/gen:23.868 - timing/ref:3.403 - timing/values:3.179 - timing/adv:0.060 - timing/update_critic:8.959 - timing/update_actor:10.798 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:23 - critic/kl:0.037 - critic/kl_coeff:0.001 - critic/vf_loss:0.092 - critic/vf_clipfrac:0.001 - critic/vpred_mean:0.196 - critic/grad_norm:30.678 - critic/lr:0.000 - actor/entropy_loss:0.415 - actor/pg_loss:-0.028 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.001 - actor/grad_norm:48.414 - actor/lr:0.000 - critic/score/mean:0.227 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.219 - critic/rewards/max:0.999 - critic/rewards/min:-0.138 - critic/advantages/mean:0.000 - critic/advantages/max:2.562 - critic/advantages/min:-9.078 - critic/returns/mean:0.185 - critic/returns/max:1.002 - critic/returns/min:-0.139 - critic/values/mean:0.071 - critic/values/max:3.969 - critic/values/min:-0.338 - response_length/mean:217.648 - response_length/max:256.000 - response_length/min:6.000 - prompt_length/mean:101.512 - prompt_length/max:191.000 - prompt_length/min:67.000 - timing/gen:23.482 - timing/ref:3.394 - timing/values:3.164 - timing/adv:0.059 - timing/update_critic:8.965 - timing/update_actor:10.772 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:24 - critic/kl:0.038 - critic/kl_coeff:0.001 - critic/vf_loss:0.095 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.285 - critic/grad_norm:35.689 - critic/lr:0.000 - actor/entropy_loss:0.427 - actor/pg_loss:-0.018 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.676 - actor/lr:0.000 - critic/score/mean:0.246 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.239 - critic/rewards/max:1.000 - critic/rewards/min:-0.024 - critic/advantages/mean:-0.000 - critic/advantages/max:2.936 - critic/advantages/min:-2.398 - critic/returns/mean:0.208 - critic/returns/max:1.002 - critic/returns/min:-0.027 - critic/values/mean:0.135 - critic/values/max:1.875 - critic/values/min:-0.235 - response_length/mean:215.758 - response_length/max:256.000 - response_length/min:81.000 - prompt_length/mean:106.418 - prompt_length/max:183.000 - prompt_length/min:64.000 - timing/gen:23.394 - timing/ref:3.387 - timing/values:3.190 - timing/adv:0.059 - timing/update_critic:8.992 - timing/update_actor:10.799 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:25 - critic/kl:0.008 - critic/kl_coeff:0.001 - critic/vf_loss:0.094 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.221 - critic/grad_norm:30.657 - critic/lr:0.000 - actor/entropy_loss:0.414 - actor/pg_loss:-0.004 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.003 - actor/grad_norm:34.984 - actor/lr:0.000 - critic/score/mean:0.250 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.251 - critic/rewards/max:2.024 - critic/rewards/min:-0.022 - critic/advantages/mean:0.000 - critic/advantages/max:5.709 - critic/advantages/min:-1.715 - critic/returns/mean:0.216 - critic/returns/max:2.050 - critic/returns/min:-0.022 - critic/values/mean:0.227 - critic/values/max:0.703 - critic/values/min:-1.055 - response_length/mean:210.586 - response_length/max:256.000 - response_length/min:78.000 - prompt_length/mean:103.406 - prompt_length/max:232.000 - prompt_length/min:64.000 - timing/gen:23.530 - timing/ref:3.407 - timing/values:3.228 - timing/adv:0.058 - timing/update_critic:8.998 - timing/update_actor:10.763 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:26 - critic/kl:0.047 - critic/kl_coeff:0.001 - critic/vf_loss:0.144 - critic/vf_clipfrac:0.031 - critic/vpred_mean:0.072 - critic/grad_norm:61.861 - critic/lr:0.000 - actor/entropy_loss:0.382 - actor/pg_loss:-0.016 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.559 - actor/lr:0.000 - critic/score/mean:0.285 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.277 - critic/rewards/max:0.998 - critic/rewards/min:-0.027 - critic/advantages/mean:-0.000 - critic/advantages/max:2.520 - critic/advantages/min:-1.550 - critic/returns/mean:0.250 - critic/returns/max:1.002 - critic/returns/min:-0.027 - critic/values/mean:0.465 - critic/values/max:0.863 - critic/values/min:0.074 - response_length/mean:207.973 - response_length/max:256.000 - response_length/min:24.000 - prompt_length/mean:103.566 - prompt_length/max:194.000 - prompt_length/min:66.000 - timing/gen:23.017 - timing/ref:3.432 - timing/values:3.209 - timing/adv:0.059 - timing/update_critic:9.104 - timing/update_actor:10.786 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:27 - critic/kl:0.048 - critic/kl_coeff:0.001 - critic/vf_loss:0.216 - critic/vf_clipfrac:0.175 - critic/vpred_mean:0.408 - critic/grad_norm:89.819 - critic/lr:0.000 - actor/entropy_loss:0.340 - actor/pg_loss:-0.018 - actor/pg_clipfrac:0.003 - actor/ppo_kl:-0.000 - actor/grad_norm:2.722 - actor/lr:0.000 - critic/score/mean:0.371 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.362 - critic/rewards/max:0.999 - critic/rewards/min:-0.025 - critic/advantages/mean:-0.000 - critic/advantages/max:2.400 - critic/advantages/min:-1.734 - critic/returns/mean:0.312 - critic/returns/max:1.002 - critic/returns/min:-0.026 - critic/values/mean:-0.169 - critic/values/max:0.279 - critic/values/min:-0.660 - response_length/mean:205.676 - response_length/max:256.000 - response_length/min:13.000 - prompt_length/mean:105.074 - prompt_length/max:238.000 - prompt_length/min:66.000 - timing/gen:23.320 - timing/ref:3.407 - timing/values:3.186 - timing/adv:0.058 - timing/update_critic:9.084 - timing/update_actor:10.797 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:28 - critic/kl:0.053 - critic/kl_coeff:0.001 - critic/vf_loss:0.186 - critic/vf_clipfrac:0.369 - critic/vpred_mean:0.509 - critic/grad_norm:81.161 - critic/lr:0.000 - actor/entropy_loss:0.373 - actor/pg_loss:-0.014 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.593 - actor/lr:0.000 - critic/score/mean:0.293 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.284 - critic/rewards/max:0.999 - critic/rewards/min:-0.024 - critic/advantages/mean:0.000 - critic/advantages/max:2.735 - critic/advantages/min:-1.528 - critic/returns/mean:0.238 - critic/returns/max:1.005 - critic/returns/min:-0.024 - critic/values/mean:0.910 - critic/values/max:1.336 - critic/values/min:0.416 - response_length/mean:202.500 - response_length/max:256.000 - response_length/min:67.000 - prompt_length/mean:103.758 - prompt_length/max:201.000 - prompt_length/min:70.000 - timing/gen:23.026 - timing/ref:3.427 - timing/values:3.172 - timing/adv:0.058 - timing/update_critic:9.130 - timing/update_actor:10.794 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m validation generation end\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>system\n", - "\u001b[36m(main_task pid=28294)\u001b[0m You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>user\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Kelian has two recipes for preparing dishes, one having 20 instructions and the second one having twice as many instructions as the first one. How many instructions does Kelian have to read to prepare the two dishes? Let's think step by step and output the final answer after \"####\".<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>assistant\n", - "\u001b[36m(main_task pid=28294)\u001b[0m To determine the total number of instructions Kelian needs to read to prepare both dishes, we can follow these steps:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 1. Identify the number of instructions in the first recipe.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 2. Calculate the number of instructions in the second recipe.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 3. Add the number of instructions from both recipes together.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m Step 1: The first recipe has 20 instructions.\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m Step 2: The second recipe has twice as many instructions as the first one. Therefore, the number of instructions in the second recipe is:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ 2 \\times 20 = 40 \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m Step 3: Add the number of instructions from both recipes together:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ 20 + 40 = 60 \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m So, the total number of instructions Kelian needs to read to prepare both dishes is:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m #### 60\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m Therefore, the final answer is:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m #### 60<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:29 - critic/kl:0.059 - critic/kl_coeff:0.001 - critic/vf_loss:0.106 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.381 - critic/grad_norm:30.421 - critic/lr:0.000 - actor/entropy_loss:0.343 - actor/pg_loss:-0.013 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.577 - actor/lr:0.000 - val/test_score/openai/gsm8k:0.344 - critic/score/mean:0.344 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.333 - critic/rewards/max:0.998 - critic/rewards/min:-0.026 - critic/advantages/mean:0.000 - critic/advantages/max:2.961 - critic/advantages/min:-1.600 - critic/returns/mean:0.290 - critic/returns/max:1.005 - critic/returns/min:-0.026 - critic/values/mean:0.208 - critic/values/max:0.637 - critic/values/min:-0.371 - response_length/mean:202.914 - response_length/max:256.000 - response_length/min:60.000 - prompt_length/mean:103.797 - prompt_length/max:160.000 - prompt_length/min:67.000 - timing/gen:23.198 - timing/ref:3.438 - timing/values:3.225 - timing/adv:0.058 - timing/update_critic:9.093 - timing/update_actor:10.812 - timing/testing:53.515 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving actor checkpoint to checkpoints/verl_examples/gsm8k/actor/global_step_29\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving critic checkpoint to checkpoints/verl_examples/gsm8k/critic/global_step_29\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:30 - critic/kl:0.071 - critic/kl_coeff:0.001 - critic/vf_loss:0.141 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.063 - critic/grad_norm:54.723 - critic/lr:0.000 - actor/entropy_loss:0.323 - actor/pg_loss:-0.016 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:2.743 - actor/lr:0.000 - critic/score/mean:0.367 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.356 - critic/rewards/max:0.996 - critic/rewards/min:-0.029 - critic/advantages/mean:-0.000 - critic/advantages/max:2.632 - critic/advantages/min:-1.622 - critic/returns/mean:0.305 - critic/returns/max:1.003 - critic/returns/min:-0.029 - critic/values/mean:0.019 - critic/values/max:0.445 - critic/values/min:-0.477 - response_length/mean:190.918 - response_length/max:256.000 - response_length/min:6.000 - prompt_length/mean:102.129 - prompt_length/max:186.000 - prompt_length/min:67.000 - timing/gen:23.954 - timing/ref:3.413 - timing/values:3.233 - timing/adv:0.057 - timing/update_critic:9.046 - timing/update_actor:10.778 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:31 - critic/kl:0.067 - critic/kl_coeff:0.001 - critic/vf_loss:0.103 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.375 - critic/grad_norm:26.295 - critic/lr:0.000 - actor/entropy_loss:0.335 - actor/pg_loss:-0.007 - actor/pg_clipfrac:0.003 - actor/ppo_kl:-0.000 - actor/grad_norm:4.026 - actor/lr:0.000 - critic/score/mean:0.371 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.361 - critic/rewards/max:0.997 - critic/rewards/min:-0.032 - critic/advantages/mean:-0.000 - critic/advantages/max:2.539 - critic/advantages/min:-1.678 - critic/returns/mean:0.305 - critic/returns/max:1.002 - critic/returns/min:-0.032 - critic/values/mean:0.350 - critic/values/max:0.805 - critic/values/min:-0.217 - response_length/mean:186.094 - response_length/max:256.000 - response_length/min:59.000 - prompt_length/mean:104.117 - prompt_length/max:191.000 - prompt_length/min:71.000 - timing/gen:22.221 - timing/ref:3.396 - timing/values:3.198 - timing/adv:0.058 - timing/update_critic:9.063 - timing/update_actor:10.775 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:32 - critic/kl:0.081 - critic/kl_coeff:0.001 - critic/vf_loss:0.140 - critic/vf_clipfrac:0.203 - critic/vpred_mean:0.298 - critic/grad_norm:57.234 - critic/lr:0.000 - actor/entropy_loss:0.308 - actor/pg_loss:0.006 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:2.751 - actor/lr:0.000 - critic/score/mean:0.391 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.378 - critic/rewards/max:0.997 - critic/rewards/min:-0.028 - critic/advantages/mean:0.000 - critic/advantages/max:2.417 - critic/advantages/min:-1.948 - critic/returns/mean:0.340 - critic/returns/max:1.002 - critic/returns/min:-0.028 - critic/values/mean:0.660 - critic/values/max:1.195 - critic/values/min:0.005 - response_length/mean:181.852 - response_length/max:256.000 - response_length/min:56.000 - prompt_length/mean:103.426 - prompt_length/max:222.000 - prompt_length/min:62.000 - timing/gen:22.361 - timing/ref:3.393 - timing/values:3.252 - timing/adv:0.057 - timing/update_critic:9.115 - timing/update_actor:10.793 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:33 - critic/kl:0.079 - critic/kl_coeff:0.001 - critic/vf_loss:0.118 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.325 - critic/grad_norm:35.494 - critic/lr:0.000 - actor/entropy_loss:0.309 - actor/pg_loss:-0.006 - actor/pg_clipfrac:0.003 - actor/ppo_kl:-0.000 - actor/grad_norm:3.946 - actor/lr:0.000 - critic/score/mean:0.383 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.370 - critic/rewards/max:1.076 - critic/rewards/min:-0.030 - critic/advantages/mean:0.000 - critic/advantages/max:2.411 - critic/advantages/min:-2.163 - critic/returns/mean:0.324 - critic/returns/max:1.116 - critic/returns/min:-0.030 - critic/values/mean:0.056 - critic/values/max:0.762 - critic/values/min:-0.660 - response_length/mean:188.148 - response_length/max:256.000 - response_length/min:53.000 - prompt_length/mean:103.168 - prompt_length/max:177.000 - prompt_length/min:69.000 - timing/gen:22.639 - timing/ref:3.408 - timing/values:3.211 - timing/adv:0.058 - timing/update_critic:9.104 - timing/update_actor:10.776 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:34 - critic/kl:0.079 - critic/kl_coeff:0.001 - critic/vf_loss:0.152 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.070 - critic/grad_norm:64.022 - critic/lr:0.000 - actor/entropy_loss:0.286 - actor/pg_loss:-0.020 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.678 - actor/lr:0.000 - critic/score/mean:0.426 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.413 - critic/rewards/max:0.996 - critic/rewards/min:-0.035 - critic/advantages/mean:0.000 - critic/advantages/max:2.692 - critic/advantages/min:-2.171 - critic/returns/mean:0.350 - critic/returns/max:1.001 - critic/returns/min:-0.035 - critic/values/mean:-0.025 - critic/values/max:0.609 - critic/values/min:-0.793 - response_length/mean:185.578 - response_length/max:256.000 - response_length/min:55.000 - prompt_length/mean:100.945 - prompt_length/max:182.000 - prompt_length/min:67.000 - timing/gen:22.709 - timing/ref:3.445 - timing/values:3.195 - timing/adv:0.057 - timing/update_critic:9.043 - timing/update_actor:10.788 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:35 - critic/kl:0.085 - critic/kl_coeff:0.001 - critic/vf_loss:0.099 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.400 - critic/grad_norm:17.049 - critic/lr:0.000 - actor/entropy_loss:0.304 - actor/pg_loss:-0.003 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.894 - actor/lr:0.000 - critic/score/mean:0.445 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.432 - critic/rewards/max:0.998 - critic/rewards/min:-0.029 - critic/advantages/mean:-0.000 - critic/advantages/max:2.507 - critic/advantages/min:-2.582 - critic/returns/mean:0.389 - critic/returns/max:1.007 - critic/returns/min:-0.030 - critic/values/mean:0.469 - critic/values/max:1.203 - critic/values/min:-0.277 - response_length/mean:182.477 - response_length/max:256.000 - response_length/min:58.000 - prompt_length/mean:105.250 - prompt_length/max:256.000 - prompt_length/min:56.000 - timing/gen:22.272 - timing/ref:3.390 - timing/values:3.223 - timing/adv:0.058 - timing/update_critic:9.046 - timing/update_actor:10.785 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:36 - critic/kl:0.086 - critic/kl_coeff:0.001 - critic/vf_loss:0.094 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.395 - critic/grad_norm:14.511 - critic/lr:0.000 - actor/entropy_loss:0.256 - actor/pg_loss:-0.045 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.899 - actor/lr:0.000 - critic/score/mean:0.473 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.459 - critic/rewards/max:1.001 - critic/rewards/min:-0.026 - critic/advantages/mean:-0.000 - critic/advantages/max:2.743 - critic/advantages/min:-2.513 - critic/returns/mean:0.385 - critic/returns/max:1.001 - critic/returns/min:-0.026 - critic/values/mean:0.459 - critic/values/max:1.219 - critic/values/min:-0.252 - response_length/mean:181.793 - response_length/max:256.000 - response_length/min:62.000 - prompt_length/mean:104.781 - prompt_length/max:211.000 - prompt_length/min:63.000 - timing/gen:22.103 - timing/ref:3.386 - timing/values:3.202 - timing/adv:0.056 - timing/update_critic:9.009 - timing/update_actor:10.751 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:37 - critic/kl:0.090 - critic/kl_coeff:0.001 - critic/vf_loss:0.103 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.414 - critic/grad_norm:11.328 - critic/lr:0.000 - actor/entropy_loss:0.291 - actor/pg_loss:0.004 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.824 - actor/lr:0.000 - critic/score/mean:0.457 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.443 - critic/rewards/max:0.994 - critic/rewards/min:-0.036 - critic/advantages/mean:-0.000 - critic/advantages/max:2.719 - critic/advantages/min:-2.849 - critic/returns/mean:0.396 - critic/returns/max:1.001 - critic/returns/min:-0.036 - critic/values/mean:0.430 - critic/values/max:1.312 - critic/values/min:-0.498 - response_length/mean:178.871 - response_length/max:256.000 - response_length/min:58.000 - prompt_length/mean:102.766 - prompt_length/max:199.000 - prompt_length/min:66.000 - timing/gen:22.195 - timing/ref:3.405 - timing/values:3.201 - timing/adv:0.056 - timing/update_critic:9.014 - timing/update_actor:10.757 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:38 - critic/kl:0.093 - critic/kl_coeff:0.001 - critic/vf_loss:0.131 - critic/vf_clipfrac:0.115 - critic/vpred_mean:0.241 - critic/grad_norm:55.256 - critic/lr:0.000 - actor/entropy_loss:0.260 - actor/pg_loss:-0.011 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.001 - actor/grad_norm:2.904 - actor/lr:0.000 - critic/score/mean:0.488 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.474 - critic/rewards/max:0.996 - critic/rewards/min:-0.025 - critic/advantages/mean:0.000 - critic/advantages/max:2.554 - critic/advantages/min:-2.374 - critic/returns/mean:0.421 - critic/returns/max:1.003 - critic/returns/min:-0.025 - critic/values/mean:0.053 - critic/values/max:0.777 - critic/values/min:-0.809 - response_length/mean:178.625 - response_length/max:256.000 - response_length/min:50.000 - prompt_length/mean:105.059 - prompt_length/max:206.000 - prompt_length/min:70.000 - timing/gen:21.984 - timing/ref:3.395 - timing/values:3.208 - timing/adv:0.057 - timing/update_critic:9.015 - timing/update_actor:10.827 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m validation generation end\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>system\n", - "\u001b[36m(main_task pid=28294)\u001b[0m You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>user\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Bob has a certain number of marbles. If he receives 2 dozen more marbles, he will have 60 marbles. If he loses 10 of the marbles he has, how many marbles will Bob have? Let's think step by step and output the final answer after \"####\".<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>assistant\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Let's denote the number of marbles Bob originally has as \\( x \\).\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m First, we know that if Bob receives 2 dozen more marbles, he will have 60 marbles. Since 1 dozen is equal to 12, 2 dozen is equal to \\( 2 \\times 12 = 24 \\) marbles. Therefore, we can write the equation:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ x + 24 = 60 \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m To find \\( x \\), we subtract 24 from both sides of the equation:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ x = 60 - 24 \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ x = 36 \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m So, Bob originally has 36 marbles. If he loses 10 of the marbles, he will have:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ 36 - 10 = 26 \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m #### 26<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:39 - critic/kl:0.084 - critic/kl_coeff:0.001 - critic/vf_loss:0.132 - critic/vf_clipfrac:0.105 - critic/vpred_mean:0.500 - critic/grad_norm:54.355 - critic/lr:0.000 - actor/entropy_loss:0.267 - actor/pg_loss:-0.019 - actor/pg_clipfrac:0.004 - actor/ppo_kl:-0.001 - actor/grad_norm:2.948 - actor/lr:0.000 - val/test_score/openai/gsm8k:0.454 - critic/score/mean:0.496 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.483 - critic/rewards/max:1.289 - critic/rewards/min:-0.031 - critic/advantages/mean:-0.000 - critic/advantages/max:2.662 - critic/advantages/min:-2.130 - critic/returns/mean:0.422 - critic/returns/max:1.423 - critic/returns/min:-0.031 - critic/values/mean:0.777 - critic/values/max:1.500 - critic/values/min:-0.105 - response_length/mean:185.688 - response_length/max:256.000 - response_length/min:67.000 - prompt_length/mean:104.715 - prompt_length/max:167.000 - prompt_length/min:64.000 - timing/gen:22.555 - timing/ref:3.405 - timing/values:3.163 - timing/adv:0.057 - timing/update_critic:9.071 - timing/update_actor:10.771 - timing/testing:48.743 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving actor checkpoint to checkpoints/verl_examples/gsm8k/actor/global_step_39\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving critic checkpoint to checkpoints/verl_examples/gsm8k/critic/global_step_39\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:40 - critic/kl:0.082 - critic/kl_coeff:0.001 - critic/vf_loss:0.147 - critic/vf_clipfrac:0.002 - critic/vpred_mean:0.669 - critic/grad_norm:62.551 - critic/lr:0.000 - actor/entropy_loss:0.274 - actor/pg_loss:-0.012 - actor/pg_clipfrac:0.004 - actor/ppo_kl:-0.001 - actor/grad_norm:4.891 - actor/lr:0.000 - critic/score/mean:0.453 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.440 - critic/rewards/max:1.374 - critic/rewards/min:-0.128 - critic/advantages/mean:-0.000 - critic/advantages/max:3.228 - critic/advantages/min:-2.352 - critic/returns/mean:0.381 - critic/returns/max:1.547 - critic/returns/min:-0.128 - critic/values/mean:0.383 - critic/values/max:1.055 - critic/values/min:-0.465 - response_length/mean:189.316 - response_length/max:256.000 - response_length/min:56.000 - prompt_length/mean:104.215 - prompt_length/max:215.000 - prompt_length/min:72.000 - timing/gen:23.543 - timing/ref:3.408 - timing/values:3.206 - timing/adv:0.058 - timing/update_critic:8.990 - timing/update_actor:10.767 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:41 - critic/kl:0.100 - critic/kl_coeff:0.001 - critic/vf_loss:0.116 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.438 - critic/grad_norm:27.778 - critic/lr:0.000 - actor/entropy_loss:0.235 - actor/pg_loss:-0.017 - actor/pg_clipfrac:0.003 - actor/ppo_kl:-0.000 - actor/grad_norm:3.307 - actor/lr:0.000 - critic/score/mean:0.531 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.516 - critic/rewards/max:0.997 - critic/rewards/min:-0.027 - critic/advantages/mean:-0.000 - critic/advantages/max:2.631 - critic/advantages/min:-2.307 - critic/returns/mean:0.463 - critic/returns/max:1.003 - critic/returns/min:-0.027 - critic/values/mean:0.381 - critic/values/max:1.023 - critic/values/min:-0.402 - response_length/mean:176.273 - response_length/max:256.000 - response_length/min:51.000 - prompt_length/mean:103.426 - prompt_length/max:183.000 - prompt_length/min:69.000 - timing/gen:21.996 - timing/ref:3.393 - timing/values:3.232 - timing/adv:0.056 - timing/update_critic:9.024 - timing/update_actor:10.754 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:42 - critic/kl:0.102 - critic/kl_coeff:0.001 - critic/vf_loss:0.105 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.490 - critic/grad_norm:19.715 - critic/lr:0.000 - actor/entropy_loss:0.239 - actor/pg_loss:-0.015 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:2.923 - actor/lr:0.000 - critic/score/mean:0.504 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.488 - critic/rewards/max:0.994 - critic/rewards/min:-0.034 - critic/advantages/mean:-0.000 - critic/advantages/max:2.133 - critic/advantages/min:-2.226 - critic/returns/mean:0.432 - critic/returns/max:1.001 - critic/returns/min:-0.034 - critic/values/mean:0.594 - critic/values/max:1.148 - critic/values/min:-0.068 - response_length/mean:175.164 - response_length/max:256.000 - response_length/min:47.000 - prompt_length/mean:103.246 - prompt_length/max:201.000 - prompt_length/min:68.000 - timing/gen:21.940 - timing/ref:3.386 - timing/values:3.189 - timing/adv:0.056 - timing/update_critic:9.052 - timing/update_actor:10.768 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:43 - critic/kl:0.106 - critic/kl_coeff:0.001 - critic/vf_loss:0.100 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.462 - critic/grad_norm:23.429 - critic/lr:0.000 - actor/entropy_loss:0.222 - actor/pg_loss:-0.020 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.573 - actor/lr:0.000 - critic/score/mean:0.559 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.542 - critic/rewards/max:1.001 - critic/rewards/min:-0.034 - critic/advantages/mean:0.000 - critic/advantages/max:2.340 - critic/advantages/min:-2.101 - critic/returns/mean:0.470 - critic/returns/max:1.003 - critic/returns/min:-0.034 - critic/values/mean:0.559 - critic/values/max:1.211 - critic/values/min:-0.291 - response_length/mean:176.883 - response_length/max:256.000 - response_length/min:71.000 - prompt_length/mean:104.879 - prompt_length/max:232.000 - prompt_length/min:64.000 - timing/gen:22.334 - timing/ref:3.384 - timing/values:3.175 - timing/adv:0.058 - timing/update_critic:9.083 - timing/update_actor:10.772 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:44 - critic/kl:0.103 - critic/kl_coeff:0.001 - critic/vf_loss:0.097 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.464 - critic/grad_norm:20.556 - critic/lr:0.000 - actor/entropy_loss:0.231 - actor/pg_loss:-0.020 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.748 - actor/lr:0.000 - critic/score/mean:0.508 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.492 - critic/rewards/max:0.997 - critic/rewards/min:-0.028 - critic/advantages/mean:-0.000 - critic/advantages/max:2.425 - critic/advantages/min:-2.328 - critic/returns/mean:0.433 - critic/returns/max:1.003 - critic/returns/min:-0.028 - critic/values/mean:0.555 - critic/values/max:1.250 - critic/values/min:-0.227 - response_length/mean:172.953 - response_length/max:256.000 - response_length/min:73.000 - prompt_length/mean:105.375 - prompt_length/max:178.000 - prompt_length/min:69.000 - timing/gen:21.645 - timing/ref:3.378 - timing/values:3.219 - timing/adv:0.057 - timing/update_critic:9.054 - timing/update_actor:11.071 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:45 - critic/kl:0.110 - critic/kl_coeff:0.001 - critic/vf_loss:0.095 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.501 - critic/grad_norm:17.849 - critic/lr:0.000 - actor/entropy_loss:0.254 - actor/pg_loss:-0.023 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:4.222 - actor/lr:0.000 - critic/score/mean:0.555 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.538 - critic/rewards/max:0.992 - critic/rewards/min:-0.028 - critic/advantages/mean:-0.000 - critic/advantages/max:2.372 - critic/advantages/min:-2.466 - critic/returns/mean:0.476 - critic/returns/max:1.082 - critic/returns/min:-0.028 - critic/values/mean:0.590 - critic/values/max:1.406 - critic/values/min:-0.508 - response_length/mean:173.477 - response_length/max:256.000 - response_length/min:59.000 - prompt_length/mean:102.488 - prompt_length/max:238.000 - prompt_length/min:69.000 - timing/gen:21.763 - timing/ref:3.466 - timing/values:3.225 - timing/adv:0.057 - timing/update_critic:9.034 - timing/update_actor:10.801 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:46 - critic/kl:0.114 - critic/kl_coeff:0.001 - critic/vf_loss:0.095 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.498 - critic/grad_norm:11.832 - critic/lr:0.000 - actor/entropy_loss:0.233 - actor/pg_loss:-0.010 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.825 - actor/lr:0.000 - critic/score/mean:0.512 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.494 - critic/rewards/max:0.992 - critic/rewards/min:-0.034 - critic/advantages/mean:-0.000 - critic/advantages/max:3.366 - critic/advantages/min:-2.487 - critic/returns/mean:0.444 - critic/returns/max:1.002 - critic/returns/min:-0.034 - critic/values/mean:0.621 - critic/values/max:1.359 - critic/values/min:-0.322 - response_length/mean:173.594 - response_length/max:256.000 - response_length/min:71.000 - prompt_length/mean:104.762 - prompt_length/max:193.000 - prompt_length/min:65.000 - timing/gen:21.762 - timing/ref:3.370 - timing/values:3.120 - timing/adv:0.057 - timing/update_critic:9.026 - timing/update_actor:10.818 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:47 - critic/kl:0.114 - critic/kl_coeff:0.001 - critic/vf_loss:0.114 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.353 - critic/grad_norm:28.083 - critic/lr:0.000 - actor/entropy_loss:0.226 - actor/pg_loss:-0.002 - actor/pg_clipfrac:0.003 - actor/ppo_kl:-0.000 - actor/grad_norm:2.908 - actor/lr:0.000 - critic/score/mean:0.531 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.514 - critic/rewards/max:0.998 - critic/rewards/min:-0.044 - critic/advantages/mean:-0.000 - critic/advantages/max:2.695 - critic/advantages/min:-2.506 - critic/returns/mean:0.453 - critic/returns/max:1.001 - critic/returns/min:-0.044 - critic/values/mean:0.326 - critic/values/max:0.992 - critic/values/min:-0.559 - response_length/mean:172.051 - response_length/max:256.000 - response_length/min:57.000 - prompt_length/mean:102.000 - prompt_length/max:179.000 - prompt_length/min:67.000 - timing/gen:22.352 - timing/ref:3.478 - timing/values:3.231 - timing/adv:0.057 - timing/update_critic:9.098 - timing/update_actor:10.837 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:48 - critic/kl:0.106 - critic/kl_coeff:0.001 - critic/vf_loss:0.116 - critic/vf_clipfrac:0.010 - critic/vpred_mean:0.528 - critic/grad_norm:37.496 - critic/lr:0.000 - actor/entropy_loss:0.218 - actor/pg_loss:-0.008 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:2.889 - actor/lr:0.000 - critic/score/mean:0.500 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.483 - critic/rewards/max:0.994 - critic/rewards/min:-0.031 - critic/advantages/mean:-0.000 - critic/advantages/max:2.590 - critic/advantages/min:-2.222 - critic/returns/mean:0.422 - critic/returns/max:1.003 - critic/returns/min:-0.031 - critic/values/mean:0.652 - critic/values/max:1.352 - critic/values/min:-0.207 - response_length/mean:179.723 - response_length/max:256.000 - response_length/min:62.000 - prompt_length/mean:103.219 - prompt_length/max:180.000 - prompt_length/min:65.000 - timing/gen:22.223 - timing/ref:3.493 - timing/values:3.271 - timing/adv:0.057 - timing/update_critic:9.073 - timing/update_actor:10.817 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.000\n", - "\u001b[36m(main_task pid=28294)\u001b[0m validation generation end\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>system\n", - "\u001b[36m(main_task pid=28294)\u001b[0m You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>user\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Mr Hezekiah had 20 trucks from his store supplying fertiliser to different farmers in his hometown dispatched for delivery on a particular day. Each truck was carrying 20 tons of fertiliser packed in bags. Two hours after the trucks had departed for delivery, Mr Hezekiah got the news that a quarter of the number of lorries dispatched for delivery had mechanical failures on the road and could not deliver the fertilisers to the farmers. Calculate the total number of tons of fertiliser that reached the farmers that day? Let's think step by step and output the final answer after \"####\".<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>assistant\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Let's break down the problem step by step:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 1. **Initial number of trucks**: 20 trucks\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 2. **Fertiliser per truck**: 20 tons\n", - "\u001b[36m(main_task pid=28294)\u001b[0m 3. **Total fertiliser**: \\(20 \\text{ trucks} \\times 20 \\text{ tons/truck} = 400 \\text{ tons}\\)\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 4. **Number of trucks with mechanical failures**: \\(20 \\text{ trucks} \\times \\frac{1}{4} = 5 \\text{ trucks}\\)\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m 5. **Fertiliser delivered to farmers**: \\(400 \\text{ tons} - 5 \\text{ trucks} \\times 20 \\text{ tons/truck} = 400 \\text{ tons} - 100 \\text{ tons} = 300 \\text{ tons}\\)\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \n", - "\u001b[36m(main_task pid=28294)\u001b[0m #### Final Answer:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m #### 300<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:49 - critic/kl:0.120 - critic/kl_coeff:0.001 - critic/vf_loss:0.119 - critic/vf_clipfrac:0.009 - critic/vpred_mean:0.338 - critic/grad_norm:32.179 - critic/lr:0.000 - actor/entropy_loss:0.224 - actor/pg_loss:-0.034 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:3.854 - actor/lr:0.000 - val/test_score/openai/gsm8k:0.466 - critic/score/mean:0.484 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.466 - critic/rewards/max:0.994 - critic/rewards/min:-0.031 - critic/advantages/mean:0.000 - critic/advantages/max:3.497 - critic/advantages/min:-2.368 - critic/returns/mean:0.411 - critic/returns/max:1.003 - critic/returns/min:-0.031 - critic/values/mean:0.094 - critic/values/max:0.820 - critic/values/min:-0.879 - response_length/mean:171.238 - response_length/max:256.000 - response_length/min:66.000 - prompt_length/mean:104.562 - prompt_length/max:169.000 - prompt_length/min:68.000 - timing/gen:21.997 - timing/ref:3.408 - timing/values:3.209 - timing/adv:0.056 - timing/update_critic:8.998 - timing/update_actor:10.805 - timing/testing:47.274 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving actor checkpoint to checkpoints/verl_examples/gsm8k/actor/global_step_49\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving critic checkpoint to checkpoints/verl_examples/gsm8k/critic/global_step_49\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:50 - critic/kl:0.122 - critic/kl_coeff:0.001 - critic/vf_loss:0.129 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.658 - critic/grad_norm:41.948 - critic/lr:0.000 - actor/entropy_loss:0.220 - actor/pg_loss:-0.009 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:3.246 - actor/lr:0.000 - critic/score/mean:0.539 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.520 - critic/rewards/max:0.995 - critic/rewards/min:-0.038 - critic/advantages/mean:-0.000 - critic/advantages/max:2.751 - critic/advantages/min:-2.409 - critic/returns/mean:0.463 - critic/returns/max:1.002 - critic/returns/min:-0.038 - critic/values/mean:0.432 - critic/values/max:1.086 - critic/values/min:-0.445 - response_length/mean:172.555 - response_length/max:256.000 - response_length/min:65.000 - prompt_length/mean:102.969 - prompt_length/max:196.000 - prompt_length/min:68.000 - timing/gen:22.668 - timing/ref:3.455 - timing/values:3.235 - timing/adv:0.057 - timing/update_critic:9.077 - timing/update_actor:10.784 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:51 - critic/kl:0.130 - critic/kl_coeff:0.001 - critic/vf_loss:0.113 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.454 - critic/grad_norm:33.814 - critic/lr:0.000 - actor/entropy_loss:0.230 - actor/pg_loss:-0.012 - actor/pg_clipfrac:0.004 - actor/ppo_kl:0.001 - actor/grad_norm:10.359 - actor/lr:0.000 - critic/score/mean:0.539 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.518 - critic/rewards/max:0.993 - critic/rewards/min:-0.035 - critic/advantages/mean:0.000 - critic/advantages/max:2.335 - critic/advantages/min:-2.426 - critic/returns/mean:0.468 - critic/returns/max:1.196 - critic/returns/min:-0.039 - critic/values/mean:0.711 - critic/values/max:1.383 - critic/values/min:-0.250 - response_length/mean:169.363 - response_length/max:256.000 - response_length/min:62.000 - prompt_length/mean:104.457 - prompt_length/max:174.000 - prompt_length/min:64.000 - timing/gen:21.843 - timing/ref:3.442 - timing/values:3.255 - timing/adv:0.057 - timing/update_critic:9.358 - timing/update_actor:10.775 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:52 - critic/kl:0.124 - critic/kl_coeff:0.001 - critic/vf_loss:0.112 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.614 - critic/grad_norm:38.258 - critic/lr:0.000 - actor/entropy_loss:0.190 - actor/pg_loss:-0.015 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.922 - actor/lr:0.000 - critic/score/mean:0.559 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.539 - critic/rewards/max:0.993 - critic/rewards/min:-0.036 - critic/advantages/mean:-0.000 - critic/advantages/max:2.523 - critic/advantages/min:-2.443 - critic/returns/mean:0.494 - critic/returns/max:1.000 - critic/returns/min:-0.036 - critic/values/mean:0.703 - critic/values/max:1.430 - critic/values/min:-0.266 - response_length/mean:170.574 - response_length/max:256.000 - response_length/min:61.000 - prompt_length/mean:103.676 - prompt_length/max:217.000 - prompt_length/min:68.000 - timing/gen:21.955 - timing/ref:3.386 - timing/values:3.169 - timing/adv:0.057 - timing/update_critic:9.108 - timing/update_actor:10.752 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:53 - critic/kl:0.127 - critic/kl_coeff:0.001 - critic/vf_loss:0.104 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.467 - critic/grad_norm:27.255 - critic/lr:0.000 - actor/entropy_loss:0.189 - actor/pg_loss:-0.007 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.912 - actor/lr:0.000 - critic/score/mean:0.570 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.551 - critic/rewards/max:0.993 - critic/rewards/min:-0.042 - critic/advantages/mean:0.000 - critic/advantages/max:2.255 - critic/advantages/min:-2.197 - critic/returns/mean:0.498 - critic/returns/max:1.001 - critic/returns/min:-0.044 - critic/values/mean:0.233 - critic/values/max:0.918 - critic/values/min:-0.551 - response_length/mean:167.039 - response_length/max:256.000 - response_length/min:68.000 - prompt_length/mean:103.355 - prompt_length/max:172.000 - prompt_length/min:65.000 - timing/gen:21.915 - timing/ref:3.404 - timing/values:3.129 - timing/adv:0.057 - timing/update_critic:9.042 - timing/update_actor:10.777 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:54 - critic/kl:0.132 - critic/kl_coeff:0.001 - critic/vf_loss:0.121 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.266 - critic/grad_norm:44.310 - critic/lr:0.000 - actor/entropy_loss:0.211 - actor/pg_loss:-0.019 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:2.932 - actor/lr:0.000 - critic/score/mean:0.539 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.519 - critic/rewards/max:0.994 - critic/rewards/min:-0.033 - critic/advantages/mean:0.000 - critic/advantages/max:2.432 - critic/advantages/min:-2.403 - critic/returns/mean:0.450 - critic/returns/max:1.003 - critic/returns/min:-0.033 - critic/values/mean:0.508 - critic/values/max:1.195 - critic/values/min:-0.453 - response_length/mean:168.863 - response_length/max:256.000 - response_length/min:59.000 - prompt_length/mean:103.875 - prompt_length/max:170.000 - prompt_length/min:69.000 - timing/gen:21.853 - timing/ref:3.451 - timing/values:3.251 - timing/adv:0.056 - timing/update_critic:9.024 - timing/update_actor:10.781 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:55 - critic/kl:0.172 - critic/kl_coeff:0.001 - critic/vf_loss:0.091 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.502 - critic/grad_norm:10.118 - critic/lr:0.000 - actor/entropy_loss:0.206 - actor/pg_loss:-0.022 - actor/pg_clipfrac:0.004 - actor/ppo_kl:-0.000 - actor/grad_norm:9.115 - actor/lr:0.000 - critic/score/mean:0.559 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.533 - critic/rewards/max:0.990 - critic/rewards/min:-0.136 - critic/advantages/mean:0.000 - critic/advantages/max:2.456 - critic/advantages/min:-2.538 - critic/returns/mean:0.478 - critic/returns/max:1.003 - critic/returns/min:-0.136 - critic/values/mean:0.494 - critic/values/max:1.188 - critic/values/min:-0.424 - response_length/mean:159.633 - response_length/max:256.000 - response_length/min:8.000 - prompt_length/mean:103.645 - prompt_length/max:199.000 - prompt_length/min:63.000 - timing/gen:21.694 - timing/ref:3.410 - timing/values:3.198 - timing/adv:0.057 - timing/update_critic:9.154 - timing/update_actor:10.823 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:56 - critic/kl:0.150 - critic/kl_coeff:0.001 - critic/vf_loss:0.124 - critic/vf_clipfrac:0.001 - critic/vpred_mean:0.584 - critic/grad_norm:50.596 - critic/lr:0.000 - actor/entropy_loss:0.200 - actor/pg_loss:0.016 - actor/pg_clipfrac:0.004 - actor/ppo_kl:0.000 - actor/grad_norm:3.316 - actor/lr:0.000 - critic/score/mean:0.473 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.451 - critic/rewards/max:0.991 - critic/rewards/min:-0.037 - critic/advantages/mean:-0.000 - critic/advantages/max:2.788 - critic/advantages/min:-2.548 - critic/returns/mean:0.399 - critic/returns/max:1.001 - critic/returns/min:-0.037 - critic/values/mean:0.707 - critic/values/max:1.484 - critic/values/min:-0.285 - response_length/mean:161.859 - response_length/max:256.000 - response_length/min:56.000 - prompt_length/mean:103.473 - prompt_length/max:190.000 - prompt_length/min:67.000 - timing/gen:21.750 - timing/ref:3.452 - timing/values:3.189 - timing/adv:0.055 - timing/update_critic:8.992 - timing/update_actor:10.782 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:57 - critic/kl:0.173 - critic/kl_coeff:0.001 - critic/vf_loss:0.139 - critic/vf_clipfrac:0.064 - critic/vpred_mean:0.498 - critic/grad_norm:42.923 - critic/lr:0.000 - actor/entropy_loss:0.202 - actor/pg_loss:-0.014 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:6.863 - actor/lr:0.000 - critic/score/mean:0.551 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.524 - critic/rewards/max:0.994 - critic/rewards/min:-0.458 - critic/advantages/mean:0.000 - critic/advantages/max:2.363 - critic/advantages/min:-3.153 - critic/returns/mean:0.465 - critic/returns/max:1.001 - critic/returns/min:-0.458 - critic/values/mean:0.200 - critic/values/max:0.961 - critic/values/min:-0.668 - response_length/mean:155.484 - response_length/max:256.000 - response_length/min:58.000 - prompt_length/mean:102.148 - prompt_length/max:194.000 - prompt_length/min:65.000 - timing/gen:21.281 - timing/ref:3.433 - timing/values:3.230 - timing/adv:0.056 - timing/update_critic:9.241 - timing/update_actor:10.909 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:58 - critic/kl:0.164 - critic/kl_coeff:0.001 - critic/vf_loss:0.207 - critic/vf_clipfrac:0.044 - critic/vpred_mean:0.939 - critic/grad_norm:84.893 - critic/lr:0.000 - actor/entropy_loss:0.196 - actor/pg_loss:-0.004 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.000 - actor/grad_norm:3.283 - actor/lr:0.000 - critic/score/mean:0.547 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.525 - critic/rewards/max:0.993 - critic/rewards/min:-0.036 - critic/advantages/mean:0.000 - critic/advantages/max:2.246 - critic/advantages/min:-2.305 - critic/returns/mean:0.474 - critic/returns/max:1.001 - critic/returns/min:-0.036 - critic/values/mean:1.102 - critic/values/max:1.812 - critic/values/min:0.367 - response_length/mean:153.152 - response_length/max:256.000 - response_length/min:35.000 - prompt_length/mean:103.617 - prompt_length/max:206.000 - prompt_length/min:66.000 - timing/gen:21.591 - timing/ref:3.401 - timing/values:3.106 - timing/adv:0.056 - timing/update_critic:8.997 - timing/update_actor:10.832 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(main_task pid=28294)\u001b[0m validation generation end\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>system\n", - "\u001b[36m(main_task pid=28294)\u001b[0m You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>user\n", - "\u001b[36m(main_task pid=28294)\u001b[0m John fills a 6 foot by 4 foot pool that is 5 feet deep. It cost $.1 per cubic foot to fill. How much does it cost to fill? Let's think step by step and output the final answer after \"####\".<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m <|im_start|>assistant\n", - "\u001b[36m(main_task pid=28294)\u001b[0m First, we calculate the volume of the pool. The volume \\( V \\) of a rectangular prism is given by the formula:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ V = \\text{length} \\times \\text{width} \\times \\text{height} \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Here, the length is 6 feet, the width is 4 feet, and the height is 5 feet. So,\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ V = 6 \\times 4 \\times 5 = 120 \\text{ cubic feet} \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m Next, we calculate the cost to fill the pool. The cost is $0.1 per cubic foot, so the total cost \\( C \\) is:\n", - "\u001b[36m(main_task pid=28294)\u001b[0m \\[ C = 120 \\times 0.1 = 12 \\text{ dollars} \\]\n", - "\u001b[36m(main_task pid=28294)\u001b[0m #### 12<|im_end|>\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:59 - critic/kl:0.167 - critic/kl_coeff:0.001 - critic/vf_loss:0.227 - critic/vf_clipfrac:0.000 - critic/vpred_mean:0.058 - critic/grad_norm:97.710 - critic/lr:0.000 - actor/entropy_loss:0.178 - actor/pg_loss:-0.008 - actor/pg_clipfrac:0.003 - actor/ppo_kl:-0.000 - actor/grad_norm:3.323 - actor/lr:0.000 - val/test_score/openai/gsm8k:0.478 - critic/score/mean:0.625 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.603 - critic/rewards/max:0.990 - critic/rewards/min:-0.037 - critic/advantages/mean:0.000 - critic/advantages/max:1.931 - critic/advantages/min:-2.355 - critic/returns/mean:0.548 - critic/returns/max:1.001 - critic/returns/min:-0.037 - critic/values/mean:0.215 - critic/values/max:0.871 - critic/values/min:-0.402 - response_length/mean:148.621 - response_length/max:256.000 - response_length/min:47.000 - prompt_length/mean:102.344 - prompt_length/max:170.000 - prompt_length/min:56.000 - timing/gen:21.616 - timing/ref:3.366 - timing/values:3.169 - timing/adv:0.056 - timing/update_critic:8.991 - timing/update_actor:10.783 - timing/testing:43.049 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving actor checkpoint to checkpoints/verl_examples/gsm8k/actor/global_step_59\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m Saving critic checkpoint to checkpoints/verl_examples/gsm8k/critic/global_step_59\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m /home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.\n", - "\u001b[36m(WorkerDict pid=28545)\u001b[0m warnings.warn(\n", - "\u001b[36m(main_task pid=28294)\u001b[0m step:60 - critic/kl:0.161 - critic/kl_coeff:0.001 - critic/vf_loss:0.138 - critic/vf_clipfrac:0.208 - critic/vpred_mean:0.673 - critic/grad_norm:59.237 - critic/lr:0.000 - actor/entropy_loss:0.173 - actor/pg_loss:-0.015 - actor/pg_clipfrac:0.003 - actor/ppo_kl:0.001 - actor/grad_norm:3.070 - actor/lr:0.000 - critic/score/mean:0.621 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.598 - critic/rewards/max:0.993 - critic/rewards/min:-0.043 - critic/advantages/mean:-0.000 - critic/advantages/max:1.835 - critic/advantages/min:-2.245 - critic/returns/mean:0.562 - critic/returns/max:1.001 - critic/returns/min:-0.043 - critic/values/mean:0.293 - critic/values/max:0.789 - critic/values/min:-0.336 - response_length/mean:155.738 - response_length/max:256.000 - response_length/min:50.000 - prompt_length/mean:103.066 - prompt_length/max:176.000 - prompt_length/min:67.000 - timing/gen:22.844 - timing/ref:3.415 - timing/values:3.202 - timing/adv:0.055 - timing/update_critic:9.302 - timing/update_actor:10.783 - timing_per_token/values:0.000 - timing_per_token/ref:0.000 - timing_per_token/update_critic:0.000 - timing_per_token/update_actor:0.000 - timing_per_token/adv:0.000 - timing_per_token/gen:0.001\n", - "^C\n" - ] - } - ], - "source": [ - "!PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \\\n", - " data.train_files=$HOME/data/gsm8k/train.parquet \\\n", - " data.val_files=$HOME/data/gsm8k/test.parquet \\\n", - " data.train_batch_size=256 \\\n", - " data.max_prompt_length=512 \\\n", - " data.max_response_length=256 \\\n", - " actor_rollout_ref.model.path=$HOME/models/Qwen2.5-0.5B-Instruct \\\n", - " actor_rollout_ref.actor.optim.lr=1e-6 \\\n", - " actor_rollout_ref.actor.ppo_mini_batch_size=64 \\\n", - " actor_rollout_ref.actor.ppo_micro_batch_size=1 \\\n", - " actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \\\n", - " actor_rollout_ref.rollout.tensor_model_parallel_size=1 \\\n", - " actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \\\n", - " actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \\\n", - " critic.optim.lr=1e-5 \\\n", - " critic.model.path=$HOME/models/Qwen2.5-0.5B-Instruct \\\n", - " critic.ppo_micro_batch_size=1 \\\n", - " algorithm.kl_ctrl.kl_coef=0.001 \\\n", - " trainer.val_before_train=False \\\n", - " trainer.default_hdfs_dir=null \\\n", - " trainer.n_gpus_per_node=1 \\\n", - " trainer.nnodes=1 \\\n", - " trainer.save_freq=10 \\\n", - " trainer.test_freq=10 \\\n", - " trainer.total_epochs=15 \\\n", - " trainer.logger=\\[console\\]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zSn7lNlZ2vfL" - }, - "source": [ - "# Stop and clean up resources" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QuJ-LgdTAPkb", - "outputId": "64f2ef75-4a6d-4a62-922e-3d09b33a8a44" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Did not find any active Ray processes.\n", - "\u001b[0m" - ] - } - ], - "source": [ - "!ray stop" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.10" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "06873240926949d98e13872546c5231d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "06e1b9b5d49d4ee3ab8d1a523659bcbf": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "07f455e5e6dd45b7ba52f78bfc7ec7d6": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0ab915aba5e14e5bba7ba1c22a682b89": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0c9b8ffe4b8c4b5ca72a21cc54a1feb9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "0eeef594fb564491ad8d80f86a8fbfdc": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9fbafd9fc26748b7889b5c52600f80a8", - "placeholder": "​", - "style": "IPY_MODEL_889e01d618544f7c9a9d748730255007", - "value": " 242/242 [00:00<00:00, 15.3kB/s]" - } - }, - "144df34a87334a6d8eb13055e7a9b9e4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1e9ee1c383074f638a688b029d72bc79", - "IPY_MODEL_5cfeadb8ff394f38ac2e23f1a66beeb3", - "IPY_MODEL_0eeef594fb564491ad8d80f86a8fbfdc" - ], - "layout": "IPY_MODEL_771c5ca9460f4539b30f452dd3f36b12" - } - }, - "1491cbb53c6e4bfb9d17cf123dea83dd": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1a382959fdeb4554827397823284d2fa": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f52d7af1a82249a3aa7785476e10c2ad", - "IPY_MODEL_afcc65785fef4b71b03ac83a4b14d97f", - "IPY_MODEL_c0b19ca098a443598c662921832e8799" - ], - "layout": "IPY_MODEL_ca24445f8af44c8397f12d15d66eebf5" - } - }, - "1e9ee1c383074f638a688b029d72bc79": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fab6aab315214fcb884529a4dbf84fe5", - "placeholder": "​", - "style": "IPY_MODEL_06e1b9b5d49d4ee3ab8d1a523659bcbf", - "value": "generation_config.json: 100%" - } - }, - "252e651f687e47f3bd20518f2ac5fb9f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2749b87567ea4b6cbc4cf825e2282615": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2babfcd555104f9d8ecf98e164ec40fc": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "3447ed64518746cabb0176348fc88d96": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "35bacfb8aa4c4a25bf8ce2d13a00f2b8": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "35e124a16d2945ddbb3ade95ef2b5519": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3e1dd2fd3bb049ab83aa987d748f5b9e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "40259328dd5e4256939d7b1a3f038d98": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_866c770e39b64dfd9764de755f6a9ec5", - "max": 1671839, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_2babfcd555104f9d8ecf98e164ec40fc", - "value": 1671839 - } - }, - "412349b6e00b4994bc3f63f8405b3ec2": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_be14bccf9f114d9f839c805afef08f61", - "max": 988097824, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_52268a2d657b4e19badd66f781f68d93", - "value": 988097824 - } - }, - "4957b3690466495997721afab68ad93a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4bdbe0a8bb434bfc8e2172ecb5189705": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_35e124a16d2945ddbb3ade95ef2b5519", - "placeholder": "​", - "style": "IPY_MODEL_7de86c10755f4e0da7974bdf1815a85d", - "value": "tokenizer_config.json: 100%" - } - }, - "4d1a260957214732940766c874d3a02b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e6b66ca90c9c4b0ead5153e4a07cdc86", - "placeholder": "​", - "style": "IPY_MODEL_3e1dd2fd3bb049ab83aa987d748f5b9e", - "value": "tokenizer.json: 100%" - } - }, - "5110a596739443a8a640cfd50030644b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_07f455e5e6dd45b7ba52f78bfc7ec7d6", - "max": 659, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_fdf06125a50249b8878dbf01993306f4", - "value": 659 - } - }, - "52268a2d657b4e19badd66f781f68d93": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "538e82daa19140098a4053da6e23de45": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5c8c3c4d700540f089f671d4f5d0dd9f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5cfeadb8ff394f38ac2e23f1a66beeb3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e3848f0a11f8472fba3ecb624bc86dd9", - "max": 242, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_c7b67dd574ad4c15b36930047553e9d3", - "value": 242 - } - }, - "645fee7bcccd42a794e4aa889c1fe145": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_aa19071cede44a089d7f3b19227d51e0", - "IPY_MODEL_412349b6e00b4994bc3f63f8405b3ec2", - "IPY_MODEL_a921f9b0d3c74381b75aa60f4d1cac1c" - ], - "layout": "IPY_MODEL_b707bf4c56744f05ac9245b07f6d1788" - } - }, - "69e57962129241a689cfd2933b64127c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_4bdbe0a8bb434bfc8e2172ecb5189705", - "IPY_MODEL_b0bbbf7f9f264dfda2c0d6775567e446", - "IPY_MODEL_6c9485ecc56f4027ad8f3824554e3968" - ], - "layout": "IPY_MODEL_3447ed64518746cabb0176348fc88d96" - } - }, - "6b14f827b15f4e34be6590a5d2085b64": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "6c9485ecc56f4027ad8f3824554e3968": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b10402691cc3480693dcf49d19336c72", - "placeholder": "​", - "style": "IPY_MODEL_f0350562775a4c4ca83772a78d05122b", - "value": " 7.30k/7.30k [00:00<00:00, 497kB/s]" - } - }, - "6cd310d2188d424eb20c3bf83ac34f56": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6f3742161c4f4bcc891c82aff7ece69f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "7363ebea3a3a4f55b69b2d813c3b2fa5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e9f9be6fa1744f3380d21c451bc81555", - "placeholder": "​", - "style": "IPY_MODEL_c5024f35870446a0ae8fd747101ab719", - "value": " 7.03M/7.03M [00:01<00:00, 6.03MB/s]" - } - }, - "763679906f9248a7a5f4c8de952d98ae": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b319db13c64b43a38250342c81708f70", - "IPY_MODEL_5110a596739443a8a640cfd50030644b", - "IPY_MODEL_e93bf508749940909c3233904e898497" - ], - "layout": "IPY_MODEL_b45a42483d64410ba245feda17ae3e16" - } - }, - "771c5ca9460f4539b30f452dd3f36b12": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7920655156a44e629514673dde2b9663": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7c45a87d87f44b2384a4fd316ae36663": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7de86c10755f4e0da7974bdf1815a85d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "835a6a0a56554d158ee40ccc5ccdffc5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "866c770e39b64dfd9764de755f6a9ec5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "889e01d618544f7c9a9d748730255007": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "89a180c90767474b8e699e264620666e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a1255e85757e495a86ae366857fb64f1", - "max": 7031645, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6f3742161c4f4bcc891c82aff7ece69f", - "value": 7031645 - } - }, - "936061cb57ad445195efc0aa24dd8d66": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "9e2c1dcd2cd643bbb941d6697fcc75a0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "9fbafd9fc26748b7889b5c52600f80a8": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a1255e85757e495a86ae366857fb64f1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a726ef24d10c42bf859e4c76cebde672": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5c8c3c4d700540f089f671d4f5d0dd9f", - "placeholder": "​", - "style": "IPY_MODEL_7c45a87d87f44b2384a4fd316ae36663", - "value": "merges.txt: 100%" - } - }, - "a921f9b0d3c74381b75aa60f4d1cac1c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_06873240926949d98e13872546c5231d", - "placeholder": "​", - "style": "IPY_MODEL_936061cb57ad445195efc0aa24dd8d66", - "value": " 988M/988M [00:24<00:00, 40.9MB/s]" - } - }, - "aa19071cede44a089d7f3b19227d51e0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_252e651f687e47f3bd20518f2ac5fb9f", - "placeholder": "​", - "style": "IPY_MODEL_835a6a0a56554d158ee40ccc5ccdffc5", - "value": "model.safetensors: 100%" - } - }, - "afcc65785fef4b71b03ac83a4b14d97f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e49f1b46d8ae4c3e8f894b1f411922b9", - "max": 2776833, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_0c9b8ffe4b8c4b5ca72a21cc54a1feb9", - "value": 2776833 - } - }, - "b0bbbf7f9f264dfda2c0d6775567e446": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4957b3690466495997721afab68ad93a", - "max": 7305, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_9e2c1dcd2cd643bbb941d6697fcc75a0", - "value": 7305 - } - }, - "b10402691cc3480693dcf49d19336c72": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b319db13c64b43a38250342c81708f70": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_538e82daa19140098a4053da6e23de45", - "placeholder": "​", - "style": "IPY_MODEL_6b14f827b15f4e34be6590a5d2085b64", - "value": "config.json: 100%" - } - }, - "b45a42483d64410ba245feda17ae3e16": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b707bf4c56744f05ac9245b07f6d1788": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "be14bccf9f114d9f839c805afef08f61": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c0b19ca098a443598c662921832e8799": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c3651669cb084d86b9b8c427c665d185", - "placeholder": "​", - "style": "IPY_MODEL_35bacfb8aa4c4a25bf8ce2d13a00f2b8", - "value": " 2.78M/2.78M [00:01<00:00, 2.42MB/s]" - } - }, - "c0e97dba53284330b0fb8cefc852d552": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_4d1a260957214732940766c874d3a02b", - "IPY_MODEL_89a180c90767474b8e699e264620666e", - "IPY_MODEL_7363ebea3a3a4f55b69b2d813c3b2fa5" - ], - "layout": "IPY_MODEL_d49791321218419d8b7af314dd904777" - } - }, - "c1020ed4d8a44747838ed59287d284ed": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_a726ef24d10c42bf859e4c76cebde672", - "IPY_MODEL_40259328dd5e4256939d7b1a3f038d98", - "IPY_MODEL_ee0b85738cbf4376a6427fadbdecfad7" - ], - "layout": "IPY_MODEL_1491cbb53c6e4bfb9d17cf123dea83dd" - } - }, - "c24df93c305e42cdbaed3d6111d72010": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c3651669cb084d86b9b8c427c665d185": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c5024f35870446a0ae8fd747101ab719": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c7b67dd574ad4c15b36930047553e9d3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "ca24445f8af44c8397f12d15d66eebf5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d49791321218419d8b7af314dd904777": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ddecda628c6a4a5680b4241633153ebd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "e3848f0a11f8472fba3ecb624bc86dd9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e49f1b46d8ae4c3e8f894b1f411922b9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e6b66ca90c9c4b0ead5153e4a07cdc86": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e93bf508749940909c3233904e898497": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0ab915aba5e14e5bba7ba1c22a682b89", - "placeholder": "​", - "style": "IPY_MODEL_2749b87567ea4b6cbc4cf825e2282615", - "value": " 659/659 [00:00<00:00, 27.5kB/s]" - } - }, - "e9f9be6fa1744f3380d21c451bc81555": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ee0b85738cbf4376a6427fadbdecfad7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7920655156a44e629514673dde2b9663", - "placeholder": "​", - "style": "IPY_MODEL_c24df93c305e42cdbaed3d6111d72010", - "value": " 1.67M/1.67M [00:00<00:00, 1.93MB/s]" - } - }, - "f0350562775a4c4ca83772a78d05122b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f52d7af1a82249a3aa7785476e10c2ad": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6cd310d2188d424eb20c3bf83ac34f56", - "placeholder": "​", - "style": "IPY_MODEL_ddecda628c6a4a5680b4241633153ebd", - "value": "vocab.json: 100%" - } - }, - "fab6aab315214fcb884529a4dbf84fe5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fdf06125a50249b8878dbf01993306f4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/ray/tutorial.ipynb b/examples/ray/tutorial.ipynb deleted file mode 100644 index 37784f6f95..0000000000 --- a/examples/ray/tutorial.ipynb +++ /dev/null @@ -1,958 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0ddc582b", - "metadata": {}, - "source": [ - "# VeRL Ray API Tutorial" - ] - }, - { - "cell_type": "markdown", - "id": "71fe3b94", - "metadata": {}, - "source": [ - "## Chapter 1: Ray Basics" - ] - }, - { - "cell_type": "code", - "execution_count": 144, - "id": "1347d381", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": 145, - "id": "e75b9d44", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import ray\n", - "import torch\n", - "import warnings\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "code", - "execution_count": 146, - "id": "2e90ae00", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-11-01 17:27:19,132\tINFO worker.py:1752 -- Started a local Ray instance.\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9cc9d2ccbdfb48918c8fd6cd13a0807a", - "version_major": 2, - "version_minor": 0 - }, - "text/html": [ - "
\n", - "
\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Python version:3.9.2
Ray version:2.10.0
\n", - "\n", - "
\n", - "
\n" - ], - "text/plain": [ - "RayContext(dashboard_url='', python_version='3.9.2', ray_version='2.10.0', ray_commit='09abba26b5bf2707639bb637c208d062a47b46f6')" - ] - }, - "execution_count": 146, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(GPUAccumulator pid=224400)\u001b[0m rank 0, value: tensor([1.], device='cuda:0')\n", - "\u001b[36m(GPUAccumulator pid=225234)\u001b[0m rank 2, value: tensor([3.], device='cuda:0')\n", - "\u001b[36m(GPUAccumulator pid=225607)\u001b[0m rank 0, value: tensor([2.], device='cuda:0')\n", - "\u001b[36m(GPUAccumulator pid=226423)\u001b[0m rank 1, value: tensor([3.], device='cuda:0')\n", - "\u001b[36m(GPUAccumulator pid=226857)\u001b[0m rank 3, value: tensor([6.], device='cuda:0')\n", - "\u001b[36m(GPUAccumulatorDecorator pid=227475)\u001b[0m 10\n", - "\u001b[36m(GPUAccumulatorDecorator pid=227475)\u001b[0m rank 0, value: tensor([10.], device='cuda:0')\n", - "\u001b[36m(GPUAccumulatorDecorator pid=227655)\u001b[0m rank 1, value: tensor([11.], device='cuda:0')\n" - ] - } - ], - "source": [ - "# Build a local ray cluster. The head node and worker node are on this machine\n", - "ray.init()" - ] - }, - { - "cell_type": "markdown", - "id": "a127e4e4", - "metadata": {}, - "source": [ - "Implement an Accumulator class." - ] - }, - { - "cell_type": "code", - "execution_count": 147, - "id": "20e7b9a3", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "@ray.remote\n", - "class Accumulator:\n", - " def __init__(self):\n", - " self.value = 0\n", - " \n", - " def add(self, x):\n", - " self.value += x\n", - " \n", - " def get_value(self):\n", - " return self.value" - ] - }, - { - "cell_type": "code", - "execution_count": 148, - "id": "3b80098c", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Instantiate an accumulator. Accumulator can be viewed as a process, acting as an RPC service.\n", - "accumulator = Accumulator.remote()" - ] - }, - { - "cell_type": "code", - "execution_count": 149, - "id": "b14b1009", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\n" - ] - } - ], - "source": [ - "value_ref = accumulator.get_value.remote() # Check the current value. Note that this function returns immediately and does not actually wait for the remote execution to complete.\n", - "# Get the value\n", - "value = ray.get(value_ref)\n", - "print(value)" - ] - }, - { - "cell_type": "code", - "execution_count": 150, - "id": "513a84b3", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10\n" - ] - } - ], - "source": [ - "# Accumulate, then check the result.\n", - "accumulator.add.remote(10) # Similarly, the 'add' here will return immediately.\n", - "new_value = ray.get(accumulator.get_value.remote())\n", - "print(new_value)" - ] - }, - { - "cell_type": "markdown", - "id": "3c332fe0", - "metadata": {}, - "source": [ - "## Chapter 2: Resource Pool and RayWorkerGroup\n", - "In the previous example, it was a simple single-process worker. \n", - "In this example, we implement a worker with a GPU and form a RayWorkerGroup. Within this RayWorkerGroup, we implement a simple operation of an accumulator." - ] - }, - { - "cell_type": "code", - "execution_count": 151, - "id": "04229afb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup, merge_resource_pool\n", - "from verl.single_controller.base import Worker" - ] - }, - { - "cell_type": "code", - "execution_count": 152, - "id": "0d0dbd58", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "resource_pool = RayResourcePool([4], use_gpu=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 153, - "id": "68f6838a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "@ray.remote\n", - "class GPUAccumulator(Worker):\n", - "\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - " # The initial value of each rank is the same as the rank\n", - " self.value = torch.zeros(size=(1,), device='cuda') + self.rank\n", - "\n", - " def add(self, x):\n", - " self.value += x\n", - " print(f'rank {self.rank}, value: {self.value}')\n", - " return self.value.cpu()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 154, - "id": "23aad8fe", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[tensor([1.]), tensor([2.]), tensor([3.]), tensor([4.])]\n" - ] - } - ], - "source": [ - "# Each worker's initial value is its rank, and then each rank's value is incremented by 1, so the values obtained on each rank are [1, 2, 3, 4]\n", - "class_with_args = RayClassWithInitArgs(cls=GPUAccumulator)\n", - "worker_group = RayWorkerGroup(resource_pool, class_with_args)\n", - "print(worker_group.execute_all_sync('add', x=[1,1,1,1]))" - ] - }, - { - "cell_type": "markdown", - "id": "e6705284", - "metadata": {}, - "source": [ - "The principle of parameter passing: The input parameter is a list of length world_size, where each element in the list is dispatched respectively to each worker in the RayWorkerGroup. \n", - "The return parameter is also a list, corresponding to the return value of each worker." - ] - }, - { - "cell_type": "markdown", - "id": "d25c2412", - "metadata": {}, - "source": [ - "### GPU Resource Sharing" - ] - }, - { - "cell_type": "markdown", - "id": "f74f6d24", - "metadata": {}, - "source": [ - "RayWorkerGroups mapped to the same resource pool share the GPU. In this example, we implement three resource pools: the first occupies 4 GPUs, the second also occupies 4 GPUs, and the last occupies all 8 GPUs. Among them, the first resource pool reuses the resource pool mentioned above." - ] - }, - { - "cell_type": "code", - "execution_count": 155, - "id": "49f9c06f", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Create a new resource pool and then merge the newly created resource pool with the previous one.\n", - "resource_pool_1 = RayResourcePool([4], use_gpu=True, name_prefix='a')\n", - "resource_pool_merge = merge_resource_pool(resource_pool, resource_pool_1)" - ] - }, - { - "cell_type": "code", - "execution_count": 156, - "id": "05c2e305", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Establish a RayWorkerGroup on the newly created resource pool.\n", - "worker_group_1 = RayWorkerGroup(resource_pool_1, class_with_args)\n", - "worker_group_merge = RayWorkerGroup(resource_pool_merge, class_with_args)" - ] - }, - { - "cell_type": "code", - "execution_count": 157, - "id": "6b9b13f4", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[tensor([2.]), tensor([3.]), tensor([4.]), tensor([5.])]\n" - ] - } - ], - "source": [ - "# Run 'add' on the second set of 4 GPUs; the result should be [2, 3, 4, 5].\n", - "output_1 = worker_group_1.execute_all_sync('add', x=[2,2,2,2])\n", - "print(output_1)" - ] - }, - { - "cell_type": "code", - "execution_count": 158, - "id": "d856d030", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[tensor([3.]), tensor([4.]), tensor([5.]), tensor([6.]), tensor([7.]), tensor([8.]), tensor([9.]), tensor([10.])]\n" - ] - } - ], - "source": [ - "# Run 'add' on the merged set of 8 GPUs; the result should be [3, 4, 5, 6, 7, 8, 9, 10].\n", - "output_merge = worker_group_merge.execute_all_sync('add', x=[3,3,3,3,3,3,3,3])\n", - "print(output_merge)" - ] - }, - { - "cell_type": "code", - "execution_count": 159, - "id": "33a4628c", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4 4 8\n" - ] - } - ], - "source": [ - "print(worker_group.world_size, worker_group_1.world_size, worker_group_merge.world_size)" - ] - }, - { - "cell_type": "markdown", - "id": "3df19d13", - "metadata": {}, - "source": [ - "## Chapter 3: Data Dispatch, Execution and Collection" - ] - }, - { - "cell_type": "markdown", - "id": "acb22d9d", - "metadata": {}, - "source": [ - "In the above example, we used the `execute_all_sync` function in the RayWorkerGroup to dispatch data from the driver to each worker. This is very inconvenient for coding. \n", - "In this chapter, we use the form of function decorators to allow RayWorkerGroup to directly call functions written in the Worker, and to greatly simplify parameter passing." - ] - }, - { - "cell_type": "code", - "execution_count": 160, - "id": "35237432", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from verl.single_controller.base.decorator import register, Dispatch, Execute" - ] - }, - { - "cell_type": "code", - "execution_count": 161, - "id": "88b8ba3b", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "@ray.remote\n", - "class GPUAccumulatorDecorator(Worker):\n", - "\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - " # The initial value of each rank is the same as the rank\n", - " self.value = torch.zeros(size=(1,), device='cuda') + self.rank\n", - " \n", - " # map from a single input to all the worker\n", - " @register(Dispatch.ONE_TO_ALL)\n", - " def add(self, x):\n", - " print(x)\n", - " self.value = self.value + x\n", - " print(f'rank {self.rank}, value: {self.value}')\n", - " return self.value.cpu()" - ] - }, - { - "cell_type": "code", - "execution_count": 162, - "id": "eddaa043", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "class_with_args = RayClassWithInitArgs(cls=GPUAccumulatorDecorator)\n", - "gpu_accumulator_decorator = RayWorkerGroup(resource_pool_merge, class_with_args)" - ] - }, - { - "cell_type": "code", - "execution_count": 163, - "id": "10087c91", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[tensor([10.]), tensor([11.]), tensor([12.]), tensor([13.]), tensor([14.]), tensor([15.]), tensor([16.]), tensor([17.])]\n" - ] - } - ], - "source": [ - "# As we can see, 10 is automatically dispatched to each Worker in this RayWorkerGroup.\n", - "print(gpu_accumulator_decorator.add(x=10))" - ] - }, - { - "cell_type": "markdown", - "id": "540ee6ad", - "metadata": {}, - "source": [ - "### Custom Dispatch, Collection\n", - "Users can customize `dispatch` and `collection` function. You only need to write the `dispatch_fn` and `collect_fn` functions yourself. We also support executing RPC only on rank_zero, with specific examples provided below." - ] - }, - { - "cell_type": "code", - "execution_count": 164, - "id": "8e041270", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from verl.single_controller.base.decorator import register, Dispatch, collect_all_to_all, Execute" - ] - }, - { - "cell_type": "code", - "execution_count": 165, - "id": "43b5be31", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def two_to_all_dispatch_fn(worker_group, *args, **kwargs):\n", - " \"\"\"\n", - " Assume the input is a list of 2. Duplicate the input interleaved and pass to each worker.\n", - " \"\"\"\n", - " for arg in args:\n", - " assert len(arg) == 2\n", - " for i in range(worker_group.world_size - 2):\n", - " arg.append(arg[i % 2])\n", - " for k, v in kwargs.items():\n", - " assert len(v) == 2\n", - " for i in range(worker_group.world_size - 2):\n", - " v.append(v[i % 2])\n", - " return args, kwargs\n", - "\n", - "\n", - "@ray.remote\n", - "class TestActor(Worker):\n", - " # TODO: pass *args and **kwargs is bug prone and not very convincing\n", - " def __init__(self, x) -> None:\n", - " super().__init__()\n", - " self._x = x\n", - "\n", - " def foo(self, y):\n", - " return self._x + y\n", - "\n", - " @register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.RANK_ZERO)\n", - " def foo_rank_zero(self, x, y):\n", - " return self._x + y + x\n", - "\n", - " @register(dispatch_mode={'dispatch_fn': two_to_all_dispatch_fn, 'collect_fn': collect_all_to_all})\n", - " def foo_custom(self, x, y):\n", - " return self._x + y + x" - ] - }, - { - "cell_type": "code", - "execution_count": 166, - "id": "83ec6609", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "class_with_args = RayClassWithInitArgs(cls=TestActor, x=2)\n", - "worker_group = RayWorkerGroup(resource_pool, class_with_args)" - ] - }, - { - "cell_type": "code", - "execution_count": 167, - "id": "62c58d8a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "output_ref = worker_group.foo_custom(x=[1, 2], y=[5, 6])\n", - "assert output_ref == [8, 10, 8, 10]\n", - "\n", - "output_ref = worker_group.foo_rank_zero(x=1, y=2)\n", - "assert output_ref == 5" - ] - }, - { - "cell_type": "code", - "execution_count": 168, - "id": "14689353", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "8\n" - ] - } - ], - "source": [ - "print(gpu_accumulator_decorator.world_size)" - ] - }, - { - "cell_type": "code", - "execution_count": 169, - "id": "2c80bbf4", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Shutdown ray cluster\n", - "ray.shutdown()" - ] - }, - { - "cell_type": "markdown", - "id": "a5c8151c", - "metadata": {}, - "source": [ - "## Chapter 4: NVMegatronRayWorkerGroup" - ] - }, - { - "cell_type": "markdown", - "id": "cd5680e9", - "metadata": {}, - "source": [ - "Due to the Ray issue, we can only support max_colocate_count=1 in RayResourcePool for now. \n", - "This means that each GPU can only have one process.\n", - "We can support max_colocate > 1 when applying this pull request: https://github.com/ray-project/ray/pull/44385" - ] - }, - { - "cell_type": "markdown", - "id": "92724419", - "metadata": {}, - "source": [ - "Therefore, we need to restart the ray and initialize a new resource_pool to demonstrate the **NVMegatronRayWorkerGroup**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9b038538", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Build a local ray cluster. The head node and worker node are on this machine\n", - "ray.init()" - ] - }, - { - "cell_type": "markdown", - "id": "ebfd8798", - "metadata": {}, - "source": [ - "Finally, we implement a `NVMegatronRayWorkerGroup`, within which we create a Megatron and then run a tensor parallel (tp) split Llama mlp layer. Here, we use a complex dispatch mode, `Megatron_COMPUTE`. This dispatch mode assumes that user passes the data partitioned by DP dimension. The data is dispatched to all tp/pp ranks within the same dp group, and ultimately only collects output data from tp=0 and the last pp. In this way, for users that only write code on the driver, the Megatron behind the RPC becomes transparent." - ] - }, - { - "cell_type": "code", - "execution_count": 171, - "id": "5a032154", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/opt/tiger/Megatron-LM\n", - "/opt/tiger/Megatron-LM/megatron/__init__.py\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "import site\n", - "\n", - "\n", - "current_pythonpath = os.environ.get('PYTHONPATH', '')\n", - "\n", - "new_path = '/opt/tiger/Megatron-LM'\n", - "\n", - "if current_pythonpath:\n", - " new_pythonpath = f'{new_path}:{current_pythonpath}'\n", - "else:\n", - " new_pythonpath = new_path\n", - "\n", - "os.environ['PYTHONPATH'] = new_pythonpath\n", - "\n", - "print(new_path)\n", - "sys.path.append(new_path)\n", - "\n", - "import megatron\n", - "print(megatron.__file__)" - ] - }, - { - "cell_type": "code", - "execution_count": 172, - "id": "8c84cd5a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from verl.single_controller.base.decorator import register, Dispatch, Execute\n", - "from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup\n", - "from verl.single_controller.base.megatron.worker import MegatronWorker\n", - "from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup\n", - "from omegaconf import OmegaConf\n", - "from megatron.core import parallel_state as mpu" - ] - }, - { - "cell_type": "code", - "execution_count": 173, - "id": "1b1debcc", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "resource_pool = RayResourcePool([4], use_gpu=True, max_colocate_count=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 174, - "id": "bccbe081", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "@ray.remote\n", - "class MLPLayerWorker(MegatronWorker):\n", - " def __init__(self):\n", - " super().__init__()\n", - " rank = int(os.environ['LOCAL_RANK'])\n", - " torch.distributed.init_process_group(backend=\"nccl\")\n", - " torch.cuda.set_device(rank)\n", - "\n", - " mpu.initialize_model_parallel(\n", - " tensor_model_parallel_size=4,\n", - " pipeline_model_parallel_size=1,\n", - " virtual_pipeline_model_parallel_size=None,\n", - " pipeline_model_parallel_split_rank=None,\n", - " use_sharp=False,\n", - " context_parallel_size=1,\n", - " expert_model_parallel_size=1,\n", - " nccl_communicator_config_path=None,\n", - " )\n", - " from megatron.core import tensor_parallel\n", - " tensor_parallel.model_parallel_cuda_manual_seed(10)\n", - "\n", - "\n", - " @register(Dispatch.ONE_TO_ALL)\n", - " def init_model(self, config):\n", - " from omegaconf import OmegaConf\n", - " from verl.utils.megatron_utils import init_model_parallel_config\n", - " from verl.models.llama.megatron.layers import ParallelLlamaMLP\n", - " megatron_config = OmegaConf.create({\n", - " 'sequence_parallel': False,\n", - " 'param_dtype': 'fp32',\n", - " 'tensor_model_parallel_size': mpu.get_tensor_model_parallel_world_size(),\n", - " 'pipeline_model_parallel_rank': mpu.get_pipeline_model_parallel_rank(),\n", - " 'pipeline_model_parallel_size': mpu.get_pipeline_model_parallel_world_size(),\n", - " 'virtual_pipeline_model_parallel_rank': mpu.get_virtual_pipeline_model_parallel_rank(),\n", - " 'virtual_pipeline_model_parallel_size': mpu.get_virtual_pipeline_model_parallel_world_size()\n", - " })\n", - "\n", - " megatron_config = init_model_parallel_config(megatron_config)\n", - " self.parallel_layer = ParallelLlamaMLP(config=config, megatron_config=megatron_config)\n", - " \n", - " @register(Dispatch.ONE_TO_ALL)\n", - " def get_weights(self):\n", - " output = {}\n", - " for key, val in self.parallel_layer.named_parameters():\n", - " output[key] = val\n", - " return output\n", - " \n", - " @register(Dispatch.MEGATRON_COMPUTE)\n", - " def run_layer(self, x):\n", - " x = x.to('cuda')\n", - " y = self.parallel_layer(x)\n", - " return y" - ] - }, - { - "cell_type": "code", - "execution_count": 175, - "id": "a655271d", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "layer_cls = RayClassWithInitArgs(cls=MLPLayerWorker)\n", - "layer_worker_group = NVMegatronRayWorkerGroup(resource_pool=resource_pool,\n", - " ray_cls_with_init=layer_cls,\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": 176, - "id": "f105ebee", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4 4 1 1\n" - ] - } - ], - "source": [ - "print(layer_worker_group.world_size, layer_worker_group.tp_size, layer_worker_group.pp_size, layer_worker_group.dp_size)" - ] - }, - { - "cell_type": "code", - "execution_count": 177, - "id": "38655091", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "ffn_hidden_size = 11008\n", - "batch_size = 16\n", - "seq_len = 2048\n", - "hidden_size = 4096\n", - "\n", - "config = OmegaConf.create({\n", - " 'hidden_size': hidden_size,\n", - " 'intermediate_size': ffn_hidden_size,\n", - " 'hidden_act': 'silu',\n", - " 'pretraining_tp': 1,\n", - " 'tp': layer_worker_group.tp_size,\n", - "})" - ] - }, - { - "cell_type": "code", - "execution_count": 178, - "id": "a026efca", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "x = torch.rand(size=(seq_len, batch_size, hidden_size), dtype=torch.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 179, - "id": "f5fcaf13", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[None, None, None, None]" - ] - }, - "execution_count": 179, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "layer_worker_group.init_model(config)" - ] - }, - { - "cell_type": "code", - "execution_count": 180, - "id": "3f5cc9b4", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([2048, 16, 4096])\n" - ] - } - ], - "source": [ - "output = layer_worker_group.run_layer([x]) # This must be a list of size 1, ensuring that the input equals the data parallel (dp).\n", - "print(output[0].shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 181, - "id": "49792210", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Shutdown ray cluster\n", - "ray.shutdown()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh b/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh deleted file mode 100644 index 545107921b..0000000000 --- a/examples/remax_trainer/run_qwen2.5-3b_seq_balance.sh +++ /dev/null @@ -1,44 +0,0 @@ -set -x - -export HF_DATASETS_OFFLINE=1 -export TRANSFORMERS_OFFLINE=1 - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=remax \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=512 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=128 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=30000 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ - actor_rollout_ref.rollout.n=4 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_remax_example_gsm8k' \ - trainer.experiment_name='qwen2.5_3b_function_rm_kl1e-3' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=5 $@ diff --git a/examples/remax_trainer/run_qwen2.5-7b_seq_balance.sh b/examples/remax_trainer/run_qwen2.5-7b_seq_balance.sh deleted file mode 100644 index ccba54e40d..0000000000 --- a/examples/remax_trainer/run_qwen2.5-7b_seq_balance.sh +++ /dev/null @@ -1,44 +0,0 @@ -set -x - -export HF_DATASETS_OFFLINE=1 -export TRANSFORMERS_OFFLINE=1 - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=remax \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=24000 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ - actor_rollout_ref.rollout.n=4 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_remax_example_gsm8k' \ - trainer.experiment_name='qwen2.5_7b_function_rm_kl1e-3' \ - trainer.val_before_train=False \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=10 $@ diff --git a/examples/rloo_trainer/run_qwen2-7b.sh b/examples/rloo_trainer/run_qwen2-7b.sh deleted file mode 100644 index 368b224b77..0000000000 --- a/examples/rloo_trainer/run_qwen2-7b.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=rloo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=Qwen/Qwen2-7B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=80 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.grad_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=160 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=160 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_rloo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 $@ \ No newline at end of file diff --git a/examples/sft/gsm8k/run_deepseek_6b7.sh b/examples/sft/gsm8k/run_deepseek_6b7.sh deleted file mode 100644 index f11965a698..0000000000 --- a/examples/sft/gsm8k/run_deepseek_6b7.sh +++ /dev/null @@ -1,29 +0,0 @@ -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_deepseek_6b7.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size_per_gpu=4 \ - model.partial_pretrain=deepseek-ai/deepseek-coder-6.7b-instruct \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-deepseek-coder-6.7b-instruct \ - trainer.total_epochs=4 \ - trainer.logger=['console','wandb'] \ - trainer.default_hdfs_dir=null $@ \ No newline at end of file diff --git a/examples/sft/gsm8k/run_gemma_2b.sh b/examples/sft/gsm8k/run_gemma_2b.sh deleted file mode 100644 index 6d7917d9e8..0000000000 --- a/examples/sft/gsm8k/run_gemma_2b.sh +++ /dev/null @@ -1,31 +0,0 @@ -# Tested with 2 & 4 GPUs - -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_gemma_2b.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size_per_gpu=4 \ - model.partial_pretrain=google/gemma-2b-it \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-gemma-2b-it \ - trainer.total_epochs=2 \ - trainer.logger=['console','wandb'] \ - trainer.default_hdfs_dir=null $@ \ No newline at end of file diff --git a/examples/sft/gsm8k/run_gemma_7b.sh b/examples/sft/gsm8k/run_gemma_7b.sh deleted file mode 100644 index fdf4435bb2..0000000000 --- a/examples/sft/gsm8k/run_gemma_7b.sh +++ /dev/null @@ -1,27 +0,0 @@ -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_gemma_7b.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=prompt \ - data.response_key=answer \ - data.micro_batch_size_per_gpu=4 \ - model.partial_pretrain=google/gemma-1.1-7b-it \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-gemma-1.1-7b-it \ - trainer.total_epochs=4 \ - trainer.logger=['console','wandb'] \ - trainer.default_hdfs_dir=null $@ \ No newline at end of file diff --git a/examples/sft/gsm8k/run_qwen_05_peft.sh b/examples/sft/gsm8k/run_qwen_05_peft.sh deleted file mode 100755 index 3ba61c3a6e..0000000000 --- a/examples/sft/gsm8k/run_qwen_05_peft.sh +++ /dev/null @@ -1,38 +0,0 @@ -# Tested with 2 & 4 GPUs - -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_qwen_05_peft.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - optim.lr=1e-4 \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size_per_gpu=4 \ - model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct \ - trainer.logger=['console'] \ - trainer.total_epochs=1 \ - trainer.default_hdfs_dir=null $@ \ - model.lora_rank=32\ - model.lora_alpha=16 \ - model.target_modules=all-linear - - # Or you can do this: - # model.target_modules=[q_proj,v_proj] \ diff --git a/examples/sft/gsm8k/run_qwen_05_sp2.sh b/examples/sft/gsm8k/run_qwen_05_sp2.sh deleted file mode 100755 index a27cef1d48..0000000000 --- a/examples/sft/gsm8k/run_qwen_05_sp2.sh +++ /dev/null @@ -1,32 +0,0 @@ -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_qwen_05_sp2.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - optim.lr=1e-4 \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size=4 \ - model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct-sp2 \ - trainer.logger=['console'] \ - trainer.total_training_steps=1 \ - trainer.default_hdfs_dir=null $@ \ - ulysses_sequence_parallel_size=2 \ - use_remove_padding=true diff --git a/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh b/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh deleted file mode 100755 index 6ef528fdcc..0000000000 --- a/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh +++ /dev/null @@ -1,32 +0,0 @@ -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_qwen_05_sp2.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - optim.lr=1e-4 \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size=4 \ - model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ - model.use_liger=True \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct-sp2-liger \ - trainer.logger=['console'] \ - trainer.default_hdfs_dir=null $@ \ - ulysses_sequence_parallel_size=2 \ - use_remove_padding=true diff --git a/examples/sky/README.md b/examples/sky/README.md deleted file mode 100644 index 29e3199c90..0000000000 --- a/examples/sky/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# SkyRL-v0 Reproduction Scripts - -We provide exact scripts to reproduce our results for SkyRL-Agent-7B-v0, SkyRL-Agent-8B-v0, SkyRL-Agent-14B-v0. - -## Pre-requisite: Data preparation - -We provide the datasets we used on HuggingFace: https://huggingface.co/novasky-ai - -We used [NovaSky-AI/SkyRL-v0-293-data](https://huggingface.co/datasets/NovaSky-AI/SkyRL-v0-293-data) for training both SkyRL-Agent-8B-v0 and SkyRL-Agent-14B-v0. -We used [NovaSky-AI/SkyRL-v0-80-data](https://huggingface.co/datasets/NovaSky-AI/SkyRL-v0-80-data) (first stage) and [NovaSky-AI/SkyRL-v0-220-data](https://huggingface.co/datasets/NovaSky-AI/SkyRL-v0-220-data) (second stage) to train SkyRL-Agent-7B-v0. -Make sure to download the dataset and update the path in `DATA_PATH` in the script. - -## Setup Environment variables - -We use a [`.env`](../../.env) file to pass environment variables to all the processes created by Ray. Make sure to set `WANDB_API_KEY`, `ALLHANDS_API_KEY` and `SANDBOX_REMOTE_RUNTIME_API_URL`. - -## Running the scripts - -For SkyRL-Agent-14B-v0 Thinking mode, use [run_SkyRL_agent_qwen14b_t.sh](./run_SkyRL_agent_qwen14b_t.sh) and for SkyRL-Agent-8B-v0 Non-Thinking Mode, use [run_SkyRL_agent_qwen8b_nt.sh](./run_SkyRL_agent_qwen8b_nt.sh). For SkyRL-Agent-7B-v0, use [run_SkyRL_agent_oh7b_s1.sh](./run_SkyRL_agent_oh7b_s1.sh) and [run_SkyRL_agent_oh7b_s2.sh](./run_SkyRL_agent_oh7b_s2.sh). -The scripts are meant to be run from the git root. The default settings are meant for a 8xH200 node. For a 8xH100 node, we recommend the following modifications: - -```bash -TP_SIZE=2 -SP_SIZE=2 -NNODES=2 -``` \ No newline at end of file diff --git a/examples/sky/run_skyrl_agent_oh7b_s1.sh b/examples/sky/run_skyrl_agent_oh7b_s1.sh deleted file mode 100644 index 6b82af9d75..0000000000 --- a/examples/sky/run_skyrl_agent_oh7b_s1.sh +++ /dev/null @@ -1,88 +0,0 @@ -PROJECT_NAME='oh-7b-training' -EXPERIMENT_NAME='SkyRL-Agent-7b-v0-stage1' -DATA_PATH="" -SFT_MODEL_PATH='NovaSky-AI/SWE-Gym-OpenHands-7B-Agent' -CKPT_PATH='' - - -BATCH_SIZE=16 -MAX_NUM_ITERS=15 -NUM_TRAJ=16 -MAX_PARALLEL_AGENTS=64 -SAVE_FREQ=1 - -USE_KL_LOSS=True -KL_LOSS_COEF=0.001 -KL_LOSS_TYPE=low_var_kl -ENTROPY_COEFF=0 -CLIP_RATIO_LOW=0.2 -CLIP_RATIO_HIGH=0.2 - -GPU_MEM_UTIL=0.8 -TP_SIZE=1 -# Assumes a h200 node -# For 2xH100: change tp size -> 2, sequence parallel size -> 2, nnodes -> 2 -NNODES=1 -SP_SIZE=1 -TEMPERATURE=0.5 -TOP_P=0.95 - - - -PYTHONUNBUFFERED=1 uv run --isolated --directory . --frozen --env-file .env -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=["$DATA_PATH/train.parquet"] \ - data.val_files=["$DATA_PATH/validation.parquet"] \ - data.train_batch_size=$BATCH_SIZE \ - data.max_prompt_length=31232 \ - data.max_response_length=1536 \ - data.truncation='error' \ - actor_rollout_ref.model.path=$SFT_MODEL_PATH \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BATCH_SIZE \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE \ - actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ - actor_rollout_ref.actor.kl_loss_coef=$KL_LOSS_COEF \ - actor_rollout_ref.actor.kl_loss_type=$KL_LOSS_TYPE \ - actor_rollout_ref.actor.entropy_coeff=$ENTROPY_COEFF \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=True \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ - actor_rollout_ref.actor.clip_ratio_low=$CLIP_RATIO_LOW \ - actor_rollout_ref.actor.clip_ratio_high=$CLIP_RATIO_HIGH \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE \ - actor_rollout_ref.rollout.name=async \ - actor_rollout_ref.rollout.gpu_memory_utilization=$GPU_MEM_UTIL \ - actor_rollout_ref.rollout.n=1 \ - actor_rollout_ref.rollout.n_trajectories=$NUM_TRAJ \ - actor_rollout_ref.rollout.sampling_params.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.sampling_params.top_p=$TOP_P \ - actor_rollout_ref.rollout.top_p=$TOP_P \ - actor_rollout_ref.rollout.max_parallel_agents=$MAX_PARALLEL_AGENTS \ - actor_rollout_ref.rollout.max_iterations=$MAX_NUM_ITERS \ - actor_rollout_ref.rollout.enable_memory_saver=True \ - actor_rollout_ref.rollout.qwen3_enable_thinking=False \ - +actor_rollout_ref.rollout.max_starting_message_length=12000 \ - actor_rollout_ref.actor.masking=True \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - reward_model.reward_manager="swebench" \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name=$PROJECT_NAME \ - trainer.experiment_name=$EXPERIMENT_NAME \ - trainer.default_local_dir=$CKPT_PATH/$PROJECT_NAME/$EXPERIMENT_NAME \ - trainer.resume_mode=auto \ - trainer.max_actor_ckpt_to_keep=10 \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=$NNODES \ - trainer.save_freq=$SAVE_FREQ \ - data.dataloader_num_workers=0 \ - actor_rollout_ref.exchange_size=500000000 \ - trainer.test_freq=-1 \ - trainer.total_epochs=100 $@ \ No newline at end of file diff --git a/examples/sky/run_skyrl_agent_oh7b_s2.sh b/examples/sky/run_skyrl_agent_oh7b_s2.sh deleted file mode 100644 index edbcc5cf72..0000000000 --- a/examples/sky/run_skyrl_agent_oh7b_s2.sh +++ /dev/null @@ -1,88 +0,0 @@ -PROJECT_NAME='oh-7b-training' -EXPERIMENT_NAME='SkyRL-Agent-7b-v0-stage2' -DATA_PATH="" -SFT_MODEL_PATH='PATH_FROM_STAGE1' -CKPT_PATH='' - - -BATCH_SIZE=16 -MAX_NUM_ITERS=25 -NUM_TRAJ=16 -MAX_PARALLEL_AGENTS=64 -SAVE_FREQ=1 - -USE_KL_LOSS=True -KL_LOSS_COEF=0.001 -KL_LOSS_TYPE=low_var_kl -ENTROPY_COEFF=0 -CLIP_RATIO_LOW=0.2 -CLIP_RATIO_HIGH=0.2 - -GPU_MEM_UTIL=0.8 -TP_SIZE=1 -# Assumes a h200 node -# For 2xH100: change tp size -> 2, sequence parallel size -> 2, nnodes -> 2 -NNODES=1 -SP_SIZE=1 -TEMPERATURE=0.5 -TOP_P=0.95 - - - -PYTHONUNBUFFERED=1 uv run --isolated --directory . --frozen --env-file .env -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=["$DATA_PATH/train.parquet"] \ - data.val_files=["$DATA_PATH/validation.parquet"] \ - data.train_batch_size=$BATCH_SIZE \ - data.max_prompt_length=31232 \ - data.max_response_length=1536 \ - data.truncation='error' \ - actor_rollout_ref.model.path=$SFT_MODEL_PATH \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BATCH_SIZE \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE \ - actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ - actor_rollout_ref.actor.kl_loss_coef=$KL_LOSS_COEF \ - actor_rollout_ref.actor.kl_loss_type=$KL_LOSS_TYPE \ - actor_rollout_ref.actor.entropy_coeff=$ENTROPY_COEFF \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=True \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ - actor_rollout_ref.actor.clip_ratio_low=$CLIP_RATIO_LOW \ - actor_rollout_ref.actor.clip_ratio_high=$CLIP_RATIO_HIGH \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE \ - actor_rollout_ref.rollout.name=async \ - actor_rollout_ref.rollout.gpu_memory_utilization=$GPU_MEM_UTIL \ - actor_rollout_ref.rollout.n=1 \ - actor_rollout_ref.rollout.n_trajectories=$NUM_TRAJ \ - actor_rollout_ref.rollout.sampling_params.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.sampling_params.top_p=$TOP_P \ - actor_rollout_ref.rollout.top_p=$TOP_P \ - actor_rollout_ref.rollout.max_parallel_agents=$MAX_PARALLEL_AGENTS \ - actor_rollout_ref.rollout.max_iterations=$MAX_NUM_ITERS \ - actor_rollout_ref.rollout.enable_memory_saver=True \ - actor_rollout_ref.rollout.qwen3_enable_thinking=False \ - +actor_rollout_ref.rollout.max_starting_message_length=12000 \ - actor_rollout_ref.actor.masking=True \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - reward_model.reward_manager="swebench" \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name=$PROJECT_NAME \ - trainer.experiment_name=$EXPERIMENT_NAME \ - trainer.default_local_dir=$CKPT_PATH/$PROJECT_NAME/$EXPERIMENT_NAME \ - trainer.resume_mode=auto \ - trainer.max_actor_ckpt_to_keep=10 \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=$NNODES \ - trainer.save_freq=$SAVE_FREQ \ - data.dataloader_num_workers=0 \ - actor_rollout_ref.exchange_size=500000000 \ - trainer.test_freq=-1 \ - trainer.total_epochs=100 $@ \ No newline at end of file diff --git a/examples/sky/run_skyrl_agent_qwen14b_t.sh b/examples/sky/run_skyrl_agent_qwen14b_t.sh deleted file mode 100644 index 0cbfbace7c..0000000000 --- a/examples/sky/run_skyrl_agent_qwen14b_t.sh +++ /dev/null @@ -1,96 +0,0 @@ -PROJECT_NAME='oh-training-qwen3-14b-ppo' -EXPERIMENT_NAME='oh-training-qwen3-14b-32bsz-16r-temp0.6-clip-0.28-35turn' -DATA_PATH="" -SFT_MODEL_PATH='Qwen/Qwen3-14B' -CRITIC_MODEL_PATH=$SFT_MODEL_PATH -CKPT_PATH='' - -BATCH_SIZE=32 -MAX_NUM_ITERS=35 -NUM_TRAJ=8 -MAX_PARALLEL_AGENTS=192 -SAVE_FREQ=1 - -USE_KL_LOSS=True -KL_LOSS_COEF=0.001 -KL_LOSS_TYPE=low_var_kl -ENTROPY_COEFF=0.001 -CLIP_RATIO_LOW=0.2 -CLIP_RATIO_HIGH=0.28 - -# Assumes a h200 node -# For 2xH100: change tp size -> 2, sequence parallel size -> 2, nnodes -> 2 -GPU_MEM_UTIL=0.8 -TP_SIZE=2 -NNODES=1 -SP_SIZE=1 - - -TEMPERATURE=0.6 -TOP_P=0.95 - -PYTHONUNBUFFERED=1 uv run --isolated --directory . --frozen --env-file .env -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=["$DATA_PATH/train.parquet"] \ - data.val_files=["$DATA_PATH/validation.parquet"] \ - data.train_batch_size=$BATCH_SIZE \ - data.max_prompt_length=31744 \ - data.max_response_length=3072 \ - data.truncation='error' \ - actor_rollout_ref.model.path=$SFT_MODEL_PATH \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BATCH_SIZE \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ - actor_rollout_ref.actor.kl_loss_coef=$KL_LOSS_COEF \ - actor_rollout_ref.actor.kl_loss_type=$KL_LOSS_TYPE \ - actor_rollout_ref.actor.entropy_coeff=$ENTROPY_COEFF \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=True \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE \ - actor_rollout_ref.actor.clip_ratio_low=$CLIP_RATIO_LOW \ - actor_rollout_ref.actor.clip_ratio_high=$CLIP_RATIO_HIGH \ - critic.optim.lr=1e-5 \ - critic.ulysses_sequence_parallel_size=$SP_SIZE \ - critic.model.use_remove_padding=True \ - critic.model.path=$CRITIC_MODEL_PATH \ - critic.model.enable_gradient_checkpointing=True \ - critic.ppo_micro_batch_size_per_gpu=1 \ - critic.model.fsdp_config.param_offload=True \ - critic.model.fsdp_config.optimizer_offload=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE \ - actor_rollout_ref.rollout.name=async \ - actor_rollout_ref.rollout.gpu_memory_utilization=$GPU_MEM_UTIL \ - actor_rollout_ref.rollout.n=1 \ - actor_rollout_ref.rollout.n_trajectories=$NUM_TRAJ \ - actor_rollout_ref.rollout.sampling_params.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.sampling_params.top_p=$TOP_P \ - actor_rollout_ref.rollout.top_p=$TOP_P \ - actor_rollout_ref.rollout.max_parallel_agents=$MAX_PARALLEL_AGENTS \ - actor_rollout_ref.rollout.max_iterations=$MAX_NUM_ITERS \ - actor_rollout_ref.rollout.enable_memory_saver=True \ - +actor_rollout_ref.rollout.max_starting_message_length=12000 \ - actor_rollout_ref.rollout.remove_think_tokens=True \ - actor_rollout_ref.actor.masking=True \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - reward_model.reward_manager="swebench" \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name=$PROJECT_NAME \ - trainer.experiment_name=$EXPERIMENT_NAME \ - trainer.default_local_dir=$CKPT_PATH/$PROJECT_NAME/$EXPERIMENT_NAME \ - trainer.resume_mode=auto \ - trainer.max_actor_ckpt_to_keep=10 \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=$NNODES \ - trainer.save_freq=$SAVE_FREQ \ - data.dataloader_num_workers=0 \ - actor_rollout_ref.exchange_size=500000000 \ - trainer.test_freq=-1 \ - trainer.total_epochs=100 $@ \ No newline at end of file diff --git a/examples/sky/run_skyrl_agent_qwen8b_nt.sh b/examples/sky/run_skyrl_agent_qwen8b_nt.sh deleted file mode 100644 index 06b8eb782c..0000000000 --- a/examples/sky/run_skyrl_agent_qwen8b_nt.sh +++ /dev/null @@ -1,90 +0,0 @@ -PROJECT_NAME='oh-training-qwen3-no-thinking-8b-grpo' -EXPERIMENT_NAME='oh-training-qwen3-8b-32bsz-16r-temp0.6-25turn-easy' -DATA_PATH="" -SFT_MODEL_PATH='Qwen/Qwen3-8B' -CKPT_PATH='' - - -BATCH_SIZE=32 -MAX_NUM_ITERS=25 -NUM_TRAJ=8 -MAX_PARALLEL_AGENTS=64 -SAVE_FREQ=1 - -USE_KL_LOSS=True -KL_LOSS_COEF=0.001 -KL_LOSS_TYPE=low_var_kl -ENTROPY_COEFF=0 -CLIP_RATIO_LOW=0.2 -CLIP_RATIO_HIGH=0.2 - -GPU_MEM_UTIL=0.8 -TP_SIZE=1 -# Assumes a h200 node -# For 2xH100: change tp size -> 2, sequence parallel size -> 2, nnodes -> 2 -NNODES=1 -SP_SIZE=1 -TEMPERATURE=0.6 -TOP_P=0.95 - - - -PYTHONUNBUFFERED=1 uv run --isolated --directory . --frozen --env-file .env -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=["$DATA_PATH/train.parquet"] \ - data.val_files=["$DATA_PATH/validation.parquet"] \ - data.train_batch_size=$BATCH_SIZE \ - data.max_prompt_length=31232 \ - data.max_response_length=1536 \ - data.truncation='error' \ - actor_rollout_ref.model.path=$SFT_MODEL_PATH \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=$BATCH_SIZE \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE \ - actor_rollout_ref.actor.use_kl_loss=$USE_KL_LOSS \ - actor_rollout_ref.actor.kl_loss_coef=$KL_LOSS_COEF \ - actor_rollout_ref.actor.kl_loss_type=$KL_LOSS_TYPE \ - actor_rollout_ref.actor.entropy_coeff=$ENTROPY_COEFF \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=True \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ - actor_rollout_ref.actor.clip_ratio_low=$CLIP_RATIO_LOW \ - actor_rollout_ref.actor.clip_ratio_high=$CLIP_RATIO_HIGH \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE \ - actor_rollout_ref.rollout.name=async \ - actor_rollout_ref.rollout.gpu_memory_utilization=$GPU_MEM_UTIL \ - actor_rollout_ref.rollout.n=1 \ - actor_rollout_ref.rollout.n_trajectories=$NUM_TRAJ \ - actor_rollout_ref.rollout.sampling_params.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.temperature=$TEMPERATURE \ - actor_rollout_ref.rollout.sampling_params.top_p=$TOP_P \ - actor_rollout_ref.rollout.top_p=$TOP_P \ - actor_rollout_ref.rollout.sampling_params.top_k=20 \ - actor_rollout_ref.rollout.sampling_params.repetition_penalty=1.5 \ - actor_rollout_ref.rollout.max_parallel_agents=$MAX_PARALLEL_AGENTS \ - actor_rollout_ref.rollout.max_iterations=$MAX_NUM_ITERS \ - actor_rollout_ref.rollout.enable_memory_saver=True \ - actor_rollout_ref.rollout.qwen3_enable_thinking=False \ - +actor_rollout_ref.rollout.max_starting_message_length=12000 \ - actor_rollout_ref.actor.masking=True \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - reward_model.reward_manager="swebench" \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name=$PROJECT_NAME \ - trainer.experiment_name=$EXPERIMENT_NAME \ - trainer.default_local_dir=$CKPT_PATH/$PROJECT_NAME/$EXPERIMENT_NAME \ - trainer.resume_mode=auto \ - trainer.max_actor_ckpt_to_keep=10 \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=$NNODES \ - trainer.save_freq=$SAVE_FREQ \ - data.dataloader_num_workers=0 \ - actor_rollout_ref.exchange_size=500000000 \ - trainer.test_freq=-1 \ - trainer.total_epochs=100 $@ \ No newline at end of file diff --git a/examples/slurm/ray_on_slurm.slurm b/examples/slurm/ray_on_slurm.slurm deleted file mode 100644 index 5d4bafda33..0000000000 --- a/examples/slurm/ray_on_slurm.slurm +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=verl-ray-on-slurm -#SBATCH --nodes=2 -#SBATCH --ntasks-per-node=1 -#SBATCH --mem=200G -#SBATCH --partition=your-partition -#SBATCH --time=01:00:00 -#SBATCH --account=your-account -#SBATCH --gpus-per-node=4 -#SBATCH --cpus-per-task=64 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# load necessary modules - -# replace these information with your own -verl_workdir=/path/to/verl -train_files=/path/to/gsm8k/train.parquet -val_files=/path/to/gsm8k/test.parquet -apptainer_image_path=/path/to/verl-ngc.sif -# replace these information with your own - -# Getting the node names -nodes=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -nodes_array=("$nodes") - -head_node=${nodes_array[0]} -head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) - -# if we detect a space character in the head node IP, we'll -# convert it to an ipv4 address. This step is optional. -if [[ "$head_node_ip" == *" "* ]]; then -IFS=' ' read -ra ADDR <<<"$head_node_ip" -if [[ ${#ADDR[0]} -gt 16 ]]; then - head_node_ip=${ADDR[1]} -else - head_node_ip=${ADDR[0]} -fi -echo "IPV6 address detected. We split the IPV4 address as $head_node_ip" -fi - -port=6379 -ip_head=$head_node_ip:$port -export ip_head -echo "IP Head: $ip_head" - -# make sure we set environment variables before Ray initialization -export VLLM_ATTENTION_BACKEND=XFORMERS - -printenv - -echo "Starting HEAD at $head_node" -srun --nodes=1 --ntasks=1 -w "$head_node" \ - apptainer run --nv --bind $verl_workdir $apptainer_image_path \ - ray start --head --node-ip-address="$head_node_ip" --port=$port \ - --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & -# optional, though may be useful in certain versions of Ray < 1.0. -sleep 10 - -# number of nodes other than the head node -worker_num=$((SLURM_JOB_NUM_NODES - 1)) - -for ((i = 1; i <= worker_num; i++)); do - node_i=${nodes_array[$i]} - echo "Starting WORKER $i at $node_i" - srun --nodes=1 --ntasks=1 -w "$node_i" \ - apptainer run --nv --bind $verl_workdir $apptainer_image_path \ - ray start --address "$ip_head" --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block & - sleep 5 -done - -PYTHONUNBUFFERED=1 srun --overlap --nodes=1 --ntasks=1 -w "$head_node" \ - apptainer run --nv --bind $verl_workdir $apptainer_image_path \ - python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$train_files \ - data.val_files=$val_files \ - data.train_batch_size=256 \ - data.max_prompt_length=512 \ - data.max_response_length=256 \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=64 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \ - critic.ppo_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.logger=['console'] \ - trainer.val_before_train=False \ - trainer.default_hdfs_dir=null \ - trainer.n_gpus_per_node="${SLURM_GPUS_PER_NODE}" \ - trainer.nnodes="${SLURM_NNODES}" \ - trainer.save_freq=10 \ - trainer.test_freq=10 \ - trainer.total_epochs=15 2>&1 | tee verl_demo_slurm.log diff --git a/examples/split_placement/README.md b/examples/split_placement/README.md deleted file mode 100644 index a552972594..0000000000 --- a/examples/split_placement/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Split Placement Example -Here we introduce how to run the naive implementation of the split placement of PPO algorithm. -We will release the complete version of flexible placement in the near future. - - For quickstart, you can only follow Step 2 to modify the code and then follow Step 4 to execute the split placement example. - -### Step 1: Placing the models to different GPUs -Specify the placement and resource allocation. In the example, we place the actor and reference in the first half of the GPUs while map the critic and reward model (if any) to the second half of the GPUs. -```python -actor_rollout_ref_pool_id = 'actor_rollout_ref_pool' -critic_pool_id = 'critic_pool' -if config.trainer.nnodes // 2 == 0 and config.trainer.n_gpus_per_node // 2 > 0: - resource_pool_spec = { - actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, - critic_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, - } -else: - resource_pool_spec = { - actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), - critic_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), - } -print(f'resource_pool_spec: {resource_pool_spec}') -mapping = { - Role.ActorRollout: actor_rollout_ref_pool_id, - Role.Critic: critic_pool_id, - Role.RefPolicy: actor_rollout_ref_pool_id, -} -mapping[Role.RewardModel] = critic_pool_id -``` - -### Step 2: Make the models executed asynchronously -Based on the model placement, we need to make the models executed asynchronously. - -To do so, you need to turn off the `blocking` flag (i.e., `blocking=False`) in our decorator of some model operations. -For example, we hope the actor update and critic update can be executed in parallel, then we need to make the following modification in `fsdp_workers.py` - -``` -@register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO, blocking=False) -def update_actor(self, data: DataProto): - ... - -@register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO, blocking=False) -def update_critic(self, data: DataProto): - ... -``` - -We can also parallelize the computation of `ref_log_prob` and `values` and `rewards` in the split placement. For simplicity of the tutorial, we don't do this in this example. - -### Step 3: Execute these operation in parallel in the single controller process -To implement the parallel execution of the actor and critic update, the only thing we need to modify in the `ray_trainer.py` is to `get` the concurrent `futures` on the single controller process. - -```python -critic_output = critic_output.get() -actor_output = actor_output.get() -``` - -### Step 4: Run the split placement example - -``` -bash run_deepseek7b_llm.sh -``` diff --git a/examples/split_placement/config/ppo_trainer_split.yaml b/examples/split_placement/config/ppo_trainer_split.yaml deleted file mode 100644 index 19dd1f1df6..0000000000 --- a/examples/split_placement/config/ppo_trainer_split.yaml +++ /dev/null @@ -1,173 +0,0 @@ -data: - tokenizer: null - train_files: ~/data/rlhf/gsm8k/train.parquet - val_files: ~/data/rlhf/gsm8k/test.parquet - prompt_key: prompt - max_prompt_length: 512 - max_response_length: 512 - train_batch_size: 1024 - val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves - return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs - return_raw_chat: False - shuffle: True - -actor_rollout_ref: - hybrid_engine: True - model: - path: ~/models/deepseek-llm-7b-chat - external_lib: null - override_config: { } - enable_gradient_checkpointing: True - use_remove_padding: False - actor: - strategy: fsdp # This is for backward-compatibility - ppo_mini_batch_size: 256 - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - use_dynamic_bsz: False - ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length} - grad_clip: 1.0 - clip_ratio: 0.2 - entropy_coeff: 0.001 - use_kl_loss: False # True for GRPO - kl_loss_coef: 0.001 # for grpo - kl_loss_type: low_var_kl # for grpo - ppo_epochs: 1 - shuffle: False - ulysses_sequence_parallel_size: 1 # sp size - optim: - lr: 1e-6 - lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - fsdp_config: - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - param_offload: False - optimizer_offload: False - fsdp_size: -1 - ref: - fsdp_config: - param_offload: False - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} # sp size - rollout: - name: vllm - temperature: 1.0 - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1 - prompt_length: ${data.max_prompt_length} # not use for opensource - response_length: ${data.max_response_length} - # for vllm rollout - dtype: bfloat16 # should align with FSDP - gpu_memory_utilization: 0.5 - ignore_eos: False - enforce_eager: True - free_cache_engine: True - load_format: dummy_dtensor - tensor_model_parallel_size: 2 - max_num_batched_tokens: 8192 - max_num_seqs: 1024 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - disable_log_stats: True - enable_chunked_prefill: True # could get higher throughput - # for hf rollout - do_sample: True - # number of responses (i.e. num sample times) - n: 1 # > 1 for grpo - -critic: - strategy: fsdp - optim: - lr: 1e-5 - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - model: - path: ~/models/deepseek-llm-7b-chat - tokenizer_path: ${actor_rollout_ref.model.path} - override_config: { } - external_lib: ${actor_rollout_ref.model.external_lib} - enable_gradient_checkpointing: True - use_remove_padding: False - fsdp_config: - param_offload: False - optimizer_offload: False - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - fsdp_size: -1 - ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - forward_micro_batch_size: ${critic.ppo_micro_batch_size} - forward_micro_batch_size_per_gpu: ${critic.ppo_micro_batch_size_per_gpu} - use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - ppo_max_token_len_per_gpu: 32768 # (${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}) * 2 - forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu} - ulysses_sequence_parallel_size: 1 # sp size - ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs} - shuffle: ${actor_rollout_ref.actor.shuffle} - grad_clip: 1.0 - cliprange_value: 0.5 - -reward_model: - enable: False - strategy: fsdp - model: - input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical - path: ~/models/FsfairX-LLaMA3-RM-v0.1 - external_lib: ${actor_rollout_ref.model.external_lib} - use_remove_padding: False - fsdp_config: - min_num_params: 0 - param_offload: False - fsdp_size: -1 - micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu - micro_batch_size_per_gpu: null # set a number - max_length: null - ulysses_sequence_parallel_size: 1 # sp size - use_dynamic_bsz: ${critic.use_dynamic_bsz} - forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} - reward_manager: naive - -algorithm: - gamma: 1.0 - lam: 1.0 - adv_estimator: gae - use_kl_in_reward: False - kl_penalty: kl # how to estimate kl divergence - kl_ctrl: - type: fixed - kl_coef: 0.001 - -trainer: - total_epochs: 30 - total_training_steps: null - project_name: verl_examples - experiment_name: gsm8k - logger: [ 'console', 'wandb' ] - log_val_generations: 0 - nnodes: 1 - n_gpus_per_node: 8 - save_freq: -1 - # auto: find the last ckpt to resume. If can't find, start from scratch - resume_mode: auto # or disable or resume_path if resume_from_path is set - resume_from_path: null - test_freq: -1 - critic_warmup: 0 - default_hdfs_dir: null - default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} \ No newline at end of file diff --git a/examples/split_placement/main_ppo_split.py b/examples/split_placement/main_ppo_split.py deleted file mode 100644 index 85347be092..0000000000 --- a/examples/split_placement/main_ppo_split.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. -""" - -from verl import DataProto -import torch -from verl.utils.reward_score import gsm8k, math -from verl.trainer.ppo.ray_trainer import RayPPOTrainer - - -def _select_rm_score_fn(data_source): - if data_source == 'openai/gsm8k': - return gsm8k.compute_score - elif data_source == 'lighteval/MATH': - return math.compute_score - else: - raise NotImplementedError - - -class RewardManager(): - - def __init__(self, tokenizer, num_examine) -> None: - self.tokenizer = tokenizer - self.num_examine = num_examine # the number of batches of decoded responses to print to the console - - def __call__(self, data: DataProto, return_dict: bool = False): - """We will expand this function gradually based on the available datasets""" - - # If there is rm score, we directly return rm score. Otherwise, we compute via rm_score_fn - if 'rm_scores' in data.batch.keys(): - return data.batch['rm_scores'] - - reward_tensor = torch.zeros_like(data.batch['responses'], dtype=torch.float32) - - already_print_data_sources = {} - - for i in range(len(data)): - data_item = data[i] # DataProtoItem - - prompt_ids = data_item.batch['prompts'] - - prompt_length = prompt_ids.shape[-1] - - valid_prompt_length = data_item.batch['attention_mask'][:prompt_length].sum() - valid_prompt_ids = prompt_ids[-valid_prompt_length:] - - response_ids = data_item.batch['responses'] - valid_response_length = data_item.batch['attention_mask'][prompt_length:].sum() - valid_response_ids = response_ids[:valid_response_length] - - # decode - sequences = torch.cat((valid_prompt_ids, valid_response_ids)) - sequences_str = self.tokenizer.decode(sequences) - - ground_truth = data_item.non_tensor_batch['reward_model']['ground_truth'] - - # select rm_score - data_source = data_item.non_tensor_batch['data_source'] - compute_score_fn = _select_rm_score_fn(data_source) - - score = compute_score_fn(solution_str=sequences_str, ground_truth=ground_truth) - reward_tensor[i, valid_response_length - 1] = score - - if data_source not in already_print_data_sources: - already_print_data_sources[data_source] = 0 - - if already_print_data_sources[data_source] < self.num_examine: - already_print_data_sources[data_source] += 1 - print(sequences_str) - - if return_dict: - return {"reward_tensor": reward_tensor} - else: - return reward_tensor - - -import ray -import hydra -from split_monkey_patch import fit - - -@hydra.main(config_path='config', config_name='ppo_trainer_split', version_base=None) -def main(config): - if not ray.is_initialized(): - # this is for local ray cluster - ray.init(runtime_env={'env_vars': {'TOKENIZERS_PARALLELISM': 'true', 'NCCL_DEBUG': 'WARN'}}) - - ray.get(main_task.remote(config)) - - -@ray.remote -def main_task(config): - from verl.utils.fs import copy_to_local - from transformers import AutoTokenizer - - # print initial config - from pprint import pprint - from omegaconf import OmegaConf - pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values - OmegaConf.resolve(config) - - # download the checkpoint from hdfs - local_path = copy_to_local(config.actor_rollout_ref.model.path) - - # instantiate tokenizer - from verl.utils import hf_tokenizer - tokenizer = hf_tokenizer(local_path) - - # define worker classes - if config.actor_rollout_ref.actor.strategy == 'fsdp': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray import RayWorkerGroup - ray_worker_group_cls = RayWorkerGroup - - elif config.actor_rollout_ref.actor.strategy == 'megatron': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup - ray_worker_group_cls = NVMegatronRayWorkerGroup - - else: - raise NotImplementedError - - from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role - - role_worker_mapping = { - Role.ActorRollout: ray.remote(ActorRolloutRefWorker), - Role.Critic: ray.remote(CriticWorker), - } - - # NOTE: initialze two resource pool - actor_rollout_ref_pool_id = 'actor_rollout_ref_pool' - critic_pool_id = 'critic_pool' - if config.trainer.nnodes // 2 == 0 and config.trainer.n_gpus_per_node // 2 > 0: - resource_pool_spec = { - actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, - critic_pool_id: [config.trainer.n_gpus_per_node // 2] * config.trainer.nnodes, - } - else: - resource_pool_spec = { - actor_rollout_ref_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), - critic_pool_id: [config.trainer.n_gpus_per_node] * (config.trainer.nnodes // 2), - } - print(f'resource_pool_spec: {resource_pool_spec}') - mapping = { - Role.ActorRollout: actor_rollout_ref_pool_id, - Role.Critic: critic_pool_id, - } - - #use reference model - if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: - role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) - mapping[Role.RefPolicy] = actor_rollout_ref_pool_id - - # we should adopt a multi-source reward function here - # - for rule-based rm, we directly call a reward score - # - for model-based rm, we call a model - # - for code related prompt, we send to a sandbox if there are test cases - # - finally, we combine all the rewards together - # - The reward type depends on the tag of the data - if config.reward_model.enable: - if config.reward_model.strategy == 'fsdp': - from verl.workers.fsdp_workers import RewardModelWorker - elif config.reward_model.strategy == 'megatron': - from verl.workers.megatron_workers import RewardModelWorker - else: - raise NotImplementedError - role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) - mapping[Role.RewardModel] = critic_pool_id - - reward_fn = RewardManager(tokenizer=tokenizer, num_examine=0) - - # Note that we always use function-based RM for validation - val_reward_fn = RewardManager(tokenizer=tokenizer, num_examine=1) - - resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) - - RayPPOTrainer.fit = fit - trainer = RayPPOTrainer(config=config, - tokenizer=tokenizer, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - ray_worker_group_cls=ray_worker_group_cls, - reward_fn=reward_fn, - val_reward_fn=val_reward_fn) - trainer.init_workers() - trainer.fit() - - -if __name__ == '__main__': - main() diff --git a/examples/split_placement/run_deepseek7b_llm.sh b/examples/split_placement/run_deepseek7b_llm.sh deleted file mode 100644 index d4484138a2..0000000000 --- a/examples/split_placement/run_deepseek7b_llm.sh +++ /dev/null @@ -1,37 +0,0 @@ -set -x - -python3 main_ppo_split.py \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=deepseek-ai/deepseek-llm-7b-chat \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=4 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.path=deepseek-ai/deepseek-llm-7b-chat \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=8 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console','wandb'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_epochs=15 $@ diff --git a/examples/split_placement/split_monkey_patch.py b/examples/split_placement/split_monkey_patch.py deleted file mode 100644 index c6826771fc..0000000000 --- a/examples/split_placement/split_monkey_patch.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -An naive implementation of split placment example -""" -from pprint import pprint -from verl import DataProto -from verl.trainer.ppo.ray_trainer import compute_advantage, apply_kl_penalty, reduce_metrics, compute_data_metrics, _timer, compute_timing_metrics, AdvantageEstimator -from copy import deepcopy -import numpy as np -import torch -import uuid - - -def fit(self): - """ - The training loop of PPO. - The driver process only need to call the compute functions of the worker group through RPC to construct the PPO dataflow. - The light-weight advantage computation is done on the driver process. - """ - from verl.utils.tracking import Tracking - from omegaconf import OmegaConf - - logger = Tracking(project_name=self.config.trainer.project_name, - experiment_name=self.config.trainer.experiment_name, - default_backend=self.config.trainer.logger, - config=OmegaConf.to_container(self.config, resolve=True)) - - self.global_steps = 0 - - # load checkpoint before doing anything - self._load_checkpoint() - - # perform validation before training - # currently, we only support validation using the reward_function. - if self.val_reward_fn is not None and self.config.trainer.get('val_before_train', True): - val_metrics = self._validate() - pprint(f'Initial validation metrics: {val_metrics}') - logger.log(data=val_metrics, step=self.global_steps) - if self.config.trainer.get('val_only', False): - return - - # we start from step 1 - self.global_steps += 1 - last_val_metrics = None - - for epoch in range(self.config.trainer.total_epochs): - for batch_dict in self.train_dataloader: - metrics = {} - timing_raw = {} - - batch: DataProto = DataProto.from_single_dict(batch_dict) - - # pop those keys for generation - gen_batch = batch.pop(batch_keys=['input_ids', 'attention_mask', 'position_ids']) - is_last_step = self.global_steps >= self.total_training_steps - - with _timer('step', timing_raw): - # generate a batch - with _timer('gen', timing_raw): - gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) - - if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: - with _timer('gen_max', timing_raw): - gen_baseline_batch = deepcopy(gen_batch) - gen_baseline_batch.meta_info['do_sample'] = False - gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) - - batch = batch.union(gen_baseline_output) - reward_baseline_tensor = self.reward_fn(batch) - reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) - - batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) - - batch.batch['reward_baselines'] = reward_baseline_tensor - - del gen_baseline_batch, gen_baseline_output - - batch.non_tensor_batch['uid'] = np.array([str(uuid.uuid4()) for _ in range(len(batch.batch))], - dtype=object) - # repeat to align with repeated responses in rollout - batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) - batch = batch.union(gen_batch_output) - - # balance the number of valid tokens on each dp rank. - # Note that this breaks the order of data inside the batch. - # Please take care when you implement group based adv computation such as GRPO and rloo - self._balance_batch(batch, metrics=metrics) - - # compute global_valid tokens - batch.meta_info['global_token_num'] = torch.sum(batch.batch['attention_mask'], dim=-1).tolist() - - # recompute old_log_probs - with _timer('old_log_prob', timing_raw): - old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) - batch = batch.union(old_log_prob) - - if self.use_reference_policy: - # compute reference log_prob - with _timer('ref', timing_raw): - ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) - batch = batch.union(ref_log_prob) - - # compute values - if self.use_critic: - with _timer('values', timing_raw): - values = self.critic_wg.compute_values(batch) - batch = batch.union(values) - - with _timer('adv', timing_raw): - # compute scores. Support both model and function-based. - # We first compute the scores using reward model. Then, we call reward_fn to combine - # the results from reward model and rule-based results. - if self.use_rm: - # we first compute reward model score - reward_tensor = self.rm_wg.compute_rm_score(batch) - batch = batch.union(reward_tensor) - - # we combine with rule-based rm - reward_tensor = self.reward_fn(batch) - batch.batch['token_level_scores'] = reward_tensor - - # compute rewards. apply_kl_penalty if available - if self.config.algorithm.use_kl_in_reward: - batch, kl_metrics = apply_kl_penalty(batch, - kl_ctrl=self.kl_ctrl_in_reward, - kl_penalty=self.config.algorithm.kl_penalty) - metrics.update(kl_metrics) - else: - batch.batch['token_level_rewards'] = batch.batch['token_level_scores'] - - # compute advantages, executed on the driver process - batch = compute_advantage(batch, - adv_estimator=self.config.algorithm.adv_estimator, - gamma=self.config.algorithm.gamma, - lam=self.config.algorithm.lam, - num_repeat=self.config.actor_rollout_ref.rollout.n) - - # update critic - if self.use_critic: - with _timer('update_critic_call', timing_raw): - critic_output = self.critic_wg.update_critic(batch) - - # implement critic warmup - if self.config.trainer.critic_warmup <= self.global_steps: - # update actor - with _timer('update_actor_call', timing_raw): - actor_output = self.actor_rollout_wg.update_actor(batch) - - # NOTE: make sure you set blocking=False in update_actor and update_crtic in the worker class - with _timer('update_actor_critic', timing_raw): - critic_output = critic_output.get() - critic_output_metrics = reduce_metrics(critic_output.meta_info['metrics']) - metrics.update(critic_output_metrics) - - actor_output = actor_output.get() - actor_output_metrics = reduce_metrics(actor_output.meta_info['metrics']) - metrics.update(actor_output_metrics) - - # validate - if self.val_reward_fn is not None and self.config.trainer.test_freq > 0 and \ - (is_last_step or self.global_steps % self.config.trainer.test_freq == 0): - with _timer('testing', timing_raw): - val_metrics: dict = self._validate() - if is_last_step: - last_val_metrics = val_metrics - metrics.update(val_metrics) - - if self.config.trainer.save_freq > 0 and (is_last_step or \ - self.global_steps % self.config.trainer.save_freq == 0): - with _timer('save_checkpoint', timing_raw): - self._save_checkpoint() - - # collect metrics - metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) - metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) - - # TODO: make a canonical logger that supports various backend - logger.log(data=metrics, step=self.global_steps) - - if self.global_steps >= self.total_training_steps: - pprint(f'Final validation metrics: {last_val_metrics}') - return - - self.global_steps += 1 diff --git a/format.sh b/format.sh new file mode 100644 index 0000000000..7a0655276d --- /dev/null +++ b/format.sh @@ -0,0 +1,10 @@ +set -e + +if command -v uv >/dev/null 2>&1; then + uv pip install -q pre-commit +else + pip install -q pre-commit +fi + +# pre-commit run --all-files always runs from the root directory. +pre-commit run --all-files --config .pre-commit-config.yaml \ No newline at end of file diff --git a/learnings/training-mechanics.md b/learnings/training-mechanics.md new file mode 100644 index 0000000000..99c231eed5 --- /dev/null +++ b/learnings/training-mechanics.md @@ -0,0 +1,216 @@ +# SkyRL Training Mechanics + +## Key Configuration Flags + +### `step_wise_trajectories` (default: false) + +**Location**: `generator.step_wise_trajectories` + +**What it does**: Controls how multi-turn trajectories become training samples. + +| Setting | Sample Structure | Batch Weighting | +|---------|------------------|-----------------| +| `false` | 1 trajectory = 1 sample (all turns concatenated) | Each trajectory contributes equally | +| `true` | 1 trajectory = N samples (one per turn) | Longer trajectories contribute more samples | + +**With `step_wise=true`**: +- Each turn becomes a separate training sample +- Advantage is computed on FINAL step only, then propagated to all earlier steps +- All steps in same trajectory get the same advantage +- Loss mask per turn is trivially correct (just that turn's response) + +**Important**: There is NO double normalization. With `step_wise=true`, a 10-turn trajectory contributes 10 samples to the batch, giving it 10x the gradient contribution of a 1-turn trajectory. + +**Code locations**: +- Generator: `skyrl_train/generators/skyrl_gym_generator.py:258` +- Advantage computation: `skyrl_train/trainer.py:787-818` +- Sample assembly: `skyrl_train/generators/skyrl_gym_generator.py:700-720` + +--- + +### `retokenize_chat_history` + +**Not a config flag** - computed as: +```python +retokenize_chat_history = self.use_conversation_multi_turn and self.custom_chat_template +``` + +**What it does**: When true, re-tokenizes the entire conversation every turn instead of appending tokens incrementally. + +**Why it exists**: To get accurate loss masks via `return_assistant_tokens_mask=True` from the tokenizer. + +**Status**: Being deprecated in favor of `step_wise_trajectories`. See comment at `skyrl_gym_generator.py:218-220`: +```python +# NOTE: `custom_chat_template` was mainly for getting accurate loss masks for thinking models. +# This is no longer needed now given that step wise training is supported +``` + +--- + +### `loss_reduction` modes + +**Location**: `trainer.algorithm.loss_reduction` + +| Mode | How it works | +|------|--------------| +| `token_mean` | Mean over all valid tokens in batch | +| `sequence_mean` | Per-sample mean, then batch mean (each sample weighted equally) | +| `seq_mean_token_sum_norm` | Sum/max_len per sample, then batch mean (Dr. GRPO) | + +**Code**: `skyrl_train/utils/ppo_utils.py:881-906` + +--- + +## Chat Templates + +### What is a chat template? + +A Jinja2 template that converts a list of messages to a tokenizable string: + +```python +# Input +messages = [ + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "math4"}, +] + +# Output (after template) +"<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\nmath4<|im_end|>\n" +``` + +### `qwen3_acc_thinking.jinja2` + +**Location**: `skyrl_train/utils/templates/qwen3_acc_thinking.jinja2` + +**Purpose**: Outputs assistant content verbatim WITHOUT reformatting `` tags. + +**Problem it solves**: The official Qwen3 template parses `` tags and adds newlines: +``` +# Model generates: +reasoninganswer + +# Official template reformats to: + +reasoning + + +answer +``` + +This reformatting produces different tokens, causing off-policy training. + +**Primary use case**: Pass to vLLM inference engine to prevent reformatting: +```bash ++generator.engine_init_kwargs.chat_template=/path/to/qwen3_acc_thinking.jinja2 +``` + +--- + +## Where Templates Are Applied + +1. **L239** - Initial prompt tokenization +2. **L281** - Each turn (if `step_wise` or `retokenize_chat_history`) +3. **L519** - Adding observations to sequence +4. **L401** - Final loss mask computation (if `retokenize_chat_history`) + +**Code**: `skyrl_train/generators/skyrl_gym_generator.py` + +--- + +## Training Path Options for Thinking Models + +### Option A: `step_wise_trajectories=true` +- Each turn is separate sample +- Loss masks trivially correct +- Training dynamics change: longer trajectories = more samples + +### Option B: `step_wise=false` + `qwen3_acc_thinking` template +- Whole trajectory is one sample +- Use template in vLLM so context isn't reformatted +- Incremental token appending stays on-policy +- Training dynamics unchanged + +--- + +## Agent Loop Flow + +``` +skyrl_gym_generator.py:agent_loop() + +1. init(): Get initial prompt from env +2. apply_chat_template() → initial_input_ids +3. while not done: + a. Generate output tokens + b. env.step(output) → observation, reward, done + c. Update state (append tokens or retokenize) +4. Build final response_ids and loss_mask +``` + +**Main loop**: `skyrl_gym_generator.py:272` - `while not agent_loop_state.done:` + +--- + +## Thinking Tokens: When They Get Stripped + +### The Problem + +The official Qwen3 chat template **strips `` content from non-last assistant messages**. This is by design for inference (users don't want to see old reasoning), but breaks RL training. + +**Example - Multi-turn with official template:** +``` +Turn 1: Model generates "step 1call_tool()" +Turn 2: Model sees context WITHOUT the thinking: "call_tool()" ← thinking stripped! +Turn 3: Loss computed on tokens that don't match what model saw +``` + +This causes **off-policy training** - the model is trained on token sequences it never actually generated in that context. + +### Why It Matters for RL + +1. **On-policy requirement**: PPO/GRPO assume the policy that generated the data is the same as the one being trained +2. **Context mismatch**: If thinking is stripped from context, the model's next-turn generation was conditioned on different tokens than what we compute loss on +3. **Credit assignment**: The model can't learn from its reasoning if that reasoning disappears from context + +### Solutions + +| Approach | How | Trade-off | +|----------|-----|-----------| +| **`step_wise_trajectories=true`** | Each turn is separate sample, no cross-turn context needed | Longer trajectories get more weight in batch | +| **`qwen3_acc_thinking.jinja2`** | Custom template that preserves thinking verbatim | Need to pass to vLLM | +| **Don't use custom template** (Fleet default) | Append raw tokens without re-applying template | Works if not using `retokenize_chat_history` | + +### Fleet Tool-Use Config (Current) + +```yaml +step_wise_trajectories=false # Whole trajectory = 1 sample +custom_chat_template=null # No template reformatting +use_conversation_multi_turn=true # Multi-turn enabled +``` + +**Why this works**: Without a custom template, SkyRL appends tokens directly without re-applying any template. The model sees its own thinking in context, and loss is computed on the same tokens. Fully on-policy. + +### When Thinking DOES Get Stripped + +1. Using official Qwen3 template with `retokenize_chat_history=true` +2. Passing messages through any template that parses `` tags +3. Using `step_wise=true` with default template (each turn re-tokenized, thinking reformatted) + +### Key Insight + +The confusion arises because there are TWO places templates matter: +1. **vLLM inference** - How the engine formats the prompt for generation +2. **Training tokenization** - How we build `response_ids` for loss computation + +For on-policy training, both must produce the same tokens. + +--- + +## Key Files + +| File | Purpose | +|------|---------| +| `skyrl_train/generators/skyrl_gym_generator.py` | Agent loop, token management | +| `skyrl_train/trainer.py` | Training loop, advantage computation | +| `skyrl_train/utils/ppo_utils.py` | Loss functions, advantage estimators | +| `skyrl_train/utils/templates/` | Chat templates for thinking models | +| `skyrl_train/config/ppo_base_config.yaml` | Default configuration | diff --git a/patches/megatron_v4.patch b/patches/megatron_v4.patch deleted file mode 100644 index 17bceecc15..0000000000 --- a/patches/megatron_v4.patch +++ /dev/null @@ -1,568 +0,0 @@ -diff --git a/.gitignore b/.gitignore -index 5955b349..ade0cd51 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -7,3 +7,5 @@ build - slurm* - logs - .vscode -+tests/* -+examples/* -diff --git a/build.sh b/build.sh -new file mode 100644 -index 00000000..49d5361f ---- /dev/null -+++ b/build.sh -@@ -0,0 +1,4 @@ -+#! /bin/bash -+ -+export PYTHONPATH=$PYTHONPATH:$(pwd) -+pip3 install regex ninja -diff --git a/megatron/__init__.py b/megatron/__init__.py -index c35de282..60896b47 100644 ---- a/megatron/__init__.py -+++ b/megatron/__init__.py -@@ -2,7 +2,7 @@ - - import torch - --from .global_vars import get_args, get_retro_args -+from .global_vars import get_args, update_args, fork_args_namespace, get_retro_args - from .global_vars import get_current_global_batch_size - from .global_vars import get_num_microbatches - from .global_vars import get_signal_handler -diff --git a/megatron/arguments.py b/megatron/arguments.py -index 0ca8776e..9ef67624 100644 ---- a/megatron/arguments.py -+++ b/megatron/arguments.py -@@ -59,6 +59,16 @@ def parse_args(extra_args_provider=None, ignore_unknown_args=False): - return args - - def validate_args(args, defaults={}): -+ # Set input defaults. -+ for key in defaults: -+ if getattr(args, key, None) is not None: -+ if args.rank == 0 and defaults[key] != getattr(args, key): -+ print('WARNING: overriding default argument {key}:{v2} \ -+ with {key}:{v}'.format(key=key, v=defaults[key], -+ v2=getattr(args, key)), -+ flush=True) -+ -+ setattr(args, key, defaults[key]) - # Tensor model parallel size. - args.tensor_model_parallel_size = min( - args.tensor_model_parallel_size, args.world_size) -@@ -125,19 +135,19 @@ def validate_args(args, defaults={}): - args.recompute_granularity = 'selective' - del args.recompute_activations - -- # Set input defaults. -- for key in defaults: -- # For default to be valid, it should not be provided in the -- # arguments that are passed to the program. We check this by -- # ensuring the arg is set to None. -- if getattr(args, key, None) is not None: -- if args.rank == 0: -- print('WARNING: overriding default arguments for {key}:{v} \ -- with {key}:{v2}'.format(key=key, v=defaults[key], -- v2=getattr(args, key)), -- flush=True) -- else: -- setattr(args, key, defaults[key]) -+ # # Set input defaults. -+ # for key in defaults: -+ # # For default to be valid, it should not be provided in the -+ # # arguments that are passed to the program. We check this by -+ # # ensuring the arg is set to None. -+ # if getattr(args, key, None) is not None: -+ # if args.rank == 0: -+ # print('WARNING: overriding default arguments for {key}:{v} \ -+ # with {key}:{v2}'.format(key=key, v=defaults[key], -+ # v2=getattr(args, key)), -+ # flush=True) -+ # else: -+ # setattr(args, key, defaults[key]) - - # Batch size. - assert args.micro_batch_size is not None -diff --git a/megatron/core/pipeline_parallel/p2p_communication.py b/megatron/core/pipeline_parallel/p2p_communication.py -index 29ee34df..fa590b16 100644 ---- a/megatron/core/pipeline_parallel/p2p_communication.py -+++ b/megatron/core/pipeline_parallel/p2p_communication.py -@@ -130,32 +130,28 @@ def _batched_p2p_ops( - send_prev_op = torch.distributed.P2POp( - torch.distributed.isend, - tensor_send_prev, -- get_pipeline_model_parallel_prev_rank(), -- group, -+ get_pipeline_model_parallel_prev_rank() - ) - ops.append(send_prev_op) - if tensor_recv_prev is not None: - recv_prev_op = torch.distributed.P2POp( - torch.distributed.irecv, - tensor_recv_prev, -- get_pipeline_model_parallel_prev_rank(), -- group, -+ get_pipeline_model_parallel_prev_rank() - ) - ops.append(recv_prev_op) - if tensor_send_next is not None: - send_next_op = torch.distributed.P2POp( - torch.distributed.isend, - tensor_send_next, -- get_pipeline_model_parallel_next_rank(), -- group, -+ get_pipeline_model_parallel_next_rank() - ) - ops.append(send_next_op) - if tensor_recv_next is not None: - recv_next_op = torch.distributed.P2POp( - torch.distributed.irecv, - tensor_recv_next, -- get_pipeline_model_parallel_next_rank(), -- group, -+ get_pipeline_model_parallel_next_rank() - ) - ops.append(recv_next_op) - if len(ops) > 0: -diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py -index 992da781..2eb78d52 100644 ---- a/megatron/core/pipeline_parallel/schedules.py -+++ b/megatron/core/pipeline_parallel/schedules.py -@@ -78,6 +78,8 @@ def get_forward_backward_func(): - transformer, this is the encoder's sequence length. This is ignored if variable_seq_lengths - in the config is True. Otherwise, each microbatch in the current global batch size must use - this sequence length. -+ -+ hidden_size (int, required): hidden size of the model - - micro_batch_size (int, required): The number of sequences in a microbatch. - -@@ -287,6 +289,7 @@ def forward_backward_no_pipelining( - model: Union[torch.nn.Module, List[torch.nn.Module]], - num_microbatches: int, - seq_length: int, # unused -+ hidden_size: int, # unused - micro_batch_size: int, # unused - decoder_seq_length: int = None, # unused - forward_only: bool = False, -@@ -370,8 +373,10 @@ def forward_backward_pipelining_with_interleaving( - data_iterator: Union[Iterator, List[Iterator]], - model: Union[torch.nn.Module, List[torch.nn.Module]], - num_microbatches: int, -- seq_length: int, -- micro_batch_size: int, -+ seq_length: int = None, -+ hidden_size: int = None, -+ micro_batch_size: int = None, -+ input_shapes: list = None, - decoder_seq_length: int = None, - forward_only: bool = False, - collect_non_loss_data: bool = False, -@@ -457,7 +462,7 @@ def forward_backward_pipelining_with_interleaving( - "Interleaving is not supported with a different decoder sequence length." - ) - -- tensor_shape = [seq_length, micro_batch_size, config.hidden_size] -+ tensor_shape = [seq_length, micro_batch_size, hidden_size] - if config.sequence_parallel: - tensor_shape[0] = tensor_shape[0] // parallel_state.get_tensor_model_parallel_world_size() - -@@ -944,6 +949,7 @@ def get_tensor_shapes( - rank: int, - model_type: ModelType, - seq_length: int, -+ hidden_size: int, - micro_batch_size: int, - decoder_seq_length: int, - config, -@@ -967,12 +973,12 @@ def get_tensor_shapes( - - if model_type == ModelType.encoder_and_decoder: - if parallel_state.is_pipeline_stage_before_split(rank): -- tensor_shapes.append((seq_length, micro_batch_size, config.hidden_size)) -+ tensor_shapes.append((seq_length, micro_batch_size, hidden_size)) - else: -- tensor_shapes.append((decoder_seq_length, micro_batch_size, config.hidden_size)) -- tensor_shapes.append((seq_length, micro_batch_size, config.hidden_size)) -+ tensor_shapes.append((decoder_seq_length, micro_batch_size, hidden_size)) -+ tensor_shapes.append((seq_length, micro_batch_size, hidden_size)) - else: -- tensor_shapes.append((seq_length, micro_batch_size, config.hidden_size)) -+ tensor_shapes.append((seq_length, micro_batch_size, hidden_size)) - return tensor_shapes - - -@@ -1050,8 +1056,10 @@ def forward_backward_pipelining_without_interleaving( - data_iterator: Union[Iterator, List[Iterator]], - model: Union[torch.nn.Module, List[torch.nn.Module]], - num_microbatches: int, -- seq_length: int, -- micro_batch_size: int, -+ seq_length: int = None, -+ hidden_size: int = None, -+ micro_batch_size: int = None, -+ input_shapes: list = None, - decoder_seq_length: int = None, - forward_only: bool = False, - collect_non_loss_data: bool = False, -@@ -1127,22 +1135,34 @@ def forward_backward_pipelining_without_interleaving( - model_type = get_model_type(model) - - rank = parallel_state.get_pipeline_model_parallel_rank() -- recv_tensor_shapes = get_tensor_shapes( -- rank=rank - 1, -- model_type=model_type, -- seq_length=seq_length, -- micro_batch_size=micro_batch_size, -- decoder_seq_length=decoder_seq_length, -- config=config, -- ) -- send_tensor_shapes = get_tensor_shapes( -- rank=rank, -- model_type=model_type, -- seq_length=seq_length, -- micro_batch_size=micro_batch_size, -- decoder_seq_length=decoder_seq_length, -- config=config, -- ) -+ -+ def get_recv_tensor_shapes(microbatch_id): -+ if input_shapes: -+ return [input_shapes[microbatch_id]] -+ recv_tensor_shapes = get_tensor_shapes( -+ rank=rank - 1, -+ model_type=model_type, -+ seq_length=seq_length, -+ hidden_size=hidden_size, -+ micro_batch_size=micro_batch_size, -+ decoder_seq_length=decoder_seq_length, -+ config=config, -+ ) -+ return recv_tensor_shapes -+ -+ def get_send_tensor_shapes(microbatch_id): -+ if input_shapes: -+ return [input_shapes[microbatch_id]] -+ send_tensor_shapes = get_tensor_shapes( -+ rank=rank, -+ model_type=model_type, -+ seq_length=seq_length, -+ hidden_size=hidden_size, -+ micro_batch_size=micro_batch_size, -+ decoder_seq_length=decoder_seq_length, -+ config=config, -+ ) -+ return send_tensor_shapes - - # Input, output tensors only need to be saved when doing backward passes - input_tensors = None -@@ -1163,7 +1183,12 @@ def forward_backward_pipelining_without_interleaving( - else: - checkpoint_activations_microbatch = None - -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {i}: warmup recv_forward begin...') -+ recv_tensor_shapes = get_recv_tensor_shapes(i) # fwd recv shape - input_tensor = recv_forward(recv_tensor_shapes, config) -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {i}: warmup recv_forward end & forward begin...') - output_tensor = forward_step( - forward_step_func, - data_iterator, -@@ -1175,7 +1200,13 @@ def forward_backward_pipelining_without_interleaving( - collect_non_loss_data, - checkpoint_activations_microbatch, - ) -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: output tensor shape = {output_tensor[0].shape}, send_tensor_shapes={send_tensor_shapes}') -+ # print(f'rank {torch.cuda.current_device()}: micro batch {i}: warmup forward end & send_forward begin...') -+ send_tensor_shapes = get_send_tensor_shapes(i) # fwd send shape - send_forward(output_tensor, send_tensor_shapes, config) -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {i}: warmup send_forward end...') - - if not forward_only: - input_tensors.append(input_tensor) -@@ -1186,11 +1217,16 @@ def forward_backward_pipelining_without_interleaving( - # If all microbatches are run in warmup / cooldown phase, then no need to - # receive this tensor here. - if num_microbatches_remaining > 0: -- input_tensor = recv_forward(recv_tensor_shapes, config) -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {num_warmup_microbatches}: 1f1b recv_forward begin...') -+ recv_tensor_shapes = get_recv_tensor_shapes(num_warmup_microbatches) # fwd recv shape -+ input_tensor = recv_forward(recv_tensor_shapes, config) - - # Run 1F1B in steady state. - for i in range(num_microbatches_remaining): - last_iteration = i == (num_microbatches_remaining - 1) -+ next_forward_k = num_warmup_microbatches + i + 1 -+ backward_k = i - - # Decide to checkpoint all layers' activations of the current micro-batch - if max_outstanding_backprops is not None: -@@ -1199,7 +1235,8 @@ def forward_backward_pipelining_without_interleaving( - ) >= config.num_microbatches_with_partial_activation_checkpoints - else: - checkpoint_activations_microbatch = None -- -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {num_warmup_microbatches + i}: 1f1b recv_forward end & forward begin...') - output_tensor = forward_step( - forward_step_func, - data_iterator, -@@ -1213,12 +1250,23 @@ def forward_backward_pipelining_without_interleaving( - ) - - if forward_only: -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {num_warmup_microbatches + i}: 1f1b forward end & send forward begin...') -+ send_tensor_shapes = get_send_tensor_shapes(next_forward_k - 1) # fwd send shape - send_forward(output_tensor, send_tensor_shapes, config) - - if not last_iteration: -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {num_warmup_microbatches + i}: 1f1b send forward end & recv forward begin...') -+ recv_tensor_shapes = get_recv_tensor_shapes(next_forward_k) # fwd recv shape - input_tensor = recv_forward(recv_tensor_shapes, config) -+ else: -+ pass -+ # if torch.cuda.current_device() == 0 or torch.cuda.current_device() == 4: -+ # print(f'rank {torch.cuda.current_device()}: micro batch {num_warmup_microbatches + i}: 1f1b send forward end...') - - else: -+ send_tensor_shapes = get_send_tensor_shapes(backward_k) # bwd recv shape - output_tensor_grad = send_forward_recv_backward( - output_tensor, send_tensor_shapes, config - ) -@@ -1245,8 +1293,10 @@ def forward_backward_pipelining_without_interleaving( - - if last_iteration: - input_tensor = None -+ recv_tensor_shapes = get_recv_tensor_shapes(backward_k) # bwd send shape - send_backward(input_tensor_grad, recv_tensor_shapes, config) - else: -+ recv_tensor_shapes = get_recv_tensor_shapes(next_forward_k) # fwd recv shape - input_tensor = send_backward_recv_forward( - input_tensor_grad, recv_tensor_shapes, config - ) -@@ -1254,7 +1304,7 @@ def forward_backward_pipelining_without_interleaving( - # Run cooldown backward passes. - if not forward_only: - for i in range(num_warmup_microbatches): -- -+ backward_k = num_microbatches_remaining + i - # Enable async grad reduction in the last backward pass - # Note: If grad sync function is provided, only enable - # async grad reduction in first pipeline stage. Other -@@ -1267,12 +1317,14 @@ def forward_backward_pipelining_without_interleaving( - input_tensor = input_tensors.pop(0) - output_tensor = output_tensors.pop(0) - -+ send_tensor_shapes = get_send_tensor_shapes(backward_k) # bwd recv shape - output_tensor_grad = recv_backward(send_tensor_shapes, config) - - input_tensor_grad = backward_step( - input_tensor, output_tensor, output_tensor_grad, model_type, config - ) - -+ recv_tensor_shapes = get_recv_tensor_shapes(backward_k) # bwd send shape - send_backward(input_tensor_grad, recv_tensor_shapes, config) - - # Launch any remaining grad reductions. -diff --git a/megatron/core/utils.py b/megatron/core/utils.py -index d4e042b2..c480d14e 100644 ---- a/megatron/core/utils.py -+++ b/megatron/core/utils.py -@@ -55,8 +55,9 @@ def get_model_type(model): - return get_attr_wrapped_model(model, 'model_type') - - -+# walkaround: get_model_config to get megatron config (ModelParallelConfig) - def get_model_config(model): -- return get_attr_wrapped_model(model, 'config', allow_none=False) -+ return get_attr_wrapped_model(model, 'megatron_config', allow_none=False) - - - class GlobalMemoryBuffer: -diff --git a/megatron/global_vars.py b/megatron/global_vars.py -index b1b4b043..9e23dea5 100644 ---- a/megatron/global_vars.py -+++ b/megatron/global_vars.py -@@ -21,11 +21,48 @@ _GLOBAL_ADLR_AUTORESUME = None - _GLOBAL_TIMERS = None - _GLOBAL_SIGNAL_HANDLER = None - --def get_args(): -+DEFAULT_NAMESPACE = 'default' -+import contextlib -+ -+@contextlib.contextmanager -+def fork_args_namespace(namespace): -+ """ -+ Usage example: -+ update_args('vit', vit_config) -+ with fork_args_namespace('vit'): -+ do vit stuff here -+ """ -+ # Check if we have added the args namespace -+ if namespace not in _GLOBAL_ARGS: -+ raise Exception('args namespace {} is not added'.format(namespace)) -+ # Store current args namespace. -+ tmp = _GLOBAL_ARGS[DEFAULT_NAMESPACE] -+ # Set args namespace to the desired one -+ _GLOBAL_ARGS[DEFAULT_NAMESPACE] = _GLOBAL_ARGS[namespace] -+ # Do the stuff we wanted to do. -+ try: -+ yield -+ finally: -+ _GLOBAL_ARGS[DEFAULT_NAMESPACE] = tmp -+ -+def get_args(namespace=DEFAULT_NAMESPACE): - """Return arguments.""" - _ensure_var_is_initialized(_GLOBAL_ARGS, 'args') -- return _GLOBAL_ARGS -+ return _GLOBAL_ARGS[namespace] - -+def set_args(args): -+ global _GLOBAL_ARGS -+ if _GLOBAL_ARGS is None: -+ _GLOBAL_ARGS = {} -+ _GLOBAL_ARGS[DEFAULT_NAMESPACE] = args -+ -+def update_args(namespace, args): -+ _ensure_var_is_initialized(_GLOBAL_ARGS, 'args') -+ if namespace not in _GLOBAL_ARGS: -+ import copy -+ _GLOBAL_ARGS[namespace] = copy.deepcopy(_GLOBAL_ARGS[DEFAULT_NAMESPACE]) -+ for k, v in args.items(): -+ setattr(_GLOBAL_ARGS[namespace], k, v) - - def get_retro_args(): - """Return retro arguments.""" -@@ -87,7 +124,7 @@ def _set_signal_handler(): - - - --def set_global_variables(args, build_tokenizer=True): -+def set_global_variables(args): - """Set args, tokenizer, tensorboard-writer, adlr-autoresume, and timers.""" - - assert args is not None -@@ -96,7 +133,7 @@ def set_global_variables(args, build_tokenizer=True): - set_args(args) - - _build_num_microbatches_calculator(args) -- if build_tokenizer: -+ if args.vocab_file: - _ = _build_tokenizer(args) - _set_tensorboard_writer(args) - _set_wandb_writer(args) -@@ -107,11 +144,6 @@ def set_global_variables(args, build_tokenizer=True): - _set_signal_handler() - - --def set_args(args): -- global _GLOBAL_ARGS -- _GLOBAL_ARGS = args -- -- - def set_retro_args(retro_args): - global _GLOBAL_RETRO_ARGS - _GLOBAL_RETRO_ARGS = retro_args -diff --git a/megatron/initialize.py b/megatron/initialize.py -index fb7866ab..01999622 100644 ---- a/megatron/initialize.py -+++ b/megatron/initialize.py -@@ -39,7 +39,7 @@ def initialize_megatron( - if not allow_no_cuda: - # Make sure cuda is available. - assert torch.cuda.is_available(), "Megatron requires CUDA." -- -+ print('use open-source megatron initialize...') - # Parse arguments - args = parse_args(extra_args_provider, ignore_unknown_args) - -diff --git a/megatron/model/fused_layer_norm.py b/megatron/model/fused_layer_norm.py -index c91a674e..bcb7bd7e 100644 ---- a/megatron/model/fused_layer_norm.py -+++ b/megatron/model/fused_layer_norm.py -@@ -81,7 +81,7 @@ class MixedFusedLayerNorm(torch.nn.Module): - if self.no_persist_layer_norm: - assert FusedLayerNormAffineFunction is not None, \ - "FusedLayerNormAffineFunction is not available, please install apex from https://github.com/NVIDIA/apex" -- return FusedLayerNormAffineFunction.apply(input, weight, self.bias, self.normalized_shape, self.eps) -+ return FusedLayerNormAffineFunction.apply(input, weight, self.bias, self.normalized_shape, self.eps, False) - else: - output = FastLayerNormFN.apply(input, weight, self.bias, self.eps) - -diff --git a/megatron/optimizer/distrib_optimizer.py b/megatron/optimizer/distrib_optimizer.py -index a04ae478..b64d22a5 100644 ---- a/megatron/optimizer/distrib_optimizer.py -+++ b/megatron/optimizer/distrib_optimizer.py -@@ -366,7 +366,7 @@ class DistributedOptimizer(MixedPrecisionOptimizer): - - def __init__(self, optimizer, clip_grad, log_num_zeros_in_grad, - check_for_nan_in_grad, params_have_main_grad, fp16, -- bf16, params_dtype, grad_scaler, models): -+ bf16, params_dtype, grad_scaler, models, overlap_param_gather=False): - """ - See top of class definition for argument descriptions. - -@@ -382,8 +382,11 @@ class DistributedOptimizer(MixedPrecisionOptimizer): - check_for_nan_in_grad, params_have_main_grad, - fp16, bf16, params_dtype, grad_scaler, models) - -- assert isinstance(optimizer, Adam), \ -- "Only Adam currently supported, due to checkpointing requirements." -+ # assert isinstance(optimizer, Adam), \ -+ # "Only Adam currently supported, due to checkpointing requirements." -+ -+ if not isinstance(optimizer, Adam): -+ print("WARNING: the optimizer type is not Adam, and now Only Adam currently support checkpointing requirements!") - - # Model grad buffer ranges. - self.model_gbuf_ranges = [] -@@ -476,7 +479,7 @@ class DistributedOptimizer(MixedPrecisionOptimizer): - self.param_buffer_copied.append(False) - self.num_all_gather_handles = len(self.all_gather_handle_index_to_bucket_index_map) - -- self.overlap_param_gather = get_args().overlap_param_gather -+ self.overlap_param_gather = overlap_param_gather - if self.overlap_param_gather: - self.remove_pre_hook_handle = torch.nn.modules.module.register_module_forward_pre_hook( - self._make_forward_pre_hook()) -diff --git a/megatron/training.py b/megatron/training.py -index 36f6c52e..73664509 100644 ---- a/megatron/training.py -+++ b/megatron/training.py -@@ -430,6 +430,7 @@ def train_step(forward_step_func, data_iterator, - model=model, - num_microbatches=get_num_microbatches(), - seq_length=args.seq_length, -+ hidden_size=args.hidden_size, - micro_batch_size=args.micro_batch_size, - decoder_seq_length=args.decoder_seq_length, - forward_only=False) -diff --git a/tools/prebuild_kernels.py b/tools/prebuild_kernels.py -new file mode 100644 -index 00000000..6f891b9e ---- /dev/null -+++ b/tools/prebuild_kernels.py -@@ -0,0 +1,13 @@ -+import os -+from megatron.fused_kernels import load -+ -+ -+class FakeArgs: -+ rank = 0 -+ -+ -+# 7.0 for V100 -+# 8.0 for A100/A800 -+os.environ["TORCH_CUDA_ARCH_LIST"] = "7.0+PTX;8.0+PTX" -+ -+load(FakeArgs) -\ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d76286439f..7aae922f24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,214 +1,58 @@ -# ------------------------------- -# build-system -# ------------------------------- -[build-system] -requires = [ - "setuptools>=61.0", - "wheel" -] -build-backend = "setuptools.build_meta" - -# ------------------------------- -# project (PEP 621 metadata) -# ------------------------------- [project] -name = "verl" -# We'll mark the version as "dynamic" because it's read from the file "verl/version/version" -# (PEP 621 calls this "dynamic version"). -# The actual version is specified in the [tool.setuptools.dynamic] section below. -dynamic = ["version", "optional-dependencies", "authors", "urls"] - -description = "verl: Volcano Engine Reinforcement Learning for LLM" -license = {file = "LICENSE"} # or "Apache-2.0", if you prefer an SPDX identifier -readme = {file = "README.md", content-type = "text/markdown"} +name = "skyrl" +version = "0.1.0" +description = "A Post-Training Stack for LLMs" +authors = [ + { name = "NovaSkyAI Team"} +] +license = { text = "MIT" } +readme = "README.md" requires-python = "==3.12.*" -dependencies=[ - 'accelerate', - 'codetiming', - 'datasets', - 'dill', - 'hydra-core', - 'numpy', - 'pandas', - 'datasets', - 'peft', - 'pyarrow>=15.0.0', - 'pybind11', - 'pylatexenc', - 'ray[default]>=2.10', - 'tensordict<=0.6.2', - 'torchdata', - 'transformers', - 'wandb', - "hf_transfer", - "torchdata", - "openhands-ai", - 'sglang[all]>=0.4.6.post1', - "flash-attn@https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl", - "streamlit", - "whatthepatch", - "retry", - "evaluate", - "swebench@https://github.com/SWE-Gym/SWE-Bench-Fork.git", - "swegym@https://github.com/SWE-Gym/SWE-Bench-Package.git", - "commit0", - "func_timeout", - "sympy", - "gdown", - "matplotlib", - "seaborn", - "tabulate", - "browsergym==0.10.2", - "browsergym-webarena==0.10.2", - "browsergym-miniwob==0.10.2", - "browsergym-visualwebarena==0.10.2", - 'tensordict<=0.6.2', - 'torch-memory-saver>=0.0.5', - 'vllm>=0.7.3' +[tool.black] +line-length = 120 +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.flake8] +max-line-length = 120 +max-doc-length = 120 +extend-ignore = [ + # Default ignored errors by flake8 + "E121", "E123", "E126", "E226", "E24", "E704", + # F401 module imported but unused + "F401", + # E203 whitespace before ':' (conflict with black) + "E203", + # E231 missing whitespace after ',' (conflict with black) + "E231", + # E501 line too long (conflict with black) + "E501", + # E741 do not use variables named 'l', 'O', or 'I' + "E741", + # W503 line break before binary operator (conflict with black) + "W503", + # W504 line break after binary operator (conflict with black) + "W504", + # W505 doc line too long (conflict with black) + "W505", + # W605 invalid escape sequence 'x' (conflict with latex within docs) + "W605", ] -[tool.uv] -find-links = ["https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python"] -override-dependencies = ["ray==2.43.0", "flash-attn@https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl", "xgrammar==0.1.17"] - - -[tool.uv.sources] -openhands-ai = { path = "./SkyRL-OpenHands", editable = true } - -# ------------------------------- -# tool.setuptools - Additional config -# ------------------------------- -[tool.setuptools] -# True means `setuptools` will attempt to include all relevant files in package_data automatically. -# This corresponds to `include_package_data=True` in setup.py. -include-package-data = true - -# We read the version from a file in 'verl/version/version' -[tool.setuptools.dynamic] -version = {file = "verl/version/version"} - -# If you need to mimic `package_dir={'': '.'}`: -[tool.setuptools.package-dir] -"" = "." - -# If you need to include specific non-Python data (like YAML files or version file): -# This is the rough equivalent of package_data={'': ['version/*'], 'verl': ['trainer/config/*.yaml']} -[tool.setuptools.package-data] -verl = [ - "version/*", - "trainer/config/*.yaml" +[tool.ruff.lint] +ignore = [ + "F722" # Syntax error in annotation - ignored because this doesn't play well with jaxtyping ] - - -[tool.pylint.message_control] -disable = [ - "abstract-method", - "anomalous-backslash-in-string", - "arguments-differ", - "arguments-renamed", - "assignment-from-none", - "attribute-defined-outside-init", - "bad-str-strip-call", - "bare-except", - "broad-exception-caught", - "broad-exception-raised", - "cell-var-from-loop", - "chained-comparison", - "consider-iterating-dictionary", - "consider-using-enumerate", - "consider-using-f-string", - "consider-using-from-import", - "consider-using-generator", - "consider-using-in", - "consider-using-max-builtin", - "consider-using-set-comprehension", - "consider-using-sys-exit", - "consider-using-with", - "cyclic-import", - "dangerous-default-value", - "duplicate-code", - "eval-used", - "expression-not-assigned", - "f-string-without-interpolation", - "fixme", - "function-redefined", - "global-statement", - "global-variable-not-assigned", - "import-error", - "import-outside-toplevel", - "import-self", - "inconsistent-return-statements", - "invalid-character-zero-width-space", - "invalid-name", - "line-too-long", - "logging-fstring-interpolation", - "logging-not-lazy", - "missing-class-docstring", - "missing-final-newline", - "missing-function-docstring", - "missing-module-docstring", - "multiple-imports", - "no-else-continue", - "no-else-raise", - "no-else-return", - "no-member", - "no-self-argument", - "no-value-for-parameter", - "not-an-iterable", - "not-callable", - "notimplemented-raised", - "pointless-exception-statement", - "pointless-string-statement", - "pointless-statement", - "possibly-used-before-assignment", - "protected-access", - "raise-missing-from", - "raising-format-tuple", - "redefined-argument-from-local", - "redefined-builtin", - "redefined-outer-name", - "redundant-u-string-prefix", - "reimported", - "simplifiable-if-expression", - "simplifiable-if-statement", - "singleton-comparison", - "super-init-not-called", - "superfluous-parens", - "too-few-public-methods", - "too-many-arguments", - "too-many-boolean-expressions", - "too-many-branches", - "too-many-instance-attributes", - "too-many-lines", - "too-many-locals", - "too-many-positional-arguments", - "too-many-return-statements", - "too-many-statements", - "trailing-newlines", - "trailing-newlines", - "trailing-whitespace", - "unbalanced-tuple-unpacking", - "undefined-loop-variable", - "undefined-variable", - "ungrouped-imports", - "unidiomatic-typecheck", - "unnecessary-comprehension", - "unnecessary-lambda", - "unnecessary-lambda-assignment", - "unnecessary-pass", - "unspecified-encoding", - "unused-argument", - "unused-import", - "unused-variable", - "unused-wildcard-import", - "use-a-generator", - "use-dict-literal", - "used-before-assignment", - "useless-object-inheritance", - "useless-parent-delegation", - "useless-return", - "wildcard-import", - "wrong-import-order", - "wrong-import-position", -] \ No newline at end of file diff --git a/recipe/dapo/README.md b/recipe/dapo/README.md deleted file mode 100644 index 5c5fdbb641..0000000000 --- a/recipe/dapo/README.md +++ /dev/null @@ -1,163 +0,0 @@ -# DAPO Open-Source Implementation - -> Open-Source Algorithm Implementation & Expriement Running: [Yuxuan Tong](https://tongyx361.github.io/), [Guangming Sheng](https://hk.linkedin.com/in/guangming-sheng-b50640211) - -> [!IMPORTANT] -> **🔥 News!!!** -> - [2025/03] We published the training record of [an early version of DAPO (w/o Token-level PG Loss & Dynamic Sampling)](./run_dapo_early_qwen2.5_32b.sh), achieving 44% on AIME 2024, in [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl). - -🏠 [Homepage](https://dapo-sia.github.io/) | 📝 [Paper](https://dapo-sia.github.io/static/pdf/dapo_paper.pdf) | 🤗 [Datasets&Models@HF](https://huggingface.co/collections/BytedTsinghua-SIA/dapo-67d7f1517ee33c8aed059da0) | 🐱 [Code@GitHub](https://github.com/volcengine/verl/tree/gm-tyx/puffin/main/recipe/dapo) | 🐱 [Repo@GitHub](https://github.com/BytedTsinghua-SIA/DAPO) - -> We propose the **D**ecoupled Clip and Dynamic s**A**mpling **P**olicy **O**ptimization (DAPO) algorithm. By making our work publicly available, we provide the broader research community and society with practical access to scalable reinforcement learning, enabling all to benefit from these advancements. Our system is based on the awesome [verl](https://github.com/volcengine/verl) framework. Thanks for their great work! Applying DAPO training to Qwen2.5-32B base model proves to outperform the previous state-of-the-art DeepSeek-R1-Zero-Qwen-32B on AIME 2024, achieving **50%** accuracy with **50%** less training steps. -> -> ![dapo-main-result](https://dapo-sia.github.io/static/images/score.png) - -## Quickstart - -1. Prepare the datasets **on the Ray cluster**: - -```bash -bash prepare_dapo_data.sh # This downloads the datasets to ${HOME}/verl/data by default -``` - -2. Submit the job to the Ray cluster **from any machine**: - -```bash -cd verl # Repo root -export RAY_ADDRESS="http://${RAY_IP:-localhost}:8265" # The Ray cluster address to connect to -export WORKING_DIR="${PWD}" # The local directory to package to the Ray cluster -# Set the runtime environment like env vars and pip packages for the Ray cluster in yaml -export RUNTIME_ENV="./verl/trainer/runtime_env.yaml" -bash recipe/dapo/run_dapo_qwen2.5_32b.sh -``` - -## Reproduction Runs - -| Setup | AIME 2024 Acc. | Training Script | Training Record | -|-------|----------------------|-----------------|-----------------| -| DAPO w/o Token-level PG Loss & Dynamic Sampling | 44% | [run_dapo_early_qwen2.5_32b.sh](./run_dapo_early_qwen2.5_32b.sh) | [W&B](https://wandb.ai/verl-org/DAPO%20Reproduction%20on%20verl) | -| DAPO | 50% | [run_dapo_qwen2.5_32b.sh](./run_dapo_qwen2.5_32b.sh) | W&B (Coming soon) | - -## Configuration - -> [!NOTE] -> Most experiments in the paper, including the best-performant one, are run without Overlong Filtering because it's somehow overlapping with Overlong Reward Shaping in terms of properly learning from the longest outputs. So we don't implement it here. - -### Separated Clip Epsilons (-> Clip-Higher) - -An example configuration: - -```yaml -actor_rollout_ref: - actor: - clip_ratio_low: 0.2 - clip_ratio_high: 0.28 -``` - -`clip_ratio_low` and `clip_ratio_high` specify the $\varepsilon_{\text {low }}$ and $\varepsilon_{\text {high }}$ in the DAPO objective. - -Core relevant code: - -```python -pg_losses1 = -advantages * ratio -pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) -pg_losses = torch.maximum(pg_losses1, pg_losses2) -``` - -### Dynamic Sampling (with Group Filtering) - -An example configuration: - -```yaml -data: - gen_batch_size: 1536 - train_batch_size: 512 -algorithm: - filter_groups: - enable: True - metric: acc # score / seq_reward / seq_final_reward / ... - max_num_gen_batches: 10 # Non-positive values mean no upper limit -``` - -Setting `filter_groups.enable` to `True` will filter out groups whose outputs' `metric` are all the same, e.g., for `acc`, groups whose outputs' accuracies are all 1 or 0. - -The trainer will repeat sampling with `gen_batch_size` until there are enough qualified groups for `train_batch_size` or reaching the upper limit specified by `max_num_gen_batches`. - -Core relevant code: - -```python -prompt_bsz = self.config.data.train_batch_size -if num_prompt_in_batch < prompt_bsz: - print(f'{num_prompt_in_batch=} < {prompt_bsz=}') - num_gen_batches += 1 - max_num_gen_batches = self.config.algorithm.filter_groups.max_num_gen_batches - if max_num_gen_batches <= 0 or num_gen_batches < max_num_gen_batches: - print(f'{num_gen_batches=} < {max_num_gen_batches=}. Keep generating...') - continue - else: - raise ValueError( - f'{num_gen_batches=} >= {max_num_gen_batches=}. Generated too many. Please check your data.' - ) -else: - # Align the batch - traj_bsz = self.config.data.train_batch_size * self.config.actor_rollout_ref.rollout.n - batch = batch[:traj_bsz] -``` - -### Flexible Loss Aggregation Mode (-> Token-level Policy Gradient Loss) - -An example configuration: - -```yaml -actor_rollout_ref: - actor: - loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean" - # NOTE: "token-mean" is the default behavior -``` - -Setting `loss_agg_mode` to `token-mean` will mean the (policy gradient) loss across all the tokens in all the sequences in a mini-batch. - -Core relevant code: - -```python -if loss_agg_mode == "token-mean": - pg_loss = verl_F.masked_mean(pg_losses, eos_mask) -elif loss_agg_mode == "seq-mean-token-sum": - pg_loss = torch.sum(pg_losses * eos_mask, dim=-1) / torch.sum(eos_mask, dim=-1) - pg_loss = torch.mean(pg_loss) -elif loss_agg_mode == "seq-mean-token-mean": - pg_loss = torch.sum(pg_losses * eos_mask, dim=-1) / torch.sum(eos_mask, dim=-1) - pg_loss = torch.mean(pg_loss) -else: - raise ValueError(f"Invalid loss_agg_mode: {loss_agg_mode}") -``` - -### Overlong Reward Shaping - -An example configuration: - -```yaml -data: - max_response_length: 20480 # 16384 + 4096 -reward_model: - overlong_buffer: - enable: True - len: 4096 - penalty_factor: 1.0 -``` - -Setting `overlong_buffer.enable` to `True` will penalize the outputs whose lengths are overlong but still within the hard context limit. - -Specifically, the penalty increases linearly from `0` to `overlong_buffer.penalty_factor` when the length of the output exceeds the `max_response_length` by `0` to `overlong_buffer.len` tokens. - -Core relevant code: - -```python -if self.overlong_buffer_cfg.enable: - overlong_buffer_len = self.overlong_buffer_cfg.len - expected_len = self.max_resp_len - overlong_buffer_len - exceed_len = valid_response_length - expected_len - overlong_penalty_factor = self.overlong_buffer_cfg.penalty_factor - overlong_reward = min(-exceed_len / overlong_buffer_len * overlong_penalty_factor, 0) - reward += overlong_reward -``` diff --git a/recipe/dapo/prepare_dapo_data.sh b/recipe/dapo/prepare_dapo_data.sh deleted file mode 100644 index c6c60bf202..0000000000 --- a/recipe/dapo/prepare_dapo_data.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -uxo pipefail - -export VERL_HOME=${VERL_HOME:-"${HOME}/verl"} -export TRAIN_FILE=${TRAIN_FILE:-"${VERL_HOME}/data/dapo-math-17k.parquet"} -export TEST_FILE=${TEST_FILE:-"${VERL_HOME}/data/aime-2024.parquet"} - -mkdir -p "${VERL_HOME}/data" - -wget -O "${TRAIN_FILE}" "https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k/resolve/main/data/dapo-math-17k.parquet?download=true" - -wget -O "${TEST_FILE}" "https://huggingface.co/datasets/BytedTsinghua-SIA/AIME-2024/resolve/main/data/aime-2024.parquet?download=true" \ No newline at end of file diff --git a/recipe/dapo/run_dapo_early_qwen2.5_32b.sh b/recipe/dapo/run_dapo_early_qwen2.5_32b.sh deleted file mode 100644 index 3b1ebfce6e..0000000000 --- a/recipe/dapo/run_dapo_early_qwen2.5_32b.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -project_name='DAPO' -exp_name='DAPO-Early-Qwen2.5-32B' - -adv_estimator=grpo - -use_kl_in_reward=False -kl_coef=0.0 -use_kl_loss=False -kl_loss_coef=0.0 - -clip_ratio_low=0.2 -clip_ratio_high=0.28 - -max_prompt_length=$((1024 * 2)) -max_response_length=$((1024 * 20)) -enable_overlong_buffer=True -overlong_buffer_len=$((1024 * 4)) -overlong_penalty_factor=1.0 - -# An early version for DAPO -loss_agg_mode="seq-mean-token-sum" - -enable_filter_groups=False -gen_prompt_bsz=512 # NOTE: no filtering here -train_prompt_bsz=512 -train_prompt_mini_bsz=32 -n_resp_per_prompt=16 - -# Ray -RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} -WORKING_DIR=${WORKING_DIR:-"${PWD}"} -RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} -NNODES=${NNODES:-16} -# Paths -RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} -MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-32B"} -CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} -TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} -TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} - -# Algorithm -temperature=1.0 -top_p=1.0 -top_k=-1 # 0 for HF rollout, -1 for vLLM rollout -val_top_p=0.7 - - -# Performance Related Parameter -sp_size=8 -use_dynamic_bsz=True -actor_ppo_max_token_len=$((max_prompt_length + max_response_length)) -infer_ppo_max_token_len=$((max_prompt_length + max_response_length)) -offload=True -gen_tp=4 - -ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ - --working-dir "${WORKING_DIR}" \ - -- python3 -m recipe.dapo.src.main_dapo \ - data.train_files="${TRAIN_FILE}" \ - data.val_files="${TEST_FILE}" \ - data.prompt_key=prompt \ - data.truncation='left' \ - data.max_prompt_length=${max_prompt_length} \ - data.max_response_length=${max_response_length} \ - data.gen_batch_size=${gen_prompt_bsz} \ - data.train_batch_size=${train_prompt_bsz} \ - actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ - algorithm.adv_estimator=${adv_estimator} \ - algorithm.use_kl_in_reward=${use_kl_in_reward} \ - algorithm.kl_ctrl.kl_coef=${kl_coef} \ - actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ - actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ - actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ - actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ - algorithm.filter_groups.enable=${enable_filter_groups} \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ - actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.model.path="${MODEL_PATH}" \ - +actor_rollout_ref.model.override_config.attention_dropout=0. \ - +actor_rollout_ref.model.override_config.embd_pdrop=0. \ - +actor_rollout_ref.model.override_config.resid_pdrop=0. \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ - actor_rollout_ref.actor.optim.weight_decay=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ - actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.grad_clip=1.0 \ - actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ - actor_rollout_ref.rollout.enable_chunked_prefill=True \ - actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.temperature=${temperature} \ - actor_rollout_ref.rollout.top_p=${top_p} \ - actor_rollout_ref.rollout.top_k="${top_k}" \ - actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ - actor_rollout_ref.rollout.val_kwargs.top_p=${val_top_p} \ - actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ - actor_rollout_ref.rollout.val_kwargs.do_sample=True \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ - actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ - actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ - reward_model.reward_manager=dapo \ - reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ - reward_model.overlong_buffer.len=${overlong_buffer_len} \ - reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ - trainer.logger=['console','wandb'] \ - trainer.project_name="${project_name}" \ - trainer.experiment_name="${exp_name}" \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes="${NNODES}" \ - trainer.val_before_train=True \ - trainer.test_freq=5 \ - trainer.save_freq=5 \ - trainer.total_epochs=1 \ - trainer.default_local_dir="${CKPTS_DIR}" \ - trainer.resume_mode=auto \ No newline at end of file diff --git a/recipe/dapo/run_dapo_qwen2.5_32b.sh b/recipe/dapo/run_dapo_qwen2.5_32b.sh deleted file mode 100644 index 3943a2e89b..0000000000 --- a/recipe/dapo/run_dapo_qwen2.5_32b.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -project_name='DAPO' -exp_name='DAPO-Qwen2.5-32B' - -adv_estimator=grpo - -use_kl_in_reward=False -kl_coef=0.0 -use_kl_loss=False -kl_loss_coef=0.0 - -clip_ratio_low=0.2 -clip_ratio_high=0.28 - -max_prompt_length=$((1024 * 2)) -max_response_length=$((1024 * 20)) -enable_overlong_buffer=True -overlong_buffer_len=$((1024 * 4)) -overlong_penalty_factor=1.0 - -loss_agg_mode="token-mean" - -enable_filter_groups=True -filter_groups_metric=acc -max_num_gen_batches=10 -train_prompt_bsz=512 -gen_prompt_bsz=$((train_prompt_bsz * 3)) -n_resp_per_prompt=16 -train_prompt_mini_bsz=32 - -# Ray -RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} -WORKING_DIR=${WORKING_DIR:-"${PWD}"} -RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} -NNODES=${NNODES:-16} -# Paths -RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} -MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-32B"} -CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} -TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} -TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} - -# Algorithm -temperature=1.0 -top_p=1.0 -top_k=-1 # 0 for HF rollout, -1 for vLLM rollout - -# Performance Related Parameter -sp_size=8 -use_dynamic_bsz=True -actor_ppo_max_token_len=$((max_prompt_length + max_response_length)) -infer_ppo_max_token_len=$((max_prompt_length + max_response_length)) -offload=True -gen_tp=4 - -ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ - --working-dir "${WORKING_DIR}" \ - -- python3 -m recipe.dapo.src.main_dapo \ - data.train_files="${TRAIN_FILE}" \ - data.val_files="${TEST_FILE}" \ - data.prompt_key=prompt \ - data.truncation='left' \ - data.max_prompt_length=${max_prompt_length} \ - data.max_response_length=${max_response_length} \ - data.gen_batch_size=${gen_prompt_bsz} \ - data.train_batch_size=${train_prompt_bsz} \ - actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ - algorithm.adv_estimator=${adv_estimator} \ - algorithm.use_kl_in_reward=${use_kl_in_reward} \ - algorithm.kl_ctrl.kl_coef=${kl_coef} \ - actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ - actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ - actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ - actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ - algorithm.filter_groups.enable=${enable_filter_groups} \ - algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ - algorithm.filter_groups.metric=${filter_groups_metric} \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${actor_ppo_max_token_len} \ - actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${infer_ppo_max_token_len} \ - actor_rollout_ref.model.path="${MODEL_PATH}" \ - +actor_rollout_ref.model.override_config.attention_dropout=0. \ - +actor_rollout_ref.model.override_config.embd_pdrop=0. \ - +actor_rollout_ref.model.override_config.resid_pdrop=0. \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ - actor_rollout_ref.actor.optim.weight_decay=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ - actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.grad_clip=1.0 \ - actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=${sp_size} \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=${gen_tp} \ - actor_rollout_ref.rollout.enable_chunked_prefill=True \ - actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.temperature=${temperature} \ - actor_rollout_ref.rollout.top_p=${top_p} \ - actor_rollout_ref.rollout.top_k="${top_k}" \ - actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ - actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ - actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ - actor_rollout_ref.rollout.val_kwargs.do_sample=True \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ - actor_rollout_ref.ref.ulysses_sequence_parallel_size=${sp_size} \ - actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ - reward_model.reward_manager=dapo \ - reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ - reward_model.overlong_buffer.len=${overlong_buffer_len} \ - reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ - trainer.logger=['console','wandb'] \ - trainer.project_name="${project_name}" \ - trainer.experiment_name="${exp_name}" \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes="${NNODES}" \ - trainer.val_before_train=True \ - trainer.test_freq=5 \ - trainer.save_freq=5 \ - trainer.total_epochs=1 \ - trainer.default_local_dir="${CKPTS_DIR}" \ - trainer.resume_mode=auto \ No newline at end of file diff --git a/recipe/dapo/src/config/dapo_trainer.yaml b/recipe/dapo/src/config/dapo_trainer.yaml deleted file mode 100644 index bcafe90f69..0000000000 --- a/recipe/dapo/src/config/dapo_trainer.yaml +++ /dev/null @@ -1,223 +0,0 @@ -data: - tokenizer: null - train_files: ~/data/rlhf/gsm8k/train.parquet - val_files: ~/data/rlhf/gsm8k/test.parquet - prompt_key: prompt - reward_fn_key: data_source - max_prompt_length: 512 - max_response_length: 512 - gen_batch_size: ${data.train_batch_size} - train_batch_size: 1024 - val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves - return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs - return_raw_chat: False - shuffle: True - filter_overlong_prompts: False # for large-scale dataset, filtering overlong prompts could be timeconsuming. You should disable this and set `truncation='left' - truncation: error - image_key: images - -actor_rollout_ref: - hybrid_engine: True - model: - path: ~/models/deepseek-llm-7b-chat - external_lib: null - override_config: { } - enable_gradient_checkpointing: True - use_remove_padding: False - actor: - strategy: fsdp # This is for backward-compatibility - ppo_mini_batch_size: 256 - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - use_dynamic_bsz: False - ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length} - grad_clip: 1.0 - # pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) - clip_ratio: 0.2 # default value if clip_ratio_low and clip_ratio_high are not specified - clip_ratio_low: 0.2 - clip_ratio_high: 0.2 - clip_ratio_c: 3.0 # lower bound of the value for Dual-clip PPO from https://arxiv.org/pdf/1912.09729 - loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean" - # NOTE: "token-mean" is the default behavior - entropy_coeff: 0.001 - use_kl_loss: False # True for GRPO - use_torch_compile: True # False to disable torch compile - kl_loss_coef: 0.001 # for grpo - kl_loss_type: low_var_kl # for grpo - ppo_epochs: 1 - shuffle: False - ulysses_sequence_parallel_size: 1 # sp size - checkpoint: - contents: ['model', 'hf_model', 'optimizer', 'extra'] # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space - optim: - lr: 1e-6 - lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - weight_decay: 0.01 - fsdp_config: - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - param_offload: False - optimizer_offload: False - fsdp_size: -1 - ref: - fsdp_config: - param_offload: False - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} # sp size - rollout: - name: vllm - temperature: 1.0 - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1 - use_fire_sampling: False # https://arxiv.org/abs/2410.21236 - prompt_length: ${data.max_prompt_length} # not use for opensource - response_length: ${data.max_response_length} - # for vllm rollout - dtype: bfloat16 # should align with FSDP - gpu_memory_utilization: 0.5 - ignore_eos: False - enforce_eager: True - free_cache_engine: True - load_format: dummy_dtensor - tensor_model_parallel_size: 2 - max_num_batched_tokens: 8192 - max_model_len: null - max_num_seqs: 1024 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - disable_log_stats: True - enable_chunked_prefill: True # may get higher throughput when set to True. When activated, Please increase max_num_batched_tokens or decrease max_model_len. - # for hf rollout - do_sample: True - # number of responses (i.e. num sample times) - n: 1 # > 1 for grpo - val_kwargs: - # sampling parameters for validation - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1.0 - temperature: 0 - n: 1 - do_sample: False # default eager for validation - -critic: - rollout_n: ${actor_rollout_ref.rollout.n} - strategy: fsdp - optim: - lr: 1e-5 - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - weight_decay: 0.01 - model: - path: ~/models/deepseek-llm-7b-chat - tokenizer_path: ${actor_rollout_ref.model.path} - override_config: { } - external_lib: ${actor_rollout_ref.model.external_lib} - enable_gradient_checkpointing: True - use_remove_padding: False - fsdp_config: - param_offload: False - optimizer_offload: False - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - fsdp_size: -1 - ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - forward_micro_batch_size: ${critic.ppo_micro_batch_size} - forward_micro_batch_size_per_gpu: ${critic.ppo_micro_batch_size_per_gpu} - use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - ppo_max_token_len_per_gpu: 32768 # (${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}) * 2 - forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu} - ulysses_sequence_parallel_size: 1 # sp size - ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs} - shuffle: ${actor_rollout_ref.actor.shuffle} - grad_clip: 1.0 - cliprange_value: 0.5 - checkpoint: - contents: ['model', 'hf_model', 'optimizer', 'extra'] # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space - -reward_model: - enable: False - strategy: fsdp - model: - input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical - path: ~/models/FsfairX-LLaMA3-RM-v0.1 - external_lib: ${actor_rollout_ref.model.external_lib} - use_remove_padding: False - fsdp_config: - wrap_policy: - min_num_params: 0 - param_offload: False - fsdp_size: -1 - micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu - micro_batch_size_per_gpu: null # set a number - max_length: null - ulysses_sequence_parallel_size: 1 # sp size - use_dynamic_bsz: ${critic.use_dynamic_bsz} - forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} - reward_manager: naive - overlong_buffer: - enable: False # We try to avoid forgetting to set enable - len: 0 - penalty_factor: 0.0 - log: False - -custom_reward_function: - path: null - name: compute_score - -algorithm: - gamma: 1.0 - lam: 1.0 - adv_estimator: gae - use_kl_in_reward: False - kl_penalty: kl # how to estimate kl divergence - kl_ctrl: - type: fixed - kl_coef: 0.001 - horizon: 10000 - target_kl: 0.1 - filter_groups: - enable: False # We try to avoid forgetting to set enable - metric: null # acc / score / seq_reward / seq_final_reward / ... - max_num_gen_batches: 0 # Non-positive values mean no upper limit - -trainer: - balance_batch: True - total_epochs: 30 - total_training_steps: null - project_name: verl_examples - experiment_name: gsm8k - logger: [ 'console', 'wandb' ] - log_val_generations: 0 - nnodes: 1 - n_gpus_per_node: 8 - save_freq: -1 - # auto: find the last ckpt to resume. If can't find, start from scratch - resume_mode: auto # or disable or resume_path if resume_from_path is set - resume_from_path: null - val_before_train: True - test_freq: -1 - critic_warmup: 0 - default_hdfs_dir: null - remove_previous_ckpt_in_save: False - del_local_ckpt_after_load: False - default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} - max_actor_ckpt_to_keep: null - max_critic_ckpt_to_keep: null diff --git a/recipe/dapo/src/dapo_ray_trainer.py b/recipe/dapo/src/dapo_ray_trainer.py deleted file mode 100644 index 7fa161af33..0000000000 --- a/recipe/dapo/src/dapo_ray_trainer.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -FSDP PPO Trainer with Ray-based single controller. -This trainer supports model-agonistic model initialization with huggingface -""" - -import uuid -from pprint import pprint -from copy import deepcopy -from collections import defaultdict -from tqdm import tqdm -import numpy as np -import torch - -from verl import DataProto -from verl.trainer.ppo.ray_trainer import RayPPOTrainer, _timer, apply_kl_penalty, compute_advantage, AdvantageEstimator -from verl.trainer.ppo.metric_utils import (compute_data_metrics, compute_throughout_metrics, compute_timing_metrics, - reduce_metrics) - - -class RayDAPOTrainer(RayPPOTrainer): - """ - Note that this trainer runs on the driver process on a single CPU/GPU node. - """ - - def fit(self): - """ - The training loop of PPO. - The driver process only need to call the compute functions of the worker group through RPC to construct the PPO dataflow. - The light-weight advantage computation is done on the driver process. - """ - from verl.utils.tracking import Tracking - from omegaconf import OmegaConf - - logger = Tracking(project_name=self.config.trainer.project_name, - experiment_name=self.config.trainer.experiment_name, - default_backend=self.config.trainer.logger, - config=OmegaConf.to_container(self.config, resolve=True)) - - self.global_steps = 0 - - # load checkpoint before doing anything - self._load_checkpoint() - - # perform validation before training - # currently, we only support validation using the reward_function. - if self.val_reward_fn is not None and self.config.trainer.get('val_before_train', True): - val_metrics = self._validate() - pprint(f'Initial validation metrics: {val_metrics}') - logger.log(data=val_metrics, step=self.global_steps) - if self.config.trainer.get('val_only', False): - return - - # add tqdm - progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") - - # we start from step 1 - self.global_steps += 1 - last_val_metrics = None - - timing_raw = defaultdict(float) - batch = None - num_prompt_in_batch = 0 - num_gen_batches = 0 - for epoch in range(self.config.trainer.total_epochs): - for batch_dict in self.train_dataloader: - metrics = {} - - new_batch: DataProto = DataProto.from_single_dict(batch_dict) - num_gen_batches += 1 - # pop those keys for generation - if 'multi_modal_inputs' in new_batch.non_tensor_batch.keys(): - gen_batch = new_batch.pop( - batch_keys=['input_ids', 'attention_mask', 'position_ids'], - non_tensor_batch_keys=['raw_prompt_ids', 'multi_modal_data', 'multi_modal_inputs'], - ) - else: - gen_batch = new_batch.pop( - batch_keys=['input_ids', 'attention_mask', 'position_ids'], - non_tensor_batch_keys=['raw_prompt_ids'], - ) - - is_last_step = self.global_steps >= self.total_training_steps - - with _timer('step', timing_raw): - # generate a batch - with _timer('gen', timing_raw): - gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) - - if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: - with _timer('gen_max', timing_raw): - gen_baseline_batch = deepcopy(gen_batch) - gen_baseline_batch.meta_info['do_sample'] = False - gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) - - new_batch = new_batch.union(gen_baseline_output) - reward_baseline_tensor = self.reward_fn(new_batch) - reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) - - new_batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) - - new_batch.batch['reward_baselines'] = reward_baseline_tensor - - del gen_baseline_batch, gen_baseline_output - - new_batch.non_tensor_batch['uid'] = np.array( - [str(uuid.uuid4()) for _ in range(len(new_batch.batch))], dtype=object) - # repeat to align with repeated responses in rollout - new_batch = new_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) - new_batch = new_batch.union(gen_batch_output) - - with _timer('reward', timing_raw): - # compute scores. Support both model and function-based. - # We first compute the scores using reward model. Then, we call reward_fn to combine - # the results from reward model and rule-based results. - if self.use_rm: - # we first compute reward model score - reward_tensor = self.rm_wg.compute_rm_score(new_batch) - new_batch = new_batch.union(reward_tensor) - - # we combine with rule-based rm - reward_extra_infos_dict: dict[str, list] - try: - reward_result = self.reward_fn(new_batch, return_dict=True) - reward_tensor = reward_result['reward_tensor'] - reward_extra_infos_dict = reward_result['reward_extra_info'] - except Exception as e: - print(f'Error in reward_fn: {e}') - reward_tensor = self.reward_fn(new_batch) - reward_extra_infos_dict = {} - - new_batch.batch['token_level_scores'] = reward_tensor - - print(f'{list(reward_extra_infos_dict.keys())=}') - if reward_extra_infos_dict: - new_batch.non_tensor_batch.update({ - k: np.array(v) for k, v in reward_extra_infos_dict.items() - }) - - # compute rewards. apply_kl_penalty if available - if self.config.algorithm.use_kl_in_reward: - new_batch, kl_metrics = apply_kl_penalty(new_batch, - kl_ctrl=self.kl_ctrl_in_reward, - kl_penalty=self.config.algorithm.kl_penalty) - metrics.update( - kl_metrics) # TODO: This will be cleared if we use multiple genenration batches - else: - new_batch.batch['token_level_rewards'] = new_batch.batch['token_level_scores'] - - if not self.config.algorithm.filter_groups.enable: - batch = new_batch - else: # NOTE: When prompts after filtering is less than train batch size, we skip to the next generation batch - metric_name = self.config.algorithm.filter_groups.metric - if metric_name == "seq_final_reward": - # Turn to numpy for easier filtering - new_batch.non_tensor_batch["seq_final_reward"] = new_batch.batch['token_level_scores'].sum( - dim=-1).numpy() - elif metric_name == "seq_reward": - new_batch.non_tensor_batch["seq_reward"] = new_batch.batch['token_level_scores'].sum( - dim=-1).numpy() - - # Collect the sequence reward for each trajectory - prompt_uid2metric_vals = defaultdict(list) - for uid, metric_val in zip(new_batch.non_tensor_batch['uid'], - new_batch.non_tensor_batch[metric_name]): - prompt_uid2metric_vals[uid].append(metric_val) - - prompt_uid2metric_std = {} - for prompt_uid, metric_vals in prompt_uid2metric_vals.items(): - prompt_uid2metric_std[prompt_uid] = np.std(metric_vals) - - kept_prompt_uids = [ - uid for uid, std in prompt_uid2metric_std.items() - if std > 0 or len(prompt_uid2metric_vals[uid]) == 1 - ] - num_prompt_in_batch += len(kept_prompt_uids) - - kept_traj_idxs = [] - for idx, traj_from_prompt_uid in enumerate(new_batch.non_tensor_batch['uid']): - if traj_from_prompt_uid in kept_prompt_uids: - kept_traj_idxs.append(idx) - - new_batch = new_batch[kept_traj_idxs] - if batch is None: - batch = new_batch - else: - batch = DataProto.concat([batch, new_batch]) - - prompt_bsz = self.config.data.train_batch_size - if num_prompt_in_batch < prompt_bsz: - print(f'{num_prompt_in_batch=} < {prompt_bsz=}') - max_num_gen_batches = self.config.algorithm.filter_groups.max_num_gen_batches - if max_num_gen_batches <= 0 or num_gen_batches < max_num_gen_batches: - print(f'{num_gen_batches=}. Keep generating...') - continue - else: - raise ValueError( - f'{num_gen_batches=} >= {max_num_gen_batches=}. Generated too many. Please check your data.' - ) - else: - # Align the batch - traj_bsz = self.config.data.train_batch_size * self.config.actor_rollout_ref.rollout.n - batch = batch[:traj_bsz] - - # balance the number of valid tokens on each dp rank. - # Note that this breaks the order of data inside the batch. - # Please take care when you implement group based adv computation such as GRPO and rloo - if self.config.trainer.balance_batch: - self._balance_batch(batch, metrics=metrics) - - # compute global_valid tokens - batch.meta_info['global_token_num'] = torch.sum(batch.batch['attention_mask'], dim=-1).tolist() - - # recompute old_log_probs - with _timer('old_log_prob', timing_raw): - old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) - batch = batch.union(old_log_prob) - - if self.use_reference_policy: - # compute reference log_prob - with _timer('ref', timing_raw): - ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) - batch = batch.union(ref_log_prob) - - # compute values - if self.use_critic: - with _timer('values', timing_raw): - values = self.critic_wg.compute_values(batch) - batch = batch.union(values) - - with _timer('adv', timing_raw): - # compute advantages, executed on the driver process - batch = compute_advantage(batch, - adv_estimator=self.config.algorithm.adv_estimator, - gamma=self.config.algorithm.gamma, - lam=self.config.algorithm.lam, - num_repeat=self.config.actor_rollout_ref.rollout.n) - - # update critic - if self.use_critic: - with _timer('update_critic', timing_raw): - critic_output = self.critic_wg.update_critic(batch) - critic_output_metrics = reduce_metrics(critic_output.meta_info['metrics']) - metrics.update(critic_output_metrics) - - # implement critic warmup - if self.config.trainer.critic_warmup <= self.global_steps: - # update actor - with _timer('update_actor', timing_raw): - actor_output = self.actor_rollout_wg.update_actor(batch) - actor_output_metrics = reduce_metrics(actor_output.meta_info['metrics']) - metrics.update(actor_output_metrics) - - # validate - if self.val_reward_fn is not None and self.config.trainer.test_freq > 0 and \ - (is_last_step or self.global_steps % self.config.trainer.test_freq == 0): - with _timer('testing', timing_raw): - val_metrics: dict = self._validate() - if is_last_step: - last_val_metrics = val_metrics - metrics.update(val_metrics) - - if self.config.trainer.save_freq > 0 and (is_last_step or - self.global_steps % self.config.trainer.save_freq == 0): - with _timer('save_checkpoint', timing_raw): - self._save_checkpoint() - - # collect metrics - metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) - metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) - # TODO: implement actual tflpo and theoretical tflpo - n_gpus = self.resource_pool_manager.get_n_gpus() - metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) - timing_raw = defaultdict(float) # clear timing - - metrics["train/num_gen_batches"] = num_gen_batches - batch = None - num_prompt_in_batch = 0 - num_gen_batches = 0 - - # TODO: make a canonical logger that supports various backend - logger.log(data=metrics, step=self.global_steps) - - if is_last_step: - pprint(f'Final validation metrics: {last_val_metrics}') - progress_bar.close() - return - - progress_bar.update(1) - self.global_steps += 1 diff --git a/recipe/dapo/src/main_dapo.py b/recipe/dapo/src/main_dapo.py deleted file mode 100644 index df96d4bbe1..0000000000 --- a/recipe/dapo/src/main_dapo.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. -""" -from .dapo_ray_trainer import RayDAPOTrainer - -import os -import ray -import hydra - - -def get_custom_reward_fn(config): - import importlib.util, os - - reward_fn_config = config.get("custom_reward_function") or {} - file_path = reward_fn_config.get("path") - if not file_path: - return None - - if not os.path.exists(file_path): - raise FileNotFoundError(f"Reward function file '{file_path}' not found.") - - spec = importlib.util.spec_from_file_location("custom_module", file_path) - module = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(module) - except Exception as e: - raise RuntimeError(f"Error loading module from '{file_path}': {e}") - - function_name = reward_fn_config.get("name") - - if not hasattr(module, function_name): - raise AttributeError(f"Reward function '{function_name}' not found in '{file_path}'.") - - print(f"using customized reward function '{function_name}' from '{file_path}'") - - return getattr(module, function_name) - - -@hydra.main(config_path='config', config_name='dapo_trainer', version_base=None) -def main(config): - run_ppo(config) - - -def run_ppo(config) -> None: - # TODO(linjunrong.ocss884): this ENV is left for resolving SGLang conflict with ray devices - # isolation, will solve in the future - os.environ["ENSURE_CUDA_VISIBLE_DEVICES"] = os.environ.get('CUDA_VISIBLE_DEVICES', '') - if not ray.is_initialized(): - # this is for local ray cluster - ray.init(runtime_env={ - 'env_vars': { - 'TOKENIZERS_PARALLELISM': 'true', - 'NCCL_DEBUG': 'WARN', - 'VLLM_LOGGING_LEVEL': 'WARN' - } - }) - - runner = TaskRunner.remote() - ray.get(runner.run.remote(config)) - - -@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head -class TaskRunner: - - def run(self, config): - from verl.utils.fs import copy_to_local - # print initial config - from pprint import pprint - from omegaconf import OmegaConf - pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values - OmegaConf.resolve(config) - - # download the checkpoint from hdfs - local_path = copy_to_local(config.actor_rollout_ref.model.path) - - # instantiate tokenizer - from verl.utils import hf_tokenizer, hf_processor - tokenizer = hf_tokenizer(local_path) - processor = hf_processor(local_path, use_fast=True) # used for multimodal LLM, could be none - - # define worker classes - if config.actor_rollout_ref.actor.strategy == 'fsdp': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray import RayWorkerGroup - ray_worker_group_cls = RayWorkerGroup - - elif config.actor_rollout_ref.actor.strategy == 'megatron': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup - ray_worker_group_cls = NVMegatronRayWorkerGroup - - else: - raise NotImplementedError - - from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role - - role_worker_mapping = { - Role.ActorRollout: ray.remote(ActorRolloutRefWorker), - Role.Critic: ray.remote(CriticWorker), - Role.RefPolicy: ray.remote(ActorRolloutRefWorker) - } - - global_pool_id = 'global_pool' - resource_pool_spec = { - global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, - } - mapping = { - Role.ActorRollout: global_pool_id, - Role.Critic: global_pool_id, - Role.RefPolicy: global_pool_id, - } - - # we should adopt a multi-source reward function here - # - for rule-based rm, we directly call a reward score - # - for model-based rm, we call a model - # - for code related prompt, we send to a sandbox if there are test cases - # - finally, we combine all the rewards together - # - The reward type depends on the tag of the data - if config.reward_model.enable: - if config.reward_model.strategy == 'fsdp': - from verl.workers.fsdp_workers import RewardModelWorker - elif config.reward_model.strategy == 'megatron': - from verl.workers.megatron_workers import RewardModelWorker - else: - raise NotImplementedError - role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) - mapping[Role.RewardModel] = global_pool_id - - # reference model - if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: - role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) - mapping[Role.RefPolicy] = global_pool_id - - reward_manager_name = config.reward_model.get("reward_manager", "naive") - if reward_manager_name == 'naive': - from verl.workers.reward_manager import NaiveRewardManager - reward_manager_cls = NaiveRewardManager - elif reward_manager_name == 'prime': - from verl.workers.reward_manager import PrimeRewardManager - reward_manager_cls = PrimeRewardManager - elif reward_manager_name == 'dapo': - from verl.workers.reward_manager import DAPORewardManager - reward_manager_cls = DAPORewardManager - else: - - raise NotImplementedError - - compute_score = get_custom_reward_fn(config) - reward_fn = reward_manager_cls(tokenizer=tokenizer, - num_examine=0, - compute_score=compute_score, - reward_fn_key=config.data.reward_fn_key, - max_resp_len=config.data.max_response_length, - overlong_buffer_cfg=config.reward_model.overlong_buffer) - - # Note that we always use function-based RM for validation - val_reward_fn = reward_manager_cls(tokenizer=tokenizer, - num_examine=1, - compute_score=compute_score, - reward_fn_key=config.data.reward_fn_key, - max_resp_len=config.data.max_response_length, - overlong_buffer_cfg=config.reward_model.overlong_buffer) - resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) - - trainer = RayDAPOTrainer(config=config, - tokenizer=tokenizer, - processor=processor, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - ray_worker_group_cls=ray_worker_group_cls, - reward_fn=reward_fn, - val_reward_fn=val_reward_fn) - trainer.init_workers() - trainer.fit() - - -if __name__ == '__main__': - main() diff --git a/recipe/dapo/test_dapo_7b.sh b/recipe/dapo/test_dapo_7b.sh deleted file mode 100644 index f370fffb41..0000000000 --- a/recipe/dapo/test_dapo_7b.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -project_name='DAPO' -exp_name='DAPO-Qwen2.5-7B-Math-Test' - -adv_estimator=grpo - -use_kl_in_reward=False -kl_coef=0.0 -use_kl_loss=False -kl_loss_coef=0.0 - -clip_ratio_low=0.2 -clip_ratio_high=0.28 - -max_prompt_length=$((1024 * 2)) -max_response_length=$((1024 * 2)) -enable_overlong_buffer=True -overlong_buffer_len=512 -overlong_penalty_factor=1.0 - -loss_agg_mode="token-mean" - -enable_filter_groups=True -filter_groups_metric=acc -max_num_gen_batches=10 -train_prompt_bsz=512 -gen_prompt_bsz=$((train_prompt_bsz * 3)) -train_prompt_mini_bsz=32 -n_resp_per_prompt=16 - -# Ray -RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"} -WORKING_DIR=${WORKING_DIR:-"${PWD}"} -RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"} -NNODES=${NNODES:-4} -# Paths -RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"} -MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen2.5-Math-7B"} -CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${project_name}/${exp_name}"} -TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/dapo-math-17k.parquet"} -TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/aime-2024.parquet"} - -# Algorithm -temperature=1.0 -top_p=1.0 -top_k=-1 # 0 for HF rollout, -1 for vLLM rollout - -# Mathematically equivalent -use_dynamic_bsz=True -infer_micro_batch_size=null -train_micro_batch_size=null -offload=False - -ray job submit --no-wait --runtime-env="${RUNTIME_ENV}" \ - --working-dir "${WORKING_DIR}" \ - -- python3 -m recipe.dapo.src.main_dapo \ - data.train_files="${TRAIN_FILE}" \ - data.val_files="${TEST_FILE}" \ - data.prompt_key=prompt \ - data.truncation='left' \ - data.max_prompt_length=${max_prompt_length} \ - data.max_response_length=${max_response_length} \ - data.gen_batch_size=${gen_prompt_bsz} \ - data.train_batch_size=${train_prompt_bsz} \ - data.truncation='left' \ - actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ - actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ - actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ - actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ - actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ - algorithm.adv_estimator=${adv_estimator} \ - algorithm.use_kl_in_reward=${use_kl_in_reward} \ - algorithm.kl_ctrl.kl_coef=${kl_coef} \ - algorithm.filter_groups.enable=${enable_filter_groups} \ - algorithm.filter_groups.metric=${filter_groups_metric} \ - algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.model.path="${MODEL_PATH}" \ - +actor_rollout_ref.model.override_config.attention_dropout=0. \ - +actor_rollout_ref.model.override_config.embd_pdrop=0. \ - +actor_rollout_ref.model.override_config.resid_pdrop=0. \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ - actor_rollout_ref.actor.optim.weight_decay=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ - actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ - actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.grad_clip=1.0 \ - actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ - actor_rollout_ref.actor.loss_agg_mode=True \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ - actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.enable_chunked_prefill=True \ - actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ - actor_rollout_ref.rollout.temperature=${temperature} \ - actor_rollout_ref.rollout.top_p=${top_p} \ - actor_rollout_ref.rollout.top_k="${top_k}" \ - actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ - actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ - actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ - actor_rollout_ref.rollout.val_kwargs.do_sample=True \ - actor_rollout_ref.rollout.val_kwargs.n=1 \ - actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ - actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ - actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ - actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ - reward_model.reward_manager=dapo \ - reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ - reward_model.overlong_buffer.len=${overlong_buffer_len} \ - reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ - trainer.logger=['console','wandb'] \ - trainer.project_name="${project_name}" \ - trainer.experiment_name="${exp_name}" \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes="${NNODES}" \ - trainer.val_before_train=True \ - trainer.test_freq=2 \ - trainer.save_freq=2 \ - trainer.total_epochs=1 \ - trainer.default_local_dir="${CKPTS_DIR}" \ - trainer.resume_mode=disable \ No newline at end of file diff --git a/recipe/prime/__init__.py b/recipe/prime/__init__.py deleted file mode 100644 index b1697c70a4..0000000000 --- a/recipe/prime/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/recipe/prime/config/prime_trainer.yaml b/recipe/prime/config/prime_trainer.yaml deleted file mode 100644 index e5946bfdb3..0000000000 --- a/recipe/prime/config/prime_trainer.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# the prime config will override default ppo_trainer.yaml - -hydra: - searchpath: - - file://verl/trainer/config - -defaults: - - ppo_trainer - - _self_ - -data: - filter_accuracy: True - accuracy_lower_bound: 0.2 - accuracy_upper_bound: 0.8 - oversample_factor: 4.0 # Sample more responses than the batch size. prompts satisfying the filter will be prioritized. - filter_truncate: True - truncation: right - -actor_rollout_ref: - hybrid_engine: True - model: - use_remove_padding: True - rollout: - # number of responses (i.e. num sample times) - n: 4 - actor: - entropy_coeff: 0.001 - -reward_model: - enable: True - strategy: fsdp - model: - ref_path: ${reward_model.model.path} - use_remove_padding: True - tokenizer_path: ${actor_rollout_ref.model.path} - enable_gradient_checkpointing: ${actor_rollout_ref.model.enable_gradient_checkpointing} - ref_type: freeze - fsdp_config: - min_num_params: 0 - param_offload: ${actor_rollout_ref.actor.fsdp_config.param_offload} -# grad_offload: ${actor_rollout_ref.actor.fsdp_config.grad_offload} - optimizer_offload: ${actor_rollout_ref.actor.fsdp_config.optimizer_offload} - update: before # ``before`` for double-forward, ``after`` for single-forward - optim: - lr: 1e-6 - lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null - warmup_style: constant - total_training_steps: -1 # must be overridden by program - weight_decay: 0. - grad_clip: 10.0 - beta_train: 0.05 - loss_type: ce # currently only supports ce loss - prime_granularity: token - prime_norm: batch_norm # batch_norm or none. if set to none, the normalizer is beta_train - mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} - reward_manager: prime - -algorithm: - adv_estimator: rloo - # now supports rloo. it treats different source of reward separately. - kl_ctrl: - type: fixed - kl_coef: 0.000 - reward_gt_coef: 5 - reward_dpo_coef: 5 - -trainer: - project_name: prime - experiment_name: examples - val_before_train: False - balance_batch: False \ No newline at end of file diff --git a/recipe/prime/main_prime.py b/recipe/prime/main_prime.py deleted file mode 100644 index 0bdae571a1..0000000000 --- a/recipe/prime/main_prime.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. -""" -from .prime_ray_trainer import RayPRIMETrainer - -import ray -import hydra - - -@hydra.main(config_path='config', config_name='prime_trainer', version_base=None) -def main(config): - run_prime(config) - - -def run_prime(config, compute_score=None): - if not ray.is_initialized(): - # this is for local ray cluster - ray.init( - runtime_env={'env_vars': { - 'TOKENIZERS_PARALLELISM': 'true', - 'NCCL_DEBUG': 'WARN' - }}, - ) - - ray.get(main_task.remote(config, compute_score)) - - -@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head -def main_task(config, compute_score=None): - from verl.utils.fs import copy_local_path_from_hdfs - # print initial config - from pprint import pprint - from omegaconf import OmegaConf - pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values - OmegaConf.resolve(config) - - # download the checkpoint from hdfs - local_path = copy_local_path_from_hdfs(config.actor_rollout_ref.model.path) - - # instantiate tokenizer - from verl.utils import hf_tokenizer - tokenizer = hf_tokenizer(local_path) - - # define worker classes - if config.actor_rollout_ref.actor.strategy == 'fsdp': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray import RayWorkerGroup - ray_worker_group_cls = RayWorkerGroup - - elif config.actor_rollout_ref.actor.strategy == 'megatron': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup - ray_worker_group_cls = NVMegatronRayWorkerGroup - - else: - raise NotImplementedError - - from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role - - role_worker_mapping = { - Role.ActorRollout: ray.remote(ActorRolloutRefWorker), - } - - global_pool_id = 'global_pool' - resource_pool_spec = { - global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, - } - mapping = { - Role.ActorRollout: global_pool_id, - } - - #use reference model - if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: - role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) - mapping[Role.RefPolicy] = global_pool_id - - if config.reward_model.enable: - from .prime_fsdp_workers import PRIMERewardModelWorker - role_worker_mapping[Role.RewardModel] = ray.remote(PRIMERewardModelWorker) - mapping[Role.RewardModel] = global_pool_id - - reward_manager_name = config.reward_model.get("reward_manager", "naive") - if reward_manager_name == 'naive': - from verl.workers.reward_manager import NaiveRewardManager - reward_manager_cls = NaiveRewardManager - elif reward_manager_name == 'prime': - from verl.workers.reward_manager import PrimeRewardManager - reward_manager_cls = PrimeRewardManager - else: - raise NotImplementedError - reward_fn = reward_manager_cls(tokenizer=tokenizer, num_examine=0, compute_score=compute_score) - - # Note that we always use function-based RM for validation - val_reward_fn = reward_manager_cls(tokenizer=tokenizer, num_examine=1, compute_score=compute_score) - - resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) - - trainer = RayPRIMETrainer(config=config, - tokenizer=tokenizer, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - ray_worker_group_cls=ray_worker_group_cls, - reward_fn=reward_fn, - val_reward_fn=val_reward_fn) - trainer.init_workers() - trainer.fit() - - -if __name__ == '__main__': - main() diff --git a/recipe/prime/prime_core_algos.py b/recipe/prime/prime_core_algos.py deleted file mode 100644 index 699df1aeb9..0000000000 --- a/recipe/prime/prime_core_algos.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import verl -import verl.utils.torch_functional as verl_F - - -def compute_rloo_advantage_return(data: verl.DataProto, response_mask: torch.Tensor, n_samples, config): - # calculate rloo reward on different reward sources, and sum again - def masked_rloo(reward_tensor_original, mask_tensor): - reward_tensor = reward_tensor_original.clone() - reward_tensor[~mask_tensor] = 0 - for start_pos in range(0, reward_tensor.shape[0], n_samples): - cur_rewards_mean = torch.cat([ - reward_tensor[pos:pos + 1][mask_tensor[pos:pos + 1]].mean(dim=0, keepdim=True) - for pos in range(start_pos, start_pos + n_samples) - ], - dim=0) - cur_rewards_sum = cur_rewards_mean.sum() - cur_reward_baseline = cur_rewards_sum / (n_samples - 1) - reward_tensor[start_pos:start_pos + n_samples][ - mask_tensor[start_pos:start_pos + n_samples]] = \ - reward_tensor[start_pos:start_pos + n_samples][ - mask_tensor[start_pos:start_pos + n_samples]] * ( - n_samples / (n_samples - 1)) - cur_reward_baseline - - return reward_tensor - - reward_tensors = [] - - with torch.no_grad(): - - if 'rm_scores' in data.batch.keys() and config.algorithm.reward_dpo_coef != 0.: - reward_tensor = data.batch['rm_scores'] - reward_mask = response_mask.bool() - - reward_tensors.append(masked_rloo(reward_tensor, reward_mask) * config.algorithm.reward_dpo_coef) - - if 'acc' in data.batch.keys() and config.algorithm.reward_gt_coef != 0.: - reward_tensor = torch.zeros_like(response_mask, dtype=torch.float32) - reward_mask = torch.zeros_like(response_mask, dtype=torch.bool) - - prompt_ids = data.batch['prompts'] - prompt_length = prompt_ids.shape[-1] - valid_response_length = data.batch['attention_mask'][:, prompt_length:].sum(-1) - - reward_mask[ - torch.arange(0, valid_response_length.shape[0], dtype=torch.long, device=valid_response_length.device), - valid_response_length - 1] = True - reward_tensor[ - torch.arange(0, valid_response_length.shape[0], dtype=torch.long, device=valid_response_length.device), - valid_response_length - 1] = data.batch['acc'] - - reward_tensors.append(masked_rloo(reward_tensor, reward_mask) * config.algorithm.reward_gt_coef) - - final_reward_tensor = sum(reward_tensors) - - returns = (final_reward_tensor * response_mask).flip(dims=[-1]).cumsum(dim=-1).flip(dims=[-1]) - - advantages = returns.clone() - advantages = verl_F.masked_whiten(advantages, response_mask) - - return advantages, returns - - -def compute_ce_dpo_loss_rm(token_level_scores, acc, response_mask, beta): - cur_scores = ((token_level_scores * response_mask).sum(dim=1) * beta).sigmoid() - cur_dpo_loss = torch.nn.functional.binary_cross_entropy(cur_scores, acc) - return cur_dpo_loss - - -def compute_detach_dpo_loss_rm(token_level_scores, acc, Q_bc, acc_bc, response_mask, beta, bon_mode='none'): - # we always assume that the BoN size equals n_samples - # mode1: use acc as rm - # mode2: use Q as rm - cur_Q = (token_level_scores * response_mask).sum(dim=1) * beta - other_Q = torch.zeros_like(cur_Q) - for i in range(token_level_scores.shape[0]): - if acc[i] > 0: - Q_chosen = Q_bc[i][acc_bc[i] < acc[i]] - else: - Q_chosen = Q_bc[i][acc_bc[i] > acc[i]] - if len(Q_chosen) > 0: - other_Q[i] = Q_chosen.mean() * beta - else: - other_Q[i] = 0 - dpo_loss = -torch.log(torch.sigmoid((cur_Q - other_Q) * ((acc > 0).float() * 2 - 1))) - if bon_mode == 'none': - dpo_loss = dpo_loss.mean() - else: - weight = torch.zeros_like(dpo_loss) - n_samples = acc_bc.shape[1] - if bon_mode == 'bon_rm': - for i in range(token_level_scores.shape[0]): - weight[i] = n_samples * torch.pow((Q_bc[i] * beta <= cur_Q[i]).float().mean(), n_samples - 1) - elif bon_mode == 'bon_acc': - for i in range(token_level_scores.shape[0]): - weight[i] = n_samples * torch.pow((acc_bc[i] <= acc[i]).float().mean(), n_samples - 1) - else: - raise NotImplementedError - dpo_loss = (dpo_loss * weight).sum() - - return dpo_loss - - -def compute_dpo_accuracy(token_level_scores, acc, response_mask, n_samples): - dpo_acc = [] - for start_id in range(0, token_level_scores.shape[0], n_samples): - cur_scores = (token_level_scores[start_id:start_id + n_samples] * - response_mask[start_id:start_id + n_samples]).sum(dim=1) - - def get_upper_triangle(tensor_x): - diff_matrix = tensor_x.unsqueeze(1) - tensor_x.unsqueeze(0) - upper_tri_indices = torch.triu(torch.ones_like(diff_matrix).bool(), diagonal=1) - return diff_matrix[upper_tri_indices] - - cur_acc_diff = get_upper_triangle(acc[start_id:start_id + n_samples]) # in range [-1,1] - cur_score_diff = get_upper_triangle(cur_scores) # in R - cur_score_prediction = (cur_score_diff > 0).float() # in [0,1] - if cur_acc_diff.abs().sum() == 0: - cur_acc = torch.zeros_like(cur_score_prediction[0]) + 0.5 - else: - cur_acc = (((cur_score_diff > 0) == (cur_acc_diff > 0)).float() * - cur_acc_diff.abs()).sum() / cur_acc_diff.abs().sum() - - dpo_acc.append(cur_acc.unsqueeze(0)) - - return torch.cat(dpo_acc, dim=0).mean() - - -def compute_dpo_abs_accuracy(token_level_scores, acc, response_mask, n_samples): - return (torch.sign((token_level_scores * response_mask).sum(dim=-1)) == torch.sign(acc * 2 - 1)).float().mean() diff --git a/recipe/prime/prime_dp_rm.py b/recipe/prime/prime_dp_rm.py deleted file mode 100644 index 16c2d32f40..0000000000 --- a/recipe/prime/prime_dp_rm.py +++ /dev/null @@ -1,344 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Implement a multiprocess PPOCritic -""" -import itertools -from typing import Iterable - -import torch -import torch.distributed -from torch import nn, optim - -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP - -from .prime_core_algos import compute_ce_dpo_loss_rm, compute_detach_dpo_loss_rm -from verl import DataProto -from verl.trainer.ppo import core_algos -from verl.workers.critic import BasePPOCritic -from verl.utils.py_functional import append_to_dict -from verl.utils.torch_functional import masked_mean -from verl.utils.ulysses import ulysses_pad_and_slice_inputs, gather_outpus_and_unpad -from verl.utils.seqlen_balancing import rearrange_micro_batches, get_reverse_idx -import verl.utils.torch_functional as verl_F - -from flash_attn.bert_padding import pad_input, unpad_input, rearrange, index_first_axis - -__all__ = ['DataParallelPRIMERewardModel'] - - -class DataParallelPRIMERewardModel: - - def __init__(self, config, reward_module: nn.Module, ref_module: nn.Module, reward_optimizer: optim.Optimizer): - self.config = config - self.reward_module = reward_module - self.ref_module = ref_module - self.reward_optimizer = reward_optimizer - self.use_remove_padding = self.config.model.get('use_remove_padding', False) - print(f'Reward model use_remove_padding={self.use_remove_padding}') - - self.ulysses_sequence_parallel_size = self.config.get('ulysses_sequence_parallel_size', 1) - - def _forward_micro_batch(self, micro_batch, prompt_length): - from flash_attn.bert_padding import pad_input, unpad_input, index_first_axis, rearrange - from verl.utils.ulysses import ulysses_pad_and_slice_inputs, gather_outpus_and_unpad - - input_ids = micro_batch['input_ids'] - batch_size, seqlen = input_ids.shape - attention_mask = micro_batch['attention_mask'] - position_ids = micro_batch['position_ids'] - - num_actions = micro_batch['input_ids'].shape[-1] - prompt_length - max_positions = micro_batch['attention_mask'][:, prompt_length:].sum(-1) - - if self.use_remove_padding: - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - - # unpad the position_ids to align the rotary - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # for compute the log_prob - input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) - - # pad and slice the inputs if sp > 1 - if self.ulysses_sequence_parallel_size > 1: - input_ids_rmpad, position_ids_rmpad, pad_size = ulysses_pad_and_slice_inputs( - input_ids_rmpad, position_ids_rmpad, sp_size=self.ulysses_sequence_parallel_size) - input_ids_rmpad_rolled, _, _ = ulysses_pad_and_slice_inputs(input_ids_rmpad_rolled, None, - self.ulysses_sequence_parallel_size) - input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) - rm_output_logits = self.reward_module(input_ids=input_ids_rmpad, - attention_mask=None, - position_ids=position_ids_rmpad, - use_cache=False).logits.squeeze( - 0) # copied. I don't really know why there is a squeeze - rm_log_labels = verl_F.logprobs_from_logits(logits=rm_output_logits, labels=input_ids_rmpad_rolled) - if self.ulysses_sequence_parallel_size > 1: - rm_log_labels = gather_outpus_and_unpad(rm_log_labels, gather_dim=0, unpad_dim=0, padding_size=pad_size) - rm_log_labels = pad_input(hidden_states=rm_log_labels.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen).squeeze(-1)[:, -num_actions - 1:-1] - - else: - rm_output_logits = self.reward_module(input_ids=micro_batch['input_ids'], - attention_mask=micro_batch['attention_mask'], - position_ids=micro_batch['position_ids'], - use_cache=False).logits - rm_log_prob = torch.nn.functional.log_softmax(rm_output_logits[:, :-1, :], - dim=-1) # (batch_size, seq_length, vocab_size) - rm_log_labels = rm_log_prob.gather(dim=-1, index=micro_batch['input_ids'][:, 1:].unsqueeze(-1)).squeeze( - -1) # (batch, seq_length) - - if self.ref_module is not None: - # do not have to pad again - with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.bfloat16): - if self.ulysses_sequence_parallel_size > 1 and self.use_remove_padding: - ref_output_logits = self.ref_module(input_ids=input_ids_rmpad, - attention_mask=None, - position_ids=position_ids_rmpad, - use_cache=False).logits.squeeze(0) - ref_log_labels = verl_F.logprobs_from_logits(logits=ref_output_logits, - labels=input_ids_rmpad_rolled) - ref_log_labels = gather_outpus_and_unpad(ref_log_labels, - gather_dim=0, - unpad_dim=0, - padding_size=pad_size) - ref_log_labels = pad_input(hidden_states=ref_log_labels.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen).squeeze(-1)[:, -num_actions - 1:-1] - else: - ref_output_logits = self.ref_module(input_ids=micro_batch['input_ids'], - attention_mask=micro_batch['attention_mask'], - position_ids=micro_batch['position_ids'], - use_cache=False).logits - ref_log_prob = torch.nn.functional.log_softmax(ref_output_logits[:, :-1, :], - dim=-1) # (batch_size, seq_length, vocab_size) - ref_log_labels = ref_log_prob.gather(dim=-1, - index=micro_batch['input_ids'][:, 1:].unsqueeze(-1)).squeeze( - -1) # (batch, seq_length) - else: - ref_log_labels = micro_batch['old_log_probs'] - - ref_log_labels.to(rm_log_labels.dtype) - q = rm_log_labels[:, -num_actions:] - ref_log_labels[:, -num_actions:] # this is actually diff of q - - # trim unnecessary logprobs here - for i in range(micro_batch['input_ids'].shape[0]): - q[i, max_positions[i]:] = 0 - - # reward computation does not need gradient. only q needs - with torch.no_grad(): - - # generalized estimation of r should go before the reward filling. r means process reward for policy model, or the advantage of reward model. - lam = self.config.get('lambda', 0.) - beta = self.config.model.get('beta_train', 0.05) - if lam == 0.: - r = q * beta - else: - # reward coefficient takes no effect here - acc = micro_batch['acc'] - q_ = q * beta - r = torch.zeros_like(q) - lastgaelam = 0 - # change the last token and mask out all paddings to make this process easier if we rely on outcome reward to calculate V - for i in range(q.shape[0]): - if self.config.prime_use_gt: - q_[i, max_positions[i] - 1] = acc[i] - q_[i, :max_positions[i] - 1].sum() - q_[i, max_positions[i]:] = 0 - - for t in reversed(range(num_actions)): - delta = q_[:, t] - lastgaelam = delta + lam * lastgaelam - r[:, t] = lastgaelam - - token_level_score = torch.zeros_like(q) - - if self.config.prime_granularity == 'token': - for i in range(micro_batch['input_ids'].shape[0]): - token_level_score[i, :max_positions[i] - 1] = r[i, :max_positions[i] - 1] - elif self.config.prime_granularity == 'whole': - for i in range(micro_batch['input_ids'].shape[0]): - token_level_score[i, max_positions[i] - 1] = r[i, :max_positions[i]] - else: - raise NotImplementedError - - return token_level_score, q - - def _optimizer_step(self): - assert self.config.model.optim.grad_clip is not None - - if isinstance(self.reward_module, FSDP): - grad_norm = self.reward_module.clip_grad_norm_(self.config.model.optim.grad_clip) - else: - grad_norm = torch.nn.utils.clip_grad_norm_(self.reward_module.parameters(), - max_norm=self.config.model.optim.grad_clip) - self.reward_optimizer.step() - return grad_norm - - def prime_norm(self, token_level_scores): - if self.config.prime_norm == 'batch_norm': - reverse_cumsum = torch.cumsum(token_level_scores.flip(dims=[1]), dim=-1).flip(dims=[1]) - token_level_scores = token_level_scores / (reverse_cumsum.abs().max() + 1e-6) - return token_level_scores - - def compute_rm_score(self, data: DataProto): - self.reward_module.eval() - self.ref_module.eval() - micro_batch_size = data.meta_info['micro_batch_size'] - select_keys = ['responses', 'input_ids', 'attention_mask', 'position_ids', 'acc'] - batch = data.select(batch_keys=select_keys).batch - use_dynamic_bsz = data.meta_info['use_dynamic_bsz'] - prompt_length = data.batch['input_ids'].shape[-1] - data.batch['responses'].shape[-1] - - if use_dynamic_bsz: - # split using dynamic bsz - max_token_len = data.meta_info['max_token_len'] * self.ulysses_sequence_parallel_size - micro_batches, indices = rearrange_micro_batches(batch=batch, max_token_len=max_token_len) - else: - micro_batches = batch.split(micro_batch_size) - - rm_scores_lst = [] - q_lst = [] - for micro_batch in micro_batches: - with torch.no_grad(): - rm_score, q = self._forward_micro_batch(micro_batch, prompt_length) - rm_scores_lst.append(rm_score) - q_lst.append(q) - rm_scores = torch.concat(rm_scores_lst, dim=0) - q = torch.concat(q_lst, dim=0) - - rm_scores = self.prime_norm(rm_scores) - - if use_dynamic_bsz: - indices = list(itertools.chain.from_iterable(indices)) - assert len(indices) == rm_scores.size(0), f"{len(indices)} vs. {rm_scores.size()}" - revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) - rm_scores = rm_scores[revert_indices] - - return rm_scores, q.detach(), { - 'reward_model/reward': rm_scores.sum(dim=-1).mean().item(), - 'reward_model/raw_reward': q.sum(dim=-1).mean().item() - } - - def update_rm(self, data: DataProto): - # make sure we are in training mode - self.reward_module.train() - metrics = {} - - beta = self.config.model.get('beta_train', 0.05) - - select_keys = ['input_ids', 'responses', 'attention_mask', 'position_ids', 'acc', 'prompts'] - - for key in ['Q_bc', 'acc_bc']: - if key in data.batch.keys(): - select_keys.append(key) - - batch = data.select(batch_keys=select_keys).batch - # Split to make minibatch iterator for updating the actor - # See PPO paper for details. https://arxiv.org/abs/1707.06347 - dataloader = batch.split(self.config.mini_batch_size) - - rm_scores_lst = [] - q_lst = [] - - for batch_idx, data in enumerate(dataloader): - # split batch into micro_batches - mini_batch = data - if self.config.use_dynamic_bsz: - max_token_len = self.config.ppo_max_token_len_per_gpu * self.ulysses_sequence_parallel_size - micro_batches, _ = rearrange_micro_batches(batch=mini_batch, max_token_len=max_token_len) - else: - micro_batches = mini_batch.split(self.config.micro_batch_size_per_gpu) - self.gradient_accumulation = self.config.mini_batch_size // self.config.micro_batch_size_per_gpu - - self.reward_optimizer.zero_grad() - - for data in micro_batches: - data = data.cuda() - attention_mask = data['attention_mask'] - acc = data['acc'] - - prompt_ids = data['prompts'] - prompt_length = prompt_ids.shape[-1] - - response_mask = attention_mask[:, prompt_length:] - - rm_score, q = self._forward_micro_batch(data, prompt_length) - - rm_scores_lst.append(rm_score) - q_lst.append(q.detach()) - - if self.config.model.loss_type == 'ce': - dpo_loss = compute_ce_dpo_loss_rm(q, acc, response_mask=response_mask, beta=beta) - elif self.config.model.loss_type == 'dpo': - # the implementation of dpo is actually detached, which means we have to know the average value of w/l reward before the update. - dpo_loss = compute_detach_dpo_loss_rm(q, - acc, - Q_bc=data['Q_bc'], - acc_bc=data['acc_bc'], - response_mask=response_mask, - beta=beta) - elif self.config.model.loss_type == 'bon_acc': - # change the original distribution of each sample to BoN distribution, then update reward model - dpo_loss = compute_detach_dpo_loss_rm(q, - acc, - Q_bc=data['Q_bc'], - acc_bc=data['acc_bc'], - response_mask=response_mask, - beta=beta, - bon_mode='bon_acc') - elif self.config.model.loss_type == 'bon_rm': - dpo_loss = compute_detach_dpo_loss_rm(q, - acc, - Q_bc=data['Q_bc'], - acc_bc=data['acc_bc'], - response_mask=response_mask, - beta=beta, - bon_mode='bon_rm') - else: - raise NotImplementedError - - data = {'reward_model/dpo_loss': dpo_loss.detach().item()} - - if self.config.use_dynamic_bsz: - # relative to the dynamic bsz - loss = dpo_loss * (len(data) / self.config.ppo_mini_batch_size) - else: - loss = dpo_loss / self.gradient_accumulation - - loss.backward() - - append_to_dict(metrics, data) - - grad_norm = self._optimizer_step() - data = {'reward_model/grad_norm': grad_norm.detach().item()} - append_to_dict(metrics, data) - self.reward_optimizer.zero_grad() - - rm_scores = torch.cat(rm_scores_lst, dim=0) - q = torch.concat(q_lst, dim=0) - - rm_scores = self.prime_norm(rm_scores) - - metrics.update({ - 'reward_model/reward': rm_scores.sum(dim=-1).mean().item(), - 'reward_model/raw_reward': q.sum(dim=-1).mean().item() - }) - - return rm_scores, metrics diff --git a/recipe/prime/prime_fsdp_workers.py b/recipe/prime/prime_fsdp_workers.py deleted file mode 100644 index 9a94afc469..0000000000 --- a/recipe/prime/prime_fsdp_workers.py +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import copy -import logging -import os -import warnings - -import torch -import torch.distributed -from torch.distributed.device_mesh import init_device_mesh -import verl.utils.torch_functional as verl_F -from omegaconf import DictConfig, open_dict -from verl import DataProto -from verl.single_controller.base import Worker -from verl.single_controller.base.decorator import register, Dispatch -from verl.utils import hf_tokenizer -from verl.utils.debug import log_gpu_memory_usage -from verl.utils.fs import copy_local_path_from_hdfs -from verl.utils.fsdp_utils import get_fsdp_wrap_policy, init_fn, get_init_weight_context_manager -from verl.utils.fsdp_utils import offload_fsdp_optimizer, offload_fsdp_model_to_cpu, load_fsdp_optimizer, \ - load_fsdp_model_to_gpu -from verl.utils.import_utils import import_external_libs -from verl.utils.model import compute_position_id_with_mask -from verl.utils.flops_counter import FlopsCounter -from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager -from verl.workers.sharding_manager.fsdp_ulysses import FSDPUlyssesShardingManager - -from codetiming import Timer -from verl.workers.fsdp_workers import create_device_mesh, get_sharding_strategy -from .prime_core_algos import compute_dpo_accuracy, compute_dpo_abs_accuracy - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_PPO_LOGGING_LEVEL', 'WARN')) - - -class PRIMERewardModelWorker(Worker): - - def __init__(self, config): - super().__init__() - import torch.distributed - if not torch.distributed.is_initialized(): - torch.distributed.init_process_group(backend="nccl") - self.config = config - - # build device mesh for Ulysses Sequence Parallel - world_size = torch.distributed.get_world_size() - from torch.distributed.device_mesh import init_device_mesh - - fsdp_size = self.config.model.fsdp_config.fsdp_size - self.device_mesh = create_device_mesh(world_size=world_size, fsdp_size=fsdp_size) - - self.ulysses_device_mesh = None - self.ulysses_sequence_parallel_size = self.config.get('ulysses_sequence_parallel_size', 1) - dp = world_size // self.ulysses_sequence_parallel_size - if self.ulysses_sequence_parallel_size > 1: - self.ulysses_device_mesh = init_device_mesh('cuda', - mesh_shape=(dp, self.ulysses_sequence_parallel_size), - mesh_dim_names=['dp', 'sp']) - - self.ulysses_sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) - - # set FSDP offload params - self._is_offload_param = self.config.model.fsdp_config.param_offload - self._is_offload_optimizer = self.config.model.fsdp_config.optimizer_offload - - # normalize config - self.config.mini_batch_size //= (torch.distributed.get_world_size() // self.ulysses_sequence_parallel_size) - if self.config.micro_batch_size is not None: - self.config.micro_batch_size //= (torch.distributed.get_world_size() // self.ulysses_sequence_parallel_size) - self.config.micro_batch_size_per_gpu = self.config.micro_batch_size - assert self.config.mini_batch_size % self.config.micro_batch_size_per_gpu == 0 - - def _build_reward_ref_model_optimizer(self, config): - # the following line is necessary - from verl.utils.model import LambdaLayer, print_model_size, squeeze - from verl.utils.torch_dtypes import PrecisionType - from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy, MixedPrecision - from torch import optim - - local_path = copy_local_path_from_hdfs(config.model.path) - - tokenizer_path = copy_local_path_from_hdfs(config.model.tokenizer_path) - self.tokenizer = hf_tokenizer(tokenizer_path, trust_remote_code=config.model.get('trust_remote_code', False)) - - from omegaconf import OmegaConf - override_config = OmegaConf.to_container(self.config.model.get('override_config', OmegaConf.create())) - override_config_kwargs = { - 'bos_token_id': self.tokenizer.bos_token_id, - 'eos_token_id': self.tokenizer.eos_token_id, - 'pad_token_id': self.tokenizer.pad_token_id, - } - override_config_kwargs.update(override_config) - if self.rank == 0: - print(f'Reward model overriding config {override_config_kwargs}') - - torch_dtype = self.config.model.fsdp_config.get('model_dtype', 'fp32') - torch_dtype = PrecisionType.to_dtype(torch_dtype) - - from transformers import AutoConfig, AutoModelForCausalLM - from torch import nn - - trust_remote_code = False - reward_model_config = AutoConfig.from_pretrained(local_path, trust_remote_code=trust_remote_code) - reward_model_config.num_labels = 1 - - init_context = get_init_weight_context_manager(use_meta_tensor=not reward_model_config.tie_word_embeddings) - with init_context(), warnings.catch_warnings(): - warnings.simplefilter("ignore") - setattr(reward_model_config, 'classifier_dropout', 0.) - setattr(reward_model_config, 'hidden_dropout', '0') - reward_module = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path=local_path, - torch_dtype=torch_dtype, - config=reward_model_config, - attn_implementation='flash_attention_2', - trust_remote_code=trust_remote_code) - - if config.model.get('use_remove_padding', False) or self.ulysses_sequence_parallel_size > 1: - from verl.models.transformers.monkey_patch import apply_monkey_patch - apply_monkey_patch(model=reward_module, ulysses_sp_size=self.ulysses_sequence_parallel_size) - - # some parameters may not in torch_dtype - reward_module.to(torch_dtype) - - if config.model.get('enable_gradient_checkpointing', False): - reward_module.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant': False}) - if self.rank == 0: - print_model_size(reward_module) - - self.reward_model_config = reward_model_config - - fsdp_config = self.config.model.fsdp_config - mixed_precision_config = fsdp_config.get('mixed_precision', None) - if mixed_precision_config is not None: - param_dtype = PrecisionType.to_dtype(mixed_precision_config.get('param_dtype', 'bf16')) - reduce_dtype = PrecisionType.to_dtype(mixed_precision_config.get('reduce_dtype', 'fp32')) - buffer_dtype = PrecisionType.to_dtype(mixed_precision_config.get('buffer_dtype', 'fp32')) - else: - param_dtype = torch.bfloat16 - reduce_dtype = torch.float32 - buffer_dtype = torch.float32 - - mixed_precision = MixedPrecision(param_dtype=param_dtype, reduce_dtype=reduce_dtype, buffer_dtype=buffer_dtype) - - auto_wrap_policy = get_fsdp_wrap_policy(module=reward_module, config=self.config.model.fsdp_config.wrap_policy) - - log_gpu_memory_usage('Before reward model FSDP', logger=None) - - fsdp_mesh = self.device_mesh - sharding_strategy = get_sharding_strategy(fsdp_mesh) - - with init_context(), warnings.catch_warnings(): - warnings.simplefilter("ignore") - setattr(reward_model_config, 'classifier_dropout', 0.) - setattr(reward_model_config, 'hidden_dropout', '0') - ref_module = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path=copy_local_path_from_hdfs( - config.model.ref_path), - torch_dtype=torch_dtype, - config=reward_model_config, - attn_implementation='flash_attention_2', - trust_remote_code=trust_remote_code) - - # some parameters may not in torch_dtype - ref_module.to(torch_dtype) - - reward_module = FSDP(reward_module, - param_init_fn=init_fn, - use_orig_params=False, - auto_wrap_policy=auto_wrap_policy, - device_id=torch.cuda.current_device(), - sharding_strategy=sharding_strategy, - mixed_precision=mixed_precision, - sync_module_states=True, - forward_prefetch=False, - device_mesh=self.device_mesh, - cpu_offload=None) - - log_gpu_memory_usage('After reward FSDP', logger=None) - - ref_module = FSDP(ref_module, - param_init_fn=init_fn, - use_orig_params=False, - auto_wrap_policy=auto_wrap_policy, - device_id=torch.cuda.current_device(), - sharding_strategy=sharding_strategy, - mixed_precision=mixed_precision, - sync_module_states=True, - forward_prefetch=False, - device_mesh=self.device_mesh, - cpu_offload=None) - - reward_optimizer = optim.AdamW(reward_module.parameters(), - lr=config.model.optim.lr, - betas=config.model.optim.get('betas', (0.9, 0.999)), - weight_decay=config.model.optim.get('weight_decay', 1e-2)) - - total_steps = config.model.optim.get('total_training_steps', 0) - num_warmup_steps = int(config.model.optim.get('lr_warmup_steps', -1)) - if num_warmup_steps < 0: - num_warmup_steps_ratio = config.model.optim.get('lr_warmup_steps_ratio', 0.) - num_warmup_steps = int(num_warmup_steps_ratio * total_steps) - - print(f'Total steps: {total_steps}, num_warmup_steps: {num_warmup_steps}') - - from verl.utils.torch_functional import get_constant_schedule_with_warmup - reward_lr_scheduler = get_constant_schedule_with_warmup(optimizer=reward_optimizer, - num_warmup_steps=num_warmup_steps) - - return reward_module, ref_module, reward_optimizer, reward_lr_scheduler - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - # This is used to import external_lib into the huggingface systems - import_external_libs(self.config.model.get('external_lib', None)) - - from .prime_dp_rm import DataParallelPRIMERewardModel - self.reward_module, self.ref_module, self.reward_optimizer, self.reward_lr_scheduler = self._build_reward_ref_model_optimizer( - config=self.config) - - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.reward_module) - offload_fsdp_model_to_cpu(self.ref_module) - if self._is_offload_optimizer: - offload_fsdp_optimizer(optimizer=self.reward_optimizer) - - self.rm = DataParallelPRIMERewardModel(config=self.config, - reward_module=self.reward_module, - ref_module=self.ref_module, - reward_optimizer=self.reward_optimizer) - - self.flops_counter = FlopsCounter(self.reward_model_config) - self.checkpoint_manager = FSDPCheckpointManager(model=self.reward_module, - optimizer=self.reward_optimizer, - lr_scheduler=self.reward_lr_scheduler, - tokenizer=self.tokenizer) - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_rm_score(self, data: DataProto): - data = data.to('cuda') - - if self._is_offload_param: - load_fsdp_model_to_gpu(self.reward_module) - load_fsdp_model_to_gpu(self.ref_module) - micro_batch_size = self.config.micro_batch_size_per_gpu - data.meta_info['micro_batch_size'] = micro_batch_size - data.meta_info['max_token_len'] = self.config.forward_max_token_len_per_gpu - data.meta_info['use_dynamic_bsz'] = self.config.use_dynamic_bsz - # perform forward computation - with self.ulysses_sharding_manager: - data = self.ulysses_sharding_manager.preprocess_data(data=data) - rm_scores, q, metrics = self.rm.compute_rm_score(data=data) - - prompt_length = data.batch['prompts'].shape[-1] - response_mask = data.batch['attention_mask'][:, prompt_length:] - acc = data.batch['acc'] - - dpo_acc = compute_dpo_accuracy(rm_scores, acc, response_mask=response_mask, n_samples=data.meta_info['n']) - dpo_acc_abs = compute_dpo_abs_accuracy(rm_scores, acc, response_mask, n_samples=data.meta_info['n']) - - metrics['reward_model/dpo_acc'] = dpo_acc.detach().item() - metrics['reward_model/dpo_acc_abs'] = dpo_acc_abs.detach().item() - - output = DataProto.from_dict(tensors={'rm_scores': rm_scores, 'q': q}, meta_info={'metrics': metrics}) - output = self.ulysses_sharding_manager.postprocess_data(data=output) - - output = output.to('cpu') - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.reward_module) - offload_fsdp_model_to_cpu(self.ref_module) - return output - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def update_rm(self, data: DataProto): - data = data.to('cuda') - if self._is_offload_param: - load_fsdp_model_to_gpu(self.ref_module) - load_fsdp_model_to_gpu(self.reward_module) - if self._is_offload_optimizer: - load_fsdp_optimizer(optimizer=self.reward_optimizer, device_id=torch.cuda.current_device()) - - # perform forward computation - with self.ulysses_sharding_manager: - data = self.ulysses_sharding_manager.preprocess_data(data=data) - - rm_scores, metrics = self.rm.update_rm(data=data) - - self.reward_lr_scheduler.step() - lr = self.reward_lr_scheduler.get_last_lr()[0] - metrics['rm/lr'] = lr - - prompt_length = data.batch['prompts'].shape[-1] - response_mask = data.batch['attention_mask'][:, prompt_length:] - acc = data.batch['acc'] - - dpo_acc_before = compute_dpo_accuracy(rm_scores, - acc, - response_mask=response_mask, - n_samples=data.meta_info['n']) - dpo_acc_abs = compute_dpo_abs_accuracy(rm_scores, acc, response_mask, n_samples=data.meta_info['n']) - - metrics['reward_model/dpo_acc_before'] = dpo_acc_before.detach().item() - metrics['reward_model/dpo_acc_abs_before'] = dpo_acc_abs.detach().item() - - output = DataProto.from_dict(tensors={'rm_scores': rm_scores}, meta_info={'metrics': metrics}) - output = self.ulysses_sharding_manager.postprocess_data(data=output) - - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.reward_module) - offload_fsdp_model_to_cpu(self.ref_module) - if self._is_offload_optimizer: - offload_fsdp_optimizer(optimizer=self.reward_optimizer) - output = output.to('cpu') - return output - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): - import torch - if self._is_offload_param: - load_fsdp_model_to_gpu(self.reward_module) - - self.checkpoint_manager.save_checkpoint(local_path=local_path, - hdfs_path=hdfs_path, - global_step=global_step, - max_ckpt_to_keep=max_ckpt_to_keep) - - torch.distributed.barrier() - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.reward_module) - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def load_checkpoint(self, local_path, del_local_after_load=True): - import torch - if self._is_offload_param: - load_fsdp_model_to_gpu(self.reward_module) - - self.checkpoint_manager.load_checkpoint(local_path=local_path, del_local_after_load=del_local_after_load) - - torch.distributed.barrier() - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.reward_module) diff --git a/recipe/prime/prime_ray_trainer.py b/recipe/prime/prime_ray_trainer.py deleted file mode 100644 index 27ece75e4b..0000000000 --- a/recipe/prime/prime_ray_trainer.py +++ /dev/null @@ -1,528 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -FSDP PPO Trainer with Ray-based single controller. -This trainer supports model-agonistic model initialization with huggingface -""" - -import os -import statistics -import uuid -from copy import deepcopy -from pprint import pprint - -import numpy as np -import torch -from omegaconf import OmegaConf, open_dict - -from verl import DataProto -from verl.single_controller.ray import RayWorkerGroup -from verl.trainer.ppo.ray_trainer import RayPPOTrainer -from verl.trainer.ppo.ray_trainer import Role, WorkerType, ResourcePoolManager, reduce_metrics, _timer -from verl.trainer.ppo.metric_utils import _compute_response_info -from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path -from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn -from . import prime_core_algos - - -def compute_advantage(data: DataProto, adv_estimator, config): - if adv_estimator == 'rloo': - responses = data.batch['responses'] - response_length = responses.size(-1) - attention_mask = data.batch['attention_mask'] - response_mask = attention_mask[:, -response_length:] - advantages, returns = prime_core_algos.compute_rloo_advantage_return(data, response_mask, - config.actor_rollout_ref.rollout.n, config) - data.batch['advantages'] = advantages - data.batch['returns'] = returns - else: - raise NotImplementedError - return data - - -def compute_data_metrics(batch, use_critic=True): - - advantages = batch.batch['advantages'] - returns = batch.batch['returns'] - - max_response_length = batch.batch['responses'].shape[-1] - - prompt_mask = batch.batch['attention_mask'][:, :-max_response_length].bool() - response_mask = batch.batch['attention_mask'][:, -max_response_length:].bool() - - max_prompt_length = prompt_mask.size(-1) - - response_info = _compute_response_info(batch) - prompt_length = response_info['prompt_length'] - response_length = response_info['response_length'] - - valid_adv = torch.masked_select(advantages, response_mask) - valid_returns = torch.masked_select(returns, response_mask) - - if use_critic: - values = batch.batch['values'] - valid_values = torch.masked_select(values, response_mask) - return_diff_var = torch.var(valid_returns - valid_values) - return_var = torch.var(valid_returns) - - metrics = { - # adv - 'critic/advantages/mean': - torch.mean(valid_adv).detach().item(), - 'critic/advantages/max': - torch.max(valid_adv).detach().item(), - 'critic/advantages/min': - torch.min(valid_adv).detach().item(), - # returns - 'critic/returns/mean': - torch.mean(valid_returns).detach().item(), - 'critic/returns/max': - torch.max(valid_returns).detach().item(), - 'critic/returns/min': - torch.min(valid_returns).detach().item(), - **({ - # values - 'critic/values/mean': torch.mean(valid_values).detach().item(), - 'critic/values/max': torch.max(valid_values).detach().item(), - 'critic/values/min': torch.min(valid_values).detach().item(), - # vf explained var - 'critic/vf_explained_var': (1.0 - return_diff_var / (return_var + 1e-5)).detach().item(), - } if use_critic else {}), - - # response length - 'response_length/mean': - torch.mean(response_length).detach().item(), - 'response_length/max': - torch.max(response_length).detach().item(), - 'response_length/min': - torch.min(response_length).detach().item(), - 'response_length/clip_ratio': - torch.mean(torch.eq(response_length, max_response_length).float()).detach().item(), - # prompt length - 'prompt_length/mean': - torch.mean(prompt_length).detach().item(), - 'prompt_length/max': - torch.max(prompt_length).detach().item(), - 'prompt_length/min': - torch.min(prompt_length).detach().item(), - 'prompt_length/clip_ratio': - torch.mean(torch.eq(prompt_length, max_prompt_length).float()).detach().item(), - } - return metrics - - -def compute_timing_metrics(batch, timing_raw): - response_info = _compute_response_info(batch) - num_prompt_tokens = torch.sum(response_info['prompt_length']).item() - num_response_tokens = torch.sum(response_info['response_length']).item() - num_overall_tokens = num_prompt_tokens + num_response_tokens - - num_tokens_of_section = { - 'gen': num_response_tokens, - **{ - name: num_overall_tokens for name in ['ref', 'values', 'adv', 'update_critic', 'update_actor'] - }, - } - - return { - **{ - f'timing_s/{name}': value for name, value in timing_raw.items() - }, - **{ - f'timing_per_token_ms/{name}': timing_raw[name] * 1000 / num_tokens_of_section[name] for name in set(num_tokens_of_section.keys( - )) & set(timing_raw.keys()) - }, - } - - -class RayPRIMETrainer(RayPPOTrainer): - """ - Note that this trainer runs on the driver process on a single CPU/GPU node. - """ - - # TODO: support each role have individual ray_worker_group_cls, - # i.e., support different backend of different role - def __init__(self, - config, - tokenizer, - role_worker_mapping: dict[Role, WorkerType], - resource_pool_manager: ResourcePoolManager, - ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup, - reward_fn=None, - val_reward_fn=None): - - # assert torch.cuda.is_available(), 'cuda must be available on driver' - - super().__init__(config, tokenizer, role_worker_mapping, resource_pool_manager, ray_worker_group_cls, reward_fn, - val_reward_fn) - - self.use_critic = False - - def _validate_config(self): - super()._validate_config() - # TODO: Additional config checks can be added here - config = self.config - - def _create_dataloader(self): - from torch.utils.data import DataLoader, RandomSampler, SequentialSampler - # TODO: we have to make sure the batch size is divisible by the dp size - self.train_dataset = RLHFDataset(parquet_files=self.config.data.train_files, - tokenizer=self.tokenizer, - prompt_key=self.config.data.prompt_key, - max_prompt_length=self.config.data.max_prompt_length, - filter_prompts=True, - return_raw_chat=self.config.data.get('return_raw_chat', False), - truncation='error', - filter_overlong_prompts=self.config.data.get('filter_overlong_prompts', False), - num_workers=self.config.data.get('filter_overlong_prompts_workers', None)) - # use sampler for better ckpt resume - if self.config.data.shuffle: - train_dataloader_generator = torch.Generator() - train_dataloader_generator.manual_seed(self.config.data.get('seed', 1)) - sampler = RandomSampler(data_source=self.train_dataset, generator=train_dataloader_generator) - else: - sampler = SequentialSampler(data_source=self.train_dataset) - - self.train_dataloader = DataLoader(dataset=self.train_dataset, - batch_size=int(self.config.data.train_batch_size * - self.config.data.oversample_factor), - drop_last=True, - collate_fn=collate_fn, - sampler=sampler) - - self.val_dataset = RLHFDataset(parquet_files=self.config.data.val_files, - tokenizer=self.tokenizer, - prompt_key=self.config.data.prompt_key, - max_prompt_length=self.config.data.max_prompt_length, - filter_prompts=True, - return_raw_chat=self.config.data.get('return_raw_chat', False), - truncation='error', - filter_overlong_prompts=self.config.data.get('filter_overlong_prompts', False), - num_workers=self.config.data.get('filter_overlong_prompts_workers', None)) - self.val_dataloader = DataLoader(dataset=self.val_dataset, - batch_size=len(self.val_dataset), - shuffle=True, - drop_last=True, - collate_fn=collate_fn) - - assert len(self.train_dataloader) >= 1 - assert len(self.val_dataloader) >= 1 - - print(f'Size of train dataloader: {len(self.train_dataloader)}') - print(f'Size of val dataloader: {len(self.val_dataloader)}') - - # inject total_training_steps to actor/critic optim_config. This is hacky. - total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs - - if self.config.trainer.total_training_steps is not None: - total_training_steps = self.config.trainer.total_training_steps - - self.total_training_steps = total_training_steps - print(f'Total training steps: {self.total_training_steps}') - - OmegaConf.set_struct(self.config, True) - with open_dict(self.config): - self.config.actor_rollout_ref.actor.optim.total_training_steps = total_training_steps - self.config.critic.optim.total_training_steps = total_training_steps - - def _save_checkpoint(self): - # path: given_path + `/global_step_{global_steps}` + `/actor` - local_global_step_folder = os.path.join(self.config.trainer.default_local_dir, - f'global_step_{self.global_steps}') - print(f'local_global_step_folder: {local_global_step_folder}') - actor_local_path = os.path.join(local_global_step_folder, 'actor') - - actor_remote_path = None if self.config.trainer.default_hdfs_dir is None else os.path.join( - self.config.trainer.default_hdfs_dir, f'global_step_{self.global_steps}', 'actor') - self.actor_rollout_wg.save_checkpoint(actor_local_path, - actor_remote_path, - self.global_steps, - remove_previous_ckpt=self.config.trainer.remove_previous_ckpt_in_save) - - if self.use_rm: - reward_local_path = os.path.join(local_global_step_folder, 'reward') - reward_remote_path = None if self.config.trainer.default_hdfs_dir is None else os.path.join( - self.config.trainer.default_hdfs_dir, f'global_step_{self.global_steps}', 'reward') - self.rm_wg.save_checkpoint(reward_local_path, - reward_remote_path, - self.global_steps, - remove_previous_ckpt=self.config.trainer.remove_previous_ckpt_in_save) - - # save dataloader - dataloader_local_path = os.path.join(local_global_step_folder, 'data.pt') - import dill - torch.save(self.train_dataloader, dataloader_local_path, pickle_module=dill) - - # latest checkpointed iteration tracker (for atomic usage) - local_latest_checkpointed_iteration = os.path.join(self.config.trainer.default_local_dir, - 'latest_checkpointed_iteration.txt') - with open(local_latest_checkpointed_iteration, 'w') as f: - f.write(str(self.global_steps)) - - def _load_checkpoint(self): - if self.config.trainer.resume_mode == 'disable': - return 0 - - # load from hdfs - if self.config.trainer.default_hdfs_dir is not None: - NotImplementedError('load from hdfs is not implemented yet') - else: - checkpoint_folder = self.config.trainer.default_local_dir # TODO: check path - if not os.path.isabs(checkpoint_folder): - working_dir = os.getcwd() - checkpoint_folder = os.path.join(working_dir, checkpoint_folder) - global_step_folder = find_latest_ckpt_path(checkpoint_folder) # None if no latest - - # find global_step_folder - if self.config.trainer.resume_mode == 'auto': - if global_step_folder is None: - print('Training from scratch') - return 0 - else: - if self.config.trainer.resume_mode == "resume_path": - assert isinstance(self.config.trainer.resume_from_path, str), "resume ckpt must be str type" - assert 'global_step_' in self.config.trainer.resume_from_path, "resume ckpt must specify the global_steps" - global_step_folder = self.config.trainer.resume_from_path - if not os.path.isabs(global_step_folder): - working_dir = os.getcwd() - global_step_folder = os.path.join(working_dir, global_step_folder) - print(f'Load from checkpoint folder: {global_step_folder}') - # set global step - self.global_steps = int(global_step_folder.split('global_step_')[-1]) - - print(f'Setting global step to {self.global_steps}') - print(f'Resuming from {global_step_folder}') - - actor_path = os.path.join(global_step_folder, 'actor') - reward_path = os.path.join(global_step_folder, 'reward') - # load actor - self.actor_rollout_wg.load_checkpoint(actor_path, - del_local_after_load=self.config.trainer.del_local_ckpt_after_load) - # load rm - if self.use_rm: - self.rm_wg.load_checkpoint(reward_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load) - - # load dataloader, - # TODO: from remote not implemented yet - dataloader_local_path = os.path.join(global_step_folder, 'data.pt') - self.train_dataloader = torch.load(dataloader_local_path) - if isinstance(self.train_dataloader.dataset, RLHFDataset): - self.train_dataloader.dataset.resume_dataset_state() - - def fit(self): - """ - The training loop of PPO. - The driver process only need to call the compute functions of the worker group through RPC to construct the PPO dataflow. - The light-weight advantage computation is done on the driver process. - """ - from verl.utils.tracking import Tracking - from omegaconf import OmegaConf - - logger = Tracking(project_name=self.config.trainer.project_name, - experiment_name=self.config.trainer.experiment_name, - default_backend=self.config.trainer.logger, - config=OmegaConf.to_container(self.config, resolve=True)) - - self.global_steps = 0 - - # load checkpoint before doing anything - self._load_checkpoint() - - # perform validation before training - # currently, we only support validation using the reward_function. - if self.val_reward_fn is not None and self.config.trainer.get('val_before_train', True): - val_metrics = self._validate() - pprint(f'Initial validation metrics: {val_metrics}') - logger.log(data=val_metrics, step=self.global_steps) - if self.config.trainer.get('val_only', False): - return - - # we start from step 1 - self.global_steps += 1 - - for epoch in range(self.config.trainer.total_epochs): - for batch_dict in self.train_dataloader: - metrics = {} - timing_raw = {} - - batch: DataProto = DataProto.from_single_dict(batch_dict) - - # pop those keys for generation - gen_batch = batch.pop(batch_keys=['input_ids', 'attention_mask', 'position_ids']) - - with _timer('step', timing_raw): - # generate a batch - with _timer('gen', timing_raw): - gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) - - if self.config.algorithm.adv_estimator == 'remax': - with _timer('gen_max', timing_raw): - gen_baseline_batch = deepcopy(gen_batch) - gen_baseline_batch.meta_info['do_sample'] = False - gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) - - batch = batch.union(gen_baseline_output) - reward_baseline_tensor = self.reward_fn(batch) - reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) - - batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) - - batch.batch['reward_baselines'] = reward_baseline_tensor - - del gen_baseline_batch, gen_baseline_output - - batch.non_tensor_batch['uid'] = np.array([str(uuid.uuid4()) for _ in range(len(batch.batch))], - dtype=object) - # repeat to align with repeated responses in rollout - batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n, interleave=True) - batch = batch.union(gen_batch_output) - - # balance the number of valid tokens on each dp rank. - # Note that this breaks the order of data inside the batch. - # Please take care when you implement group based adv computation such as GRPO and rloo - # self._balance_batch(batch, metrics=metrics) - - # compute global_valid tokens - batch.meta_info['global_token_num'] = torch.sum(batch.batch['attention_mask'], dim=-1).tolist() - - # verify - with _timer('verify', timing_raw): - scores = self.reward_fn.verify(batch) - metrics['acc'] = statistics.mean(scores) - - # filter the batch. 1/oversample_factor samples will be kept. If there is a filter, prompts passing it will be prioritized. - - batch = self.filter_and_downsample(scores, batch) - batch.meta_info['n'] = self.config.actor_rollout_ref.rollout.n - n_samples = self.config.actor_rollout_ref.rollout.n - - # recompute old_log_probs - with _timer('old_log_prob', timing_raw): - old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) - batch = batch.union(old_log_prob) - - if self.use_reference_policy: - # compute reference log_prob - with _timer('ref', timing_raw): - ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) - batch = batch.union(ref_log_prob) - - with _timer('adv', timing_raw): - - if self.use_rm: - update_style = self.config.reward_model.model.get('update', 'none') - if update_style == 'none': # only run forward - reward_output = self.rm_wg.compute_rm_score(batch) - elif update_style == 'after': # update and directly return the reward - reward_output = self.rm_wg.update_rm(batch) - elif update_style == 'before': # update reward model, and then run forward - reward_output = self.rm_wg.update_rm(batch) - if 'metrics' in reward_output.meta_info.keys(): - reward_output_metrics = reduce_metrics(reward_output.meta_info['metrics']) - metrics.update(reward_output_metrics) - - reward_output = self.rm_wg.compute_rm_score(batch) - elif update_style == 'reverse': # run forward to calculate statistics, then update reward model - reward_output = self.rm_wg.compute_rm_score(batch) - # broadcast q and acc tensor to each result - bc_td = DataProto.from_dict( - tensors={ - 'Q_bc': - reward_output.batch['q'].sum(dim=-1).view(-1, n_samples).unsqueeze( - 1).expand(-1, n_samples, -1).reshape(-1, n_samples), - 'acc_bc': - batch.batch['acc'].view(-1, n_samples).unsqueeze(1).expand( - -1, n_samples, -1).reshape(-1, n_samples) - }) - batch = batch.union(bc_td) - reward_output = self.rm_wg.update_rm(batch) - else: - raise NotImplementedError - batch = batch.union(reward_output) - if 'metrics' in reward_output.meta_info.keys(): - reward_output_metrics = reduce_metrics(reward_output.meta_info['metrics']) - metrics.update(reward_output_metrics) - - # compute advantages, executed on the driver process - batch = compute_advantage(batch, - adv_estimator=self.config.algorithm.adv_estimator, - config=self.config) - - # update actor - with _timer('update_actor', timing_raw): - actor_output = self.actor_rollout_wg.update_actor(batch) - actor_output_metrics = reduce_metrics(actor_output.meta_info['metrics']) - metrics.update(actor_output_metrics) - - # validate - if self.val_reward_fn is not None and self.config.trainer.test_freq > 0 and \ - self.global_steps % self.config.trainer.test_freq == 0: - with _timer('testing', timing_raw): - val_metrics: dict = self._validate() - metrics.update(val_metrics) - - if self.config.trainer.save_freq > 0 and \ - self.global_steps % self.config.trainer.save_freq == 0: - with _timer('save_checkpoint', timing_raw): - self._save_checkpoint() - - # collect metrics - metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) - metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) - - # TODO: make a canonical logger that supports various backend - logger.log(data=metrics, step=self.global_steps) - - self.global_steps += 1 - - if self.global_steps >= self.total_training_steps: - - # perform validation after training - if self.val_reward_fn is not None: - val_metrics = self._validate() - pprint(f'Final validation metrics: {val_metrics}') - logger.log(data=val_metrics, step=self.global_steps) - if self.config.trainer.save_freq > 0 and \ - (self.global_steps - 1) % self.config.trainer.save_freq != 0: - with _timer('save_checkpoint', timing_raw): - self._save_checkpoint() - return - - def filter_and_downsample(self, scores, batch: DataProto): - """ - downsample the batch according to oversample_factor - samples passing the filters will be prioritized - """ - n_samples = int(self.config.actor_rollout_ref.rollout.n) - reward_matrix = torch.tensor(scores).reshape(-1, n_samples) - - filter_mask = torch.ones((reward_matrix.shape[0]), dtype=torch.bool) - - if self.config.data.filter_accuracy: - acc_tensor = torch.mean(reward_matrix, dim=-1) - filter_mask[(acc_tensor > self.config.data.accuracy_upper_bound) | - (acc_tensor < self.config.data.accuracy_lower_bound)] = False - - if self.config.data.filter_truncate: - length_matrix = batch.batch['attention_mask'][:, -batch.batch['responses'].shape[-1]:].sum(dim=-1).reshape( - -1, n_samples) - length_tensor = torch.max(length_matrix, dim=-1)[0] - filter_mask[length_tensor >= self.config.data.max_response_length - 1] = False - - reorder_index = torch.argsort(filter_mask, descending=True) - reorder_index = (reorder_index.unsqueeze(-1) * n_samples + torch.arange(0, n_samples).unsqueeze(0)).view(-1) - batch.reorder(reorder_index[:int(len(batch) // - self.config.data.oversample_factor)]) # this operation is inplace - - return batch diff --git a/recipe/prime/run_prime_qwen.sh b/recipe/prime/run_prime_qwen.sh deleted file mode 100644 index 2b68f91661..0000000000 --- a/recipe/prime/run_prime_qwen.sh +++ /dev/null @@ -1,63 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -gsm8k_train_path=$HOME/data/gsm8k/train.parquet -gsm8k_test_path=$HOME/data/gsm8k/test.parquet -math_train_path=$HOME/data/math/train.parquet -math_test_path=$HOME/data/math/test.parquet - -train_files="['$gsm8k_train_path', '$math_train_path']" -test_files="['$gsm8k_test_path', '$math_test_path']" - -model_path=PRIME-RL/Eurus-2-7B-SFT -# model_path=Qwen/Qwen2.5-0.5B-Instruct - -python3 -m recipe.prime.main_prime \ - data.train_files="$train_files" \ - data.val_files="$test_files" \ - data.train_batch_size=64 \ - data.val_batch_size=6312 \ - data.max_prompt_length=1024 \ - data.max_response_length=3072 \ - data.filter_overlong_prompts=True \ - data.filter_accuracy=True \ - data.accuracy_lower_bound=0.2 \ - data.accuracy_upper_bound=0.8 \ - data.oversample_factor=4 \ - actor_rollout_ref.model.path=$model_path \ - actor_rollout_ref.actor.optim.lr=5e-7 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=64 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=True \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.n=4 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ - algorithm.adv_estimator=rloo \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - reward_model.model.path=$model_path \ - reward_model.micro_batch_size_per_gpu=1 \ - reward_model.model.update=before \ - reward_model.model.beta_train=0.05 \ - reward_model.model.optim.lr=1e-6 \ - reward_model.model.optim.grad_clip=10.0 \ - reward_model.model.input_tokenizer=null \ - reward_model.mini_batch_size=64 \ - trainer.val_before_train=False \ - trainer.logger=['console','wandb'] \ - trainer.project_name='prime_example' \ - trainer.experiment_name='Eurus-2-7B-SFT-gsm8k' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=64 \ - trainer.test_freq=64 \ - trainer.total_epochs=15 $@ diff --git a/recipe/r1/__init__.py b/recipe/r1/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/recipe/r1/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/recipe/r1/config/evaluation.yaml b/recipe/r1/config/evaluation.yaml deleted file mode 100644 index d0e2cbd93a..0000000000 --- a/recipe/r1/config/evaluation.yaml +++ /dev/null @@ -1,10 +0,0 @@ -data: - path: /tmp/math_Qwen2-7B-Instruct.parquet - prompt_key: prompt - response_key: responses - data_source_key: data_source - reward_model_key: reward_model - -custom_reward_function: - path: null - name: compute_score diff --git a/recipe/r1/data_process.py b/recipe/r1/data_process.py deleted file mode 100644 index 93107865c0..0000000000 --- a/recipe/r1/data_process.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Preprocess the dataset to parquet format -""" - -import os -from datasets import load_dataset, concatenate_datasets -from functools import partial - -from verl.utils.hdfs_io import copy, makedirs -import argparse - - -def example_map_fn(example, idx, process_fn, data_source, ability, split): - question, solution = process_fn(example) - data = { - "data_source": data_source, - "prompt": [{ - "role": "user", - "content": question - }], - "ability": ability, - "reward_model": { - "style": "rule", - "ground_truth": solution - }, - "extra_info": { - 'split': split, - 'index': idx - } - } - return data - - -def build_aime2024_dataset(): - - def process_aime2024(example): - return example["Problem"], str(example["Answer"]) - - data_source = 'Maxwell-Jia/AIME_2024' - print(f"Loading the {data_source} dataset from huggingface...", flush=True) - dataset = load_dataset(data_source, split="train") - map_fn = partial(example_map_fn, - process_fn=process_aime2024, - data_source=data_source, - ability="English", - split="test") - dataset = dataset.map(map_fn, with_indices=True, remove_columns=dataset.column_names) - return dataset - - -def build_gpqa_dimond_dataset(): - import random - GPQA_QUERY_TEMPLATE = "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\n{Question}\n\nA) {A}\nB) {B}\nC) {C}\nD) {D}" - - def process_gpqa_diamond(example): - choices = [example["Incorrect Answer 1"], example["Incorrect Answer 2"], example["Incorrect Answer 3"]] - random.shuffle(choices) - gold_index = random.randint(0, 3) - choices.insert(gold_index, example["Correct Answer"]) - query_prompt = GPQA_QUERY_TEMPLATE.format(A=choices[0], - B=choices[1], - C=choices[2], - D=choices[3], - Question=example["Question"]) - gold_choice = "ABCD"[gold_index] - return query_prompt, gold_choice - - data_source = 'Idavidrein/gpqa' - print(f"Loading the {data_source} dataset from huggingface...", flush=True) - - dataset = load_dataset(data_source, "gpqa_diamond", split="train") - map_fn = partial(example_map_fn, - process_fn=process_gpqa_diamond, - data_source=data_source, - ability="Math", - split="test") - dataset = dataset.map(map_fn, with_indices=True, remove_columns=dataset.column_names) - return dataset - - -def build_cnmo2024_dataset(): - - def process_cnmo2024(example): - return example["question"], example["answer"] - - data_source = 'opencompass/LiveMathBench' - print(f"Loading the {data_source} dataset from huggingface...", flush=True) - - dataset_en = load_dataset(data_source, "v202412_CNMO_en", split="test") - map_fn_en = partial(example_map_fn, - process_fn=process_cnmo2024, - data_source='opencompass/cnmo2024_en', - ability="Math", - split="test") - dataset_en = dataset_en.map(map_fn_en, with_indices=True, remove_columns=dataset_en.column_names) - - dataset_zh = load_dataset(data_source, "v202412_CNMO_cn", split="test") - map_fn_zh = partial(example_map_fn, - process_fn=process_cnmo2024, - data_source='opencompass/cnmo2024_zh', - ability="Math", - split="test") - dataset_zh = dataset_zh.map(map_fn_zh, with_indices=True, remove_columns=dataset_zh.column_names) - - dataset = concatenate_datasets([dataset_en, dataset_zh]) - return dataset - - -def build_livecodebench_dataset(): - import json, pickle, zlib, base64 - - def process_livecodebench(example): - # Construct Query Prompt - # From https://github.com/LiveCodeBench/LiveCodeBench/blob/998c52d394b836f15fff3b9a29866191108ff81b/lcb_runner/prompts/code_generation.py#L140 - query_prompt = ( - "You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests.\n\n" - f"Question: {example['question_content']}\n\n") - if example["starter_code"]: - query_prompt += ( - "You will use the following starter code to write the solution to the problem and enclose your code within delimiters.\n" - f"```python\n{example['starter_code']}\n```") - else: - query_prompt += ( - "Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows. Ensure that when the python program runs, it reads the inputs, runs the algorithm and writes output to STDOUT." - f"```python\n# YOUR CODE HERE\n```") - - # Construct test cases - public_test_cases = json.loads(example["public_test_cases"]) - try: - private_test_cases = json.loads(example["private_test_cases"]) - except: - private_test_cases = json.loads( - pickle.loads(zlib.decompress(base64.b64decode(example["private_test_cases"].encode("utf-8"))))) - full_test_cases = public_test_cases + private_test_cases - - metadata = json.loads(example["metadata"]) - test_cases = { - "inputs": [t["input"] for t in full_test_cases], - "outputs": [t["output"] for t in full_test_cases], - "fn_name": metadata.get("func_name", None), - } - text_cases_compressed = base64.b64encode(zlib.compress(pickle.dumps(json.dumps(test_cases)))).decode("utf-8") - return query_prompt, text_cases_compressed - - data_source = 'livecodebench/code_generation_lite' - print(f"Loading the {data_source} dataset from huggingface...", flush=True) - dataset = load_dataset(data_source, split="test") - # R1 Evaluation use LiveCodeBench 24.08-25.01 - dataset = dataset.filter(lambda line: "2024-08-00T00:00:00" <= line["contest_date"] < "2025-01-00T00:00:00") - map_fn = partial(example_map_fn, - process_fn=process_livecodebench, - data_source=data_source, - ability="Code", - split="test") - - dataset = dataset.map(map_fn, with_indices=True, remove_columns=dataset.column_names, num_proc=8) - return dataset - - -TASK2DATA = { - "aime2024": build_aime2024_dataset, - "gpqa_diamond": build_gpqa_dimond_dataset, - "cnmo2024": build_cnmo2024_dataset, - "livecodebench": build_livecodebench_dataset, -} -SUPPORTED_TASKS = TASK2DATA.keys() - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--local_dir', default='~/data/r1') - parser.add_argument('--hdfs_dir', default=None) - parser.add_argument('--tasks', default="all") - - args = parser.parse_args() - - if args.tasks.lower() == "all": - args.tasks = SUPPORTED_TASKS - else: - args.tasks = [task.strip() for task in args.tasks.split(',') if task.strip()] - for task in args.tasks: - if task not in SUPPORTED_TASKS: - raise NotImplementedError(f"{task} has not been supported.") - - datasets = [] - for task in args.tasks: - datasets.append(TASK2DATA[task]()) - test_dataset = concatenate_datasets(datasets) - - local_dir = args.local_dir - hdfs_dir = args.hdfs_dir - - test_dataset.to_parquet(os.path.join(local_dir, 'test.parquet')) - - if hdfs_dir is not None: - makedirs(hdfs_dir) - - copy(src=local_dir, dst=hdfs_dir) diff --git a/recipe/r1/main_eval.py b/recipe/r1/main_eval.py deleted file mode 100644 index 40e871cf86..0000000000 --- a/recipe/r1/main_eval.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Offline evaluate the performance of a generated file using reward model and ground truth verifier. -The input is a parquet file that contains N generated sequences and (optional) the ground truth. - -""" - -import hydra -from verl.utils.fs import copy_to_local -import pandas as pd -import numpy as np -from tqdm import tqdm -from collections import defaultdict -import ray - - -def get_custom_reward_fn(config): - import importlib.util, os - - reward_fn_config = config.get("custom_reward_function") or {} - file_path = reward_fn_config.get("path") - if not file_path: - return None - - if not os.path.exists(file_path): - raise FileNotFoundError(f"Reward function file '{file_path}' not found.") - - spec = importlib.util.spec_from_file_location("custom_module", file_path) - module = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(module) - except Exception as e: - raise RuntimeError(f"Error loading module from '{file_path}': {e}") - - function_name = reward_fn_config.get("name") - - if not hasattr(module, function_name): - raise AttributeError(f"Reward function '{function_name}' not found in '{file_path}'.") - - print(f"using customized reward function '{function_name}' from '{file_path}'") - - return getattr(module, function_name) - - -@ray.remote -def process_item(reward_fn, data_source, response_lst, reward_data): - ground_truth = reward_data['ground_truth'] - score_lst = [reward_fn(data_source, r, ground_truth) for r in response_lst] - return data_source, np.mean(score_lst) - - -@hydra.main(config_path='config', config_name='evaluation', version_base=None) -def main(config): - local_path = copy_to_local(config.data.path) - dataset = pd.read_parquet(local_path) - prompts = dataset[config.data.prompt_key] - responses = dataset[config.data.response_key] - data_sources = dataset[config.data.data_source_key] - reward_model_data = dataset[config.data.reward_model_key] - - total = len(dataset) - - # Initialize Ray - if not ray.is_initialized(): - ray.init() - - # evaluate test_score based on data source - data_source_reward = defaultdict(list) - compute_score = get_custom_reward_fn(config) - - # Create remote tasks - remote_tasks = [ - process_item.remote(compute_score, data_sources[i], responses[i], reward_model_data[i]) for i in range(total) - ] - - # Process results as they come in - with tqdm(total=total) as pbar: - while len(remote_tasks) > 0: - # Use ray.wait to get completed tasks - done_ids, remote_tasks = ray.wait(remote_tasks) - for result_id in done_ids: - data_source, score = ray.get(result_id) - data_source_reward[data_source].append(score) - pbar.update(1) - - metric_dict = {} - for data_source, rewards in data_source_reward.items(): - metric_dict[f'test_score/{data_source}'] = np.mean(rewards) - - print(metric_dict) - - -if __name__ == '__main__': - main() diff --git a/recipe/r1/reward_score.py b/recipe/r1/reward_score.py deleted file mode 100644 index 2fe2e28e19..0000000000 --- a/recipe/r1/reward_score.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def reward_func(data_source, solution_str, ground_truth, extra_info=None): - if data_source in ['Maxwell-Jia/AIME_2024', "opencompass/cnmo2024_en", "opencompass/cnmo2024_zh"]: - from recipe.r1.tasks import math - return math.compute_score(solution_str, ground_truth) - elif data_source == 'Idavidrein/gpqa': - from recipe.r1.tasks import gpqa - return gpqa.compute_score(solution_str, ground_truth) - elif data_source in ['livecodebench/code_generation_lite', 'livecodebench/code_generation']: - from recipe.r1.tasks import livecodebench - return livecodebench.compute_score(solution_str, ground_truth) - else: - raise NotImplementedError diff --git a/recipe/r1/run_r1_distill_qwen.sh b/recipe/r1/run_r1_distill_qwen.sh deleted file mode 100644 index a1aa9edccc..0000000000 --- a/recipe/r1/run_r1_distill_qwen.sh +++ /dev/null @@ -1,33 +0,0 @@ -MODEL_PATH=Qwen/DeepSeek-R1-Distill-Qwen-1.5B -DATA_PATH=/workspace/datasets/r1_bench - -# Eval Data Process -python3 -m recipe.r1.data_process \ - --local_dir $DATA_PATH \ - --tasks all - -# Generation -python3 -m verl.trainer.main_generation \ - trainer.nnodes=1 \ - trainer.n_gpus_per_node=8 \ - data.path=$DATA_PATH/test.parquet \ - data.prompt_key=prompt \ - data.batch_size=1024 \ - data.n_samples=8 \ - data.output_path=$DATA_PATH/test-output-8.parquet \ - model.path=$MODEL_PATH \ - rollout.temperature=0.6 \ - rollout.top_p=0.95 \ - rollout.prompt_length=1024 \ - rollout.response_length=32768 \ - rollout.tensor_model_parallel_size=1 \ - rollout.gpu_memory_utilization=0.9 \ - rollout.max_num_batched_tokens=65536 - -# Evaluation -python3 -m recipe.r1.main_eval \ - data.path=$DATA_PATH/test-output-8.parquet \ - data.prompt_key=prompt \ - data.response_key=responses \ - custom_reward_function.path=recipe/r1/reward_score.py \ - custom_reward_function.name=reward_func diff --git a/recipe/r1/tasks/__init__.py b/recipe/r1/tasks/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/recipe/r1/tasks/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/recipe/r1/tasks/gpqa.py b/recipe/r1/tasks/gpqa.py deleted file mode 100644 index 65b37e9166..0000000000 --- a/recipe/r1/tasks/gpqa.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -# Extraction Template from https://github.com/openai/simple-evals/blob/90e3e821cabba2aeb6be651dcb662b253df04225/common.py#L25 -ANSWER_PATTERN_MULTICHOICE = r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?" - - -def compute_score(solution_str, ground_truth) -> float: - match = re.search(ANSWER_PATTERN_MULTICHOICE, solution_str) - extracted_answer = match.group(1) if match else None - score = 1.0 if extracted_answer == ground_truth else 0.0 - return score diff --git a/recipe/r1/tasks/livecodebench.py b/recipe/r1/tasks/livecodebench.py deleted file mode 100644 index 38c26e96fd..0000000000 --- a/recipe/r1/tasks/livecodebench.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import multiprocessing -import json -import pickle -import zlib -import base64 - -# Reuse `run_test` for convenience -from verl.utils.reward_score.prime_code.testing_util import run_test - - -def _temp_run(in_outs, generation, debug, result, metadata_list, timeout): - res, metadata = run_test(in_outs, test=generation, debug=debug, timeout=timeout) - result.append(res) - metadata_list.append(metadata) - - -def check_correctness(in_outs, generation, timeout, debug=True): - """Check correctness of code generation with a global timeout. - The global timeout is to catch some extreme/rare cases not handled by the timeouts - inside `run_test`""" - - manager = multiprocessing.Manager() - result = manager.list() - metadata_list = manager.list() - p = multiprocessing.Process( - target=_temp_run, - args=(in_outs, generation, debug, result, metadata_list, timeout), - ) - p.start() - p.join(timeout=(timeout + 1) * len(in_outs["inputs"]) + 5) - if p.is_alive(): - p.kill() - if not result: - # consider that all tests failed - result = [[-1 for i in range(len(in_outs["inputs"]))]] - if debug: - print(f"global timeout") - return result[0], metadata_list[0] - - -def compute_score(completion, test_cases): - solution = completion.split('```python')[-1].split('```')[0] - - # extract test cases - try: - in_outs = json.loads(test_cases) - except: - in_outs = json.loads(pickle.loads(zlib.decompress(base64.b64decode(test_cases.encode("utf-8"))))) - - success = False - try: - res, metadata = check_correctness(in_outs=in_outs, generation=solution, timeout=6, debug=False) - success = all(map(lambda x: x == True, res)) - except Exception as e: - pass - - return success diff --git a/recipe/r1/tasks/math.py b/recipe/r1/tasks/math.py deleted file mode 100644 index 83a83bb046..0000000000 --- a/recipe/r1/tasks/math.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - from math_verify.metric import math_metric - from math_verify.parser import LatexExtractionConfig, ExprExtractionConfig -except ImportError: - print("To use Math-Verify, please install it first by running `pip install math-verify`.") - - -def compute_score(model_output: str, ground_truth: str) -> bool: - verify_func = math_metric( - gold_extraction_target=(LatexExtractionConfig(),), - pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig()), - ) - ret_score = 0. - - # Wrap the ground truth in \boxed{} format for verification - ground_truth_boxed = "\\boxed{" + ground_truth + "}" - try: - ret_score, _ = verify_func([ground_truth_boxed], [model_output]) - except Exception as e: - pass - - return ret_score diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f9af7396c8..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -# requirements.txt records the full set of dependencies for development -accelerate -codetiming -datasets -dill -flash-attn -hydra-core -liger-kernel -numpy -pandas -peft -pyarrow>=15.0.0 -pybind11 -pylatexenc -ray[data,train,tune,serve] -tensordict==0.5.0 -torchdata==0.11.0 -transformers -vllm<=0.6.3 -wandb \ No newline at end of file diff --git a/requirements_sglang.txt b/requirements_sglang.txt deleted file mode 100644 index 150f9d7c18..0000000000 --- a/requirements_sglang.txt +++ /dev/null @@ -1,22 +0,0 @@ -# requirements.txt records the full set of dependencies for development -accelerate -codetiming -datasets -dill -flash-attn -hydra-core -numpy -pandas -datasets -peft -pyarrow>=15.0.0 -pybind11 -pylatexenc -ray[default]>=2.10 -tensordict<=0.6.2 -torchdata -torchvision -transformers -wandb -sglang[all]==0.4.4.post3 -torch-memory-saver>=0.0.5 \ No newline at end of file diff --git a/scripts/configure-runner-watchdog.sh b/scripts/configure-runner-watchdog.sh new file mode 100644 index 0000000000..983556b2cd --- /dev/null +++ b/scripts/configure-runner-watchdog.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Configure systemd watchdog for GitHub Actions runner service +# This ensures the runner automatically restarts if it crashes or becomes unresponsive +# +# Usage: sudo ./configure-runner-watchdog.sh + +set -euo pipefail + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + +log() { echo -e "${GREEN}[INFO]${NC} $1"; } +error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; } + +# Must run as root +if [ "$EUID" -ne 0 ]; then + error "Please run as root (sudo)" +fi + +# Find the runner service +SERVICE_NAME=$(systemctl list-units --type=service --all | grep "actions.runner" | awk '{print $1}' | head -1) + +if [ -z "$SERVICE_NAME" ]; then + error "Could not find GitHub Actions runner service" +fi + +log "Found runner service: $SERVICE_NAME" + +# Create override directory +OVERRIDE_DIR="/etc/systemd/system/${SERVICE_NAME}.d" +mkdir -p "$OVERRIDE_DIR" + +# Create watchdog override +OVERRIDE_FILE="$OVERRIDE_DIR/override.conf" +cat > "$OVERRIDE_FILE" << 'EOF' +[Service] +# Automatically restart if the service fails +Restart=always +RestartSec=10 + +# Kill process if it doesn't respond within 5 minutes +WatchdogSec=300 + +# Limit restart attempts: 5 times within 10 minutes +StartLimitIntervalSec=600 +StartLimitBurst=5 + +# After hitting limit, wait 30 minutes before allowing restarts again +StartLimitAction=none + +# Send SIGKILL after 90 seconds if graceful shutdown fails +TimeoutStopSec=90 +EOF + +log "Created watchdog configuration: $OVERRIDE_FILE" + +# Reload systemd +log "Reloading systemd..." +systemctl daemon-reload + +# Restart the runner service to apply changes +log "Restarting runner service..." +systemctl restart "$SERVICE_NAME" + +sleep 3 + +# Verify service is running +if systemctl is-active --quiet "$SERVICE_NAME"; then + log "Runner service restarted successfully with watchdog enabled" + echo "" + echo "Watchdog configuration:" + echo " - Auto-restart on failure: YES" + echo " - Restart delay: 10 seconds" + echo " - Watchdog timeout: 5 minutes" + echo " - Max restarts: 5 within 10 minutes" +else + error "Runner service failed to restart" +fi + +# Show current status +echo "" +log "Current service status:" +systemctl status "$SERVICE_NAME" --no-pager diff --git a/scripts/format.sh b/scripts/format.sh deleted file mode 100755 index 4572411bb0..0000000000 --- a/scripts/format.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -pip3 install --upgrade yapf -python3 -m yapf -ir -vv --style ./.style.yapf verl tests examples recipe diff --git a/scripts/model_merger.py b/scripts/model_merger.py deleted file mode 100644 index 06ae644c08..0000000000 --- a/scripts/model_merger.py +++ /dev/null @@ -1,426 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Tuple, Dict -import re -import os -import torch -import argparse -from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForTokenClassification, AutoModelForVision2Seq, AutoTokenizer -from concurrent.futures import ThreadPoolExecutor -from torch.distributed._tensor import DTensor, Shard, Placement -from safetensors.torch import load_file - -try: - from verl.utils.megatron_utils import get_model_checkpoint_path, get_hf_model_checkpoint_path -except (ModuleNotFoundError, ImportError): - get_hf_model_checkpoint_path = None - get_hf_model_checkpoint_path = None - -parser = argparse.ArgumentParser() -parser.add_argument('--backend', type = str, required=True, help="The backend of the model") -parser.add_argument('--tie-word-embedding', action='store_true', help="Whether to tie word embedding weights") -parser.add_argument('--is-value-model', action='store_true', help="Whether the model loaded as value model") -parser.add_argument('--hf_model_path', type = str, required=True, help="The path for the huggingface model") -parser.add_argument('--local_dir', type = str, required=True, help="The path for your saved model. For megatron, point to the base dir of model, rng, optimizer checkpoints, commonly be `config.default_local_dir/global_step_\{global_step\}`.") -parser.add_argument('--target_dir', required=False, default="tmp", type = str, help="The path for the target model") -parser.add_argument("--hf_upload_path", default=False, type = str, help="The path of the huggingface repo to upload") -parser.add_argument("--test", action="store_true", help="test correctness of hf_model") -parser.add_argument("--test_hf_dir", type = str, required=False, help="test correctness of hf_model, , with hf_model in checkpoint.contents") -args = parser.parse_args() -os.makedirs(args.target_dir, exist_ok=True) -if args.test: - assert args.test_hf_dir is not None, f'You must run verl save checkpoint first, with hf_model in checkpoint.contents, and provide the directory here' - -def merge_by_placement(tensors: List[torch.Tensor], placement: Placement): - if placement.is_replicate(): - return tensors[0] - elif placement.is_partial(): - raise NotImplementedError("Partial placement is not supported yet") - elif placement.is_shard(): - return torch.cat(tensors, dim=placement.dim).contiguous() - else: - raise ValueError(f"Unsupported placement: {placement}") - - -def upload_model_to_huggingface(hf_path): - # Push to hugging face - from huggingface_hub import HfApi - api = HfApi() - api.create_repo(repo_id=args.hf_upload_path, private=False, exist_ok=True) - api.upload_folder( - folder_path=hf_path, - repo_id=args.hf_upload_path, - repo_type="model" - ) - - -def convert_fsdp_checkpoints_to_hfmodels(): - local_dir = args.local_dir - - # copy rank zero to find the shape of (dp, fsdp) - rank = 0 - world_size = 0 - for filename in os.listdir(local_dir): - match = re.match(r"model_world_size_(\d+)_rank_0\.pt", filename) - if match: - world_size = match.group(1) - break - assert world_size, "No model file with the proper format" - - state_dict = torch.load(os.path.join(local_dir, f'model_world_size_{world_size}_rank_{rank}.pt'), map_location='cpu') - pivot_key = sorted(list(state_dict.keys()))[0] - weight = state_dict[pivot_key] - assert isinstance(weight, torch.distributed._tensor.DTensor) - # get sharding info - device_mesh = weight.device_mesh - mesh = device_mesh.mesh - mesh_dim_names = device_mesh.mesh_dim_names - - print(f'Got device mesh {mesh}, mesh_dim_names {mesh_dim_names}') - - assert mesh_dim_names in ( - ('fsdp',), - ), f'Unsupported mesh_dim_names {mesh_dim_names}' - - if 'tp' in mesh_dim_names: - # fsdp * tp - total_shards = mesh.shape[-1] * mesh.shape[-2] - mesh_shape = (mesh.shape[-2], mesh.shape[-1]) - else: - # fsdp - total_shards = mesh.shape[-1] - mesh_shape = (mesh.shape[-1],) - - print(f'Processing model shards with {total_shards} {mesh_shape} in total') - - model_state_dict_lst = [] - model_state_dict_lst.append(state_dict) - model_state_dict_lst.extend([""] * (total_shards - 1)) - - def process_one_shard(rank): - model_path = os.path.join(local_dir, f'model_world_size_{world_size}_rank_{rank}.pt') - state_dict = torch.load(model_path, map_location='cpu', weights_only=False) - model_state_dict_lst[rank] = state_dict - return state_dict - - with ThreadPoolExecutor(max_workers=min(32, os.cpu_count())) as executor: - for rank in range(1, total_shards): - executor.submit(process_one_shard, rank) - state_dict = {} - param_placements: Dict[str, List[Placement]] = {} - keys = set(model_state_dict_lst[0].keys()) - for key in keys: - state_dict[key] = [] - for model_state_dict in model_state_dict_lst: - try: - tensor = model_state_dict.pop(key) - except: - print("-"*30) - print(model_state_dict) - if isinstance(tensor, DTensor): - state_dict[key].append(tensor._local_tensor.bfloat16()) - placements = tuple(tensor.placements) - # replicated placement at dp dimension can be discarded - if mesh_dim_names[0] == 'dp': - placements = placements[1:] - if key not in param_placements: - param_placements[key] = placements - else: - assert param_placements[key] == placements - else: - state_dict[key] = tensor.bfloat16() - - del model_state_dict_lst - - for key in sorted(state_dict): - if not isinstance(state_dict[key], list): - print(f"No need to merge key {key}") - continue - # merge shards - placements: Tuple[Shard] = param_placements[key] - if len(mesh_shape) == 1: - # 1-D list, FSDP without TP - assert len(placements) == 1 - shards = state_dict[key] - state_dict[key] = merge_by_placement(shards, placements[0]) - else: - # 2-D list, FSDP + TP - raise NotImplementedError("FSDP + TP is not supported yet") - - print('Writing to local disk') - if args.target_dir is None: - hf_path = os.path.join(local_dir, 'huggingface') - else: - hf_path = args.target_dir - config = AutoConfig.from_pretrained(args.hf_model_path) - tokenizer = AutoTokenizer.from_pretrained(args.hf_model_path) - - if 'ForTokenClassification' in config.architectures[0]: - auto_model = AutoModelForTokenClassification - elif 'ForCausalLM' in config.architectures[0]: - auto_model = AutoModelForCausalLM - elif 'ForConditionalGeneration' in config.architectures[0]: - auto_model = AutoModelForVision2Seq - else: - raise NotImplementedError(f'Unknown architecture {config["architectures"]}') - - with torch.device('meta'): - model = auto_model.from_config(config, torch_dtype=torch.bfloat16) - model.to_empty(device='cpu') - - print(f'Saving model to {hf_path}') - model.save_pretrained(hf_path, state_dict=state_dict) - tokenizer.save_pretrained(hf_path, state_dict=state_dict) - del state_dict - del model - if args.hf_upload_path: - upload_model_to_huggingface(hf_path) - - -def get_tp_pp_rank_from_sharded_dir(sharded_dir): - match = re.match(r"mp_rank_(\d\d)_(\d\d\d)", sharded_dir) - tp_rank = int(match.group(1)) - pp_rank = int(match.group(2)) - return tp_rank, pp_rank - -def check_megatron_checkpoint_path(model_path): - sharded_dirs = sorted(os.listdir(model_path)) - tp_size = 0 - pp_size = 0 - for sharded_dir in sharded_dirs: - match = re.match(r"mp_rank_(\d\d)_(\d\d\d)", sharded_dir) - assert match, f"Invalid sharded dir {sharded_dir}" - assert f"model.pt" in os.listdir(os.path.join(model_path, sharded_dir)), f"model.pt not found in {sharded_dir}" - tp_rank = int(match.group(1)) - pp_rank = int(match.group(2)) - if tp_size < tp_rank + 1: - tp_size = tp_rank + 1 - if pp_size < pp_rank + 1: - pp_size = pp_rank + 1 - return sharded_dirs, tp_size, pp_size - -def convert_megatron_checkpoints_to_hfmodes(): - local_path = args.local_dir - - model_ckpt_path = get_model_checkpoint_path(local_path) - hf_model_ckpt_path = get_hf_model_checkpoint_path(local_path) - sharded_dirs, tp_size, pp_size = check_megatron_checkpoint_path(model_ckpt_path) - mp_size = len(sharded_dirs) - - model_state_dict_lst = [] - for i in range(pp_size): - model_state_dict_lst.append([]) - for j in range(tp_size): - model_state_dict_lst[i].append("") - - print(f'sharded_dirs: {sharded_dirs}, tp_size: {tp_size}, pp_size: {pp_size}, mp_size: {mp_size}') - - def process_one_shard(shard_dir): - model_path = os.path.join(model_ckpt_path, shard_dir, "model.pt") - state_dict = torch.load(model_path, map_location='cpu', weights_only=False) - tp_rank, pp_rank = get_tp_pp_rank_from_sharded_dir(shard_dir) - model_state_dict_lst[pp_rank][tp_rank] = state_dict - - # with ThreadPoolExecutor(max_workers=min(32, os.cpu_count())) as executor: - # for rank in range(1, mp_size): - # executor.submit(process_one_shard, sharded_dirs[rank]) - for sharded_dir in sharded_dirs: - process_one_shard(sharded_dir) - - state_dict = {} - config = AutoConfig.from_pretrained(args.hf_model_path) - if args.test: - ref_state_dict = load_file(os.path.join(args.test_hf_dir, 'model.safetensors')) - - def merge_across_tp(key, tp_data): - if "linear_fc1.weight" in key: - # if the tensor is gate and proj - gate_lst = [] - up_lst = [] - for infer_param in tp_data: - gate, up = infer_param.chunk(2) - gate_lst.append(gate) - up_lst.append(up) - gate = torch.cat(gate_lst, dim=0) - up = torch.cat(up_lst, dim=0) - tp_data = [gate, up] - elif "self_attention.linear_qkv." in key and 'layer_norm' not in key: - # if the tensor is qkv, for each param on tp, split into q, k, v - # concat q, k, v separately. - q_lst = [] - k_lst = [] - v_lst = [] - assert config.num_attention_heads % config.num_key_value_heads == 0 - num_q_per_kv = config.num_attention_heads // config.num_key_value_heads - assert tp_data[0].shape[0] % (num_q_per_kv + 2) == 0 - kv_size_per_tp = tp_data[0].shape[0] // (num_q_per_kv + 2) - split_size = [kv_size_per_tp * num_q_per_kv, kv_size_per_tp, kv_size_per_tp] - for infer_param in tp_data: - num_query_groups_per_partition = config.num_key_value_heads // tp_size - for chunk in infer_param.chunk(num_query_groups_per_partition): - split_size = [ - kv_size_per_tp * num_q_per_kv // num_query_groups_per_partition, - kv_size_per_tp // num_query_groups_per_partition, - kv_size_per_tp // num_query_groups_per_partition - ] - q, k, v = chunk.split(split_size) - q_lst.append(q) - k_lst.append(k) - v_lst.append(v) - q = torch.cat(q_lst, dim=0) - k = torch.cat(k_lst, dim=0) - v = torch.cat(v_lst, dim=0) - - tp_data = [q,k,v] - - elif "layer_norm" in key or "layernorm" in key or "output_layer" in key and args.is_value_model: - tp_data = tp_data[0] - else: - dim = 0 - if "linear_fc2.weight" in key or "self_attention.linear_proj" in key: - dim = 1 - tp_data = torch.cat(tp_data, dim=dim) - - - return tp_data - - vpp_size = len(model_state_dict_lst[0][0]) - layers_cum = 0 - for vpp_rank in range(vpp_size): - for pp_rank in range(pp_size): - layers_handled = 0 - keys = model_state_dict_lst[pp_rank][0][vpp_rank].keys() - for key in keys: - if "extra_state" in key: - continue - if args.tie_word_embedding and ("output_layer" in key): - print(f'skip lm_head and reward_head loading because of tie_word_embeddings') - continue - new_key = key - if "decoder.layers." in key: - local_layer_no = int(key.split('.')[2]) - layers_handled = max(local_layer_no, layers_handled) - global_layer_no = local_layer_no + layers_cum - new_key_list=key.split('.') - new_key_list[2] = str(global_layer_no) - new_key = '.'.join(new_key_list) - - tp_data = [model_state_dict_lst[pp_rank][tp_rank][vpp_rank][key] for tp_rank in range(tp_size)] - merged = merge_across_tp(new_key, tp_data) - if not isinstance(merged,list): - state_dict[new_key] = merged - elif len(merged)==3: - # split qkv - for n,d in zip(['q','k','v'], merged): - state_dict[new_key.replace("linear_qkv",f"linear_{n}")] = d - elif len(merged)==2: - # split gate up - state_dict[new_key.replace("linear_fc1","gate_proj")] = merged[0] - state_dict[new_key.replace("linear_fc1","up_proj")] = merged[1] - layers_cum += layers_handled+1 # zero based - - del model_state_dict_lst - - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("self_attention.linear_qkv.layer_norm_weight", "input_layernorm.weight"), - ("self_attention.linear_qkv.layer_norm_bias", "input_layernorm.bias"), - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", "self_attn.o_proj"), - ("pre_mlp_layernorm", "post_attention_layernorm"), - ("mlp.linear_fc1.layer_norm_weight", "post_attention_layernorm.weight"), - ("mlp.linear_fc1.layer_norm_bias", "post_attention_layernorm.bias"), - ("mlp.linear_fc1", "mlp.gate_up_proj"), - ("mlp.linear_fc2", "mlp.down_proj"), - ("decoder.final_layernorm", "model.norm"), - ("output_layer", "lm_head"), - ("self_attention.linear_q", "self_attn.q_proj"), - ("self_attention.linear_k", "self_attn.k_proj"), - ("self_attention.linear_v", "self_attn.v_proj"), - ] - - if args.test: - - for original_name, loaded_weight in state_dict.items(): - name = _replace_name(original_name, params_mapping) - if not name or name.endswith(".bias") and name not in ref_state_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - if args.tie_word_embedding and "lm_head.weight" in name: - continue - if name not in ref_state_dict: - raise RuntimeError(f'key: {name} not exist in state_dict') - param = ref_state_dict[name] - assert loaded_weight.dtype == param.dtype - torch.testing.assert_close(loaded_weight, param, atol=1e-4, rtol=1e-4) - - print('Writing to local disk') - if args.target_dir is None: - hf_path = os.path.join(args.local_dir, 'huggingface') - else: - hf_path = args.target_dir - - if 'ForTokenClassification' in config.architectures[0]: - auto_model = AutoModelForTokenClassification - elif 'ForCausalLM' in config.architectures[0]: - auto_model = AutoModelForCausalLM - elif 'ForConditionalGeneration' in config.architectures[0]: - auto_model = AutoModelForVision2Seq - else: - raise NotImplementedError(f'Unknown architecture {config["architectures"]}') - - with torch.device('meta'): - model = auto_model.from_config(config, torch_dtype=torch.bfloat16) - model.to_empty(device='cpu') - - print(f'Saving model to {hf_path}') - model.save_pretrained(hf_path, state_dict=state_dict) - del state_dict - del model - if args.hf_upload_path: - upload_model_to_huggingface(hf_path) - - -def _replace_name(megatron_name, name_mapping): - for m_name, v_name in name_mapping: - if m_name not in megatron_name: - continue - if "layers" in megatron_name: # deal with decoder layers - megatron_name = megatron_name.replace("decoder", "model") - megatron_name_list = megatron_name.split(".") - if "layer_norm_weight" in megatron_name_list or "layer_norm_bias" in megatron_name_list: - param_name_list = megatron_name_list[:3] - param_name_list.append(v_name) - param_name = ".".join(param_name_list) - else: - param_name_list = megatron_name_list[:3] - weight_or_bias = megatron_name_list[-1] - param_name_list.append(v_name) - param_name_list.append(weight_or_bias) - param_name = ".".join(param_name_list) - return param_name - else: - param_name = megatron_name.replace(m_name, v_name) - return param_name - -if __name__ == '__main__': - if args.backend == "fsdp": - convert_fsdp_checkpoints_to_hfmodels() - elif args.backend == "megatron": - convert_megatron_checkpoints_to_hfmodes() - else: - raise NotImplementedError(f"{args.backend} not supported") diff --git a/scripts/runner-health-check.sh b/scripts/runner-health-check.sh new file mode 100644 index 0000000000..9498e156bd --- /dev/null +++ b/scripts/runner-health-check.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# Health check script for GitHub Actions self-hosted runners +# Runs via cron to monitor runner health and auto-restart on failure +# +# Install: Add to crontab with: */5 * * * * /opt/runner-health-check.sh +# Requires: SLACK_WEBHOOK_URL environment variable (optional) + +set -euo pipefail + +RUNNER_DIR="${RUNNER_DIR:-$HOME/actions-runner}" +RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" +LOG_FILE="/var/log/runner-health-check.log" +SLACK_WEBHOOK="${SLACK_WEBHOOK_URL:-}" +MAX_RESTART_ATTEMPTS=3 +RESTART_ATTEMPTS_FILE="/tmp/runner_restart_attempts" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | sudo tee -a "$LOG_FILE" >/dev/null +} + +notify_slack() { + local message="$1" + local emoji="${2:-warning}" + if [ -n "$SLACK_WEBHOOK" ]; then + curl -sf -X POST "$SLACK_WEBHOOK" \ + -H 'Content-type: application/json' \ + -d "{\"text\":\":${emoji}: [${RUNNER_NAME}] ${message}\"}" >/dev/null 2>&1 || true + fi +} + +get_restart_attempts() { + if [ -f "$RESTART_ATTEMPTS_FILE" ]; then + cat "$RESTART_ATTEMPTS_FILE" + else + echo "0" + fi +} + +increment_restart_attempts() { + local current=$(get_restart_attempts) + echo $((current + 1)) > "$RESTART_ATTEMPTS_FILE" +} + +reset_restart_attempts() { + echo "0" > "$RESTART_ATTEMPTS_FILE" +} + +check_runner_service() { + # Find the runner service (name varies by runner name) + local service_name=$(systemctl list-units --type=service --all | grep "actions.runner" | awk '{print $1}' | head -1) + + if [ -z "$service_name" ]; then + log "ERROR: Could not find runner service" + return 1 + fi + + if systemctl is-active --quiet "$service_name"; then + return 0 + else + return 1 + fi +} + +check_github_registration() { + # Check if runner is registered as online with GitHub + # Look for "Listening for Jobs" in recent logs (last 5 min) + local recent_logs=$(journalctl -u "actions.runner.*" --since "5 minutes ago" 2>/dev/null) + + # Signs of healthy connection + if echo "$recent_logs" | grep -q "Listening for Jobs\|Running job\|Successfully renew job"; then + return 0 + fi + + # Signs of disconnection + if echo "$recent_logs" | grep -q "Unable to read data from the transport\|SocketException\|Operation canceled"; then + log "Detected GitHub connection errors in logs" + return 1 + fi + + # No recent activity - check if service started recently + local uptime=$(systemctl show "actions.runner.*" --property=ActiveEnterTimestamp 2>/dev/null | cut -d= -f2) + if [ -n "$uptime" ]; then + local uptime_secs=$(( $(date +%s) - $(date -d "$uptime" +%s 2>/dev/null || echo 0) )) + if [ "$uptime_secs" -lt 300 ]; then + # Service started less than 5 min ago, give it time + return 0 + fi + fi + + # No activity for 5+ min - suspicious + log "No GitHub activity detected in last 5 minutes" + return 1 +} + +check_github_connectivity() { + if curl -sf --max-time 10 https://api.github.com >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +check_disk_space() { + local usage=$(df "$RUNNER_DIR" | tail -1 | awk '{print $5}' | sed 's/%//') + if [ "$usage" -gt 90 ]; then + return 1 + fi + return 0 +} + +cleanup_disk() { + log "Disk usage high, running cleanup..." + local before=$(df "$RUNNER_DIR" | tail -1 | awk '{print $5}') + # Clean old SkyPilot logs (keep last 2 days) + find ~/.sky/logs -type f -mtime +2 -delete 2>/dev/null || true + find ~/.sky/logs -type d -empty -delete 2>/dev/null || true + # Clean pip cache + pip cache purge 2>/dev/null || true + # Clean apt cache + sudo apt-get clean 2>/dev/null || true + # Clean tmp files older than 1 day + find /tmp -type f -mtime +1 -not -name "runner_*" -delete 2>/dev/null || true + # Clean old journal logs + sudo journalctl --vacuum-time=2d 2>/dev/null || true + local after=$(df "$RUNNER_DIR" | tail -1 | awk '{print $5}') + log "Disk cleanup: $before -> $after" +} + +restart_runner() { + local service_name=$(systemctl list-units --type=service --all | grep "actions.runner" | awk '{print $1}' | head -1) + log "Restarting runner service: $service_name" + sudo systemctl restart "$service_name" + sleep 5 + + if check_runner_service; then + log "Runner restarted successfully" + reset_restart_attempts + notify_slack "Runner restarted successfully" "white_check_mark" + return 0 + else + log "Runner restart failed" + return 1 + fi +} + +main() { + local issues=() + + # Check 1: Runner service status + if ! check_runner_service; then + log "ALERT: Runner service is not running" + issues+=("service_down") + fi + + # Check 2: GitHub connectivity (network level) + if ! check_github_connectivity; then + log "ALERT: Cannot reach GitHub API" + issues+=("network_issue") + fi + + # Check 3: GitHub registration (runner is actually connected) + if check_runner_service && ! check_github_registration; then + log "ALERT: Runner service running but disconnected from GitHub" + issues+=("github_disconnected") + fi + + # Check 4: Disk space — auto-cleanup if above 90% + if ! check_disk_space; then + log "ALERT: Disk usage above 90%" + issues+=("disk_full") + cleanup_disk + if ! check_disk_space; then + notify_slack "Disk usage still above 90% after cleanup - manual intervention needed" "warning" + else + log "Disk cleanup resolved the issue" + notify_slack "Disk usage was above 90%, auto-cleanup freed space" "white_check_mark" + fi + fi + + # Handle issues + if [[ " ${issues[*]} " =~ " service_down " ]] || [[ " ${issues[*]} " =~ " github_disconnected " ]]; then + local attempts=$(get_restart_attempts) + local issue_type="Service down" + [[ " ${issues[*]} " =~ " github_disconnected " ]] && issue_type="GitHub disconnected" + + if [ "$attempts" -lt "$MAX_RESTART_ATTEMPTS" ]; then + increment_restart_attempts + notify_slack "${issue_type} - attempting restart (attempt $((attempts+1))/$MAX_RESTART_ATTEMPTS)" "rotating_light" + + if restart_runner; then + log "Runner recovered" + else + log "Runner restart attempt failed" + fi + else + notify_slack "CRITICAL: Runner failed after $MAX_RESTART_ATTEMPTS restart attempts - manual intervention needed" "red_circle" + log "CRITICAL: Max restart attempts reached" + fi + elif [ ${#issues[@]} -eq 0 ]; then + # All healthy - reset restart counter + reset_restart_attempts + log "Health check passed" + fi +} + +main "$@" diff --git a/scripts/setup-gha-runner.sh b/scripts/setup-gha-runner.sh new file mode 100755 index 0000000000..2fc63fa2f1 --- /dev/null +++ b/scripts/setup-gha-runner.sh @@ -0,0 +1,230 @@ +#!/bin/bash +# Setup script for GitHub Actions self-hosted runner with SkyPilot +# Usage: ./setup-gha-runner.sh +# +# Prerequisites: +# - Ubuntu 22.04+ EC2 instance +# - Cloud credentials (Lambda, RunPod, Vast, Nebius) available as env vars or files +# +# This script will: +# 1. Install system dependencies +# 2. Install Python and SkyPilot +# 3. Configure cloud credentials +# 4. Configure swap (OOM prevention) +# 5. Install and start GitHub Actions runner + +set -e + +RUNNER_NAME="${1:-fleet-runner-$(hostname)}" +GITHUB_TOKEN="${2:-}" +REPO_URL="https://github.com/fleet-ai/SkyRL" +RUNNER_VERSION="2.321.0" +RUNNER_LABELS="fleet-runner" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { echo -e "${GREEN}[INFO]${NC} $1"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; } + +# Check if running as root +if [ "$EUID" -eq 0 ]; then + error "Do not run as root. Run as a regular user with sudo access." +fi + +# Check for GitHub token +if [ -z "$GITHUB_TOKEN" ]; then + echo "" + echo "GitHub Runner Token required." + echo "Get it from: https://github.com/fleet-ai/SkyRL/settings/actions/runners/new" + echo "" + read -p "Enter GitHub Runner Token: " GITHUB_TOKEN +fi + +log "Setting up GitHub Actions runner: $RUNNER_NAME" + +# 1. Install system dependencies +log "Installing system dependencies..." +sudo apt-get update +sudo apt-get install -y \ + curl \ + git \ + jq \ + python3 \ + python3-pip \ + python3-venv \ + build-essential \ + libssl-dev \ + libffi-dev \ + unzip + +# Install AWS CLI (needed for S3 data validation) +log "Installing AWS CLI..." +curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip +cd /tmp && unzip -q awscliv2.zip && sudo ./aws/install +aws --version + +# 2. Install SkyPilot +log "Installing SkyPilot..." +pip3 install --user "skypilot-nightly[lambda,runpod,vast,nebius,primeintellect,aws]" +# Also install cloud-specific dependencies +pip3 install --user runpod 2>/dev/null || true +pip3 install --user prime 2>/dev/null || true +export PATH="$HOME/.local/bin:$PATH" +echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc + +# Create symlinks so GHA runner service can find sky and runpod +sudo ln -sf "$HOME/.local/bin/sky" /usr/local/bin/sky +sudo ln -sf "$HOME/.local/bin/runpod" /usr/local/bin/runpod +sky --version + +# 3. Configure cloud credentials +log "Configuring cloud credentials..." + +# Lambda Cloud +if [ -n "$LAMBDA_API_KEY" ]; then + mkdir -p ~/.lambda_cloud + echo "api_key = $LAMBDA_API_KEY" > ~/.lambda_cloud/lambda_keys + log "Lambda Cloud configured" +fi + +# RunPod +if [ -n "$RUNPOD_API_KEY" ]; then + pip3 install --user runpod + runpod config "$RUNPOD_API_KEY" || true + log "RunPod configured" +fi + +# Vast.ai +if [ -n "$VAST_API_KEY" ]; then + mkdir -p ~/.config/vastai + echo "$VAST_API_KEY" > ~/.config/vastai/vast_api_key + log "Vast.ai configured" +fi + +# Nebius +if [ -n "$NEBIUS_CREDENTIALS_JSON" ]; then + mkdir -p ~/.nebius + echo "$NEBIUS_CREDENTIALS_JSON" > ~/.nebius/credentials.json + log "Nebius configured" +fi + +# AWS (for S3 trajectory storage) +if [ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY" ]; then + mkdir -p ~/.aws + cat > ~/.aws/credentials << EOF +[default] +aws_access_key_id = $AWS_ACCESS_KEY_ID +aws_secret_access_key = $AWS_SECRET_ACCESS_KEY +EOF + log "AWS configured" +fi + +# Verify SkyPilot can see clouds +log "Verifying cloud access..." +sky check || warn "Some clouds may not be configured" + +# 4. Configure swap (prevents OOM from crashing the instance) +log "Configuring 4GB swap..." +if [ ! -f /swapfile ]; then + sudo fallocate -l 4G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + grep -q swapfile /etc/fstab || echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab + log "Swap configured (4GB)" +else + log "Swap file already exists, ensuring it's active" + sudo swapon /swapfile 2>/dev/null || true +fi + +# 5. Install GitHub Actions runner +log "Installing GitHub Actions runner..." +RUNNER_DIR="$HOME/actions-runner" +mkdir -p "$RUNNER_DIR" +cd "$RUNNER_DIR" + +# Download runner +curl -o actions-runner-linux-x64.tar.gz -L \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" +tar xzf actions-runner-linux-x64.tar.gz +rm actions-runner-linux-x64.tar.gz + +# Configure runner +./config.sh \ + --url "$REPO_URL" \ + --token "$GITHUB_TOKEN" \ + --name "$RUNNER_NAME" \ + --labels "$RUNNER_LABELS" \ + --unattended \ + --replace + +# 6. Install and start as service +log "Installing runner as service..." +sudo ./svc.sh install +sudo ./svc.sh start + +# 7. Configure systemd watchdog for auto-restart +log "Configuring systemd watchdog..." +SERVICE_NAME=$(systemctl list-units --type=service --all | grep "actions.runner" | awk '{print $1}' | head -1) +if [ -n "$SERVICE_NAME" ]; then + OVERRIDE_DIR="/etc/systemd/system/${SERVICE_NAME}.d" + sudo mkdir -p "$OVERRIDE_DIR" + sudo tee "$OVERRIDE_DIR/override.conf" > /dev/null << 'WATCHDOG_EOF' +[Service] +KillMode=control-group +Restart=always +RestartSec=10 +WatchdogSec=300 +StartLimitIntervalSec=600 +StartLimitBurst=5 +TimeoutStopSec=90 +WATCHDOG_EOF + sudo systemctl daemon-reload + log "Watchdog configured: auto-restart enabled" +fi + +# 8. Install health check cron job +log "Installing health check cron job..." +HEALTH_SCRIPT="/opt/runner-health-check.sh" +sudo curl -sf -o "$HEALTH_SCRIPT" \ + "https://raw.githubusercontent.com/fleet-ai/SkyRL/main/scripts/runner-health-check.sh" +sudo chmod +x "$HEALTH_SCRIPT" + +# Add cron job (runs every 5 minutes) +CRON_LINE="*/5 * * * * RUNNER_NAME=$RUNNER_NAME RUNNER_DIR=$RUNNER_DIR $HEALTH_SCRIPT" +(crontab -l 2>/dev/null | grep -v "runner-health-check"; echo "$CRON_LINE") | crontab - +log "Health check cron installed" + +# Create log file +sudo touch /var/log/runner-health-check.log +sudo chmod 644 /var/log/runner-health-check.log + +# Restart service to apply watchdog +sudo systemctl restart "$SERVICE_NAME" 2>/dev/null || true +sleep 3 + +# Verify runner is running +if sudo ./svc.sh status | grep -q "active (running)"; then + log "Runner installed and running successfully!" +else + warn "Runner service may not be running. Check with: sudo ./svc.sh status" +fi + +echo "" +echo "============================================" +echo "Runner setup complete!" +echo "Name: $RUNNER_NAME" +echo "Labels: $RUNNER_LABELS" +echo "Directory: $RUNNER_DIR" +echo "" +echo "Useful commands:" +echo " Check status: sudo $RUNNER_DIR/svc.sh status" +echo " View logs: sudo journalctl -u actions.runner.fleet-ai-SkyRL.$RUNNER_NAME -f" +echo " Stop runner: sudo $RUNNER_DIR/svc.sh stop" +echo " Start runner: sudo $RUNNER_DIR/svc.sh start" +echo "============================================" diff --git a/setup.py b/setup.py deleted file mode 100644 index ebfc1c4c9b..0000000000 --- a/setup.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# setup.py is the fallback installation script when pyproject.toml does not work -from setuptools import setup, find_packages -import os - -version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__))) - -with open(os.path.join(version_folder, 'verl/version/version')) as f: - __version__ = f.read().strip() - -TEST_REQUIRES = ['pytest', 'yapf', 'py-spy'] -# PRIME_REQUIRES = ['pyext'] -GEO_REQUIRES = ['mathruler'] -GPU_REQUIRES = ['liger-kernel', 'flash-attn'] -MATH_REQUIRES = ['math-verify'] # Add math-verify as an optional dependency -VLLM_REQUIRES = ['tensordict<=0.6.2', 'vllm<=0.8.2'] -# SGLANG_REQUIRES = [ -# 'tensordict<=0.6.2', -# 'sglang[all]==0.4.4.post3', -# 'torch-memory-saver>=0.0.5' -# ] - -extras_require = { - 'test': TEST_REQUIRES, - # 'prime': PRIME_REQUIRES, - 'geo': GEO_REQUIRES, - 'gpu': GPU_REQUIRES, - 'math': MATH_REQUIRES, - 'vllm': VLLM_REQUIRES, - # 'sglang': SGLANG_REQUIRES, -} - -from pathlib import Path -this_directory = Path(__file__).parent -long_description = (this_directory / "README.md").read_text() - -setup( - name='verl', - version=__version__, - package_dir={'': '.'}, - packages=find_packages(where='.'), - url='https://github.com/volcengine/verl', - license='Apache 2.0', - author='Bytedance - Seed - MLSys', - author_email='zhangchi.usc1992@bytedance.com, gmsheng@connect.hku.hk', - description='verl: Volcano Engine Reinforcement Learning for LLM', - extras_require=extras_require, - package_data={'': ['version/*'], - 'verl': ['trainer/config/*.yaml'],}, - include_package_data=True, - long_description=long_description, - long_description_content_type='text/markdown' -) \ No newline at end of file diff --git a/skyrl-agent/.env.template b/skyrl-agent/.env.template new file mode 100644 index 0000000000..3637cc7c28 --- /dev/null +++ b/skyrl-agent/.env.template @@ -0,0 +1,41 @@ +VLLM_USE_V1=1 +SKYRL_LD_LIBRARY_PATH_EXPORT=1 # skyrl + efa +FI_PROVIDER=efa +NCCL_DEBUG=INFO # fine to always be on, negligible perf impact +LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libnccl.so:/usr/lib/x86_64-linux-gnu/libstdc++.so.6" +WANDB_API_KEY="" + +# SWE training +ALLHANDS_API_KEY="sycao-sandbox-remote" +SANDBOX_REMOTE_RUNTIME_API_URL="" + +# LLM Judge +OPENAI_API_KEY="" +API_KEY="" +API_BASE=https://api.openai.com/v1 +MATH_LLM_JUDGE_URL="" +STEM_LLM_JUDGE_URL="" +SANDBOX_FUSION_URL="" + +# Web Research +GOOGLE_SEARCH_KEY="" +JINA_API_KEYS="" +WEB_SUMMARY_API_BASE="" +WEB_SUMMARY_MODEL="" +SKYAGENT_WEB_CACHE_DIR="$HOME/data/web_cache" +SEARCH_BLOCKLIST_DOMAINS="huggingface.co,github.com,gitlab.com,chegg.com,coursehero.com,studocu.com,brainly.com,quizlet.com" +SEARCH_BLOCKLIST_KEYWORDS="gpqa,Chemistry-GPQA,gpqa,Chemistry-GPQA,gpqa-diamond" +SEARCH_NEGATIVE_FILTERS=true +WEB_BLOCKLIST_DOMAINS="huggingface.co,github.com,gitlab.com,chegg.com,coursehero.com,studocu.com,brainly.com,quizlet.com" +WEB_BLOCKLIST_KEYWORDS="gpqa,Chemistry-GPQA,gpqa,Chemistry-GPQA,gpqa-diamond" + +# Search-R1 & BrowseComp-Plus +LOCAL_SEARCH_URL="" +LOCAL_SEARCH_TOPK=3 +FAISS_INDEX_PATH="" +FAISS_EMBEDDING_MODEL_NAME="Qwen/Qwen3-Embedding-8B" +FAISS_EMBEDDING_API_URL="" + +# Backend +TINKER_API_KEY="" + diff --git a/skyrl-agent/.gitignore b/skyrl-agent/.gitignore new file mode 100644 index 0000000000..85dc10ee0c --- /dev/null +++ b/skyrl-agent/.gitignore @@ -0,0 +1,11 @@ +__pycache__/ +*.txt +*.pyc +outputs/* +**/*.egg-info/ +*.py[cod] +*.pyo +.venv +*.log +*.wandb +tinker_output* \ No newline at end of file diff --git a/skyrl-agent/README.md b/skyrl-agent/README.md new file mode 100644 index 0000000000..ffac6953cf --- /dev/null +++ b/skyrl-agent/README.md @@ -0,0 +1,78 @@ +
+ +SkyRL-Agent + +Training and evaluating modern AI agents with modular tasks, tools, and backends. + +[![arXiv](https://img.shields.io/badge/arXiv-2511.16108-b31b1b.svg)](https://arxiv.org/pdf/2511.16108) +[![HF Model](https://img.shields.io/badge/HuggingFace-SA--SWE--32B-orange.svg)](https://huggingface.co/NovaSky-AI/SA-SWE-32B) + +
+SkyRL Agent Overview +
+ +
+ +## News 📰✨ + +- 🚀 Initial public release with SWE, MemAgent (step-wise training), and Web Research examples! + +## Why SkyRL-Agent + +- Unified interface for agentic tasks and training backends +- Pluggable tools (browser, search, code execution, finish, etc.) +- Efficient and flexible async dispatching strategies +- Works with OpenAI-compatible serving (vLLM/others), VERL, SkyRL-Train, Tinker. Switch the backend with one line configuration! + +

+ Dispatcher Flow +

+ +## Quickstart + +```bash +git clone --recurse-submodules https://github.com/NovaSky-AI/SkyRL.git +# our working directory +cd skyrl-agent +``` + +Then head to the `examples/` folder to run tasks (training and inference). Each task’s script/YAML documents its own knobs and environment requirements. + +## Results & Profiling (glimpses) + +

+ GPU Utilization +    + Reward Usage Mix +

+ +## Roadmap + +- [ ] OSWorld Integration +- [ ] Simplify SWE agent training code path +- [ ] More training recipes +- [ ] Evaluation harness unification + +## Acknowledgements +Huge thanks to these projects: +- [VERL](https://github.com/volcengine/verl) +- [OpenHands](https://github.com/OpenHands/OpenHands) +- [LocAgent](https://github.com/gersteinlab/LocAgent) +- [R2E-Gym](https://github.com/R2E-Gym/R2E-Gym) +- [Tinker Cookbook](https://github.com/thinking-machines-lab/tinker-cookbook) +- [rLLM](https://github.com/rllm-org/rllm) +- [WebThinker](https://github.com/RUC-NLPIR/WebThinker) +- [WebSailor](https://github.com/Alibaba-NLP/DeepResearch) +- [ARPO](https://github.com/dvlab-research/ARPO) +- [MemAgent](https://github.com/BytedTsinghua-SIA/MemAgent) + +## Citation + +```bibtex +@article{cao2025skyrl, + title={SkyRL-Agent: Efficient RL Training for Multi-turn LLM Agent}, + author={Cao, Shiyi and Li, Dacheng and Zhao, Fangzhou and Yuan, Shuo and Hegde, Sumanth R and Chen, Connor and Ruan, Charlie and Griggs, Tyler and Liu, Shu and Tang, Eric and others}, + journal={arXiv preprint arXiv:2511.16108}, + year={2025} +} +``` \ No newline at end of file diff --git a/skyrl-agent/assets/gpu_util_plot.png b/skyrl-agent/assets/gpu_util_plot.png new file mode 100644 index 0000000000..237ce8ba8f Binary files /dev/null and b/skyrl-agent/assets/gpu_util_plot.png differ diff --git a/skyrl-agent/assets/new-cropped.png b/skyrl-agent/assets/new-cropped.png new file mode 100644 index 0000000000..48b8fac71d Binary files /dev/null and b/skyrl-agent/assets/new-cropped.png differ diff --git a/skyrl-agent/assets/reward_usage_none_combo_plot.png b/skyrl-agent/assets/reward_usage_none_combo_plot.png new file mode 100644 index 0000000000..5c24c70bbd Binary files /dev/null and b/skyrl-agent/assets/reward_usage_none_combo_plot.png differ diff --git a/skyrl-agent/assets/skyrl-agent-dispatch-new.png b/skyrl-agent/assets/skyrl-agent-dispatch-new.png new file mode 100644 index 0000000000..7d7178519d Binary files /dev/null and b/skyrl-agent/assets/skyrl-agent-dispatch-new.png differ diff --git a/skyrl-agent/assets/skyrl-agent-dispatch-svg.svg b/skyrl-agent/assets/skyrl-agent-dispatch-svg.svg new file mode 100644 index 0000000000..43dbb68bdc --- /dev/null +++ b/skyrl-agent/assets/skyrl-agent-dispatch-svg.svg @@ -0,0 +1,3 @@ + + +
INIT 0
INIT 1
INIT 3
Async Batch
INIT 0
LLM 0
Env
INIT 1
Env
LLM 0
Env
INIT 2
LLM 2
LLM 1
LLM 1
LLM 1
INIT 3
Reward 0
Reward 1
LLM 3
Low GPU utilization period
INIT 2
Env
LLM 3
Reward 3
Env
LLM 2
Env
LLM 2
Env
LLM 2
Reward 2
INIT 0
LLM 0
Env
INIT 1
Env
LLM 0
Env
INIT 3
LLM 2
LLM 1
LLM 1
LLM 1
INIT 2
Async Pipeline
Reward 0
Reward 1
LLM 3
Env
LLM 3
Reward 3
Env
LLM 2
Env
LLM 2
Env
LLM 2
Reward 2
INIT 4
INIT 5
LLM 5
LLM 4
LLM 0
LLM 1
Env
Env
LLM 2
Env
LLM 0
LLM 3
Env
Env
LLM 1
LLM 2
Env
LLM 3
Env
Reward 0
LLM 1
Reward 1
LLM 2
LLM 3
Reward 2
Reward 3
Env
LLM 3
LLM 4
Env
Env
LLM 4
Env
...
...
...
Time
Env
LLM 3
LLM 2
Async Batch (Bounded)
 \ No newline at end of file diff --git a/skyrl-agent/assets/skyrl-agent-svg.svg b/skyrl-agent/assets/skyrl-agent-svg.svg new file mode 100644 index 0000000000..f79e1f2ba5 --- /dev/null +++ b/skyrl-agent/assets/skyrl-agent-svg.svg @@ -0,0 +1,3 @@ + + +
observation
@record_transitions
Tool Execution Runtime (K8s/Ray/Remote Server)
llm.generate()
adv_calc(),
forward_backward(), optimizer_step()
post_process()
1
2
3
T4
Task Datasets
T9
T8
T7
Tools
History Message
T5
T6
Runtime Initialization
Agent Run
Reward Calculation
input_ids
llm response
input_ids, response_ids, reward, mask
Agent Loop Dataflow
Training Data Flow
command, args
SkyRL-train, VeRL, Tinker,...
SkyRL-Agent
Agent
Tx
x-th Trajectory
Transition Buffer
T1
T2
T3
 \ No newline at end of file diff --git a/skyrl-agent/data/browsecomp-plus.py b/skyrl-agent/data/browsecomp-plus.py new file mode 100644 index 0000000000..f32a964ff9 --- /dev/null +++ b/skyrl-agent/data/browsecomp-plus.py @@ -0,0 +1,120 @@ +import pandas as pd +import numpy as np +import json +import argparse + + +def convert_browsecomp_plus_to_skyagent(jsonl_file_path, output_path): + """Convert BrowseComp Plus JSONL dataset to SkyRL-Agent format + + Note: We only keep minimal information (question and answer), + dropping gold_docs, negative_docs, and evidence_docs as redundant. + """ + print("Loading BrowseComp Plus dataset...") + + # Load JSONL file line by line + records = [] + with open(jsonl_file_path, "r", encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + try: + data = json.loads(line.strip()) + records.append(data) + if line_num % 100 == 0: + print(f"Loaded {line_num} records...") + except json.JSONDecodeError as e: + print(f"Warning: Skipping malformed line {line_num}: {e}") + continue + + print(f"Loaded {len(records)} records") + + # Convert to SkyRL-Agent format + skyagent_records = [] + + for idx, record in enumerate(records): + # Extract minimal fields only - question and answer + query_id = record.get("query_id", str(idx)) + question = record.get("query", "") + answer = record.get("answer", "") + + # Create SkyRL-Agent record with minimal information + skyagent_record = { + "id": f"browsecomp_plus_{query_id}", + "question": question, # Copied but ignored by SkyRL-Agent + "golden_answers": ( + [answer] if isinstance(answer, str) else list(answer) + ), # Copied but ignored by SkyRL-Agent + "data_source": "browsecomp_plus", + "prompt": [{"content": f"Answer the given question: {question}", "role": "user"}], + "ability": "fact-reasoning", + "reward_model": { + "ground_truth": {"target": np.array([answer]) if isinstance(answer, str) else np.array(answer)}, + "style": "rule", + }, + "extra_info": { + "original_query_id": query_id, + # Dropping: gold_docs, negative_docs, evidence_docs (redundant) + }, + "metadata": None, + } + + skyagent_records.append(skyagent_record) + + if (idx + 1) % 1000 == 0: + print(f"Processed {idx + 1} records...") + + # Convert to DataFrame + skyagent_df = pd.DataFrame(skyagent_records) + + print(f"Conversion complete! Shape: {skyagent_df.shape}") + print(f"Columns: {list(skyagent_df.columns)}") + + # Save to parquet + skyagent_df.to_parquet(output_path, index=False) + print(f"Saved to {output_path}") + + # Show sample + print("\n" + "=" * 80) + print("Sample Converted Record (first record):") + print("=" * 80) + sample = skyagent_df.iloc[0] + for col in skyagent_df.columns: + print(f"\n{col}:") + if col in ["prompt", "reward_model", "extra_info"]: + print(f" {sample[col]}") + else: + val = str(sample[col]) + if len(val) > 200: + print(f" {val[:200]}...") + else: + print(f" {val}") + + # Show statistics + print("\n" + "=" * 80) + print("Dataset Statistics:") + print("=" * 80) + print(f"Total records: {len(skyagent_df)}") + print("Data source: browsecomp_plus") + print(f"Average question length: {skyagent_df['question'].str.len().mean():.1f} characters") + print( + f"Average answer length: {skyagent_df['golden_answers'].apply(lambda x: len(str(x[0])) if x else 0).mean():.1f} characters" + ) + + return skyagent_df + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert BrowseComp Plus JSONL dataset to SkyRL-Agent parquet format") + parser.add_argument("-i", "--input", required=True, help="Path to the BrowseComp Plus JSONL file") + parser.add_argument("-o", "--output", required=True, help="Path to the output parquet file") + args = parser.parse_args() + jsonl_file_path = args.input + output_path = args.output + + print("BrowseComp Plus to SkyRL-Agent Converter") + print("=" * 80) + print("Note: Only keeping minimal information (question + answer)") + print("Dropping: gold_docs, negative_docs, evidence_docs (redundant)") + print("=" * 80) + print() + + convert_browsecomp_plus_to_skyagent(jsonl_file_path, output_path) diff --git a/skyrl-agent/data/deep_research.py b/skyrl-agent/data/deep_research.py new file mode 100644 index 0000000000..f09393d568 --- /dev/null +++ b/skyrl-agent/data/deep_research.py @@ -0,0 +1,49 @@ +import argparse +import os +import shutil +from pathlib import Path +from typing import List, Tuple + +from huggingface_hub import hf_hub_download + + +# (remote path in repo, desired local filename) +FILES: List[Tuple[str, str]] = [ + ("train/train.parquet", "textbook_stem_5k_with_instance.parquet"), + ("validation/validation.parquet", "hle_webthinker_converted.parquet"), +] + + +def download_files(repo: str, output_dir: str, files: List[Tuple[str, str]]) -> None: + os.makedirs(output_dir, exist_ok=True) + for remote_path, local_name in files: + local_tmp = hf_hub_download( + repo_id=repo, + filename=remote_path, + repo_type="dataset", + local_dir=None, + ) + target = os.path.join(output_dir, local_name) + shutil.copyfile(local_tmp, target) + print(f"Saved {remote_path} -> {target}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download Deep Research parquet files.") + parser.add_argument( + "--repo", + default="NovaSky-AI/Deep-Research", + help="Hugging Face dataset repo id.", + ) + parser.add_argument( + "--output-dir", + default=str(Path(__file__).resolve().parent / "DeepResearch-Data"), + help="Destination directory for parquet files (defaults next to this script).", + ) + args = parser.parse_args() + + download_files(args.repo, args.output_dir, FILES) + + +if __name__ == "__main__": + main() diff --git a/skyrl-agent/data/memagent.py b/skyrl-agent/data/memagent.py new file mode 100644 index 0000000000..4fa0777848 --- /dev/null +++ b/skyrl-agent/data/memagent.py @@ -0,0 +1,276 @@ +""" +Unified converter for MemAgent-style datasets. +Generates SkyRL-Agent formatted training and evaluation parquet files. + +Default behavior (hardcoded sources like original scripts): + - Train URL: + https://huggingface.co/datasets/BytedTsinghua-SIA/hotpotqa/resolve/main/hotpotqa_train_32k.parquet + - Eval URLs: + https://huggingface.co/datasets/BytedTsinghua-SIA/hotpotqa/resolve/main/eval_{num_docs}.json + for num_docs in [100, 200, 400, 800, 1600, 3200, 6400, 12800, 25600] + +Run to produce BOTH train and eval outputs into a directory: + python data/memagent.py --output-dir ./outputs +""" + +import argparse +import json +import os +import re +from multiprocessing import Pool, cpu_count +from typing import Any, Dict, Tuple + +import numpy as np +import pandas as pd +import requests +from transformers import AutoTokenizer +from pathlib import Path + +# Global tokenizer for multiprocessing +tokenizer = None + + +def init_tokenizer(): + """Initialize tokenizer in each worker process.""" + global tokenizer + if tokenizer is None: + print("Loading Qwen tokenizer in worker process...") + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B", trust_remote_code=True) + + +def _np_to_list_inplace(record: Dict[str, Any]) -> Dict[str, Any]: + for key, value in list(record.items()): + if isinstance(value, np.ndarray): + record[key] = value.tolist() + return record + + +# ========== Optional chunking utility (currently unused but kept) ========== +def chunk_documents_by_tokens(context: str, max_tokens: int = 4000) -> list[str]: + """Split context by 'Document X:' and group up to max_tokens using tokenizer lengths.""" + global tokenizer + if tokenizer is None: + init_tokenizer() + + doc_pattern = r"(Document \d+:)" + parts = re.split(doc_pattern, context or "") + + documents = [] + for i in range(1, len(parts), 2): + if i + 1 < len(parts): + doc = parts[i] + "\n" + parts[i + 1] + documents.append(doc.strip()) + elif i < len(parts): + documents.append(parts[i].strip()) + + chunks: list[str] = [] + current_chunk: list[str] = [] + current_tokens = 0 + + for doc in documents: + doc_tokens = len(tokenizer.encode(doc)) + if current_tokens + doc_tokens > max_tokens and current_chunk: + chunks.append("\n\n".join(current_chunk)) + current_chunk = [doc] + current_tokens = doc_tokens + else: + current_chunk.append(doc) + current_tokens += doc_tokens + + if current_chunk: + chunks.append("\n\n".join(current_chunk)) + return chunks + + +# ========== Training conversion ========== +def _process_train_row(args: Tuple[int, pd.Series]) -> Dict[str, Any]: + idx, row = args + try: + extra_info = row.get("extra_info", {}) if isinstance(row, dict) else row["extra_info"] + except Exception: + extra_info = {} + original_index = extra_info.get("index", idx) if isinstance(extra_info, dict) else idx + + try: + reward_model = row.get("reward_model") if isinstance(row, dict) else row["reward_model"] + except Exception: + reward_model = {} + ground_truth = (reward_model or {}).get("ground_truth", []) + + question = "" + try: + if isinstance(extra_info, dict): + question = extra_info.get("question", "") + except Exception: + pass + + try: + data_source = ( + ("ruler_" + str(row.get("data_source"))) if isinstance(row, dict) else ("ruler_" + str(row["data_source"])) + ) + except Exception: + data_source = "ruler_hotpotqa" + + try: + prompt = row.get("prompt") if isinstance(row, dict) else row["prompt"] + except Exception: + prompt = [{"role": "user", "content": question}] + + try: + ability = row.get("ability") if isinstance(row, dict) else row["ability"] + except Exception: + ability = "memory" + + try: + context = row.get("context") if isinstance(row, dict) else row["context"] + except Exception: + context = "" + + record = { + "id": f"ruler_hotpotqa_{original_index}", + "question": question, + "golden_answers": ground_truth, + "data_source": data_source, + "prompt": prompt, + "ability": ability, + "reward_model": reward_model, + "context": context, + "extra_info": extra_info, + "metadata": None, + } + return _np_to_list_inplace(record) + + +def convert_train_to_skyagent(input_parquet: str, output_parquet: str, processes: int | None = None) -> pd.DataFrame: + """Convert training parquet to SkyRL-Agent format parquet.""" + print(f"Loading training dataset from: {input_parquet}") + df = pd.read_parquet(input_parquet) + print(f"Loaded: shape={df.shape}, columns={list(df.columns)}") + + rows_data = [(idx, row) for idx, row in df.iterrows()] + nproc = processes or cpu_count() + print(f"Processing with {nproc} worker processes...") + + with Pool(processes=nproc, initializer=init_tokenizer) as pool: + sky_records: list[Dict[str, Any]] = [] + for i, rec in enumerate(pool.imap(_process_train_row, rows_data, chunksize=10)): + sky_records.append(rec) + if (i + 1) % 100 == 0: + print(f"Processed {i + 1:,} / {len(df):,} records ({(i+1)/len(df)*100:.1f}%)") + + sky_df = pd.DataFrame(sky_records) + print(f"Converted training: shape={sky_df.shape}") + _save_parquet(sky_df, output_parquet) + return sky_df + + +# ========== Eval conversion ========== +def _process_eval_record(args: Tuple[int, Dict[str, Any]]) -> Dict[str, Any]: + idx, record = args + original_index = record.get("index", idx) + question = record.get("input", "") + answers = record.get("answers", []) + context = record.get("context", "") + num_docs = record.get("num_docs", 0) + + prompt = [{"content": question, "role": "user"}] + reward_model = {"ground_truth": answers, "type": "exact_match"} + extra_info = {"question": question, "index": original_index, "num_docs": num_docs} + + sky = { + "id": f"ruler_hotpotqa_{original_index}", + "question": question, + "golden_answers": answers, + "data_source": "ruler_hotpotqa", + "prompt": prompt, + "ability": "memory", + "reward_model": reward_model, + "context": context, + "extra_info": extra_info, + "metadata": None, + } + return _np_to_list_inplace(sky) + + +def _load_eval_json(input_path: str) -> list[Dict[str, Any]]: + """Load eval JSON from local path or HTTP URL; tolerate /blob/ -> /resolve/.""" + if input_path.startswith("http://") or input_path.startswith("https://"): + url = input_path.replace("/blob/", "/resolve/") + print(f"Downloading eval JSON: {url}") + resp = requests.get(url) + resp.raise_for_status() + data = resp.json() + else: + with open(input_path, "r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, list): + data = [data] + return data + + +def convert_eval_to_skyagent(input_json: str, output_parquet: str, processes: int | None = None) -> pd.DataFrame: + """Convert evaluation JSON to SkyRL-Agent format parquet.""" + data = _load_eval_json(input_json) + print(f"Loaded eval records: {len(data):,}") + if data: + print(f"Sample eval keys: {list(data[0].keys())}") + + records_data = [(idx, rec) for idx, rec in enumerate(data)] + nproc = processes or cpu_count() + print(f"Processing with {nproc} worker processes...") + + with Pool(processes=nproc, initializer=init_tokenizer) as pool: + sky_records: list[Dict[str, Any]] = [] + for i, rec in enumerate(pool.imap(_process_eval_record, records_data, chunksize=10)): + sky_records.append(rec) + if (i + 1) % 100 == 0: + print(f"Processed {i + 1:,} / {len(data):,} records ({(i+1)/len(data)*100:.1f}%)") + + sky_df = pd.DataFrame(sky_records) + print(f"Converted eval: shape={sky_df.shape}") + _save_parquet(sky_df, output_parquet) + return sky_df + + +# ========== IO helpers ========== +def _save_parquet(df: pd.DataFrame, output_parquet: str) -> None: + os.makedirs(os.path.dirname(output_parquet) or ".", exist_ok=True) + df.to_parquet(output_parquet, index=False, engine="pyarrow", row_group_size=len(df)) + print(f"Saved to {output_parquet}") + # quick verify + df2 = pd.read_parquet(output_parquet) + print(f"Verified load: shape={df2.shape}") + + +# ========== CLI ========== +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="MemAgent data converter to SkyRL-Agent format (train + eval)") + p.add_argument("--output-dir", required=True, help="Directory to write train and eval parquet files") + p.add_argument("--processes", type=int, default=None, help="Number of worker processes (default: cpu_count)") + p.add_argument("--skip-train", action="store_true", help="Skip train conversion") + p.add_argument("--skip-eval", action="store_true", help="Skip eval conversion") + return p.parse_args() + + +def main(): + args = parse_args() + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + # Hardcoded sources as in the original scripts + train_url = "https://huggingface.co/datasets/BytedTsinghua-SIA/hotpotqa/resolve/main/hotpotqa_train_32k.parquet" + eval_sizes = [100, 200, 400, 800, 1600, 3200, 6400, 12800, 25600] + + if not args.skip_train: + train_out = str(out_dir / "ruler_hotpotqa_train_32k_skyagent.parquet") + convert_train_to_skyagent(train_url, train_out, processes=args.processes) + + if not args.skip_eval: + for num_docs in eval_sizes: + input_json = f"https://huggingface.co/datasets/BytedTsinghua-SIA/hotpotqa/resolve/main/eval_{num_docs}.json" + output_parquet = str(out_dir / f"ruler_hotpotqa_eval_{num_docs}_skyagent.parquet") + convert_eval_to_skyagent(input_json, output_parquet, processes=args.processes) + + +if __name__ == "__main__": + main() diff --git a/skyrl-agent/data/search-r1.py b/skyrl-agent/data/search-r1.py new file mode 100644 index 0000000000..e050848e16 --- /dev/null +++ b/skyrl-agent/data/search-r1.py @@ -0,0 +1,79 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the nq dataset to parquet format +""" + +import os +import datasets + +import argparse + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--local_dir", default="./data/nq_search") + parser.add_argument("--hdfs_dir", default=None) + parser.add_argument("--template_type", type=str, default="base") + + args = parser.parse_args() + + data_source = "nq" + + dataset = datasets.load_dataset("RUC-NLPIR/FlashRAG_datasets", "nq") + + train_dataset = dataset["train"] + test_dataset = dataset["test"] + + # add a row to each data item that represents a unique id + def make_map_fn(split): + + def process_fn(example, idx): + example["question"] = example["question"].strip() + if example["question"][-1] != "?": + example["question"] += "?" + question = example["question"] + solution = { + "target": example["golden_answers"], + } + + data = { + "data_source": data_source, + "question": question, + "golden_answers": example["golden_answers"], + "prompt": [ + { + "role": "user", + "content": question, + } + ], + "ability": "fact-reasoning", + "reward_model": {"style": "rule", "ground_truth": solution}, + "extra_info": { + "split": split, + "index": idx, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) + + local_dir = args.local_dir + hdfs_dir = args.hdfs_dir + + train_dataset.to_parquet(os.path.join(local_dir, "train.parquet")) + test_dataset.to_parquet(os.path.join(local_dir, "test.parquet")) diff --git a/skyrl-agent/data/swe_data.py b/skyrl-agent/data/swe_data.py new file mode 100644 index 0000000000..bc1a9106d5 --- /dev/null +++ b/skyrl-agent/data/swe_data.py @@ -0,0 +1,75 @@ +import os +from datasets import load_dataset, Dataset +import argparse +from tqdm import tqdm +from collections import defaultdict + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=str, default="R2E-Gym/R2E-Gym-Subset") + parser.add_argument("--output", type=str, default="dataset/r2e-all") + args = parser.parse_args() + + def extract_func_part(instance_id: str) -> str: + parts = instance_id.split(".") + if len(parts) >= 3: + func_part = parts[2].split("__")[0] + return func_part + return None + + method_set = set() + + input_dataset = load_dataset(args.input, trust_remote_code=True) + image_name_count = defaultdict(int) + mapping = {"docker_image": "instance_id", "repo_name": "repo", "commit_hash": "base_commit"} + for split in ["train"]: + output_dataset = [] + cur_dataset = input_dataset[split] + idx = -1 + for data_entry in tqdm(cur_dataset): + for old_key, new_key in mapping.items(): + if old_key in data_entry: + data_entry[new_key] = data_entry.pop(old_key) + + if not data_entry["problem_statement"]: + continue + + image_name = data_entry["repo"] + + cur_data = { + "prompt": data_entry["problem_statement"], + "data_source": "r2e-gym", + "ability": "coding", + "instance": data_entry, + } + + image_name_count[image_name] += 1 + output_dataset.append(cur_data) + + print(f"Selected {len(output_dataset)} instances out of {len(cur_dataset)}") + + output_dataset = Dataset.from_list(output_dataset) + + os.makedirs(args.output, exist_ok=True) + output_dataset.to_parquet(os.path.join(args.output, f"{split}.parquet")) + + # For validation set, keep the original logic + input_dataset_name = "princeton-nlp/SWE-bench_Verified" + input_dataset = load_dataset(input_dataset_name) + for split in ["validation"]: + output_dataset = [] + cur_dataset = input_dataset["test"] + for data_entry in tqdm(cur_dataset): + cur_data = { + "prompt": data_entry["problem_statement"], + "data_source": "swe-bench", + "ability": "coding", + "instance": data_entry, + } + output_dataset.append(cur_data) + + print(f"Selected {len(output_dataset)} validation instances") + output_dataset = Dataset.from_list(output_dataset) + + output_dataset.to_parquet(os.path.join(args.output, f"{split}.parquet")) diff --git a/skyrl-agent/examples/README.md b/skyrl-agent/examples/README.md new file mode 100644 index 0000000000..e520a85dac --- /dev/null +++ b/skyrl-agent/examples/README.md @@ -0,0 +1,114 @@ +## Examples + +This directory contains runnable examples for multiple tasks. Each section outlines how to set up any required services, prepare datasets, and launch training or inference. + +### 1) SWE Training + +- Setup remote runtime/server: + - Refer to the [SkyRL-OpenHands](https://github.com/NovaSky-AI/SkyRL-OpenHands) documentation to set up a remote sandbox server and cache train/eval images. + - After setup, configure your the remote server URL and API key in the environment file (i.e., `.env`). + +- Prepare dataset: + - Run the dataset preparation script: + ```bash + python ./data/swe_data.py --output SWE_DATA_PATH + ``` + +- Launch training (modify the corresponding path in the script first): + - VERL-based: + ```bash + bash ./examples/run_verl/verl_oh.sh + ``` + - SkyRL-Train-based: + ```bash + bash ./examples/run_skyrl/skyrl_swe.sh + ``` + +- Run inference (demo): +Launch an OpenAI API-compatible serving (e.g., vLLM or similar), then configure the `api_url` in the corresponding YAML (typically under the backend config) to point to your serving endpoint. Then run: + ```bash + python ./examples/run_openai/test_vllm_oh_demo.py + ``` + +### 2) MemAgent Training + +- Prepare dataset: + ```bash + python ./data/memagent.py --output-dir MEM_DATA_DIR + ``` + +- Configure API: + - Set the OpenAI API key in the environment file (i.e., `.env`); by default we use GPT-5-nano as the LLM judge for reward calculation. + +- Backend note: + - MemAgent currently supports only the Tinker backend for step-wise training. Set your Tinker API key in the environment file (i.e., `.env`). + +- Launch training (modify the corresponding path in the script first): + ```bash + bash ./examples/run_tinker/tinker_memagent.sh + ``` + +### 3) Deep Research (web_research_hle.sh) + +- Quick setup: `uv venv && uv sync`. + +- Required `.env`: `WANDB_API_KEY`, `GOOGLE_SEARCH_KEY` (Serper key), `JINA_API_KEYS`, `WEB_SUMMARY_API_BASE`, `WEB_SUMMARY_MODEL` (e.g., `Qwen/Qwen3-32B`), `SKYAGENT_WEB_CACHE_DIR`, `STEM_LLM_JUDGE_URL`; optional blocklists. + +- Dataset: + ```bash + python ./data/deep_research.py --output-dir DR_DATA_DIR + ``` + +- Web summary (required): + - Point `WEB_SUMMARY_API_BASE` to your remote OpenAI-compatible endpoint (e.g., `http://host:port/v1`). + - Keep the model name in `WEB_SUMMARY_MODEL`. +- Optional router (for load-balancing/failover): + ```bash + SUMMARY_UPSTREAMS=http://host:port/v1 \ + SUMMARY_MODEL=Qwen/Qwen3-32B \ + PORT=8080 \ + bash services/run_router.sh + ``` + then set `WEB_SUMMARY_API_BASE=http://:8080/v1`. + +- Optional: `TRAIN_OUTPUT_DIR`, `ROLLOUT_DIR`, `VAL_ROLLOUT_DIR` for storage paths. + +- Run: + ```bash + bash ./examples/run_verl/web_research_hle.sh + ``` + +### 4) OSWorld + +Placeholder for now. + +### 5) BrowseComp-Plus (Dense Retrieval) + +- Prepare dataset/index. First download the decrypted dataset following [official instruction](https://github.com/texttron/BrowseComp-Plus?tab=readme-ov-file#-downloading-the-dataset). Then run: + ```bash + python ./data/browsecomp-plus.py --input DECRYPTED_JSON_PATH --output BC_DATA + ``` +- Download Pre-built Index for `Qwen/Qwen3-Embedding-8B`: + ```bash + huggingface-cli download Tevatron/browsecomp-plus-indexes --repo-type=dataset --include="qwen3-embedding-8b/*" --local-dir FAISS_INDEX_PATH + ``` + +- Serve embedding model: + - Start an OpenAI-compatible embedding server using `Qwen/Qwen3-Embedding-8B` as the embedding model. For example: + ```bash + vllm serve Qwen/Qwen3-Embedding-8B \ + --port 8000 \ + --task embed \ + --max-model-len 8192 \ + --tensor-parallel-size 1 \ + --dtype float16 + ``` + - Configure your `.env` with: + - `FAISS_EMBEDDING_API_URL` (embedding server base URL) + - `FAISS_EMBEDDING_MODEL_NAME` (e.g., `Qwen/Qwen3-Embedding-8B` model name used by your server) + - `FAISS_INDEX_PATH` (file path to the downloaded pre-built index) + +- Launch eval (modify the corresponding path in the script first): + ```bash + bash ./examples/run_verl/verl_browsecomp.sh + ``` diff --git a/skyrl-agent/examples/run_openai/openai_react.yaml b/skyrl-agent/examples/run_openai/openai_react.yaml new file mode 100644 index 0000000000..0c44325ea5 --- /dev/null +++ b/skyrl-agent/examples/run_openai/openai_react.yaml @@ -0,0 +1,42 @@ +agent_cls: skyrl_agent.agents.react.ReActAgent + +task: skyrl_agent.tasks.general_react.utils.GeneralReactTask + +tools: ["finish", "faiss_search"] + + +data: + data_source_key: data_source + +generator: + infer_backend: openai_server + backend_config: + model_name: Qwen/Qwen3-32B + api_url: http://localhost:8000 + model_max_len: 40960 + num_trajectories: 1 + max_iterations: 30 + max_prompt_length: 32768 + sampling_params: + temperature: 0.5 + top_p: 0.95 + max_tokens: 32768 + val_config: + num_trajectories: 1 + sampling_params: + temperature: 1 + top_p: 0.95 + max_tokens: 32768 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: true + qwen3_acc_thinking: true + early_step_threshold: 7 + enable_turn_reminder: true + +dispatcher: + type: async_batch + scheduler: naive + max_parallel_agents: 1 + max_eval_parallel_agents: 1 + diff --git a/skyrl-agent/examples/run_openai/run_openai_react.py b/skyrl-agent/examples/run_openai/run_openai_react.py new file mode 100644 index 0000000000..433152425f --- /dev/null +++ b/skyrl-agent/examples/run_openai/run_openai_react.py @@ -0,0 +1,27 @@ +import os +from skyrl_agent import AutoAgentRunner +from pathlib import Path +from transformers import AutoTokenizer +import datasets +import asyncio + +os.environ["OPENAI_API_KEY"] = "sc" # dummy key, assumes an unath'ed vLLM service running locally +model = "Qwen/Qwen3-32B" + +tokenizer = AutoTokenizer.from_pretrained(model) +dataset = "datasets/browsecomp-plus/browsecomp-plus-skyagent.parquet" +# read a few samples from the dataset +dataset = datasets.load_dataset("parquet", data_files=dataset)["train"].select(range(10)) +print(dataset[0]) + +yaml_path = str(Path(__file__).parent / "openai_react.yaml") + +agent_generator = AutoAgentRunner.from_task( + yaml_path, + # no explicit inference engine with OpenAI + infer_engine=None, + tokenizer=tokenizer, +) + +output = asyncio.run(agent_generator.run(dataset, val_mode=True)) +print(output["rewards"]) diff --git a/skyrl-agent/examples/run_openai/test_vllm_oh.yaml b/skyrl-agent/examples/run_openai/test_vllm_oh.yaml new file mode 100644 index 0000000000..9521e4cc97 --- /dev/null +++ b/skyrl-agent/examples/run_openai/test_vllm_oh.yaml @@ -0,0 +1,53 @@ +agent_cls: skyrl_agent.agents.oh_codeact.OHCodeActAgent + +task: skyrl_agent.tasks.swebench.utils.SWEBenchTask + +tools: + enable_bash_tool: false + enable_browsing: false + enable_llm_editor: false + enable_editor: true + enable_jupyter: false + enable_cmd: true + enable_think: false + enable_finish: true + enable_mcp: false + enable_search: true + +data: + instance_key: instance + instance_id_key: instance.instance_id + data_source_key: data_source + +generator: + infer_backend: openai_server + backend_config: + # TODO: model_name need not be in config + model_name: Qwen/Qwen3-32B + api_url: http://localhost:8000 + model_max_len: 40960 + num_trajectories: 1 + max_iterations: 50 + max_prompt_length: 32768 + sampling_params: + temperature: 1 + top_p: 1 + max_tokens: 32768 + # not needed for inference + val_config: + num_trajectories: 1 + sampling_params: + temperature: 1 + top_p: 0.95 + max_tokens: 32768 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: true + qwen3_acc_thinking: true + +dispatcher: + type: async_pipeline + scheduler: naive + max_parallel_agents: 32 + max_eval_parallel_agents: 32 + diff --git a/skyrl-agent/examples/run_openai/test_vllm_oh_demo.py b/skyrl-agent/examples/run_openai/test_vllm_oh_demo.py new file mode 100644 index 0000000000..47402e2a27 --- /dev/null +++ b/skyrl-agent/examples/run_openai/test_vllm_oh_demo.py @@ -0,0 +1,33 @@ +from skyrl_agent import AutoAgentRunner +from transformers import AutoTokenizer +from datasets import load_dataset +import asyncio +import os + +DOCKER_IMAGE_PREFIX = os.environ.get("EVAL_DOCKER_IMAGE_PREFIX", "xingyaoww/") + + +def get_instance_docker_image(instance_id) -> str: + image_name = "sweb.eval.x86_64." + instance_id + image_name = image_name.replace("__", "_s_") # to comply with docker image naming convention + name = (DOCKER_IMAGE_PREFIX.rstrip("/") + "/" + image_name).lower() + return name + + +os.environ["OPENAI_API_KEY"] = "sc" +model = "Qwen/Qwen3-32B" +tokenizer = AutoTokenizer.from_pretrained(model) +dataset_file = "/data/sycao/r2e-all/train.parquet" +dataset = load_dataset("parquet", data_files=dataset_file)["train"] +print(f"Loaded dataset with {len(dataset)} instances") + +agent_generator = AutoAgentRunner.from_task("./examples/test_vllm_oh.yaml", infer_engine=None, tokenizer=tokenizer) + +output = asyncio.run(agent_generator.run(dataset, val_mode=True)) + +print(output["rewards"]) +print(output["rollout_metrics"]) +print("Output keys: ", list(output.keys())) + + +print("Average reward: ", sum(output["rewards"]) / len(output["rewards"])) diff --git a/skyrl-agent/examples/run_skyrl/run_skyrl_swe.sh b/skyrl-agent/examples/run_skyrl/run_skyrl_swe.sh new file mode 100644 index 0000000000..c581a0b992 --- /dev/null +++ b/skyrl-agent/examples/run_skyrl/run_skyrl_swe.sh @@ -0,0 +1,85 @@ +set -x + +DATA_DIR="/mnt/shared_storage/datasets/r2e-all" +TRAIN_DATA="${DATA_DIR}/train.parquet" +VAL_DATA="${DATA_DIR}/validation.parquet" + +CKPT_DIR=$HOME/ckpts +EXPORT_DIR=$HOME/exports + + +MODEL=Qwen/Qwen3-32B +NNODES=2 +SP_SIZE=4 +TP_SIZE=4 +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=4 +BATCH_SIZE=64 +LOGGER=wandb +INFERENCE_BACKEND=vllm +seed=1 + +# export LD_LIBRARY_PATH="/opt/amazon/efa/lib:$LD_LIBRARY_PATH" + +uv run --isolated --env-file .env --extra skyrl-train \ + --with vllm==0.9.2 \ + --with transformers==4.53.0 \ + --with torch==2.7.0 \ + --with "flash-attn@https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.7cxx11abiTRUE-cp312-cp312-linux_x86_64.whl" \ + -m skyrl_agent.integrations.skyrl_train.skyrl_train_main \ + data.train_data="['$TRAIN_DATA']" \ + data.val_data="['$VAL_DATA']" \ + trainer.algorithm.advantage_estimator="loop" \ + trainer.policy.model.path=$MODEL \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + trainer.placement.policy_num_nodes=$NNODES \ + trainer.placement.ref_num_nodes=$NNODES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + +generator.task="./examples/run_skyrl/skyrl_swe.yaml" \ + trainer.epochs=10 \ + trainer.seed=$seed \ + trainer.eval_batch_size=128 \ + trainer.eval_before_train=false \ + trainer.eval_interval=10 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$BATCH_SIZE \ + trainer.policy_mini_batch_size=$BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=2 \ + trainer.max_ckpts_to_keep=20 \ + trainer.max_prompt_length=8000 \ + generator.sampling_params.max_generate_length=32768 \ + generator.enforce_eager=false \ + generator.enable_prefix_caching=true \ + trainer.algorithm.policy_loss_type="dual_clip" \ + trainer.policy.optimizer_config.lr=1e-6 \ + trainer.policy.sequence_parallel_size=$SP_SIZE \ + trainer.ref.sequence_parallel_size=$SP_SIZE \ + trainer.algorithm.use_kl_loss=false \ + trainer.algorithm.kl_loss_coef=0.001 \ + trainer.algorithm.eps_clip_low=0.2 \ + trainer.algorithm.eps_clip_high=0.28 \ + trainer.algorithm.loss_reduction="seq_mean_token_sum_norm" \ + trainer.algorithm.grpo_norm_by_std=false \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=null \ + generator.n_samples_per_prompt=8 \ + generator.eval_n_samples_per_prompt=1 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="skyagent-32b-r2e-skyrl" \ + trainer.run_name="skyagent-skyrl-32b-r2e-4500-loop-tool" \ + trainer.ckpt_path="$CKPT_DIR" \ + trainer.export_path="$EXPORT_DIR" \ + trainer.dump_data_batch=true \ + generator.max_num_batched_tokens=16384 \ + $@ diff --git a/skyrl-agent/examples/run_skyrl/skyrl_swe.yaml b/skyrl-agent/examples/run_skyrl/skyrl_swe.yaml new file mode 100644 index 0000000000..5704065871 --- /dev/null +++ b/skyrl-agent/examples/run_skyrl/skyrl_swe.yaml @@ -0,0 +1,51 @@ +agent_cls: skyrl_agent.agents.oh_codeact.OHCodeActAgent + +task: skyrl_agent.tasks.swebench.utils.SWEBenchTask + +tools: + enable_bash_tool: true + enable_browsing: false + enable_llm_editor: false + enable_editor: true + enable_jupyter: false + enable_cmd: true + enable_think: false + enable_finish: true + enable_mcp: false + enable_search: true + +data: + instance_key: instance + instance_id_key: instance.instance_id + data_source_key: data_source + +generator: + infer_backend: skyrl-train + backend_config: null + num_trajectories: 8 + max_iterations: 50 + max_prompt_length: 32768 + sampling_params: + temperature: 1 + top_p: 1 + top_k: -1 + max_tokens: 32768 + val_config: + num_trajectories: 1 + sampling_params: + temperature: 0 + top_p: 0.95 + max_tokens: 32768 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: true + qwen3_acc_thinking: true + +dispatcher: + type: async_pipeline + scheduler: naive + max_parallel_agents: 96 + max_eval_parallel_agents: 96 + val_config: + max_parallel_agents: 32 + max_eval_parallel_agents: 32 diff --git a/skyrl-agent/examples/run_skyrl/skyrl_web_research_hle.sh b/skyrl-agent/examples/run_skyrl/skyrl_web_research_hle.sh new file mode 100644 index 0000000000..14c5df4a84 --- /dev/null +++ b/skyrl-agent/examples/run_skyrl/skyrl_web_research_hle.sh @@ -0,0 +1,90 @@ +# Colocated GRPO training+generation for Qwen3-8B on deep search. + +set -x + +# Basic environment setup +export PYTHONUNBUFFERED=1 +export RUST_BACKTRACE=1 +export HYDRA_FULL_ERROR=1 +# increase ulimit for ray if applicable +# ulimit -n 65535 + +DATA_DIR=$HOME/data/web_research +CKPT_DIR=$HOME/ckpts +# save eval results and dump training batch +EXPORT_DIR=$HOME/exports + +TRAIN_DATA="$DATA_DIR/textbook_stem_5k_with_instance.parquet" +VAL_DATA="$DATA_DIR/hle_webthinker_converted.parquet" + +# Set a writable, shared web cache directory for the web_browser tool +# change if needed +export SKYAGENT_WEB_CACHE_DIR="$DATA_DIR/skyagent_web_cache" +mkdir -p "$SKYAGENT_WEB_CACHE_DIR" + +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout +seed=1 # for exact reproducibility with verl + +INFERENCE_BACKEND="vllm" + +# Load environment variables from .env file +if [ -f .env ]; then + set -a # automatically export all variables + source .env + set +a +fi + +# Set a writable, shared web cache directory for the web_browser tool +mkdir -p "${SKYAGENT_WEB_CACHE_DIR:-$DATA_DIR/web_cache}" + +uv run --isolated --env-file .env --extra skyrl-train -m skyrl_agent.integrations.skyrl_train.skyrl_train_main \ + data.train_data="['$TRAIN_DATA']" \ + data.val_data="['$VAL_DATA']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-8B" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + +generator.task="./examples/run_skyrl/skyrl_web_research_hle.yaml" \ + trainer.epochs=10 \ + trainer.seed=$seed \ + trainer.eval_batch_size=128 \ + trainer.eval_before_train=true \ + trainer.eval_interval=50 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=64 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=1 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=31232 \ + generator.enforce_eager=true \ + trainer.algorithm.policy_loss_type="dual_clip" \ + trainer.policy.optimizer_config.lr=5e-6 \ + trainer.policy.sequence_parallel_size=2 \ + trainer.ref.sequence_parallel_size=2 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.eps_clip_low=0.4 \ + trainer.algorithm.eps_clip_high=0.4 \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=null \ + generator.n_samples_per_prompt=8 \ + generator.eval_n_samples_per_prompt=1 \ + generator.gpu_memory_utilization=0.75 \ + trainer.logger="$LOGGER" \ + trainer.project_name="skyagent_websearch" \ + trainer.run_name="train-64n8-withoss20b-skyrl-train" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$CKPT_DIR" \ + trainer.export_path="$EXPORT_DIR" \ + trainer.dump_data_batch=true \ + $@ \ No newline at end of file diff --git a/skyrl-agent/examples/run_skyrl/skyrl_web_research_hle.yaml b/skyrl-agent/examples/run_skyrl/skyrl_web_research_hle.yaml new file mode 100644 index 0000000000..e77aa37e35 --- /dev/null +++ b/skyrl-agent/examples/run_skyrl/skyrl_web_research_hle.yaml @@ -0,0 +1,42 @@ +agent_cls: skyrl_agent.agents.react.ReActAgent + +task: skyrl_agent.tasks.web_research_task.WebResearchTask + +tools: ["search_engine", "web_browser", "finish"] + +data: + data_source_key: data_source + instance_id_key: extra_info.index + # Intentionally omit instance_key to pass full record (dict) to the task + +generator: + infer_backend: skyrl-train + backend_config: null + # Keep this in sync with +skyrl_agent.num_trajectories at launch + num_trajectories: 8 + max_iterations: 20 + # Keep prompt/response lengths modest for initial smoke tests; tune per model/context + max_prompt_length: 31232 + sampling_params: + temperature: 1.0 + top_p: 1.0 + max_tokens: 31232 + stop: [] + val_config: + num_trajectories: 1 + sampling_params: + temperature: 1 + top_p: 0.95 + max_tokens: 31232 + stop: [] + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: true + qwen3_acc_thinking: true + profile_tools: true + debug_log: false + +dispatcher: + type: async_batch + max_parallel_agents: 200 + max_eval_parallel_agents: 200 \ No newline at end of file diff --git a/skyrl-agent/examples/run_tinker/tinker_memagent.sh b/skyrl-agent/examples/run_tinker/tinker_memagent.sh new file mode 100644 index 0000000000..c73a345f35 --- /dev/null +++ b/skyrl-agent/examples/run_tinker/tinker_memagent.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -x + +# ============================================================================= +# Tinker RL Training for MemAgent Task +# ============================================================================= +# This script demonstrates how to train a model on ruler/hotpotqa using: +# - GRPO (Group Relative Policy Optimization) for advantages +# - PPO loss for stable training +# - MemAgent tool with multi-turn interactions +# ============================================================================= + +# Data paths +DATASET_FILE="/mnt/shared_storage/dacheng/ruler_hotpotqa_train_32k_skyagent.parquet" + +EVAL_DATASET_FILE="/mnt/shared_storage/dacheng/ruler_hotpotqa_eval_800_skyagent.parquet" + +# Output directory +NAME="${NAME:-nov24_qwen3_8b_ruler_max8k_external_min0turns_max30turns_memagent_tinker_lr4e_5_rank128}" +OUTPUT_DIR="/mnt/local_storage/checkpoints/${NAME}" +mkdir -p "$OUTPUT_DIR" + +# Model configuration +MODEL_NAME="${MODEL_NAME:-Qwen/Qwen3-8B}" +LORA_RANK="${LORA_RANK:-128}" + +# Training hyperparameters +BATCH_SIZE="${BATCH_SIZE:-32}" +LEARNING_RATE="${LEARNING_RATE:-4e-5}" +MAX_STEPS="${MAX_STEPS:-12}" +SAVE_EVERY="${SAVE_EVERY:-5}" +EVAL_EVERY="${EVAL_EVERY:-10}" + +# RL configuration +LOSS_FN="${LOSS_FN:-ppo}" +GROUP_SIZE="${GROUP_SIZE:-8}" # Should match num_trajectories in YAML +NORMALIZE_ADVANTAGES="${NORMALIZE_ADVANTAGES:-false}" + +# Logging +WANDB_PROJECT="${WANDB_PROJECT:-tinker-reasoning}" +WANDB_NAME="${WANDB_NAME:-${NAME}}" + +# Task configuration +TASK_YAML="./examples/run_tinker/tinker_memagent.yaml" + +echo "================================================" +echo "Tinker RL Training Configuration - MemAgent" +echo "================================================" +echo "Model: $MODEL_NAME" +echo "Dataset: $DATASET_FILE" +echo "Task YAML: $TASK_YAML" +echo "Batch Size: $BATCH_SIZE" +echo "Group Size (GRPO): $GROUP_SIZE" +echo "Max Steps: $MAX_STEPS" +echo "Output: $OUTPUT_DIR" +echo "================================================" + +# Run training +uv run --isolated --extra tinker --env-file .env -m skyrl_agent.integrations.tinker.tinker_train \ + model_name="$MODEL_NAME" \ + skyrl_agent_task_yaml="$TASK_YAML" \ + dataset_file="$DATASET_FILE" \ + eval_dataset_file="$EVAL_DATASET_FILE" \ + batch_size="$BATCH_SIZE" \ + learning_rate="$LEARNING_RATE" \ + lora_rank="$LORA_RANK" \ + max_steps="$MAX_STEPS" \ + save_every="$SAVE_EVERY" \ + loss_fn="$LOSS_FN" \ + group_size="$GROUP_SIZE" \ + normalize_advantages="$NORMALIZE_ADVANTAGES" \ + wandb_project="$WANDB_PROJECT" \ + wandb_name="$WANDB_NAME" \ + log_dir="$OUTPUT_DIR" \ + "$@" + +echo "================================================" +echo "Training completed!" +echo "Checkpoints saved to: ${OUTPUT_DIR}/${WANDB_NAME}_*" +echo "================================================" diff --git a/skyrl-agent/examples/run_tinker/tinker_memagent.yaml b/skyrl-agent/examples/run_tinker/tinker_memagent.yaml new file mode 100644 index 0000000000..030537daed --- /dev/null +++ b/skyrl-agent/examples/run_tinker/tinker_memagent.yaml @@ -0,0 +1,36 @@ +agent_cls: skyrl_agent.agents.react.ReActAgent + +task: skyrl_agent.tasks.general_react.utils.GeneralReactTask + +tools: ["finish", "next_with_summary"] + +data: + data_source_key: data_source + +generator: + infer_backend: tinker + backend_config: null + num_trajectories: 8 # Number of trajectories per prompt for GRPO + max_iterations: 30 + max_prompt_length: 8192 + sampling_params: + temperature: 1.0 + top_p: 1 + max_tokens: 8192 + val_config: + num_trajectories: 1 + sampling_params: + temperature: 0.7 + top_p: 0.95 + max_tokens: 8192 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: false + qwen3_acc_thinking: false + profile_tools: true + +dispatcher: + type: async_batch + max_parallel_agents: 32 # Adjust based on your resources + max_eval_parallel_agents: 32 + diff --git a/skyrl-agent/examples/run_tinker/tinker_reasoning_search_r1.sh b/skyrl-agent/examples/run_tinker/tinker_reasoning_search_r1.sh new file mode 100644 index 0000000000..1afbd68dec --- /dev/null +++ b/skyrl-agent/examples/run_tinker/tinker_reasoning_search_r1.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -x + +# ============================================================================= +# Tinker RL Training for Math Reasoning Task +# ============================================================================= +# This script demonstrates how to train a model on math reasoning using: +# - GRPO (Group Relative Policy Optimization) for advantages +# - PPO loss for stable training +# ============================================================================= + +# Data paths +DATA_DIR="${DATA_DIR:-/home/ec2-user/data/nq_search}" +DATASET_FILE="${DATASET_FILE:-${DATA_DIR}/train.parquet}" +EVAL_DATASET_FILE="${EVAL_DATASET_FILE:-${DATA_DIR}/test.parquet}" + +# Output directory +OUTPUT_DIR="${OUTPUT_DIR:-./tinker_outputs/search-r1}" +mkdir -p "$OUTPUT_DIR" + +# Model configuration +MODEL_NAME="${MODEL_NAME:-Qwen/Qwen3-4B-Instruct-2507}" +LORA_RANK="${LORA_RANK:-32}" + +# Training hyperparameters +BATCH_SIZE="${BATCH_SIZE:-512}" +EVAL_BATCH_SIZE="${EVAL_BATCH_SIZE:-1024}" +LEARNING_RATE="${LEARNING_RATE:-4e-5}" +MAX_STEPS="${MAX_STEPS:-100}" +SAVE_EVERY="${SAVE_EVERY:-2}" +EVAL_EVERY="${EVAL_EVERY:-10}" + +# RL configuration +LOSS_FN="${LOSS_FN:-ppo}" +GROUP_SIZE="${GROUP_SIZE:-8}" # Should match num_trajectories in YAML +NORMALIZE_ADVANTAGES="${NORMALIZE_ADVANTAGES:-false}" + +# Logging +WANDB_PROJECT="${WANDB_PROJECT:-tinker-reasoning}" +WANDB_NAME="${WANDB_NAME:-tinker-qwen3-4b-search-r1-grpo}" +RESUME_EXP_NAME="${RESUME_EXP_NAME:-tinker-qwen3-4b-search-r1-grpo_1030T19:52:09}" + +# Task configuration +TASK_YAML="./examples/run_tinker/tinker_reasoning_search_r1.yaml" + +echo "================================================" +echo "Tinker RL Training Configuration" +echo "================================================" +echo "Model: $MODEL_NAME" +echo "Dataset: $DATASET_FILE" +echo "Task YAML: $TASK_YAML" +echo "Batch Size: $BATCH_SIZE" +echo "Group Size (GRPO): $GROUP_SIZE" +echo "Max Steps: $MAX_STEPS" +echo "Output: $OUTPUT_DIR" +echo "================================================" + +# Run training +uv run --isolated --extra tinker --env-file .env -m skyrl_agent.integrations.tinker.tinker_train \ + model_name="$MODEL_NAME" \ + skyrl_agent_task_yaml="$TASK_YAML" \ + dataset_file="$DATASET_FILE" \ + eval_dataset_file="$EVAL_DATASET_FILE" \ + batch_size="$BATCH_SIZE" \ + eval_batch_size="$EVAL_BATCH_SIZE" \ + learning_rate="$LEARNING_RATE" \ + lora_rank="$LORA_RANK" \ + max_steps="$MAX_STEPS" \ + save_every="$SAVE_EVERY" \ + eval_every="$EVAL_EVERY" \ + loss_fn="$LOSS_FN" \ + group_size="$GROUP_SIZE" \ + normalize_advantages="$NORMALIZE_ADVANTAGES" \ + wandb_project="$WANDB_PROJECT" \ + wandb_name="$WANDB_NAME" \ + resume_exp_name="$RESUME_EXP_NAME" \ + log_dir="$OUTPUT_DIR" \ + "$@" + +echo "================================================" +echo "Training completed!" +echo "Checkpoints saved to: ${OUTPUT_DIR}/tinker_output/${WANDB_NAME}_*" +echo "================================================" + diff --git a/skyrl-agent/examples/run_tinker/tinker_reasoning_search_r1.yaml b/skyrl-agent/examples/run_tinker/tinker_reasoning_search_r1.yaml new file mode 100644 index 0000000000..7bcfc99970 --- /dev/null +++ b/skyrl-agent/examples/run_tinker/tinker_reasoning_search_r1.yaml @@ -0,0 +1,35 @@ +agent_cls: skyrl_agent.agents.react.ReActAgent + +task: skyrl_agent.tasks.general_react.utils.GeneralReactTask + +tools: ["em_finish", "local_search"] + +data: + data_source_key: data_source + +generator: + infer_backend: tinker + backend_config: null + num_trajectories: 8 # Number of trajectories per prompt for GRPO + max_iterations: 5 + max_prompt_length: 8000 + sampling_params: + temperature: 1.0 + top_p: 1 + max_tokens: 8000 + val_config: + num_trajectories: 1 + sampling_params: + temperature: 0 + top_p: 0.95 + max_tokens: 8000 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: false + qwen3_acc_thinking: false + +dispatcher: + type: async_batch + max_parallel_agents: 32 # Adjust based on your resources + max_eval_parallel_agents: 32 + diff --git a/skyrl-agent/examples/run_verl/verl_browsecomp.sh b/skyrl-agent/examples/run_verl/verl_browsecomp.sh new file mode 100644 index 0000000000..27d8116574 --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_browsecomp.sh @@ -0,0 +1,73 @@ +set -x + +train_data="/mnt/shared_storage/browsecomp_plus/browsecomp_plus_train.parquet" +test_data="/mnt/shared_storage/browsecomp_plus/browsecomp_plus_skyagent.parquet" +# test_data="/mnt/shared_storage/browsecomp_plus/browsecomp_plus_test.parquet" + + +# MODEL=Qwen/Qwen3-8B +# MODEL=Qwen/Qwen3-32B +MODEL=/mnt/cluster_storage/ckpt/skyrl-agent-swe/step_125 +NNODES=1 +SP_SIZE=4 +TP_SIZE=4 + + + uv run --isolated --extra verl --env-file .env -m skyrl_agent.integrations.verl.verl_main_ppo \ + algorithm.adv_estimator=loop \ + data.train_files=$train_data \ + data.val_files=$test_data \ + data.dataloader_num_workers=0 \ + data.train_batch_size=64 \ + data.val_batch_size=830 \ + data.max_prompt_length=8000 \ + data.max_response_length=32768 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=true \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.actor.loss_agg_mode=seq-mean-token-sum \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.use_dynamic_bsz=False \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.clip_ratio_high=0.28 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + algorithm.norm_adv_by_std_in_grpo=False \ + trainer.val_before_train=True \ + trainer.val_only=True \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='skyagent-32b-r2e' \ + trainer.experiment_name='browsecomp-eval-base' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=$NNODES \ + trainer.max_actor_ckpt_to_keep=10 \ + trainer.save_freq=1 \ + trainer.default_local_dir=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/browsecomp-eval \ + trainer.test_freq=20 \ + trainer.total_epochs=15 \ + +skyrl_agent.task_yaml="./examples/run_verl/verl_browsecomp.yaml" \ + +skyrl_agent.num_trajectories=1 \ + +trainer.remote_anyscale_upload=True \ + +trainer.remote_upload_dir=remote_ckpts/skyagent-32b-r2e-verl/browsecomp-eval $@ diff --git a/skyrl-agent/examples/run_verl/verl_browsecomp.yaml b/skyrl-agent/examples/run_verl/verl_browsecomp.yaml new file mode 100644 index 0000000000..697a73d65a --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_browsecomp.yaml @@ -0,0 +1,39 @@ +agent_cls: skyrl_agent.agents.react.ReActAgent + +task: skyrl_agent.tasks.general_react.utils.GeneralReactTask + +tools: ["finish", "faiss_search"] + + +data: + data_source_key: data_source + +generator: + infer_backend: verl + backend_config: null + num_trajectories: 1 + max_iterations: 30 + max_prompt_length: 32768 + sampling_params: + temperature: 0.5 + top_p: 0.95 + max_tokens: 32768 + val_config: + num_trajectories: 1 + sampling_params: + temperature: 1 + top_p: 0.95 + max_tokens: 32768 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: true + qwen3_acc_thinking: true + early_step_threshold: 7 + enable_turn_reminder: true + +dispatcher: + type: async_batch + scheduler: naive + max_parallel_agents: 1 + max_eval_parallel_agents: 1 + diff --git a/skyrl-agent/examples/run_verl/verl_oh.sh b/skyrl-agent/examples/run_verl/verl_oh.sh new file mode 100644 index 0000000000..35bb534abf --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_oh.sh @@ -0,0 +1,78 @@ +set -x + +DATA_DIR="/mnt/shared_storage/datasets/r2e-all" +train_data="${DATA_DIR}/train.parquet" +test_data="${DATA_DIR}/validation.parquet" + + +# MODEL=Qwen/Qwen3-8B +MODEL=Qwen/Qwen3-32B +NNODES=4 +SP_SIZE=4 +TP_SIZE=4 +# NNODES=1 +# SP_SIZE=2 +# TP_SIZE=2 + + + uv run --isolated --frozen --extra verl --env-file .env -m skyrl_agent.integrations.verl.verl_main_ppo \ + algorithm.adv_estimator=loop \ + data.train_files=$train_data \ + data.val_files=$test_data \ + data.dataloader_num_workers=0 \ + data.train_batch_size=64 \ + data.max_prompt_length=8000 \ + data.max_response_length=32768 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=true \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.actor.loss_agg_mode=seq-mean-token-sum \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.use_dynamic_bsz=False \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.clip_ratio_high=0.28 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + algorithm.norm_adv_by_std_in_grpo=False \ + trainer.val_before_train=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='skyagent-32b-r2e' \ + trainer.experiment_name='skyagent-verl-30b-r2e-4500-loop-sep27' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=$NNODES \ + trainer.max_actor_ckpt_to_keep=10 \ + trainer.save_freq=1 \ + trainer.default_local_dir=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-30b-r2e-4500-loop-sep27 \ + trainer.test_freq=20 \ + trainer.total_epochs=15 \ + trainer.rollout_data_dir=/mnt/local_storage/rollouts/verl_grpo_skyagent/qwen2.5_1.5b_function_rm \ + trainer.validation_data_dir=/mnt/local_storage/rollouts/verl_grpo_skyagent/qwen2.5_1.5b_function_rm_val \ + +skyrl_agent.task_yaml="./examples/run_verl/verl_oh.yaml" \ + +skyrl_agent.num_trajectories=8 $@ + # +trainer.remote_anyscale_upload=True \ + # +trainer.remote_upload_dir=remote_ckpts/skyagent-32b-r2e-verl/skyagent-verl-30b-r2e-4500-loop-sep27 $@ + # trainer.resume_from_path=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-32b-r2e-4500-loop-aug20/global_step_14 \ + # trainer.resume_mode=resume_path\ + # trainer.resume_from_path=/mnt/local_storage/ckpts/skyagent-32b-r2e-verl/skyagent-verl-32b-r2e-4500-acc-thinking-mini4-aug15/global_step_26 \ diff --git a/skyrl-agent/examples/run_verl/verl_oh.yaml b/skyrl-agent/examples/run_verl/verl_oh.yaml new file mode 100644 index 0000000000..7c67accfc7 --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_oh.yaml @@ -0,0 +1,51 @@ +agent_cls: skyrl_agent.agents.oh_codeact.OHCodeActAgent + +task: skyrl_agent.tasks.swebench.utils.SWEBenchTask + +tools: + enable_bash_tool: true + enable_browsing: false + enable_llm_editor: false + enable_editor: true + enable_jupyter: false + enable_cmd: true + enable_think: false + enable_finish: true + enable_mcp: false + enable_search: true + +data: + instance_key: instance + instance_id_key: instance.instance_id + data_source_key: data_source + +generator: + infer_backend: verl + backend_config: null + num_trajectories: 8 + max_iterations: 50 # 50 + max_prompt_length: 32768 + sampling_params: + temperature: 1 + top_p: 1 + top_k: -1 + min_p: 0 + max_tokens: 32768 + val_config: + num_trajectories: 1 + sampling_params: + temperature: 1 + top_p: 0.95 + max_tokens: 32768 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: true + qwen3_acc_thinking: true + +dispatcher: + type: async_pipeline + max_parallel_agents: 96 + max_eval_parallel_agents: 96 + val_config: + max_parallel_agents: 32 + max_eval_parallel_agents: 32 diff --git a/skyrl-agent/examples/run_verl/verl_search_r1.sh b/skyrl-agent/examples/run_verl/verl_search_r1.sh new file mode 100644 index 0000000000..93a707cad7 --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_search_r1.sh @@ -0,0 +1,69 @@ +set -x + +train_data="/mnt/shared_storage/data/nq_search/train.parquet" +test_data="/mnt/shared_storage/data/nq_search/test.parquet" + + +MODEL=Qwen/Qwen3-4B-Instruct-2507 +NNODES=1 +SP_SIZE=1 +TP_SIZE=1 + + + uv run --isolated --extra verl --env-file .env -m skyrl_agent.integrations.verl.verl_main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=$train_data \ + data.val_files=$test_data \ + data.dataloader_num_workers=0 \ + data.train_batch_size=512 \ + data.val_batch_size=1024 \ + data.max_prompt_length=8000 \ + data.max_response_length=8000 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=true \ + actor_rollout_ref.model.path=$MODEL \ + actor_rollout_ref.actor.loss_agg_mode=seq-mean-token-sum \ + actor_rollout_ref.actor.optim.lr=2e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=512 \ + actor_rollout_ref.actor.use_dynamic_bsz=False \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.clip_ratio_high=0.28 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=$SP_SIZE \ + actor_rollout_ref.rollout.tensor_model_parallel_size=$TP_SIZE \ + actor_rollout_ref.rollout.enforce_eager=False \ + actor_rollout_ref.rollout.free_cache_engine=True \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + algorithm.use_kl_in_reward=False \ + algorithm.norm_adv_by_std_in_grpo=False \ + trainer.val_before_train=True \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='skyagent-4b-searchr1' \ + trainer.experiment_name='skyagent-4b-searchr1-qaem' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=$NNODES \ + trainer.max_actor_ckpt_to_keep=10 \ + trainer.save_freq=10 \ + trainer.default_local_dir=/mnt/local_storage/ckpts/skyagent-4b-searchr1-verl/verl-4b-searchr1-qaem \ + trainer.test_freq=10 \ + trainer.total_epochs=15 \ + +skyrl_agent.task_yaml="./examples/run_verl/verl_search_r1.yaml" \ + +skyrl_agent.num_trajectories=8 \ + +trainer.remote_anyscale_upload=True \ + +trainer.remote_upload_dir=remote_ckpts/skyagent-4b-searchr1-verl/verl-4b-searchr1-qaem $@ diff --git a/skyrl-agent/examples/run_verl/verl_search_r1.yaml b/skyrl-agent/examples/run_verl/verl_search_r1.yaml new file mode 100644 index 0000000000..9daa5c30f8 --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_search_r1.yaml @@ -0,0 +1,35 @@ +agent_cls: skyrl_agent.agents.react.ReActAgent + +task: skyrl_agent.tasks.general_react.utils.GeneralReactTask + +tools: ["em_finish", "local_search"] + +data: + data_source_key: data_source + +generator: + infer_backend: verl + backend_config: null + num_trajectories: 8 # Number of trajectories per prompt for GRPO + max_iterations: 5 + max_prompt_length: 8000 + sampling_params: + temperature: 1.0 + top_p: 1 + max_tokens: 8000 + val_config: + num_trajectories: 1 + sampling_params: + temperature: 0 + top_p: 0.95 + max_tokens: 8000 + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: false + qwen3_acc_thinking: false + +dispatcher: + type: async_batch + max_parallel_agents: 32 # Adjust based on your resources + max_eval_parallel_agents: 32 + diff --git a/skyrl-agent/examples/run_verl/verl_web_research_hle.sh b/skyrl-agent/examples/run_verl/verl_web_research_hle.sh new file mode 100755 index 0000000000..f01fffb99a --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_web_research_hle.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -x + +# Basic environment setup +export PYTHONUNBUFFERED=1 +export RUST_BACKTRACE=1 +export HYDRA_FULL_ERROR=1 +# increase ulimit for ray if needed +# ulimit -n 65535 + + +# Default data/checkpoint directories (override via env if needed) +DATA_DIR=${DATA_DIR:-"$HOME/data/web_research"} +mkdir -p "$DATA_DIR" + +DATA_FILE=${DATA_FILE:-"$DATA_DIR/textbook_stem_5k_with_instance.parquet"} +VAL_FILE=${VAL_FILE:-"$DATA_DIR/hle_webthinker_converted.parquet"} + +TRAIN_OUTPUT_DIR=${TRAIN_OUTPUT_DIR:-"$DATA_DIR/ckpts"} +ROLLOUT_DIR=${ROLLOUT_DIR:-"$DATA_DIR/rollouts/trainwithoss20b"} +VAL_ROLLOUT_DIR=${VAL_ROLLOUT_DIR:-"$DATA_DIR/rollouts/web_research_val_tigerb_hlestart_70ckpt"} +mkdir -p "$TRAIN_OUTPUT_DIR" "$ROLLOUT_DIR" "$VAL_ROLLOUT_DIR" + +# Load environment variables from .env file +if [ -f .env ]; then + set -a # automatically export all variables + source .env + set +a +fi + +# Set a writable, shared web cache directory for the web_browser tool +mkdir -p "${SKYAGENT_WEB_CACHE_DIR:-$DATA_DIR/web_cache}" + + + +# Optimize HuggingFace model loading +# export HF_HUB_OFFLINE=1 # Commented out to allow model download if needed +# export TRANSFORMERS_OFFLINE=1 # Commented out to allow online checks +# export HF_HOME=~/.cache/huggingface # Use default cache location where models exist + +uv run --active --extra verl --env-file .env -m skyrl_agent.integrations.verl.verl_main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files="${DATA_FILE}" \ + data.val_files="${VAL_FILE}" \ + data.max_prompt_length=8192 \ + data.prompt_key=prompt \ + data.return_raw_chat=true \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.train_batch_size=64 \ + actor_rollout_ref.actor.strategy=fsdp2 \ + actor_rollout_ref.model.path=Qwen/Qwen3-8B \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.prompt_length=8192 \ + actor_rollout_ref.rollout.response_length=32000 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=4 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.optim.lr=5e-6 \ + actor_rollout_ref.actor.clip_ratio_low=0.4 \ + actor_rollout_ref.actor.clip_ratio_high=0.4 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=11000 \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=14000 \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=14000 \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=8 \ + actor_rollout_ref.ref.strategy=fsdp2 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ + algorithm.use_kl_in_reward=False \ + trainer.val_before_train=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='skyagent_websearch' \ + trainer.experiment_name='train-64n8-withoss20b-verl' \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=5 \ + trainer.test_freq=50 \ + data.val_batch_size=500 \ + trainer.total_epochs=10 \ + trainer.ray_wait_register_center_timeout=900 \ + trainer.default_local_dir="${TRAIN_OUTPUT_DIR}" \ + trainer.rollout_data_dir="${ROLLOUT_DIR}" \ + trainer.validation_data_dir="${VAL_ROLLOUT_DIR}" \ + +skyrl_agent.task_yaml="./examples/run_verl/verl_web_research_hle.yaml" \ + +skyrl_agent.num_trajectories=8 "$@" diff --git a/skyrl-agent/examples/run_verl/verl_web_research_hle.yaml b/skyrl-agent/examples/run_verl/verl_web_research_hle.yaml new file mode 100644 index 0000000000..dd71f98190 --- /dev/null +++ b/skyrl-agent/examples/run_verl/verl_web_research_hle.yaml @@ -0,0 +1,42 @@ +agent_cls: skyrl_agent.agents.react.ReActAgent + +task: skyrl_agent.tasks.web_research_task.WebResearchTask + +tools: ["search_engine", "web_browser", "finish"] + +data: + data_source_key: data_source + instance_id_key: extra_info.index + # Intentionally omit instance_key to pass full record (dict) to the task + +generator: + infer_backend: verl + backend_config: null + # Keep this in sync with +skyrl_agent.num_trajectories at launch + num_trajectories: 8 + max_iterations: 20 + # Keep prompt/response lengths modest for initial smoke tests; tune per model/context + max_prompt_length: 31232 + sampling_params: + temperature: 1.0 + top_p: 1.0 + max_tokens: 31232 + stop: [] + val_config: + num_trajectories: 1 + sampling_params: + temperature: 1 + top_p: 0.95 + max_tokens: 31232 + stop: [] + remove_think_tokens: false + vision_is_active: false + qwen3_enable_thinking: true + qwen3_acc_thinking: true + profile_tools: true + debug_log: false + +dispatcher: + type: async_batch + max_parallel_agents: 200 + max_eval_parallel_agents: 200 diff --git a/skyrl-agent/pyproject.toml b/skyrl-agent/pyproject.toml new file mode 100644 index 0000000000..bfc368dbd3 --- /dev/null +++ b/skyrl-agent/pyproject.toml @@ -0,0 +1,222 @@ +# ------------------------------- +# build-system +# ------------------------------- +[build-system] +requires = [ + "setuptools>=61.0", + "wheel" +] +build-backend = "setuptools.build_meta" + +# ------------------------------- +# project (PEP 621 metadata) +# ------------------------------- +[project] +name = "skyrl_agent" +version = "0.0.1" + +description = "SkyRL-Agent: Training and evaluating AI agents in complex environments." +license = {file = "LICENSE"} # or "Apache-2.0", if you prefer an SPDX identifier +readme = {file = "README.md", content-type = "text/markdown"} +requires-python = "==3.12.*" + +dependencies=[ + 'codetiming', + 'datasets', + 'dill', + 'faiss-cpu>=1.11.0.post1', + 'hydra-core', + 'numpy', + 'pandas', + 'datasets', + 'pyarrow>=15.0.0', + 'pybind11', + 'pylatexenc', + 'ray[default]>=2.10', + 'tevatron', + 'transformers', + "hf_transfer", + "openhands-ai", + "streamlit", + "whatthepatch", + "retry", + "evaluate", + "swebench@https://github.com/SWE-Gym/SWE-Bench-Fork.git", + "swegym@https://github.com/SWE-Gym/SWE-Bench-Package.git", + "func_timeout", + "sympy", + "gdown", + "matplotlib", + "seaborn", + "tabulate", + "browsergym==0.13.3", + "browsergym-webarena==0.13.3", + "browsergym-miniwob==0.13.3", + "browsergym-visualwebarena==0.13.3", + "json5", + "math_verify", + "jsonlines", + "daytona-api-client==0.20.1", + "polars", + "datasets==4.0.0", + "loguru", + "wandb", + "vllm>=0.8.5", + "together", + "litellm", +] + +[project.optional-dependencies] +skyrl-train = ["skyrl-train[vllm]", 'torchdata'] +verl = ["verl[vllm,gpu]","vllm==0.8.5"] +tinker = ["tinker", "chz"] + + +[tool.uv] +override-dependencies = ["ray==2.48.0", "xgrammar==0.1.17", "opentelemetry-api==1.26.0", "opentelemetry-sdk==1.26.0", "opentelemetry-exporter-otlp==1.26.0", "opentelemetry-exporter-otlp-proto-grpc==1.26.0"] +conflicts = [ + [ + { extra = "skyrl-train" }, + { extra = "verl" }, + { extra = "tinker" }, + ], +] + +[tool.uv.sources] +openhands-ai = { git = "https://github.com/NovaSky-AI/SkyRL-OpenHands.git", rev = "main" } +tevatron = { git = "https://github.com/texttron/tevatron.git", rev = "main" } +verl = { git = "https://github.com/NovaSky-AI/verl-fork", rev = "main" } +skyrl-train = { path = "./skyrl-train", editable = true } + +[tool.uv.extra-build-dependencies] +flash-attn = [{requirement = "torch", match-runtime = true}] + +[tool.uv.extra-build-variables] +flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE"} + +# ------------------------------- +# tool.setuptools - Additional config +# ------------------------------- +[tool.setuptools.packages.find] +include = ["skyrl_agent*"] + +# We read the version from a file in 'skyagent/version/version' +[tool.setuptools.dynamic] +version = {attr = "skyrl_agent.__version__"} + +# If you need to mimic `package_dir={'': '.'}`: +[tool.setuptools.package-dir] +"" = "." + + +[tool.pylint.message_control] +disable = [ + "abstract-method", + "anomalous-backslash-in-string", + "arguments-differ", + "arguments-renamed", + "assignment-from-none", + "attribute-defined-outside-init", + "bad-str-strip-call", + "bare-except", + "broad-exception-caught", + "broad-exception-raised", + "cell-var-from-loop", + "chained-comparison", + "consider-iterating-dictionary", + "consider-using-enumerate", + "consider-using-f-string", + "consider-using-from-import", + "consider-using-generator", + "consider-using-in", + "consider-using-max-builtin", + "consider-using-set-comprehension", + "consider-using-sys-exit", + "consider-using-with", + "cyclic-import", + "dangerous-default-value", + "duplicate-code", + "eval-used", + "expression-not-assigned", + "f-string-without-interpolation", + "fixme", + "function-redefined", + "global-statement", + "global-variable-not-assigned", + "import-error", + "import-outside-toplevel", + "import-self", + "inconsistent-return-statements", + "invalid-character-zero-width-space", + "invalid-name", + "line-too-long", + "logging-fstring-interpolation", + "logging-not-lazy", + "missing-class-docstring", + "missing-final-newline", + "missing-function-docstring", + "missing-module-docstring", + "multiple-imports", + "no-else-continue", + "no-else-raise", + "no-else-return", + "no-member", + "no-self-argument", + "no-value-for-parameter", + "not-an-iterable", + "not-callable", + "notimplemented-raised", + "pointless-exception-statement", + "pointless-string-statement", + "pointless-statement", + "possibly-used-before-assignment", + "protected-access", + "raise-missing-from", + "raising-format-tuple", + "redefined-argument-from-local", + "redefined-builtin", + "redefined-outer-name", + "redundant-u-string-prefix", + "reimported", + "simplifiable-if-expression", + "simplifiable-if-statement", + "singleton-comparison", + "super-init-not-called", + "superfluous-parens", + "too-few-public-methods", + "too-many-arguments", + "too-many-boolean-expressions", + "too-many-branches", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-positional-arguments", + "too-many-return-statements", + "too-many-statements", + "trailing-newlines", + "trailing-newlines", + "trailing-whitespace", + "unbalanced-tuple-unpacking", + "undefined-loop-variable", + "undefined-variable", + "ungrouped-imports", + "unidiomatic-typecheck", + "unnecessary-comprehension", + "unnecessary-lambda", + "unnecessary-lambda-assignment", + "unnecessary-pass", + "unspecified-encoding", + "unused-argument", + "unused-import", + "unused-variable", + "unused-wildcard-import", + "use-a-generator", + "use-dict-literal", + "used-before-assignment", + "useless-object-inheritance", + "useless-parent-delegation", + "useless-return", + "wildcard-import", + "wrong-import-order", + "wrong-import-position", +] diff --git a/skyrl-agent/services/router_simple.py b/skyrl-agent/services/router_simple.py new file mode 100644 index 0000000000..eeb8d140eb --- /dev/null +++ b/skyrl-agent/services/router_simple.py @@ -0,0 +1,542 @@ +import os +import json +import asyncio +import itertools +import hashlib +import time +import random +from typing import Optional, List, Dict, Any + +from fastapi import FastAPI, Request, Response +from starlette.responses import StreamingResponse, JSONResponse +import httpx + + +""" +Simple Web Summary router (no keys). It only handles requests whose JSON +body has model == SUMMARY_MODEL, and forwards them to one of fixed +summary servers using either Rendezvous Hash (HRW) sticky routing (per +trajectory) or round-robin (when no sticky key), with the ability to +hard-disable failed upstreams (auto or manual) and skip them for all +subsequent requests. + + - Configure via SUMMARY_UPSTREAMS (e.g. http://localhost:8000/v1) + +Standby (used when primaries are unavailable): + - Configure via SUMMARY_STANDBY_UPSTREAMS + +All other requests are rejected, so your LLM judge and unrelated traffic +stay untouched unless explicitly pointed here. + +Usage +- Configure via env (optional): + - SUMMARY_MODEL: defaults to "Qwen/Qwen3-32B" + - SUMMARY_UPSTREAMS: comma-separated upstream bases (e.g. "http://host1:8000/v1,http://host2:8000/v1"). + - SUMMARY_STANDBY_UPSTREAMS: comma-separated standby bases. + - ROUTER_ENFORCE_MODEL: "1" to enforce model match (default), "0" to forward all. +- Run: uvicorn router_simple:app --host 0.0.0.0 --port 8080 +- Point only the Web Summary client to: http://localhost:8080/v1 + (set WEB_SUMMARY_API_BASE to this). Judge stays on your current base. +""" + +app = FastAPI() +client = httpx.AsyncClient(timeout=None) + +# Only handle Web Summary requests; identified by model in JSON body +SUMMARY_MODEL = os.getenv("SUMMARY_MODEL", "Qwen/Qwen3-32B") +ENFORCE_MODEL = os.getenv("ROUTER_ENFORCE_MODEL", "1").strip().lower() in ("1", "true", "yes", "on") +DISABLE_STICKY = os.getenv("SUMMARY_DISABLE_STICKY", "1").strip().lower() in ("1", "true", "yes", "on") + + +# Fixed summary servers (provided by user). Stored without trailing /v1 +def _normalize_base(u: str) -> str: + b = (u or "").strip().rstrip("/") + if b.endswith("/v1"): + b = b[:-3] + return b + + +def _parse_upstreams_env(env_name: str, default_bases: List[str]) -> List[str]: + raw = os.getenv(env_name, "") + if not raw.strip(): + return default_bases + parts = [p.strip() for p in raw.split(",") if p.strip()] + bases = [_normalize_base(p) for p in parts] + return bases or default_bases + + +_DEFAULT_SUMMARY_UPSTREAMS_BASE: List[str] = [] + +_SUMMARY_UPSTREAMS_BASE: List[str] = _parse_upstreams_env( + "SUMMARY_UPSTREAMS", + _DEFAULT_SUMMARY_UPSTREAMS_BASE, +) + +if not _SUMMARY_UPSTREAMS_BASE: + raise RuntimeError( + "No SUMMARY_UPSTREAMS configured. Set the environment variable to a comma-separated list of upstream base URLs." + ) + +# Standby upstreams (not part of sticky hashing; used only when needed) +_DEFAULT_STANDBY_UPSTREAMS_BASE: List[str] = [] + +_STANDBY_UPSTREAMS_BASE: List[str] = _parse_upstreams_env( + "SUMMARY_STANDBY_UPSTREAMS", + _DEFAULT_STANDBY_UPSTREAMS_BASE, +) + +# Weighted bias settings: send a portion of traffic to a preferred upstream +_BIAS_TARGET_BASE = _normalize_base(os.getenv("SUMMARY_BIAS_TARGET", "")) +_BIAS_TARGET_BASE2 = _normalize_base(os.getenv("SUMMARY_BIAS_TARGET2", "")) +try: + _BIAS_RATIO = float(os.getenv("SUMMARY_BIAS_RATIO", "0.0")) +except Exception: + _BIAS_RATIO = 0.0 +try: + _BIAS_RATIO2 = float(os.getenv("SUMMARY_BIAS_RATIO2", "0.0")) +except Exception: + _BIAS_RATIO2 = 0.0 + +_rr_iter = itertools.cycle(_SUMMARY_UPSTREAMS_BASE) +_rr_lock = asyncio.Lock() + +# Ensure bias targets are part of upstreams even if env forgot them +if _BIAS_TARGET_BASE and _BIAS_TARGET_BASE not in _SUMMARY_UPSTREAMS_BASE: + _SUMMARY_UPSTREAMS_BASE.append(_BIAS_TARGET_BASE) +if _BIAS_TARGET_BASE2 and _BIAS_TARGET_BASE2 not in _SUMMARY_UPSTREAMS_BASE: + _SUMMARY_UPSTREAMS_BASE.append(_BIAS_TARGET_BASE2) + +# Optional smooth weighted round-robin for non-sticky requests +USE_SWRR = os.getenv("SUMMARY_USE_SWRR", "1").strip().lower() in ("1", "true", "yes", "on") +_swrr_weights: Dict[str, int] = {} +_swrr_current: Dict[str, int] = {} +_swrr_known_set: set[str] = set() + + +def _rebuild_swrr(primaries: List[str]) -> None: + global _swrr_weights, _swrr_current, _swrr_known_set + actives = [p for p in primaries if p not in _disabled] + _swrr_weights = {} + _swrr_current = {} + _swrr_known_set = set(actives) + if not actives: + return + scale = 100 # percent basis + total = int(round(scale)) + # compute weights for up to two preferred targets + w1 = int(round(_BIAS_RATIO * scale)) if _BIAS_TARGET_BASE in actives else 0 + w2 = int(round(_BIAS_RATIO2 * scale)) if _BIAS_TARGET_BASE2 in actives else 0 + rem = max(0, total - w1 - w2) + others = [p for p in actives if p not in (_BIAS_TARGET_BASE, _BIAS_TARGET_BASE2)] + if others: + base_w = rem // len(others) + extra = rem - base_w * len(others) + for i, p in enumerate(others): + w = base_w + (1 if i < extra else 0) + _swrr_weights[p] = max(0, w) + if _BIAS_TARGET_BASE in actives: + _swrr_weights[_BIAS_TARGET_BASE] = max(1, w1) + if _BIAS_TARGET_BASE2 in actives: + _swrr_weights[_BIAS_TARGET_BASE2] = max(1, w2) + if not _swrr_weights: + # fallback: equal weights if nothing assigned + base_w = total // len(actives) + extra = total - base_w * len(actives) + for i, p in enumerate(actives): + w = base_w + (1 if i < extra else 0) + _swrr_weights[p] = max(1, w) + for p in actives: + _swrr_current[p] = 0 + + +def _swrr_pick(primaries: List[str]) -> Optional[str]: + actives = [p for p in primaries if p not in _disabled] + if set(actives) != _swrr_known_set: + _rebuild_swrr(actives) + if not actives: + return None + if not _swrr_weights: + _rebuild_swrr(actives) + total_w = sum(_swrr_weights.get(p, 0) for p in actives) + if total_w <= 0: + return random.choice(actives) + best_p = None + best_val = None + for p in actives: + _swrr_current[p] = _swrr_current.get(p, 0) + _swrr_weights.get(p, 0) + v = _swrr_current[p] + if best_val is None or v > best_val: + best_val = v + best_p = p + if best_p is None: + return random.choice(actives) + _swrr_current[best_p] = _swrr_current.get(best_p, 0) - total_w + return best_p + + +async def _choose_summary_upstream() -> str: + async with _rr_lock: + return next(_rr_iter) + + +def _is_summary_request(body_obj) -> bool: + return isinstance(body_obj, dict) and body_obj.get("model") == SUMMARY_MODEL + + +def _choose_sticky_upstream(key: str) -> str: + # stable choice based on sha1(key) + digest = hashlib.sha1(key.encode("utf-8")).digest() + idx = int.from_bytes(digest[:4], "big") % len(_SUMMARY_UPSTREAMS_BASE) + return _SUMMARY_UPSTREAMS_BASE[idx] + + +# Hard-disable state and simple metrics +_disabled: Dict[str, Dict[str, Any]] = {} +_fail_counts: Dict[str, int] = {} +_metrics: Dict[str, Dict[str, Any]] = {} +FAIL_THRESHOLD = int(os.getenv("ROUTER_FAIL_THRESHOLD", "2")) +LOAD_BETA = float(os.getenv("ROUTER_LOAD_BETA", "0.5")) +_sticky_map: Dict[str, str] = {} +_in_flight: Dict[str, int] = {} + + +def _active_primaries() -> List[str]: + return [b for b in _SUMMARY_UPSTREAMS_BASE if b not in _disabled] + + +def _next_active_rr() -> Optional[str]: + # Get next round-robin base that is not disabled + for _ in range(len(_SUMMARY_UPSTREAMS_BASE)): + base = next(_rr_iter) + if base not in _disabled: + return base + return None + + +def _any_primary_disabled() -> bool: + return any(b in _disabled for b in _SUMMARY_UPSTREAMS_BASE) + + +def _active_set_for_sticky() -> List[str]: + primaries = _active_primaries() + # If any primary is disabled, include standby to share the load + if len(primaries) < len(_SUMMARY_UPSTREAMS_BASE): + for sb in _STANDBY_UPSTREAMS_BASE: + if sb not in _disabled and sb not in primaries: + primaries.append(sb) + return primaries + + +def _hrw_order(sticky_key: str, servers: List[str], load_aware: bool) -> List[str]: + scored = [] + for b in servers: + h = hashlib.sha1() + h.update(sticky_key.encode("utf-8")) + h.update(b.encode("utf-8")) + score = int.from_bytes(h.digest()[:8], "big") + if load_aware: + load = float(_in_flight.get(b, 0)) + denom = 1.0 + LOAD_BETA * load + adj = score / denom if denom > 0 else float(score) + scored.append((adj, b)) + else: + scored.append((float(score), b)) + scored.sort(key=lambda x: x[0], reverse=True) + return [b for _, b in scored] + + +def _ordered_candidates(sticky_key: Optional[str]) -> List[str]: + primaries = _active_primaries() + ordered: List[str] = [] + if primaries: + if sticky_key: + # Rendezvous Hash (HRW) over active primaries + ordered = _hrw_order(sticky_key, primaries, load_aware=False) + else: + if USE_SWRR: + pick = _swrr_pick(primaries) + if pick is None: + ordered = primaries + else: + rest = [b for b in primaries if b != pick] + ordered = [pick] + rest + else: + # Weighted bias for non-sticky requests (randomized, up to two preferred targets) + bias1 = _BIAS_TARGET_BASE in primaries + bias2 = _BIAS_TARGET_BASE2 in primaries + others = [b for b in primaries if b not in (_BIAS_TARGET_BASE, _BIAS_TARGET_BASE2)] + p = random.random() + if bias1 and p < _BIAS_RATIO: + ordered = [_BIAS_TARGET_BASE] + ([b for b in primaries if b != _BIAS_TARGET_BASE]) + elif bias2 and p < (_BIAS_RATIO + _BIAS_RATIO2): + ordered = [_BIAS_TARGET_BASE2] + ([b for b in primaries if b != _BIAS_TARGET_BASE2]) + else: + # Keep a simple rotation start to avoid starving any server + start = _next_active_rr() + if start and start in others: + sidx = others.index(start) + ordered = others[sidx:] + others[:sidx] + else: + ordered = others if others else primaries + # Put bias targets at the end as fallbacks if present + if bias1 and _BIAS_TARGET_BASE not in ordered: + ordered += [_BIAS_TARGET_BASE] + if bias2 and _BIAS_TARGET_BASE2 not in ordered: + ordered += [_BIAS_TARGET_BASE2] + # Append standby (also skip disabled if ever set via admin) + # Standby will be fully included for sticky path via _active_set_for_sticky when any primary is disabled + ordered += [b for b in _STANDBY_UPSTREAMS_BASE if b not in _disabled] + return ordered + + +def _mark_ok(base: str, status: int) -> None: + _fail_counts[base] = 0 + m = _metrics.setdefault(base, {"ok": 0, "err": 0, "last_status": None, "last_error": None, "disabled": False}) + m["ok"] += 1 + m["last_status"] = status + m["last_error"] = None + m["disabled"] = base in _disabled + + +def _mark_err(base: str, error: str) -> None: + cnt = _fail_counts.get(base, 0) + 1 + _fail_counts[base] = cnt + m = _metrics.setdefault(base, {"ok": 0, "err": 0, "last_status": None, "last_error": None, "disabled": False}) + m["err"] += 1 + m["last_error"] = error + # Hard-disable once threshold reached + if cnt >= FAIL_THRESHOLD: + _disabled[base] = { + "disabled_at": time.time(), + "reason": f"auto_fail_threshold_{FAIL_THRESHOLD}", + "last_error": error, + } + m["disabled"] = True + + +@app.get("/health") +async def health(): + return { + "ok": True, + "service": "router_simple", + "expected_model": SUMMARY_MODEL, + "enforce_model": ENFORCE_MODEL, + "upstreams": _SUMMARY_UPSTREAMS_BASE, + "standby": _STANDBY_UPSTREAMS_BASE, + "disabled": list(_disabled.keys()), + } + + +@app.get("/metrics") +async def metrics(): + # include in_flight in metrics snapshot + snapshot: Dict[str, Dict[str, Any]] = {} + for b in set(_SUMMARY_UPSTREAMS_BASE + _STANDBY_UPSTREAMS_BASE): + m = _metrics.setdefault( + b, {"ok": 0, "err": 0, "last_status": None, "last_error": None, "disabled": (b in _disabled)} + ) + m["disabled"] = b in _disabled + m["in_flight"] = int(_in_flight.get(b, 0)) + snapshot[b] = dict(m) + snapshot["sticky_map_size"] = {"count": len(_sticky_map)} + return snapshot + + +@app.get("/admin/status") +async def admin_status(): + return { + "active": _active_primaries(), + "standby": [b for b in _STANDBY_UPSTREAMS_BASE if b not in _disabled], + "disabled": _disabled, + "fail_counts": _fail_counts, + "sticky_disabled": DISABLE_STICKY, + } + + +@app.post("/admin/disable") +@app.get("/admin/disable") +async def admin_disable(base: str): + base = _normalize_base(base) + _disabled[base] = {"disabled_at": time.time(), "reason": "manual"} + m = _metrics.setdefault(base, {"ok": 0, "err": 0, "last_status": None, "last_error": None, "disabled": False}) + m["disabled"] = True + return {"ok": True, "disabled": list(_disabled.keys())} + + +@app.post("/admin/enable") +@app.get("/admin/enable") +async def admin_enable(base: str): + base = _normalize_base(base) + _disabled.pop(base, None) + _fail_counts[base] = 0 + m = _metrics.setdefault(base, {"ok": 0, "err": 0, "last_status": None, "last_error": None, "disabled": False}) + m["disabled"] = False + return {"ok": True, "disabled": list(_disabled.keys())} + + +@app.post("/admin/clear_sticky") +@app.get("/admin/clear_sticky") +async def admin_clear_sticky(): + _sticky_map.clear() + return {"ok": True, "sticky_map_size": len(_sticky_map)} + + +@app.api_route("/v1/{rest:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"]) +async def proxy(rest: str, request: Request): + raw = await request.body() + body_obj = None + if raw: + try: + body_obj = json.loads(raw.decode("utf-8")) + except Exception: + body_obj = None + + # Determine sticky key from header or body + sticky_key = None + try: + sticky_key = request.headers.get("x-trajectory-id") or None + except Exception: + sticky_key = None + if sticky_key is None and isinstance(body_obj, dict): + meta = body_obj.get("metadata") or {} + if isinstance(meta, dict): + sticky_key = meta.get("trajectory_id") or meta.get("traj_id") + if sticky_key is None: + sticky_key = body_obj.get("trajectory_id") or body_obj.get("traj_id") + if sticky_key is None: + # allow OpenAI chat.completions 'user' as sticky key + u = body_obj.get("user") + if isinstance(u, str) and u: + sticky_key = u + + # Determine candidate list based on sticky and active set + candidates: List[str] = [] + if (not ENFORCE_MODEL) or _is_summary_request(body_obj) or (request.method == "GET" and rest.startswith("models")): + if sticky_key and (not DISABLE_STICKY): + actives = _active_set_for_sticky() + mapped = _sticky_map.get(sticky_key) + if mapped in actives: + # stick to existing mapping; use HRW order for failover on remaining actives + failover_order = [b for b in _hrw_order(sticky_key, actives, load_aware=False) if b != mapped] + candidates = [mapped] + failover_order + else: + # Weighted bias for sticky mapping creation + bias1 = _BIAS_TARGET_BASE in actives + bias2 = _BIAS_TARGET_BASE2 in actives + others = [b for b in actives if b not in (_BIAS_TARGET_BASE, _BIAS_TARGET_BASE2)] + p = random.random() + if bias1 and p < _BIAS_RATIO: + _sticky_map[sticky_key] = _BIAS_TARGET_BASE + failover_order = [ + b for b in _hrw_order(sticky_key, actives, load_aware=True) if b != _BIAS_TARGET_BASE + ] + candidates = [_BIAS_TARGET_BASE] + failover_order + elif bias2 and p < (_BIAS_RATIO + _BIAS_RATIO2): + _sticky_map[sticky_key] = _BIAS_TARGET_BASE2 + failover_order = [ + b for b in _hrw_order(sticky_key, actives, load_aware=True) if b != _BIAS_TARGET_BASE2 + ] + candidates = [_BIAS_TARGET_BASE2] + failover_order + else: + pool = others if (others) else actives + ordered = _hrw_order(sticky_key, pool, load_aware=True) + if ordered: + _sticky_map[sticky_key] = ordered[0] + failover_order = [ + b for b in _hrw_order(sticky_key, actives, load_aware=True) if b != ordered[0] + ] + candidates = [ordered[0]] + failover_order + else: + candidates = [] + else: + candidates = _ordered_candidates(None) + if not candidates: + return JSONResponse( + { + "error": "router_simple: request not handled (not the Web Summary model)", + "expected_model": SUMMARY_MODEL, + }, + status_code=400, + ) + + forward_path = rest + + headers = dict(request.headers) + headers.pop("host", None) # let httpx set Host for upstream + params = dict(request.query_params) + + want_stream = (isinstance(body_obj, dict) and body_obj.get("stream") is True) or ( + "text/event-stream" in request.headers.get("accept", "") + ) + + FAILOVER_STATUS = {408, 429, 500, 502, 503, 504} + tried: List[str] = [] + last_error: Optional[str] = None + + for idx, up in enumerate(candidates): + if up in _disabled: + continue + tried.append(up) + url = up.rstrip("/") + "/v1/" + forward_path.lstrip("/") + try: + timeout = httpx.Timeout(connect=2.0, read=None, write=10.0, pool=None) + async with client.stream( + request.method, url, params=params, headers=headers, content=raw, timeout=timeout + ) as resp: + if resp.status_code in FAILOVER_STATUS: + _mark_err(up, error=f"status_{resp.status_code}") + # Hard disable on threshold, then continue to next + continue + ctype = resp.headers.get("content-type", "") + if want_stream or ctype.startswith("text/event-stream"): + # Track in-flight for streaming until generator completes + _in_flight[up] = _in_flight.get(up, 0) + 1 + + async def gen(): + try: + async for chunk in resp.aiter_bytes(): + yield chunk + finally: + _in_flight[up] = max(0, _in_flight.get(up, 1) - 1) + + _mark_ok(up, resp.status_code) + return StreamingResponse( + gen(), + status_code=resp.status_code, + media_type=ctype, + headers={ + "X-Upstream-Server": up, + "X-Failover-Attempts": str(idx), + "X-Tried-Upstreams": ",".join(tried), + }, + ) + # Non-streaming: count in-flight during body read + _in_flight[up] = _in_flight.get(up, 0) + 1 + try: + content = await resp.aread() + finally: + _in_flight[up] = max(0, _in_flight.get(up, 1) - 1) + passthru = { + k: v + for k, v in resp.headers.items() + if k.lower() in ["content-type", "cache-control", "x-request-id"] + } + # annotate which upstream handled the request + passthru["X-Upstream-Server"] = up + passthru["X-Failover-Attempts"] = str(idx) + passthru["X-Tried-Upstreams"] = ",".join(tried) + _mark_ok(up, resp.status_code) + return Response(content=content, status_code=resp.status_code, headers=passthru) + except httpx.HTTPError as e: + last_error = str(e) + _mark_err(up, error=last_error) + continue + + # All attempts failed or all candidates disabled + detail = { + "error": "All upstreams failed or disabled", + "tried": tried, + "last_error": last_error, + "disabled": list(_disabled.keys()), + } + return JSONResponse(detail, status_code=502) diff --git a/skyrl-agent/services/run_router.sh b/skyrl-agent/services/run_router.sh new file mode 100755 index 0000000000..c9927bde25 --- /dev/null +++ b/skyrl-agent/services/run_router.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Simple launcher for the local router. No keys involved. +# Edit routes inside router_simple.py to point to your upstreams. + +PORT=${PORT:-8080} +HOST=${HOST:-0.0.0.0} + +echo "Starting router on http://${HOST}:${PORT}/v1" +cd "$(dirname "$0")" +exec uvicorn router_simple:app --host "${HOST}" --port "${PORT}" + +# curl -s http://127.0.0.1:8080/health | jq \ No newline at end of file diff --git a/skyrl-agent/services/test_remote_services.py b/skyrl-agent/services/test_remote_services.py new file mode 100644 index 0000000000..8e3c41f38d --- /dev/null +++ b/skyrl-agent/services/test_remote_services.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +""" +Test the STEM Judge and Web Summary services that run on remote servers. + +Updates: +- Force the service addresses to be configured through environment variables (no default fallback): + - STEM: STEM_LLM_JUDGE_URL (e.g. http://localhost:8004 or a base that already includes /v1) + - Web Summary: WEB_SUMMARY_API_BASE or EB_SUMMARY_API_BASE (e.g. http://localhost:8080/v1) +- Automatically derive the /health and /v1/chat/completions endpoints based on whether the base already ends with /v1. +- Keep the model identifiers fixed to the user-provided IDs without probing or falling back: + - Web Summary: Qwen/Qwen3-32B + - STEM Judge: openai/gpt-oss-20b +""" + +import requests +import json +import time +from datetime import datetime +import os +from urllib.parse import urlparse, urlunparse +import re + +# Fixed model identifiers provided by the user +WEB_SUMMARY_MODEL_NAME = "Qwen/Qwen3-32B" # Updated to the actually available model +STEM_MODEL_NAME = "openai/gpt-oss-20b" + +# Read service base URLs from environment variables without falling back to defaults +_stem_env = os.getenv("STEM_LLM_JUDGE_URL") +STEM_BASE = _stem_env.rstrip("/") if _stem_env else None + +_web_env = os.getenv("WEB_SUMMARY_API_BASE", os.getenv("EB_SUMMARY_API_BASE")) +WEB_SUMMARY_BASE = _web_env.rstrip("/") if _web_env else None + +# When environment variables are not set, try the following candidates automatically +WEB_SUMMARY_CANDIDATES = [] # Provide candidates via env vars if discovery is desired + +STEM_CANDIDATES = [] # Provide candidates via env vars if discovery is desired + + +def _strip_v1_from_path(path: str) -> str: + """Remove a trailing /v1 or /v1/ fragment from the path.""" + return re.sub(r"/v1/?$", "", path.rstrip("/")) or "/" + + +def build_health_url(base_url: str) -> str: + """Build the health endpoint, replacing a trailing /v1 with the root.""" + parsed = urlparse(base_url) + base_path = parsed.path or "/" + root_path = _strip_v1_from_path(base_path) + # Ensure there is only a single slash + health_path = (root_path.rstrip("/") + "/health") if root_path != "/" else "/health" + return urlunparse((parsed.scheme, parsed.netloc, health_path, "", "", "")) + + +def build_completions_url(base_url: str) -> str: + """Build the /v1/chat/completions endpoint without duplicating /v1.""" + parsed = urlparse(base_url) + base_path = parsed.path or "" + if re.search(r"/v1/?$", base_path): + path = base_path.rstrip("/") + "/chat/completions" + else: + path = base_path.rstrip("/") + "/v1/chat/completions" + return urlunparse((parsed.scheme, parsed.netloc, path, "", "", "")) + + +def test_health_endpoint(service_name, url): + """Exercise a service health-check endpoint.""" + print(f"\n{'='*60}") + print(f"Testing {service_name} health endpoint") + print(f"URL: {url}") + print(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("-" * 60) + + try: + response = requests.get(url, timeout=3) + if response.status_code == 200: + print(f"✅ Status: OK (HTTP {response.status_code})") + print(f"Response body: {response.text[:200]}") + return True + else: + print(f"⚠️ Status: Unexpected (HTTP {response.status_code})") + print(f"Response body: {response.text[:200]}") + return False + except requests.exceptions.RequestException as e: + print(f"❌ Error: {str(e)}") + return False + + +def try_health_on_base(base_url: str, timeout: int = 3) -> bool: + """Try calling /health for the given base URL.""" + url = build_health_url(base_url) + try: + resp = requests.get(url, timeout=timeout) + return resp.status_code == 200 + except requests.exceptions.RequestException: + return False + + +def discover_base(service_name: str, candidates: list) -> str: + """Probe the candidate list and return the first healthy base URL, if any.""" + print(f"\nStarting {service_name} base URL discovery...") + for i, base in enumerate(candidates, 1): + health_url = build_health_url(base) + print(f" [{i:02d}] Trying: {health_url}", end=" ") + ok = try_health_on_base(base) + if ok: + print("=> ✅ Available") + return base.rstrip("/") + else: + print("=> ❌ Unavailable") + print(f"Unable to discover a reachable {service_name} service") + return None + + +def test_stem_judge_inference(): + """Run inference checks against the STEM Judge service.""" + print(f"\n{'='*60}") + print("Testing STEM Judge inference") + print("-" * 60) + + if not STEM_BASE: + # Auto-discover when the base URL is not preset + base = discover_base("STEM Judge", STEM_CANDIDATES) + if not base: + print("❌ Unable to discover a STEM service, skipping inference tests") + return + globals()["STEM_BASE"] = base + + url = build_completions_url(STEM_BASE) + + # Example test cases + test_cases = [ + { + "name": "Simple arithmetic (correct answer)", + "question": "1+1=?", + "ground_truth": "2", + "student_answer": "2", + "expected": "Yes", + }, + { + "name": "Simple arithmetic (incorrect answer)", + "question": "1+1=?", + "ground_truth": "2", + "student_answer": "3", + "expected": "No", + }, + ] + + for i, test_case in enumerate(test_cases, 1): + print(f"\nTest case {i}: {test_case['name']}") + print("-" * 40) + + # Build the judge prompt + prompt = f"""### Question: {test_case['question']} + +### Ground Truth Answer: {test_case['ground_truth']} + +### Student Answer: {test_case['student_answer']} + +If correct, output "Final Decision: Yes" else "Final Decision: No".""" + + # Build the OpenAI-compatible request payload + payload = { + "model": STEM_MODEL_NAME, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0, + "max_tokens": 100, + } + + print(f"Sending request to: {url}") + print(f"Expected result: Final Decision: {test_case['expected']}") + + try: + start_time = time.time() + response = requests.post(url, headers={"Content-Type": "application/json"}, json=payload, timeout=30) + elapsed_time = time.time() - start_time + + if response.status_code == 200: + result = response.json() + if "choices" in result and len(result["choices"]) > 0: + content = result["choices"][0]["message"]["content"] + print(f"✅ Inference succeeded (elapsed: {elapsed_time:.2f} seconds)") + print(f"Model response: {content}") + + # Verify the response matches expectations + if f"Final Decision: {test_case['expected']}" in content: + print("✅ Result matches expectation") + else: + print("⚠️ Result may not match expectation") + else: + print("⚠️ Response format is unexpected") + print(f"Response: {json.dumps(result, indent=2)}") + else: + print(f"❌ Request failed (HTTP {response.status_code})") + print(f"Error details: {response.text[:500]}") + + except requests.exceptions.RequestException as e: + print(f"❌ Request error: {str(e)}") + except json.JSONDecodeError as e: + print(f"❌ JSON decode error: {str(e)}") + print(f"Raw response: {response.text[:500]}") + + +def test_web_summary(): + """Run inference checks against the Web Summary service.""" + print(f"\n{'='*60}") + print("Testing Web Summary service") + print("-" * 60) + + if not WEB_SUMMARY_BASE: + # Auto-discover when the base URL is not preset + base = discover_base("Web Summary", WEB_SUMMARY_CANDIDATES) + if not base: + print("❌ Unable to discover a Web Summary service, skipping inference tests") + return + globals()["WEB_SUMMARY_BASE"] = base + + # Start by testing the health endpoint + health_url = build_health_url(WEB_SUMMARY_BASE) + if test_health_endpoint("Web Summary", health_url): + print("\nRunning Web Summary inference checks...") + + # Then exercise the inference endpoint + url = build_completions_url(WEB_SUMMARY_BASE) + + test_content = """ + This is a test article about artificial intelligence. + AI has made significant progress in recent years. + Machine learning models can now perform various tasks. + """ + + # Always use the specified model name + payload = { + "model": WEB_SUMMARY_MODEL_NAME, + "messages": [{"role": "user", "content": f"Please summarize the following text:\n\n{test_content}"}], + "temperature": 0.7, + "max_tokens": 150, + } + + print(f"Sending test request to: {url}") + print(f"Using model: {WEB_SUMMARY_MODEL_NAME}") + + try: + start_time = time.time() + response = requests.post(url, headers={"Content-Type": "application/json"}, json=payload, timeout=30) + elapsed_time = time.time() - start_time + + if response.status_code == 200: + result = response.json() + if "choices" in result and len(result["choices"]) > 0: + content = result["choices"][0]["message"]["content"] + print(f"✅ Inference succeeded (elapsed: {elapsed_time:.2f} seconds)") + print(f"Summary result: {content[:500]}") + else: + print("⚠️ Response format is unexpected") + print(f"Response: {json.dumps(result, indent=2)[:500]}") + else: + print(f"⚠️ Inference endpoint might be unavailable (HTTP {response.status_code})") + print(f"Response: {response.text[:500]}") + + except requests.exceptions.RequestException as e: + print(f"❌ Request error: {str(e)}") + except json.JSONDecodeError as e: + print(f"❌ JSON decode error: {str(e)}") + + +def main(): + """Entry point for ad-hoc remote service validation.""" + global STEM_BASE, WEB_SUMMARY_BASE + print("=" * 60) + print("Remote service testing utility") + print(f"STEM base URL: {STEM_BASE or 'Not configured (will discover automatically)'}") + print(f"Web Summary base URL: {WEB_SUMMARY_BASE or 'Not configured (will discover automatically)'}") + print("=" * 60) + + # Discover base URLs when they are not pre-configured + if not WEB_SUMMARY_BASE: + WEB_SUMMARY_BASE = discover_base("Web Summary", WEB_SUMMARY_CANDIDATES) + if not STEM_BASE: + STEM_BASE = discover_base("STEM Judge", STEM_CANDIDATES) + + # Test health-check endpoints + web_summary_health = False + stem_judge_health = False + + if WEB_SUMMARY_BASE: + print(f" Web Summary health URL: {build_health_url(WEB_SUMMARY_BASE)}") + web_summary_health = test_health_endpoint("Web Summary", build_health_url(WEB_SUMMARY_BASE)) + if STEM_BASE: + print(f" STEM Judge health URL: {build_health_url(STEM_BASE)}") + stem_judge_health = test_health_endpoint("STEM Judge", build_health_url(STEM_BASE)) + + # Run inference tests when the health-check passes + if stem_judge_health: + test_stem_judge_inference() + else: + if STEM_BASE: + print("\n⚠️ Skipping STEM Judge inference test (health check failed)") + + if web_summary_health: + test_web_summary() + else: + if WEB_SUMMARY_BASE: + print("\n⚠️ Skipping Web Summary inference test (health check failed)") + + # Summarize the checks at the end + print(f"\n{'='*60}") + print("Test summary") + print("-" * 60) + print(f"Web Summary health check: {'✅ Passed' if web_summary_health else '❌ Failed or skipped'}") + print(f"STEM Judge health check: {'✅ Passed' if stem_judge_health else '❌ Failed or skipped'}") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/skyrl-agent/skyrl-train b/skyrl-agent/skyrl-train new file mode 120000 index 0000000000..b345938d2f --- /dev/null +++ b/skyrl-agent/skyrl-train @@ -0,0 +1 @@ +../skyrl-train/ \ No newline at end of file diff --git a/skyrl-agent/skyrl_agent/__init__.py b/skyrl-agent/skyrl_agent/__init__.py new file mode 100644 index 0000000000..ba4fb3fc39 --- /dev/null +++ b/skyrl-agent/skyrl_agent/__init__.py @@ -0,0 +1 @@ +from .auto import AutoAgentRunner diff --git a/skyrl-agent/skyrl_agent/agents/__init__.py b/skyrl-agent/skyrl_agent/agents/__init__.py new file mode 100644 index 0000000000..8885b05111 --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/__init__.py @@ -0,0 +1,2 @@ +from .base import AgentRunner +from .mapping import AGENT_GENERATOR_REGISTRY, AGENT_TRAJECTORY_REGISTRY diff --git a/skyrl-agent/skyrl_agent/agents/base.py b/skyrl-agent/skyrl_agent/agents/base.py new file mode 100644 index 0000000000..b4d801defd --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/base.py @@ -0,0 +1,691 @@ +from typing import Any, Dict, TypedDict, List, Optional, Type, Callable +from collections import defaultdict +from abc import ABC, abstractmethod +import os +import copy +from omegaconf import OmegaConf, DictConfig +import pandas as pd +from loguru import logger +from skyrl_agent.tasks.base import BaseTask +from transformers import AutoTokenizer +from dataclasses import dataclass + +from skyrl_agent.integrations.base import ( + build_backend, + build_generator_input, + build_generator_output, + _import_object, + AsyncInferBackend, +) +from skyrl_agent.dispatcher.dispatchers import DISPATCHER_REGISTRY, DispatcherType +from skyrl_agent.config.configuration_utils import TASK_CONFIG_REGISTRY, get_field_from_config, TrajectoryConfig +from skyrl_agent.functional.chat_template import chat_template, chat_template_qwen3_thinking +from skyrl_agent.functional.utils import transitions_to_training_data +from .mapping import AGENT_TRAJECTORY_REGISTRY + + +class CompleterOutput: + pass + + +@dataclass +class RuntimeConfig: + runtime_initializer: Optional[Callable] + instruction_getter: Callable + config_builder: Optional[Callable] + completer: Optional[Callable] + evaluator: Callable + + @classmethod + def from_dict(cls, cfg: DictConfig): + def safe_import(cfg, key): + try: + val = cfg.get(key, None) + return _import_object(val) if val else None + except AttributeError: + return None + + runtime_initializer = safe_import(cfg, "initializer") + config_builder = safe_import(cfg, "config_builder") + instruction_getter = safe_import(cfg, "instruction_getter") # If optional; else raise if missing + completer = safe_import(cfg, "completer") + evaluator = safe_import(cfg, "evaluator") + return cls( + runtime_initializer=runtime_initializer, + config_builder=config_builder, + instruction_getter=instruction_getter, + completer=completer, + evaluator=evaluator, + ) + + +@dataclass +class TrajectoryResult(TypedDict): + instance_id: str + trajectory_id: str + messages: List[Dict[str, str]] + state: Any + results: Optional[CompleterOutput] + error: Optional[str] + finish: bool + finish_reason: str + reward: Optional[bool] + eval_error: Optional[str] + + +class BaseTrajectory(ABC): + + def __init__( + self, + cfg: TrajectoryConfig, + data: Dict[str, Any], + infer_engine: AsyncInferBackend, + tokenizer: AutoTokenizer, + task: BaseTask, + val_mode: bool = False, + ) -> None: + super().__init__() + + self.cfg = cfg + self.data = data + self.infer_engine = infer_engine + self.tokenizer = tokenizer + self.task = task + self.val_mode = val_mode + self.agent_cls = _import_object(cfg.agent_cls) + + self.result: TrajectoryResult = None + + @abstractmethod + async def initialize_trajectory(self): + pass + + @abstractmethod + async def generate_trajectory(self): + pass + + @abstractmethod + async def evaluate_trajectory(self): + pass + + +# TODO(csy): also specify whether loss_mask, attention_mask, etc. are needed -- for training or eval +class AgentRunner: + def __init__(self, cfg: Dict[str, Any], infer_engine: Any, tokenizer: Any) -> None: + """ + Initialize the CodeActGenerator with the given configuration. + + Args: + generation_config: Configuration dictionary containing parameters like max_prompt_length, max_response_length, etc. + """ + self.cfg = cfg + + # infer engine + self.infer_engine = build_backend( + cfg.generator.infer_backend, + infer_engine=infer_engine, + tokenizer=tokenizer, + cfg=cfg.generator.backend_config, + ) + self.tokenizer = tokenizer + self.traj_cls: Type[BaseTrajectory] = _import_object(AGENT_TRAJECTORY_REGISTRY.get(cfg.agent_cls)) + self.task: BaseTask = _import_object(cfg.task)() + + # metadata + self.trajectories: Dict[str, Dict[str, BaseTrajectory]] = {} + + # Will be set in subclasses + self.agent_config = None + + @classmethod + def from_task(cls, task: str, infer_engine: Any, tokenizer: Any): + # Resolve task name or path + if os.path.exists(task): + config_path = task + elif task in TASK_CONFIG_REGISTRY: + config_path = TASK_CONFIG_REGISTRY[task] + else: + raise ValueError( + f"Unknown task '{task}'. Must be a YAML path or one of: {list(TASK_CONFIG_REGISTRY.keys())}" + ) + + cfg = OmegaConf.load(config_path) + + return cls(cfg, infer_engine, tokenizer) + + def _get_data(self, content) -> Dict[str, Any]: + """Process input data into trajectory input.""" + data_cfg = self.cfg.get("data", {}) + + # Resolve instance payload + instance = None + if data_cfg.get("instance_key"): + try: + instance = get_field_from_config(data_cfg.get("instance_key"), content) + except ValueError: + instance = None + + # Resolve instance_id; rely on configured key and downstream default to batch_id if missing + instance_id = None + if data_cfg.get("instance_id_key"): + try: + instance_id = get_field_from_config(data_cfg.get("instance_id_key"), content) + except ValueError: + instance_id = None + + # Resolve data_source with default fallback + data_source = "default" + if data_cfg.get("data_source_key"): + try: + data_source = get_field_from_config(data_cfg.get("data_source_key"), content) + except ValueError: + data_source = "default" + + return { + "instance": instance if instance is not None else content, + "instance_id": instance_id, + "data_source": data_source, + } + + def _initialize_trajectories(self, val_mode: bool = False): + for batch_id, content in enumerate(self.batch): + data = self._get_data(content) + instance_id: str = data["instance_id"] if data["instance_id"] else batch_id + self.trajectories[instance_id] = {} + sampling_params = ( + self.cfg.generator.val_config.sampling_params if val_mode else self.cfg.generator.sampling_params + ) + sampling_params = OmegaConf.to_container(sampling_params, resolve=True) # e.g. converts ListConfig to list + num_trajectories = ( + self.cfg.generator.val_config.num_trajectories if val_mode else self.cfg.generator.num_trajectories + ) + + # Simple generator-level toggles (defaults) + profile_tools = bool(self.cfg.generator.get("profile_tools", False)) + debug_log = bool(self.cfg.generator.get("debug_log", False)) + enable_turn_reminder = bool(self.cfg.generator.get("enable_turn_reminder", False)) + + for traj_id in range(num_trajectories): + traj_cfg = TrajectoryConfig( + instance_id=instance_id, + trajectory_id=traj_id, + max_prompt_length=self.cfg.generator.max_prompt_length, + sampling_params=sampling_params, + vision_is_active=self.cfg.generator.vision_is_active, + qwen3_enable_thinking=self.cfg.generator.qwen3_enable_thinking, + qwen3_acc_thinking=self.cfg.generator.qwen3_acc_thinking, + max_iterations=self.cfg.generator.max_iterations, + tools=self.cfg.tools, + agent_cls=self.cfg.agent_cls, + profile_tools=profile_tools, + debug_log=debug_log, + early_step_threshold=self.cfg.generator.get("early_step_threshold", 0), + enable_turn_reminder=enable_turn_reminder, + ) + traj: BaseTrajectory = self.traj_cls( + cfg=traj_cfg, + data=data, + tokenizer=self.tokenizer, + infer_engine=self.infer_engine, + task=self.task, + val_mode=val_mode, + ) + self.trajectories[instance_id][traj_id] = traj + + def _post_process_results(self, return_tensors=False, val_mode: bool = False) -> Dict[str, Any]: + """ + Post-process the results to convert them into the appropriate output format. + + Returns: + A dictionary containing the processed results. + """ + raw_reward_list = [] + all_results = {} + matched_results = [] + instance_list = [] + error_list = [] + resolved_list = [] + traj_reward_list = [] + has_finish_action_list = [] + finish_reason_list = [] + + num_trajectories = ( + self.cfg.generator.val_config.num_trajectories if val_mode else self.cfg.generator.num_trajectories + ) + + for instance_id in self.trajectories: + for trajectory_id in self.trajectories[instance_id]: + all_results.setdefault(instance_id, {})[trajectory_id] = self.trajectories[instance_id][ + trajectory_id + ].result + + for batch_idx, content in enumerate(self.batch): + data = self._get_data(content) + instance = pd.Series(data["instance"]) + instance_id = data["instance_id"] if data["instance_id"] else batch_idx + instance["instance_id"] = instance_id # safe mutation + trajectories = all_results.get(instance_id, {}) + matched_results.extend(trajectories.values()) + instance_list.extend([instance] * len(trajectories)) + + assert len(matched_results) == num_trajectories * len( + self.batch + ), f"Expected number of results {num_trajectories * len(self.batch)}, got {len(matched_results)}" + + # Group results by instance_id for message handling + results_by_instance = {} + for i, (instance, result) in enumerate(zip(instance_list, matched_results)): + instance_id = instance["instance_id"] + results_by_instance.setdefault(instance_id, []).append((i, result)) + + global_fallback_set = None + for results in results_by_instance.values(): + if all(res.get("messages") for _, res in results): + global_fallback_set = [copy.deepcopy(res) for _, res in results] + break + # get reward before handling empty messages + for idx, result in enumerate(matched_results): + reward = result.get("reward", False) + raw_reward_list.append(reward) + raw_reward = sum(raw_reward_list) / len(raw_reward_list) + num_empty_messages = sum(1 for res in matched_results if not res.get("messages", [])) + # Handle empty messages by copying from another trajectory of the same instance + for instance_id, results in results_by_instance.items(): + # Look for a non-empty base result + fallback = next((res for _, res in results if res.get("messages")), None) + if not fallback: + if global_fallback_set: + logger.warning( + f"[WARN] No local fallback for instance_id {instance_id}, using global fallback set." + ) + for j, (idx, res) in enumerate(results): + # Use corresponding global fallback result (same trajectory index) + fallback_res = global_fallback_set[j % len(global_fallback_set)] + print(f"Empty messages for instance_id {instance_id}, trajectory {idx}. Using global fallback.") + for key, value in fallback_res.items(): + matched_results[idx][key] = copy.deepcopy(value) + matched_results[idx]["finish_reason"] = "error_runtime" + + else: + logger.error(f"[FATAL] No fallback (local/global) for instance_id {instance_id}. Skipping.") + continue + else: + for idx, res in results: + if not res.get("messages", []): + print(f"Empty messages for instance_id {instance_id}, trajectory {idx}. Using local fallback.") + for key, value in fallback.items(): + matched_results[idx][key] = copy.deepcopy(value) + matched_results[idx]["finish_reason"] = "error_runtime" + + # error evaluation mainly due to timeout during tool execution + mask_out_reason = [ + "CONTEXT_WINDOW_EXCEEDED", + "error_runtime", + "error_evaluation", + "max_iterations_reached", + "BAD_LLM_RESPONSE", + "stuck_in_a_loop", + "cmd_timeout", + ] + # Get training data + + # backward compatibility for old format + # TODO(csy): remove this after oh_agent is updated + all_messages = [] + all_prompts = [] + all_responses = [] + + # step-level + prompt_input_ids = [] + response_ids = [] + response_assistant_mask = [] + logprobs = [] + steps_per_trajectory = [] + reward_list = [] + is_last_episode_list = [] + traj_idx_list = [] + step_finish_reason_list = [] + + num_turns = [] # assistant-based turns + for result in matched_results: + current_traj_id = f"{result.get('instance_id')}-traj{result.get('trajectory_id')}" + messages = result.get("messages", []) + # Count assistant messages as turns to match actual steps + num_turns.append(sum(1 for m in messages if m.get("role") == "assistant")) + # trajectory-level results + error_list.append(result.get("error", None)) + resolved_list.append(result.get("reward", False)) + traj_reward_list.append(result.get("reward", False)) + has_finish_action_list.append(result.get("finish", False)) + finish_reason_list.append(result.get("finish_reason", None)) + + transitions = result.get("transitions", []) + # backward compatibility for old format + # TODO(csy): remove this after oh_agent is updated + if not transitions: + logger.info( + f"No transitions found for instance_id {instance_id}, trajectory_id {trajectory_id}. Using messages instead." + ) + all_messages.append(messages) + starting_index = 0 + for i, msg in enumerate(messages): + if msg["role"] == "assistant": + starting_index = i + break + if starting_index == 0: + # If we don't find an assistant, all messages are prompts and there are no responses + print( + f'ERROR: Found no assistant message. len(messages) == {len(messages)} and roles are {[msg["role"] for msg in messages]}' + ) + starting_index = len(messages) + prompt = messages[:starting_index] + all_prompts.append(prompt) + response = messages[starting_index:] + all_responses.append(response) + # filter bad trajectories + if messages and messages[-1]["role"] == "assistant": + finish_reason = result.get("finish_reason", None) + if finish_reason not in mask_out_reason: + if not ( + "" in messages[-1]["content"] and "" in messages[-1]["content"] + ): + print( + f"[WARN] Last message does not contain finish function call. Marking finish_reason {finish_reason} as BAD_LLM_RESPONSE. Content: {messages[-1]['content']}" + ) + result["finish_reason"] = "BAD_LLM_RESPONSE" + + if messages and messages[-1]["role"] == "user": + finish_reason = result.get("finish_reason", None) + if finish_reason not in mask_out_reason: + print( + f"[WARN] Last message is from user but it's not in mask_out_reason. Marking finish_reason {finish_reason} as error_runtime. Content: {messages[-1]['content']}" + ) + result["finish_reason"] = "error_runtime" + continue + + # step-level results + data_list = transitions_to_training_data(transitions) + for data in data_list: + prompt_input_ids.append(data.input_tokens) + response_ids.append(data.response_tokens) + logprobs.append(data.response_logprobs) + response_assistant_mask.append(data.response_mask) + is_last_episode_list.append(False) + is_last_episode_list[-1] = True + steps_per_trajectory.append(len(data_list)) + reward_list.extend([result.get("reward", False)] * len(data_list)) + step_finish_reason_list.extend([result.get("finish_reason", None)] * len(data_list)) + traj_idx_list.extend([current_traj_id] * len(data_list)) + + # backward compatibility for old format + # TODO(csy): remove this after oh_agent is updated + if all_messages: + # Encode messages, get assitant mask and position ids + prompt_encodings = self.tokenizer.apply_chat_template( + all_prompts, + # return_tensors="pt", + add_generation_prompt=True, + return_dict=True, + # padding=True + ) + prompt_input_ids = prompt_encodings["input_ids"] + + response_encodings = self.tokenizer.apply_chat_template( + all_responses, + chat_template=chat_template_qwen3_thinking if self.cfg.generator.remove_think_tokens else chat_template, + return_assistant_tokens_mask=True, + add_generation_prompt=False, + return_dict=True, + ) + + response_ids = response_encodings["input_ids"] + response_assistant_mask = response_encodings["assistant_masks"] + # to be compatible with new format + logprobs = [None] * len(response_ids) + step_finish_reason_list = finish_reason_list + reward_list = resolved_list + + max_response_length = self.cfg.generator.max_prompt_length + truncated_ids = [] + truncated_masks = [] + truncated_logprobs = [] + + for idx, (ids, mask, logprob, reason) in enumerate( + zip(response_ids, response_assistant_mask, logprobs, step_finish_reason_list) + ): + # Check if truncation is needed + first_nonzero = mask.index(1) if 1 in mask else len(mask) + ids = ids[first_nonzero:] + mask = mask[first_nonzero:] + if logprob is not None: + logprob = logprob[first_nonzero:] + + if len(ids) > max_response_length: + if reason not in mask_out_reason: + logger.warning( + f"[WARN] Response length {len(ids)} > max_response_length={max_response_length} " + f"but finish_reason='{reason}' not in mask_out_reason={mask_out_reason}. " + ) + # modify reason to CONTEXT_WINDOW_EXCEEDED + finish_reason_list[idx] = "CONTEXT_WINDOW_EXCEEDED" + # Truncate tokens and masks + ids = ids[:max_response_length] + mask = mask[:max_response_length] + if logprob is not None: + logprob = logprob[:max_response_length] + + truncated_ids.append(ids) + truncated_masks.append(mask) + truncated_logprobs.append(logprob) + + response_ids = truncated_ids + response_assistant_mask = truncated_masks + logprobs = truncated_logprobs + + loss_mask = [ + [0] * len(mask) if (reason in mask_out_reason) else mask + for mask, reason in zip(response_assistant_mask, step_finish_reason_list) + ] + + rollout_metrics = {} + # Compute assistant-based turn average and record metric + avg_turn_assistant = (sum(num_turns) / len(num_turns)) if len(num_turns) > 0 else 0.0 + rollout_metrics["rollout_metrics/avg_turn_assistant"] = avg_turn_assistant + + # Note: no backward-compat key kept (removed per request) + + total_per_instance = defaultdict(int) + resolved_per_instance = defaultdict(int) + for instance, reward in zip(instance_list, resolved_list): + instance_id = instance["instance_id"] + total_per_instance[instance_id] += 1 + if reward > 0: + resolved_per_instance[instance_id] += 1 + + # Track how many instances have resolution rate 0% or 100% + num_resolved_0 = 0 + num_resolved_1 = 0 + + # Print ratio and update counts + for instance in sorted(total_per_instance): + total = total_per_instance[instance] + resolved = resolved_per_instance[instance] + + if resolved == 0: + num_resolved_0 += 1 + elif resolved == total: + num_resolved_1 += 1 + + rollout_metrics["rollout_metrics/num_all_resolved"] = num_resolved_1 + rollout_metrics["rollout_metrics/num_none_resolved"] = num_resolved_0 + rollout_metrics["rollout_metrics/finish_tool_ratio"] = sum( + 1 for reason in finish_reason_list if reason == "FINISH_TOOL" + ) / len(finish_reason_list) + rollout_metrics["rollout_metrics/context_exceed_ratio"] = sum( + 1 for reason in finish_reason_list if reason == "CONTEXT_WINDOW_EXCEEDED" + ) / len(finish_reason_list) + # Ratio of trajectories stopped by iteration cap; avoid 'max' in key to prevent max-reduction + rollout_metrics["rollout_metrics/iter_cap_ratio"] = sum( + 1 for reason in finish_reason_list if reason == "max_iterations_reached" + ) / len(finish_reason_list) + rollout_metrics["rollout_metrics/stuck_in_a_loop_ratio"] = sum( + 1 for reason in finish_reason_list if reason == "stuck_in_a_loop" + ) / len(finish_reason_list) + rollout_metrics["rollout_metrics/error_runtime"] = sum( + 1 for reason in finish_reason_list if reason == "error_runtime" + ) / len(finish_reason_list) + rollout_metrics["rollout_metrics/error_evaluation"] = sum( + 1 for reason in finish_reason_list if reason == "error_evaluation" + ) / len(finish_reason_list) + rollout_metrics["rollout_metrics/num_mask_out"] = sum(1 for mask in loss_mask if all(m == 0 for m in mask)) + rollout_metrics["rollout_metrics/num_mask_non_zero_reward"] = sum( + 1 for mask, reward in zip(loss_mask, resolved_list) if all(m == 0 for m in mask) and reward > 0 + ) + rollout_metrics["rollout_metrics/bad_llm_response"] = sum( + 1 for reason in finish_reason_list if reason == "BAD_LLM_RESPONSE" + ) / len(finish_reason_list) + rollout_metrics["rollout_metrics/raw_reward"] = raw_reward + rollout_metrics["rollout_metrics/num_empty_messages"] = num_empty_messages + + # Optional aggregation of tool-call profiling if available + try: + cfg_profile = None + for path in ("generator.profile_tools", "skyrl_agent.profile_tools"): + try: + cfg_profile = OmegaConf.select(self.cfg, path) + except Exception: + cfg_profile = None + if cfg_profile is not None: + break + profile_enabled = bool(cfg_profile) + tool_calls_per_traj = [] + tool_calls_per_traj_nf = [] # exclude finish + tool_name_totals = defaultdict(int) + if profile_enabled: + for res in matched_results: + state = res.get("state") or {} + prof = state.get("tool_profile") if isinstance(state, dict) else None + if prof and isinstance(prof, dict): + total = prof.get("tool_calls_total") + by_name = prof.get("tool_calls_by_name") or {} + if isinstance(total, int): + tool_calls_per_traj.append(total) + if isinstance(by_name, dict): + # accumulate per-tool totals + for k, v in by_name.items(): + try: + tool_name_totals[k] += int(v) + except Exception: + pass + # compute no-finish per-traj sum + try: + nf_sum = sum(int(v) for k, v in by_name.items() if k != "finish") + tool_calls_per_traj_nf.append(nf_sum) + except Exception: + pass + + def emit_distribution(prefix: str, vals: list[int]): + if not vals: + return + n = len(vals) + s = sorted(vals) + mean_val = sum(vals) / n + mnv = s[0] + mxv = s[-1] + rollout_metrics[f"{prefix}_total"] = int(sum(vals)) + rollout_metrics[f"{prefix}_per_traj_mean"] = float(mean_val) + rollout_metrics[f"{prefix}_per_traj_min"] = float(mnv) + rollout_metrics[f"{prefix}_per_traj_max"] = float(mxv) + + # Emit distributions for overall and no-finish variants + emit_distribution("rollout_metrics/tool_calls", tool_calls_per_traj) + emit_distribution("rollout_metrics/tool_calls_no_finish", tool_calls_per_traj_nf) + + for name, cnt in tool_name_totals.items(): + rollout_metrics[f"rollout_metrics/tool_name/{name}"] = int(cnt) + except Exception: + pass + + print("rollout metrics:", rollout_metrics) + + print(f"Finish reason: {finish_reason_list}") + # Create tensor dictionary + output = { + "prompt_token_ids": prompt_input_ids, + "response_ids": response_ids, + "rewards": reward_list, + "traj_rewards": traj_reward_list, + "loss_masks": loss_mask, + "episode_nums": steps_per_trajectory, + "is_last_episode": is_last_episode_list, + "traj_idx": traj_idx_list, + "stop_reasons": None, + "rollout_logprobs": logprobs, + "rollout_metrics": rollout_metrics, + } + + return output + + async def run(self, input_batch: Any, val_mode: bool = False) -> Any: + """ + Generate trajectories for the given prompts using the configured agents. + + Args: + prompts: A dictionary containing training instances. + val_mode: Whether we're running validation. + + Returns: + Results converted to the appropriate output format based on infer backend. + """ + self.batch = build_generator_input(self.cfg.generator.infer_backend, input_batch).input_batch + + if val_mode: + num_trajectories = self.cfg.generator.val_config.num_trajectories + sampling_params = self.cfg.generator.val_config.sampling_params + else: + sampling_params = self.cfg.generator.sampling_params + num_trajectories = self.cfg.generator.num_trajectories + + # Initialize agents and other components + self._initialize_trajectories(val_mode=val_mode) + + generator_dispatcher: DispatcherType | None = DISPATCHER_REGISTRY.get(self.cfg.dispatcher.type) + if not generator_dispatcher: + raise ValueError(f"Unknown generator type: {self.cfg.dispatcher.type}") + else: + logger.info(f"Using generator dispatcher: {self.cfg.dispatcher.type}") + init_fn = "initialize_trajectory" + run_fn = "generate_trajectory" + eval_fn = "evaluate_trajectory" + if val_mode: + max_parallel_agents = self.cfg.dispatcher.get("val_config", {}).get( + "max_parallel_agents", self.cfg.dispatcher.max_parallel_agents + ) + max_eval_parallel_agents = self.cfg.dispatcher.get("val_config", {}).get( + "max_eval_parallel_agents", self.cfg.dispatcher.max_eval_parallel_agents + ) + dispatcher_cfg = { + "sampling_params": sampling_params, + "max_parallel_agents": max_parallel_agents, + "max_eval_parallel_agents": max_eval_parallel_agents, + "num_instances": len(self.batch), + "num_trajectories": num_trajectories, + } + else: + dispatcher_cfg = { + "sampling_params": sampling_params, + "max_parallel_agents": self.cfg.dispatcher.max_parallel_agents, + "max_eval_parallel_agents": self.cfg.dispatcher.max_eval_parallel_agents, + "num_instances": len(self.batch), + "num_trajectories": num_trajectories, + } + await generator_dispatcher( + dispatcher_cfg, self.trajectories, init_fn=init_fn, run_fn=run_fn, eval_fn=eval_fn + ) + + output = self._post_process_results(val_mode=val_mode) + + # reset after run + self.trajectories = {} + + return build_generator_output(self.cfg.generator.infer_backend, output).result diff --git a/skyrl-agent/skyrl_agent/agents/mapping.py b/skyrl-agent/skyrl_agent/agents/mapping.py new file mode 100644 index 0000000000..4609eae33c --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/mapping.py @@ -0,0 +1,9 @@ +AGENT_GENERATOR_REGISTRY = { + "skyrl_agent.agents.oh_codeact.OHCodeActAgent": "skyrl_agent.agents.base.AgentRunner", + "skyrl_agent.agents.react.ReActAgent": "skyrl_agent.agents.base.AgentRunner", +} + +AGENT_TRAJECTORY_REGISTRY = { + "skyrl_agent.agents.oh_codeact.OHCodeActAgent": "skyrl_agent.agents.oh_codeact.CodeActTrajectory", + "skyrl_agent.agents.react.ReActAgent": "skyrl_agent.agents.react.ReActTrajectory", +} diff --git a/skyrl-agent/skyrl_agent/agents/oh_codeact/__init__.py b/skyrl-agent/skyrl_agent/agents/oh_codeact/__init__.py new file mode 100644 index 0000000000..b8c2021506 --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/oh_codeact/__init__.py @@ -0,0 +1,2 @@ +from .codeact_agent import OHCodeActAgent +from .codeact_runner import CodeActTrajectory diff --git a/skyrl-agent/skyrl_agent/agents/oh_codeact/codeact_agent.py b/skyrl-agent/skyrl_agent/agents/oh_codeact/codeact_agent.py new file mode 100644 index 0000000000..a8ff422ce4 --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/oh_codeact/codeact_agent.py @@ -0,0 +1,308 @@ +from typing import Any, List, Optional +import json +from pathlib import Path +from datetime import datetime +from uuid import uuid4 +import traceback +import copy + +from skyrl_agent.functional.function_calling import convert_str_to_completion_format +from skyrl_agent.functional.chat_template import get_templates_path +from skyrl_agent.config.configuration_utils import TrajectoryConfig +from skyrl_agent.integrations.base import AsyncInferBackend +from skyrl_agent.dispatcher.async_utils import call_async_from_sync + +import openhands.agenthub.codeact_agent.function_calling as codeact_function_calling +from openhands.agenthub.codeact_agent import CodeActAgent +from openhands.controller.agent import Agent +from openhands.controller.state.state import State +from openhands.core.config import LLMConfig, AgentConfig +from openhands.core.logger import openhands_logger as logger +from openhands.core.message import Message +from openhands.core.exceptions import ( + FunctionCallNotExistsError, + FunctionCallValidationError, + LLMMalformedActionError, + LLMNoActionError, + LLMResponseError, +) +from openhands.events.event import Event +from openhands.events.action import ( + Action, + AgentFinishAction, + MessageAction, +) +from openhands.memory.condenser.condenser import Condensation, View + +from openhands.llm.fn_call_converter import ( + convert_fncall_messages_to_non_fncall_messages, + convert_non_fncall_messages_to_fncall_messages, +) +from openhands.llm.llm import LLM +from openhands.core.config.condenser_config import ( + NoOpCondenserConfig, +) + + +class OHCodeActAgent(CodeActAgent): + """ + An online implementation of CodeActAgent that leverages infer's asynchronous capabilities + for a single agent instance. + """ + + def __init__( + self, + traj_config: TrajectoryConfig, + infer_engine: Optional[AsyncInferBackend] = None, + tokenizer: Optional[Any] = None, + ) -> None: + """ + Initialize a single OHCodeActAgent instance. + """ + # dummy value to let openhands tracks the name + llm = LLM(LLMConfig(model="dummy", max_message_chars=128000)) + + self.agent_config = AgentConfig( + enable_jupyter=traj_config.tools.get("enable_jupyter", False), + enable_browsing=traj_config.tools.get("enable_browsing", False), + enable_llm_editor=traj_config.tools.get("enable_llm_editor", False), + enable_editor=traj_config.tools.get("enable_editor", False), + enable_think=traj_config.tools.get("enable_think", False), + enable_search=traj_config.tools.get("enable_search", False), + condenser=NoOpCondenserConfig(), + enable_prompt_extensions=traj_config.tools.get("enable_prompt_extensions", False), + ) + super().__init__(llm, self.agent_config) + + self.tokenizer = tokenizer + self.max_prompt_length = traj_config.max_prompt_length + self.step_count = 0 + self.infer_engine = infer_engine + self.sampling_params = traj_config.sampling_params + + # Store instance and trajectory IDs separately + self.instance_id = traj_config.instance_id + self.trajectory_id = traj_config.trajectory_id + self.qwen3_enable_thinking = traj_config.qwen3_enable_thinking + self.qwen3_acc_thinking = traj_config.qwen3_acc_thinking + + # will be set in _initialize_runtime_for_agent + self.runtime = None + self.instruction = None + self.app_config = None + + self.agent_id = uuid4().hex + # record the messages ourselves + self.messages = [] + self.prompt_token_len = 0 + self.response_token_len = 0 + + def _encode_prompt(self, messages): + if self.qwen3_acc_thinking: + # Use the Qwen3 thinking mode chat template + assert self.qwen3_enable_thinking, "Qwen3 thinking mode should for accumulating thinking." + chat_template = get_templates_path() / "qwen3_acc_thinking.jinja2" + input_ids = self.tokenizer.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + enable_thinking=self.qwen3_enable_thinking, + chat_template=chat_template.read_text(), + ) + else: + input_ids = self.tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=True, enable_thinking=self.qwen3_enable_thinking + ) + return input_ids + + def close(self): + """Close the agent runtime.""" + if self.runtime: + # remove all threads in event stream + self.runtime.event_stream.close() + self.runtime.close() + + def step(self, state: State) -> Action: + """Generate a response using batched infer.""" + self.step_count += 1 + print(f"instance id {self.instance_id}, trajectory {self.trajectory_id}, step {self.step_count}") + if self.pending_actions: + return self.pending_actions.popleft() + + # if we're done, go back + latest_user_message = state.get_last_user_message() + if latest_user_message and latest_user_message.content.strip() == "/exit": + return AgentFinishAction() + + # prepare what we want to send to the LLM + condensed_history: list[Event] = [] + match self.condenser.condensed_history(state): + case View(events=events): + condensed_history = events + + case Condensation(action=condensation_action): + return condensation_action + + logger.debug(f"Processing {len(condensed_history)} events from a total of {len(state.history)} events") + + initial_user_message = self._get_initial_user_message(state.history) + messages = self._get_messages(condensed_history, initial_user_message) + messages = self.llm.format_messages_for_llm(messages) + messages = convert_fncall_messages_to_non_fncall_messages( + messages, self.tools, add_in_context_learning_example=False + ) + + # the first step, use the constructed messages from openhands + if len(self.messages) == 0: + self.messages = messages + else: + obs = messages[-1] + assert obs["role"] == "user", f"The last message should always be a user message, but got {obs['role']}" + remaining_steps = self.app_config.max_iterations - self.step_count + 1 + if remaining_steps > 1: + obs["content"] += f"\nSteps remaining: {remaining_steps}." + else: + obs[ + "content" + ] += "\nThis is your last step, make sure to use the finish tool to submit your final answer." + self.messages.append(obs) + print(f"Obs: {obs['content']}") + + response_str = None + try: + input_ids = self._encode_prompt(self.messages) + if len(self.messages) == 2: + # system + first user message + self.prompt_token_len = len(input_ids) + else: + self.response_token_len = len(input_ids) - self.prompt_token_len + + if self.response_token_len >= self.max_prompt_length - 3000: + # modify the last user message to inform context window exceeded + self.messages[-1][ + "content" + ] += "\nNote: You are running out of tokens, submit your solution through finish tool now." + input_ids = self._encode_prompt(self.messages) + self.response_token_len = len(input_ids) - self.prompt_token_len + if self.response_token_len >= self.max_prompt_length: + return AgentFinishAction(thought="CONTEXT_WINDOW_EXCEEDED") + + sampling_params = copy.deepcopy(self.sampling_params) + sampling_params["max_tokens"] = self.max_prompt_length - self.response_token_len + + response_str, meta_info = call_async_from_sync( + self.infer_engine.async_generate_ids, + input_ids=input_ids, + sampling_params=sampling_params, + request_id=self.agent_id, + ) + stop_reason = meta_info.get("finish_reason") + print( + f"instance id {self.instance_id}, trajectory {self.trajectory_id}, response {response_str} stop reason {stop_reason}" + ) + + if not response_str: + # If we got an empty response (possible error), return a message action + return AgentFinishAction(thought="BAD_LLM_RESPONSE") + else: + # Convert to actions + message = [ + { + "role": "assistant", + "content": response_str, + } + ] + self.messages.append({"role": "assistant", "content": response_str}) + if stop_reason == "length": + return AgentFinishAction(thought="CONTEXT_WINDOW_EXCEEDED") + fn_call_messages = convert_non_fncall_messages_to_fncall_messages(message, self.tools) + actions = codeact_function_calling.response_to_actions( + convert_str_to_completion_format(fn_call_messages), mcp_tool_names=list(self.mcp_tools.keys()) + ) + print(f"Take action: {[type(action) for action in actions]}") + if len(actions) > 1: + logger.warning("Multiple actions detected, only the first action will be executed.") + # for action in actions: + self.pending_actions.append(actions[0]) + + except ( + LLMMalformedActionError, + LLMNoActionError, + LLMResponseError, + FunctionCallValidationError, + FunctionCallNotExistsError, + ): + raise + + except Exception as e: + logger.error(f"Error in agent step: {str(e)}") + logger.debug(f"{traceback.format_exc()}") + # Handle errors gracefully by creating a message action + self.pending_actions.append( + MessageAction( + content=f"An error: {str(e)} encountered. Please try a different approach.", + ) + ) + + # Return the first pending action + if not self.pending_actions: + # Fallback in case of empty actions + return AgentFinishAction() + + return self.pending_actions.popleft() + + def get_final_messages(self, state: State) -> List[Message]: + """Get the final messages for this agent.""" + condensed_history: list[Event] = [] + match self.condenser.condensed_history(state): + case View(events=events): + condensed_history = events + + case Condensation(action=condensation_action): + return condensation_action + + logger.debug(f"Processing {len(condensed_history)} events from a total of {len(state.history)} events") + + initial_user_message = self._get_initial_user_message(state.history) + messages = self._get_messages(condensed_history, initial_user_message) + messages = self.llm.format_messages_for_llm(messages) + messages = convert_fncall_messages_to_non_fncall_messages( + messages, self.tools, add_in_context_learning_example=False + ) + + import wandb + + current_step = wandb.run.step if wandb.run else 1 + run_name = wandb.run.name if wandb.run else "no_run" + logger.info(f"Detected run name: {run_name}") + if (current_step == 1) or (current_step % 10 == 0): + instance_dir = ( + Path(f"./trace/{run_name}/step{current_step}") / str(self.instance_id) / str(self.trajectory_id) + ) + instance_dir.mkdir(exist_ok=True, parents=True) + + # Generate a unique filename using a timestamp with microsecond resolution + timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f") + trace_file = instance_dir / f"trace_{timestamp}.json" + + with open(trace_file, "w") as f: + result_json = json.dumps(self.messages, default=lambda x: str(x)) + f.write(result_json) + + return self.messages + + def _is_last_action_finish(self, state: State): + finish_reason = None + mask_out_reason = ["CONTEXT_WINDOW_EXCEEDED", "BAD_LLM_RESPONSE", "NO_FUNCTION_CALL", "cmd_timeout"] + if state and state.history: + last_action = next( + (event for event in reversed(state.history) if isinstance(event, Action)), + None, + ) + if isinstance(last_action, AgentFinishAction): + finish_reason = last_action.thought if last_action.thought in mask_out_reason else "FINISH_TOOL" + return True, finish_reason + return False, finish_reason + + +Agent.register("OHCodeActAgent", OHCodeActAgent) diff --git a/skyrl-agent/skyrl_agent/agents/oh_codeact/codeact_runner.py b/skyrl-agent/skyrl_agent/agents/oh_codeact/codeact_runner.py new file mode 100644 index 0000000000..c4fb0c3632 --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/oh_codeact/codeact_runner.py @@ -0,0 +1,281 @@ +from typing import Any, Dict, Callable +import pandas as pd +import traceback + +from skyrl_agent.agents.oh_codeact.codeact_agent import OHCodeActAgent +from skyrl_agent.dispatcher.async_utils import call_sync_from_async +from skyrl_agent.config.configuration_utils import TrajectoryConfig +from skyrl_agent.agents.base import BaseTrajectory, TrajectoryResult, TrajectoryConfig, AsyncInferBackend, AutoTokenizer + +from openhands.core.main import run_controller +from openhands.controller.state.state import State +from openhands.events.action import ( + Action, + MessageAction, +) +from openhands.core.logger import openhands_logger as logger +from openhands.core.exceptions import ( + AgentRuntimeBuildError, + AgentRuntimeDisconnectedError, + AgentRuntimeError, + AgentRuntimeNotFoundError, + AgentRuntimeNotReadyError, + AgentRuntimeTimeoutError, + AgentRuntimeUnavailableError, +) + +from dataclasses import dataclass + +from skyrl_agent.tasks.swebench.utils import SWEBenchTask + + +@dataclass +class TaskHandle: + instance_id: str + trajectory_id: int + batch_id: int + + +def is_fatal_evaluation_error(error: str | None) -> bool: + if not error: + return False + + FATAL_EXCEPTIONS = [ + AgentRuntimeError, + AgentRuntimeBuildError, + AgentRuntimeTimeoutError, + AgentRuntimeUnavailableError, + AgentRuntimeNotReadyError, + AgentRuntimeDisconnectedError, + AgentRuntimeNotFoundError, + ConnectionError, + ] + + if any(exception.__name__ in error for exception in FATAL_EXCEPTIONS): + logger.error(f"Fatal evaluation error detected: {error}") + return True + + return False + + +def codeact_user_response( + state: State, + encapsulate_solution: bool = False, + try_parse: Callable[[Action], str] | None = None, +) -> str: + encaps_str = ( + ( + "Please encapsulate your final answer (answer ONLY) within and .\n" + "For example: The answer to the question is 42 .\n" + ) + if encapsulate_solution + else "" + ) + msg = ( + "No function call detected. You should always include one action in your response.\n" + 'If you think you have solved the task, please use the "finish" tool to finish the interaction.\n' + ) + + if state.history: + # check if the last action has an answer, if so, early exit + if try_parse is not None: + last_action = next( + (event for event in reversed(state.history) if isinstance(event, Action)), + None, + ) + ans = try_parse(last_action) + if ans is not None: + return "/exit" + + # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up + user_msgs = [event for event in state.history if isinstance(event, MessageAction) and event.source == "user"] + if len(user_msgs) >= 2: + # let the agent know that it can give up when it has tried 3 times + return msg + "You should at least take one actions.\n" + return msg + + +class CodeActTrajectory(BaseTrajectory): + def __init__( + self, + cfg: TrajectoryConfig, + data: Dict[str, Any], + infer_engine: AsyncInferBackend, + tokenizer: AutoTokenizer, + task: SWEBenchTask, + val_mode: bool, + ) -> None: + super().__init__(cfg, data, infer_engine, tokenizer, task, val_mode) + assert isinstance(task, SWEBenchTask) + + async def initialize_trajectory(self): + """Initialize the runtime for a specific agent.""" + # only swebench task supported, redundant but makes linter happy + assert isinstance(self.task, SWEBenchTask) + + batch_id = self.cfg.instance_id + trajectory_id = self.cfg.trajectory_id + + # data = self._get_data(data) + data = self.data + instance_id = data["instance_id"] if data["instance_id"] else batch_id + instance = pd.Series(data["instance"]) + data_source = data["data_source"] + self.agent = OHCodeActAgent(traj_config=self.cfg, infer_engine=self.infer_engine, tokenizer=self.tokenizer) + + init_successful = False + try: + + runtime = await self.task.initialize_runtime( + instance, data_source, self.agent.config, self.cfg.max_iterations + ) + + app_config = self.task.get_config(instance, data_source, self.agent.config, self.cfg.max_iterations) + + # Store the runtime and instruction + self.agent.runtime = runtime + self.agent.instruction = self.task.get_instruction(instance, data_source) + self.agent.app_config = app_config + + init_successful = True + logger.info(f"Successfully initialized runtime for instance {instance_id}") + except Exception as e: + logger.error(f"Failed to initialize runtime for instance {instance_id}: {str(e)}") + self.agent.runtime = None + + return_val = { + "instance_id": instance_id, + "trajectory_id": trajectory_id, + "messages": [], + "state": None, + "results": None, + "error": str(e), + "finish": False, + "finish_reason": "error_initialization", + } + + self.result = return_val + finally: + if not init_successful: + logger.info( + f"Init failed. Running cleanup for init agent task for instance {instance_id}, trajectory {trajectory_id}" + ) + if "runtime" in locals() and runtime: + runtime.event_stream.close() + runtime.close() + + async def generate_trajectory(self) -> None: + # only swebench task supported, redundant but makes linter happy + assert isinstance(self.task, SWEBenchTask) + + data = self.data + instance_id = data["instance_id"] if data["instance_id"] else self.cfg.instance_id + trajectory_id = self.cfg.trajectory_id + instance = pd.Series(data["instance"]) + data_source = data["data_source"] + agent = self.agent + runtime = agent.runtime + state = None + + try: + if not runtime: + raise Exception(f"Runtime not initialized for instance {instance_id}, trajectory {trajectory_id}") + + state = await run_controller( + config=agent.app_config, + initial_user_action=agent.instruction, + runtime=runtime, + agent=agent, + fake_user_response_fn=codeact_user_response, + ) + if state and is_fatal_evaluation_error(state.last_error): + raise Exception("Fatal error: " + state.last_error) + + final_messages = agent.get_final_messages(state) + result = await call_sync_from_async(self.task.complete_runtime, runtime, instance, data_source) + + finish, finish_reason = agent._is_last_action_finish(state) + if state and state.last_error: + if "RuntimeError: Agent reached maximum iteration in headless mode" in state.last_error: + finish_reason = "max_iterations_reached" + elif "Agent got stuck in a loop" in state.last_error: + finish_reason = "stuck_in_a_loop" + + if "finish_reason" in result: + finish_reason = result["finish_reason"] + + return_val = TrajectoryResult( + { + "instance_id": instance_id, + "trajectory_id": trajectory_id, + "messages": final_messages, + "state": state, + "results": result, + "error": state.last_error if state and state.last_error else None, + "finish": finish, + "finish_reason": finish_reason, + } + ) + except Exception as e: + logger.error(f"Run error {instance_id}: {e}") + logger.debug(f"Full Traceback: {traceback.format_exc()}") + final_messages = agent.get_final_messages(state) if state else [] + if not final_messages or len(final_messages) == 0: + logger.debug( + f"Final messages are non-existent (or empty) for instance {instance_id}, trajectory {trajectory_id}" + ) + return_val = TrajectoryResult( + { + "instance_id": instance_id, + "trajectory_id": trajectory_id, + "messages": final_messages, + "state": state, + "results": None, + "error": str(e), + "finish": False, + "finish_reason": "error_runtime", + } + ) + finally: + logger.info(f"Running cleanup for run agent task for instance {instance_id}, trajectory {trajectory_id}") + self._cleanup_agent() + + self.result = return_val + + async def evaluate_trajectory(self) -> None: + # only swebench task supported, redundant but makes linter happy + assert isinstance(self.task, SWEBenchTask) + + batch_id = self.cfg.instance_id + trajectory_id = self.cfg.trajectory_id + data = self.data + instance_id = data["instance_id"] if data["instance_id"] else batch_id + instance = pd.Series(data["instance"]) + data_source = data["data_source"] + + try: + # TODO: Why does "result" have a "results" entry? we should flatten + results = self.result.get("results", None) + if not results: + raise Exception(f"No results found for instance {instance_id}, trajectory {trajectory_id}") + if "reward" in results: + self.result["reward"] = results["reward"] + return + + eval_results = await self.task.evaluate_result(instance, results, instance_id, trajectory_id, data_source) + self.result["reward"] = eval_results + + logger.info( + f"Successfully evaluated instance {instance_id}, trajectory {trajectory_id} with reward {eval_results}" + ) + except Exception as e: + logger.error(f"Failed to evaluate traj {trajectory_id} for instance {instance_id}: {str(e)}") + self.result["reward"] = False + self.result["eval_error"] = str(e) + self.result["finish_reason"] = "error_evaluation" if "No git patch found" not in str(e) else "no_git_patch" + + def _cleanup_agent(self): + try: + self.agent.close() + except Exception as e: + logger.warning(f"Error closing agent {self.cfg.instance_id}, trajectory {self.cfg.trajectory_id}: {str(e)}") diff --git a/skyrl-agent/skyrl_agent/agents/react/__init__.py b/skyrl-agent/skyrl_agent/agents/react/__init__.py new file mode 100644 index 0000000000..1873c2637b --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/react/__init__.py @@ -0,0 +1,4 @@ +from .react_agent import ReActAgent +from .react_runner import ReActTrajectory + +__all__ = ["ReActAgent", "ReActTrajectory"] diff --git a/skyrl-agent/skyrl_agent/agents/react/messages.py b/skyrl-agent/skyrl_agent/agents/react/messages.py new file mode 100644 index 0000000000..8795d363c0 --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/react/messages.py @@ -0,0 +1,75 @@ +"""Reusable message templates for ReAct agent guidance.""" + +TOOL_CALL_PARSE_ERROR_GUIDANCE = ( + "Tool call parsing failed with error:\n" + "{error}\n\n" + "Please use the correct format:\n" + "\n" + "value\n" + "\n\n" + "Remember: Always end with the finish tool using \\boxed{} format." +) + + +NO_TOOL_CALL_DETECTED_GUIDANCE = ( + "No valid tool call detected. Please use the correct format:\n" + "\n" + "value\n" + "\n\n" + "Remember to call the finish tool with \\boxed{} format:\n" + "\n" + "\\boxed{YOUR_ANSWER}\n" + "" +) + + +TOOL_INVOCATION_ERROR_GUIDANCE = ( + "Tool call failed due to invalid arguments. Please correct and retry.\n" + "Examples:\n" + '- search_engine: {"query": ["term1", "term2"]}\n' + '- web_browser: {"url": ["https://example.com"], "goal": "extract key facts"}' +) + + +STEP_REMAINING_REMINDER = "\nSteps remaining: {remaining_steps}." + + +STEP_REMAINING_EARLY_REMINDER = "\nDo not stop now, conduct at least {min_steps} more actions." + + +STEP_LAST_REMINDER = "\nThis is your last step, make sure to use the finish tool to submit your final answer." + + +def get_turn_reminder_text( + step_count: int, + remaining_steps: int, + early_step_threshold: int = 0, +) -> str: + """Generate turn reminder text based on step count and remaining steps. + + Args: + step_count: Current step count (1-indexed) + remaining_steps: Number of steps remaining (max_iterations - step_count + 1) + early_step_threshold: Step count threshold for early reminder (default: 0) + + Returns: + Formatted reminder text to append to the last message + """ + if remaining_steps > 1 and step_count > early_step_threshold: + return STEP_REMAINING_REMINDER.format(remaining_steps=remaining_steps) + elif remaining_steps > 1 and step_count <= early_step_threshold: + min_steps = early_step_threshold - step_count + return STEP_REMAINING_EARLY_REMINDER.format(min_steps=min_steps) + else: + return STEP_LAST_REMINDER + + +__all__ = [ + "TOOL_CALL_PARSE_ERROR_GUIDANCE", + "NO_TOOL_CALL_DETECTED_GUIDANCE", + "TOOL_INVOCATION_ERROR_GUIDANCE", + "STEP_REMAINING_REMINDER", + "STEP_REMAINING_EARLY_REMINDER", + "STEP_LAST_REMINDER", + "get_turn_reminder_text", +] diff --git a/skyrl-agent/skyrl_agent/agents/react/react_agent.py b/skyrl-agent/skyrl_agent/agents/react/react_agent.py new file mode 100644 index 0000000000..61cd2f5a85 --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/react/react_agent.py @@ -0,0 +1,558 @@ +import json +import copy +from typing import Any, List, Dict +from collections import defaultdict +from uuid import uuid4 +import traceback +from skyrl_agent.functional.utils import ( + Transition, + record_transition, + StepResult, + StepException, + ContextWindowExceeded, + ParseError, + NoToolCall, + ToolExecutionFailed, +) + +from skyrl_agent.functional.history import ( + MessageHistory, + MessageEncoder, + parse_tool_call, + extract_tool_info, + check_truncated_tool_call, + format_output_preview, +) +from skyrl_agent.config.configuration_utils import TrajectoryConfig +from skyrl_agent.integrations.base import AsyncInferBackend +from skyrl_agent.tools.base import TOOL_REGISTRY +from skyrl_agent.dispatcher.async_utils import call_sync_from_async +from .messages import ( + TOOL_CALL_PARSE_ERROR_GUIDANCE, + NO_TOOL_CALL_DETECTED_GUIDANCE, + TOOL_INVOCATION_ERROR_GUIDANCE, + get_turn_reminder_text, +) +from skyrl_agent.functional.function_calling import convert_fncall_messages_to_non_fncall_messages + + +class ReActAgent: + def __init__( + self, + traj_config: TrajectoryConfig, + infer_engine: AsyncInferBackend, + tokenizer: Any, + ) -> None: + self.tokenizer = tokenizer + self.infer_engine = infer_engine + self.sampling_params = traj_config.sampling_params + + self.max_prompt_length = traj_config.max_prompt_length + self.qwen3_enable_thinking = traj_config.qwen3_enable_thinking + self.qwen3_acc_thinking = traj_config.qwen3_acc_thinking + + self.instance_id = traj_config.instance_id + self.trajectory_id = traj_config.trajectory_id + self.max_iterations = traj_config.max_iterations + self.early_step_threshold = traj_config.early_step_threshold + self.enable_turn_reminder = getattr(traj_config, "enable_turn_reminder", False) + + self.step_count = 0 + self.history = MessageHistory() + self.tools = {} + self.tool_params = [] + self.transitions: List[Transition] = [] # Record transitions per LLM call + + self.agent_id = uuid4().hex + self._register_tools(traj_config.tools) + + # Message encoder + self.message_encoder = MessageEncoder( + tokenizer, qwen3_enable_thinking=self.qwen3_enable_thinking, qwen3_acc_thinking=self.qwen3_acc_thinking + ) + + self.prompt_token_len = 0 + self.response_token_len = 0 + + # Debug and profiling flags/counters + self._debug = bool(traj_config.debug_log) + self._profile_enabled = bool(traj_config.profile_tools) + self._tool_calls_total: int = 0 + self._tool_calls_by_name: Dict[str, int] = defaultdict(int) + + def _register_tools(self, tools: List[str]) -> None: + """Register a list of tool instances.""" + print(f"[Register Tools] {tools}") + for name in tools: + if name not in TOOL_REGISTRY: + raise ValueError(f"Unknown tool '{name}'. Must be one of: {list(TOOL_REGISTRY)}") + tool = TOOL_REGISTRY[name]() + # Enforce unique function names per agent for ALL tools + if tool.name in self.tools: + raise ValueError( + f"Duplicate tool function name '{tool.name}' for this agent. " + f"Tool function names must be unique per agent." + ) + self.tools[tool.name] = tool + self.tool_params.append(tool.get_tool_param()) + + @record_transition + async def _generate_with_recording(self, input_ids, sampling_params, request_id): + """LLM generation wrapper that records transitions. + + This method is decorated to automatically capture: + - input_ids: tokens fed to the LLM + - output_tokens: tokens generated by the LLM + - logprobs: log probabilities of generated tokens + """ + return await self.infer_engine.async_generate_ids( + input_ids=input_ids, + sampling_params=sampling_params, + request_id=request_id, + ) + + def _prepare_llm_input(self) -> tuple[List[int], Dict]: + """Prepare input_ids and sampling params for LLM using incremental encoding. + + When history is reset, retokenizes everything. Otherwise, performs incremental + encoding by appending new messages to existing tokens. + + Returns: + Tuple of (input_ids, sampling_params) + """ + + # Check if history was reset - store flag before clearing it + history_was_reset = self.history.was_reset() + if history_was_reset: + self.prompt_token_len = 0 + self.history.clear_reset_flag() + + # Add turn reminder to history (optional via config) + if self.enable_turn_reminder: + remaining_steps = self.max_iterations - self.step_count + 1 + reminder_text = get_turn_reminder_text( + self.step_count, + remaining_steps, + early_step_threshold=self.early_step_threshold, + ) + self.history.add_turn_reminder(reminder_text) + + # Determine if we should retokenize everything or use incremental encoding + is_prompt = len(self.history) == 2 # system + user (with reminder appended) + should_retokenize = is_prompt or history_was_reset or not self.transitions + + if should_retokenize: + # Retokenize everything (first message or after history reset) + input_ids = self.message_encoder.encode_messages( + self.history.messages, + self.tool_params, + is_first_message=True, + ) + self.prompt_token_len = len(input_ids) + else: + # Incremental encoding: append new messages to existing tokens + # TODO(@csy): Handle nested agent scenarios + # When tools spawn subagents that also use LLM generation, self.transitions[-1] + # may belong to a subagent rather than this agent, breaking incremental encoding. + # Proposed solution: Filter transitions by agent_id: + # own_transitions = [t for t in self.transitions if getattr(t, 'agent_id', None) == self.agent_id] + # last_transition = own_transitions[-1] if own_transitions else None + # If last_transition is None, fallback to retokenizing everything. + last_transition = self.transitions[-1] + if not last_transition.ac.token_ids: + # Retokenize the action response_str if serving endpoints do not return token_ids + message = [{"role": "assistant", "content": last_transition.ac.text}] + last_transition.ac.token_ids = self.message_encoder.encode_messages( + message, self.tool_params, add_generation=False + ) + + # Encode only the new observation message(s) + # Simple default behavior: assuming one env observation message per step + # Can be overridden by subclasses + new_obs_ids = self.message_encoder.encode_messages( + [self.history.messages[-1]], self.tool_params, add_generation=True + ) + + # Build input_ids incrementally: previous observation + previous action + new observation + input_ids = last_transition.ob.input_ids + last_transition.ac.token_ids + new_obs_ids + + self.response_token_len = len(input_ids) - self.prompt_token_len + + # Prepare sampling params + sampling_params = copy.deepcopy(self.sampling_params) + sampling_params["max_tokens"] = self.max_prompt_length - self.response_token_len + + return input_ids, sampling_params + + def _prepare_llm_input_deprecated(self) -> tuple[List[int], Dict]: + """[DEPRECATED] Prepare input_ids and sampling params for LLM. + + This is an old version that performs retokenization at every turn. + This method is deprecated and will be removed in a future version. + Use `_prepare_llm_input()` instead. + + Returns: + Tuple of (input_ids, sampling_params) + """ + + # Check if history was reset - store flag before clearing it + history_was_reset = self.history.was_reset() + if history_was_reset: + self.prompt_token_len = 0 + self.history.clear_reset_flag() + + # Track token lengths + # Encode messages to input_ids + if self.enable_turn_reminder: + remaining_steps = self.max_iterations - self.step_count + 1 + reminder_text = get_turn_reminder_text( + self.step_count, + remaining_steps, + early_step_threshold=self.early_step_threshold, + ) + self.history.add_turn_reminder(reminder_text) + + input_ids = self.message_encoder.encode_messages( + self.history.messages, + self.tool_params, + is_first_message=True, + ) + # Set prompt_token_len on first message (initial setup) or after history reset + is_prompt = len(self.history) == 2 # system + user (possibly with reminder appended) + if is_prompt or history_was_reset: + self.prompt_token_len = len(input_ids) + + self.response_token_len = len(input_ids) - self.prompt_token_len + + # Prepare sampling params + sampling_params = copy.deepcopy(self.sampling_params) + sampling_params["max_tokens"] = self.max_prompt_length - self.response_token_len + + return input_ids, sampling_params + + def _handle_parse_error(self, error: str) -> None: + """Handle tool call parsing error and raise ParseError.""" + print(f"[Agent Step Error] Converter failed to parse tool call: {error}") + guidance = TOOL_CALL_PARSE_ERROR_GUIDANCE.format(error=error) + + self.history.add_tool_error(error) + self.history.add_user_guidance(guidance) + + raise ParseError() + + def _handle_no_tool_call(self, response_str: str) -> None: + """Handle case when no tool call is detected and raise NoToolCall.""" + print(f"[Agent Step {self.step_count}] No tool call found in response") + + # Check if response was likely truncated during a tool call + if check_truncated_tool_call(response_str): + print("[ERROR] Tool call appears incomplete - likely truncated!") + print(f"[ERROR] Last 500 chars: {response_str[-500:]}") + + self.history.add_user_guidance(NO_TOOL_CALL_DETECTED_GUIDANCE) + raise NoToolCall() + + async def _execute_tool(self, tool_name: str, tool_args: Dict, tool_call_id: str) -> Any: + """Execute a tool and return output. + + Raises: + ToolExecutionFailed: If tool execution fails + """ + tool = self.tools[tool_name] + + try: + output = await call_sync_from_async( + tool.call, + tool_args, + agent=self, + trajectory_id=self.trajectory_id, + ) + + # Record profiling stats if enabled + if self._profile_enabled: + try: + self._tool_calls_total += 1 + if tool_name: + self._tool_calls_by_name[tool_name] += 1 + except Exception: + pass + + return output + + except Exception as e: + # Tool invocation failed + error_str = str(e) + try: + self.history.add_tool_error(error_str, tool_call_id) + except Exception: + self.history.add_tool_error("Tool failed with an exception.", tool_call_id) + + self.history.add_user_guidance(TOOL_INVOCATION_ERROR_GUIDANCE) + raise ToolExecutionFailed() + + def _append_tool_output(self, output: Any, tool_call_id: str) -> None: + """Append tool output to message history. + + Args: + output: Tool output to append + tool_call_id: ID of the tool call + """ + try: + self.history.add_tool_response(output, tool_call_id) + + if self._debug: + preview = format_output_preview(output) + print(f"[Tool Output Preview] {preview}") + + except Exception as e: + print(f"[Agent Step Error] Error appending tool output to messages: {str(e)}") + self.history.add_tool_error(str(e), tool_call_id) + + async def step(self): + """Execute one agent step: LLM generation -> tool call -> tool execution. + + Returns: + Tuple of (done, finish_reason, result) + """ + self.step_count += 1 + print(f"[Agent Step {self.step_count}] instance={self.instance_id} traj={self.trajectory_id}") + + result = None + + try: + # 1. Prepare LLM input + input_ids, sampling_params = self._prepare_llm_input() + + # Check context window + if self.response_token_len >= self.max_prompt_length: + print("[Agent Step] Stopping reason: context_window_exceeded. Stopping agent.") + raise ContextWindowExceeded() + + # 2. Generate LLM response + response_str, meta_info = await self._generate_with_recording( + input_ids=input_ids, + sampling_params=sampling_params, + request_id=self.agent_id, + ) + stop_reason = meta_info["finish_reason"] + print(f"[Agent Step {self.step_count}] LLM response: {response_str}. Stop reason: {stop_reason}") + + # Add assistant message to history + self.history.add_assistant(response_str) + + # Check if generation stopped due to length + if stop_reason == "length": + print(f"[Agent Step] Stopping reason: {stop_reason}. Stopping agent.") + raise ContextWindowExceeded() + + # 3. Parse tool call from response + tool_call, parse_error = parse_tool_call(response_str, self.tool_params) + + # Handle parse error + if parse_error: + self._handle_parse_error(parse_error) + + # Handle no tools scenario + if not self.tools: + print(f"[Agent Step {self.step_count}] No tools provided, returning response.") + result = StepResult.finished("FINISH", response_str) + + # Handle no tool call detected + elif tool_call is None: + self._handle_no_tool_call(response_str) + + else: + # 4. Extract tool information + tool_name, tool_args = extract_tool_info(tool_call) + tool_call_id = tool_call.get("id") + + # Validate tool exists + if tool_name not in self.tools: + self.history.add_user_guidance(json.dumps({"error": f"Tool '{tool_name}' not found."})) + result = StepResult.continuing(response_str) + else: + # 5. Execute tool + output = await self._execute_tool(tool_name, tool_args, tool_call_id) + + # 6. Check if finish tool was called + if tool_name == "finish": + print(f"[Agent Step {self.step_count}] Finish tool called. Stopping agent.") + result = StepResult.finished("FINISH_TOOL", output) + else: + # Continue agent loop + result = StepResult.continuing(response_str) + + # 7. Append tool output to history only if output is not None + # Some tools (like next_with_summary) embed feedback in user message + # and return None to skip adding tool output + if output is not None: + print(f"[Tool Output step {self.step_count}] {output}") + self._append_tool_output(output, tool_call_id) + else: + print(f"[Tool Output step {self.step_count}] No output (feedback embedded in user message)") + + except StepException as e: + # Handle expected control flow exceptions + result = e.step_result + + except Exception as e: + # Handle unexpected errors + print(f"[Agent Step Error] Error during step: {str(e)}") + result = StepResult.finished(f"error: {str(e)}", None) + + # Single exit point + return result.to_tuple() + + async def run(self, instruction: List[Dict], instance: Dict | None = None) -> List[str]: + """Run the agent till the end with the provided user input. + Optionally accepts an instance payload for tools (stored on self.instance). + """ + self.instance = instance + self._init_message(instruction) + result = None + finish_reason = None + while self.step_count < self.max_iterations: + try: + done, finish_reason, result = await self.step() + if done: + break + except Exception as e: + finish_reason = f"error: {str(e)}" + print(f"[Agent Run Error] Exception during step: {str(e)}") + # traceback + print(traceback.format_exc()) + break + else: # If we exit the loop without hitting a break, it means we reached max iterations + finish_reason = "max_iterations_reached" + + return finish_reason, result + + def get_messages(self) -> List[dict]: + return convert_fncall_messages_to_non_fncall_messages(self.history.messages, self.tool_params) + + def get_transitions(self) -> List[Transition]: + """Return the list of transitions recorded during agent execution. + + Each transition contains: + - ob: Observation with input_ids (tokens fed to LLM) + - ac: TokensWithLogprobs with output_tokens, logprobs, and generated text + - reward: Float reward value (default 0.0, can be updated based on outcomes) + - episode_done: Boolean indicating if episode finished + - metrics: Dict with finish_reason, response_length, and other metadata + """ + return self.transitions + + def _init_message(self, instruction: List[Dict]) -> None: + """Initialize the agent's message history with the provided instruction. + + Automatically collects system prompt prefixes from registered tools and prepends them + to the system message if present. + """ + if not isinstance(instruction, list): + raise ValueError("Instruction must be a list of messages.") + + for msg in instruction: + if not isinstance(msg, dict) or "role" not in msg or "content" not in msg: + raise ValueError("Each message must be a dictionary with 'role' and 'content'.") + + # Collect system prompt prefixes from registered tools + tool_prefixes = [] + for tool_name, tool in self.tools.items(): + prefix = tool.get_system_prompt_prefix() + if prefix: + tool_prefixes.append(prefix) + + # Prepend tool prefixes to system message if any exist + if tool_prefixes: + processed_instruction = copy.deepcopy(instruction) + # Combine all tool prefixes + combined_prefix = "\n\n---\n\n".join(tool_prefixes) + + # Find the first system message + system_msg_found = False + for msg in processed_instruction: + if msg.get("role") == "system": + # Prepend tool prefixes to existing system message + msg["content"] = combined_prefix + "\n\n---\n\n" + msg["content"] + system_msg_found = True + break + + # If no system message exists, create one at the beginning + if not system_msg_found: + processed_instruction.insert(0, {"role": "system", "content": combined_prefix}) + + self.history.initialize(processed_instruction) + else: + self.history.initialize(instruction) + + # Expose profiling snapshot for upstream aggregation + def get_tool_profile(self) -> Dict[str, Any]: + if not self._profile_enabled: + return None + try: + return { + "tool_calls_total": int(self._tool_calls_total), + "tool_calls_by_name": dict(self._tool_calls_by_name), + } + except Exception: + return None + + +if __name__ == "__main__": + # Example usage for testing + from skyrl_agent.config.configuration_utils import TrajectoryConfig + from skyrl_agent.integrations.openai import OpenAIBackend, OpenAIBackendConfig + from transformers import AutoTokenizer + import asyncio + + # Load tokenizer and model + model_name = "Qwen/Qwen2.5-1.5B-Instruct" + tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Define trajectory configuration + traj_config = TrajectoryConfig( + instance_id="test_instance", + trajectory_id="test_trajectory", + sampling_params={ + "temperature": 0.7, + "top_p": 0.95, + "max_tokens": 2048, + }, + max_prompt_length=12048, + qwen3_enable_thinking=True, + tools=["finish", "code_interpreter"], + max_iterations=5, + agent_cls="skyrl_agent.agents.react.ReActAgent", # Use ReActAgent for testing + ) + + backend_config = OpenAIBackendConfig( + model_name=model_name, + # change this to your desired url and port + api_url="http://localhost:8000", + ) + # TODO: model_name need not be in config + infer_engine = OpenAIBackend(infer_engine=None, cfg=backend_config) + + # Create the ReAct agent + # Test for with tools + agent = ReActAgent( + traj_config=traj_config, + infer_engine=infer_engine, + tokenizer=tokenizer, + ) + + # Define a sample instruction + instruction = [ + {"content": "Please reason step by step, and put your final answer within \\boxed{}.", "role": "system"}, + { + "content": "Points $A,B,C,D,E$ and $F$ lie, in that order, on $\\overline{AF}$, dividing it into five segments, each of length 1. Point $G$ is not on line $AF$. Point $H$ lies on $\\overline{GD}$, and point $J$ lies on $\\overline{GF}$. The line segments $\\overline{HC}, \\overline{JE},$ and $\\overline{AG}$ are parallel. Find $HC/JE$.", + "role": "user", + }, + ] + + # Run the agent + finish_reason, result = asyncio.run(agent.run(instruction)) + + print(agent.get_messages()) + print(f"Finish Reason: {finish_reason}") + print(f"Result: {result}") diff --git a/skyrl-agent/skyrl_agent/agents/react/react_runner.py b/skyrl-agent/skyrl_agent/agents/react/react_runner.py new file mode 100644 index 0000000000..d1f8d7489d --- /dev/null +++ b/skyrl-agent/skyrl_agent/agents/react/react_runner.py @@ -0,0 +1,68 @@ +import pandas as pd + +from skyrl_agent.agents.react.react_agent import ReActAgent + +from skyrl_agent.agents.base import BaseTrajectory + + +class ReActTrajectory(BaseTrajectory): + async def initialize_trajectory(self): + pass + + async def generate_trajectory(self) -> None: + data = self.data + instance_id = data["instance_id"] if data["instance_id"] else self.cfg.instance_id + instance = pd.Series(data["instance"]) + # self.agent = ReActAgent(traj_config=self.cfg, infer_engine=self.infer_engine, tokenizer=self.tokenizer) + self.agent: ReActAgent = self.agent_cls( + traj_config=self.cfg, + infer_engine=self.infer_engine, + tokenizer=self.tokenizer, + ) + # sys + user messages + instruction = self.task.get_instruction(instance) + + finish_reason, result = await self.agent.run(instruction, instance) + # Optional tool profile snapshot (env-gated inside agent) + tool_profile = None + try: + tool_profile = self.agent.get_tool_profile() + except Exception: + tool_profile = None + self.result = { + "instance_id": instance_id, + "trajectory_id": self.cfg.trajectory_id, + "messages": self.agent.get_messages(), + "transitions": self.agent.get_transitions(), + "results": result, + "finish_reason": finish_reason, + "state": {"tool_profile": tool_profile} if tool_profile is not None else {}, + } + + async def evaluate_trajectory(self) -> None: + instance_id = self.cfg.instance_id + trajectory_id = self.cfg.trajectory_id + data = self.data + instance_id = data["instance_id"] if data["instance_id"] else self.cfg.instance_id + instance = data["instance"] + # print(f"[react_runner] instance_id={instance_id} original instance type: {type(instance).__name__}") + # if isinstance(instance, dict): + # # print(f"[react_runner] instance keys: {list(instance.keys())}") + # if not isinstance(instance, (dict, pd.Series)): + # # print(f"[react_runner] Converting to Series for instance_id={instance_id}") + # instance = pd.Series(instance) + result = self.result.get("results") + + try: + eval_result = await self.task.evaluate_result( + result, + instance, + data["data_source"], + instance_id, + trajectory_id, + ) + self.result["reward"] = eval_result + except Exception as e: + print(f"Error evaluating result: {e}") + self.result["reward"] = 0 + self.result["eval_error"] = str(e) diff --git a/skyrl-agent/skyrl_agent/auto.py b/skyrl-agent/skyrl_agent/auto.py new file mode 100644 index 0000000000..694374d917 --- /dev/null +++ b/skyrl-agent/skyrl_agent/auto.py @@ -0,0 +1,32 @@ +# skyagent/auto.py +from typing import Type +import os +from omegaconf import OmegaConf +from importlib import import_module +from skyrl_agent.agents.mapping import AGENT_GENERATOR_REGISTRY +from skyrl_agent.agents import AgentRunner + + +def _import_object(path: str): + module_path, class_name = path.rsplit(".", 1) + return getattr(import_module(module_path), class_name) + + +class AutoAgentRunner: + @classmethod + def from_task(cls, task_yaml: str, infer_engine, tokenizer) -> AgentRunner: + # Load config + + if not os.path.exists(task_yaml): + raise FileNotFoundError(f"Task YAML not found: {task_yaml}") + cfg = OmegaConf.load(task_yaml) + + runner_path = AGENT_GENERATOR_REGISTRY.get(cfg.agent_cls, None) + if not runner_path: + raise ValueError( + f"`AgentRunner` class for agent {cfg.agent_cls} is not specified. Please ensure that the agent is present in the registry" + ) + + runner_cls: Type[AgentRunner] = _import_object(runner_path) + + return runner_cls.from_task(task_yaml, infer_engine, tokenizer) diff --git a/verl/workers/agentic/__init__.py b/skyrl-agent/skyrl_agent/config/__init__.py similarity index 100% rename from verl/workers/agentic/__init__.py rename to skyrl-agent/skyrl_agent/config/__init__.py diff --git a/skyrl-agent/skyrl_agent/config/configuration_utils.py b/skyrl-agent/skyrl_agent/config/configuration_utils.py new file mode 100644 index 0000000000..a39206c6f6 --- /dev/null +++ b/skyrl-agent/skyrl_agent/config/configuration_utils.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass +from typing import Any, Optional + + +# TODO(csy): a smarter way? +def get_field_from_config(key_path, context): + parts = key_path.split(".") + value = context + try: + for part in parts: + value = value[part] + except (KeyError, TypeError): + raise ValueError(f"Path '{key_path}' not found in context.") + return value + + +@dataclass +class TrajectoryConfig: + instance_id: int + trajectory_id: int + max_prompt_length: int = 1024 + sampling_params: Optional[Any] = None + vision_is_active: bool = False + qwen3_enable_thinking: bool = True + qwen3_acc_thinking: bool = True + max_iterations: int = 5 + tools: Optional[list] = None + agent_cls: str = "skyrl_agent.agents.react.ReActAgent" + profile_tools: bool = False + debug_log: bool = False + early_step_threshold: int = 0 # Step count threshold for early reminder + enable_turn_reminder: bool = False + + +# DEPR +@dataclass +class AgentConfig: + max_iterations: int = 5 + tools: Optional[list] = None + + +TASK_CONFIG_REGISTRY = {"swe_bench": "swe_bench.yaml"} diff --git a/verl/utils/checkpoint/upload_utils.py b/skyrl-agent/skyrl_agent/dispatcher/__init__.py similarity index 100% rename from verl/utils/checkpoint/upload_utils.py rename to skyrl-agent/skyrl_agent/dispatcher/__init__.py diff --git a/skyrl-agent/skyrl_agent/dispatcher/async_utils.py b/skyrl-agent/skyrl_agent/dispatcher/async_utils.py new file mode 100644 index 0000000000..5c5856b27f --- /dev/null +++ b/skyrl-agent/skyrl_agent/dispatcher/async_utils.py @@ -0,0 +1,91 @@ +import asyncio +from concurrent import futures +from concurrent.futures import ThreadPoolExecutor +from typing import Callable, Coroutine, Iterable, List + +GENERAL_TIMEOUT: int = 15 +EXECUTOR = ThreadPoolExecutor() + + +async def call_sync_from_async(fn: Callable, *args, **kwargs): + """ + Shorthand for running a function in the default background thread pool executor + and awaiting the result. The nature of synchronous code is that the future + returned by this function is not cancellable + """ + loop = asyncio.get_event_loop() + coro = loop.run_in_executor(None, lambda: fn(*args, **kwargs)) + result = await coro + return result + + +def call_async_from_sync(corofn: Callable, timeout: float = GENERAL_TIMEOUT, *args, **kwargs): + """ + Shorthand for running a coroutine in the default background thread pool executor + and awaiting the result + """ + + if corofn is None: + raise ValueError("corofn is None") + if not asyncio.iscoroutinefunction(corofn): + raise ValueError("corofn is not a coroutine function") + + async def arun(): + coro = corofn(*args, **kwargs) + result = await coro + return result + + def run(): + loop_for_thread = asyncio.new_event_loop() + try: + asyncio.set_event_loop(loop_for_thread) + return asyncio.run(arun()) + finally: + loop_for_thread.close() + + future = EXECUTOR.submit(run) + futures.wait([future], timeout=timeout or None) + result = future.result() + return result + + +async def call_coro_in_bg_thread(corofn: Callable, timeout: float = GENERAL_TIMEOUT, *args, **kwargs): + """Function for running a coroutine in a background thread.""" + await call_sync_from_async(call_async_from_sync, corofn, timeout, *args, **kwargs) + + +async def wait_all(iterable: Iterable[Coroutine], timeout: int = GENERAL_TIMEOUT) -> List: + """ + Shorthand for waiting for all the coroutines in the iterable given in parallel. Creates + a task for each coroutine. + Returns a list of results in the original order. If any single task raised an exception, this is raised. + If multiple tasks raised exceptions, an AsyncException is raised containing all exceptions. + """ + tasks = [asyncio.create_task(c) for c in iterable] + if not tasks: + return [] + _, pending = await asyncio.wait(tasks, timeout=timeout) + if pending: + for task in pending: + task.cancel() + raise asyncio.TimeoutError() + results = [] + errors = [] + for task in tasks: + try: + results.append(task.result()) + except Exception as e: + errors.append(e) + if errors: + if len(errors) == 1: + raise errors[0] + raise AsyncException(errors) + return [task.result() for task in tasks] + + +class AsyncException(Exception): + def __init__(self, exceptions): + self.exceptions = exceptions + + def __str__(self): + return "\n".join(str(e) for e in self.exceptions) diff --git a/skyrl-agent/skyrl_agent/dispatcher/dispatchers.py b/skyrl-agent/skyrl_agent/dispatcher/dispatchers.py new file mode 100644 index 0000000000..580334f924 --- /dev/null +++ b/skyrl-agent/skyrl_agent/dispatcher/dispatchers.py @@ -0,0 +1,184 @@ +import asyncio +from typing import Callable, Any, Dict +from loguru import logger + +DoFnType = Callable[[int, int], Any] # batch_idx, trajectory_id +DispatcherType = Callable[[DoFnType, DoFnType, DoFnType], Any] + +# Dispatcher Registry +DISPATCHER_REGISTRY: Dict[str, DispatcherType] = {} + + +def register_dispatcher(name): + def decorator(fn): + DISPATCHER_REGISTRY[name] = fn + return fn + + return decorator + + +# Async Pipeline Dispatcher (Producer-Consumer Pipelining) +@register_dispatcher("async_pipeline") +async def async_pipeline_dispatcher( + cfg, trajectories: Dict[str, Dict[str, Any]], init_fn: str, run_fn: str, eval_fn: str +): + async def pipeline(): + """Pipeline dispatcher for async processing of init, run, and eval functions.""" + # Initialize queues + init_queue = asyncio.Queue() + run_queue = asyncio.Queue() + eval_queue = asyncio.Queue() + + # Get the generator instance from the init function + max_parallel_agents = cfg["max_parallel_agents"] + max_eval_parallel_agents = cfg.get("max_eval_parallel_agents", max_parallel_agents) + + num_instances = cfg["num_instances"] + num_trajectories = cfg["num_trajectories"] + total_instances = num_instances + + max_eval_parallel_agents = min(total_instances * num_trajectories, max_eval_parallel_agents) + max_parallel_agents = min(total_instances * num_trajectories, max_parallel_agents) + + logger.info( + f"Using max_parallel_agents of {max_parallel_agents} for {total_instances} instances with {num_trajectories} trajectories each" + ) + logger.info( + f"Using max_eval_parallel_agents of {max_eval_parallel_agents} for {total_instances} instances with {num_trajectories} trajectories each" + ) + + # Fill the init queue with tasks + for trajectory_id in range(num_trajectories): + for instance_id in trajectories.keys(): + await init_queue.put((instance_id, trajectory_id)) + + async def initialize_one(): + while True: + instance_id, trajectory_id = await init_queue.get() + await getattr(trajectories[instance_id][trajectory_id], init_fn)() + await run_queue.put((instance_id, trajectory_id)) + init_queue.task_done() + + async def run_one(): + while True: + instance_id, trajectory_id = await run_queue.get() + await getattr(trajectories[instance_id][trajectory_id], run_fn)() + await eval_queue.put((instance_id, trajectory_id)) + run_queue.task_done() + + async def eval_one(): + while True: + instance_id, trajectory_id = await eval_queue.get() + await getattr(trajectories[instance_id][trajectory_id], eval_fn)() + eval_queue.task_done() + + # Create tasks for initialization, running and evaluation + init_tasks = [asyncio.create_task(initialize_one()) for _ in range(max_parallel_agents)] + run_tasks = [asyncio.create_task(run_one()) for _ in range(max_parallel_agents)] + eval_tasks = [asyncio.create_task(eval_one()) for _ in range(max_eval_parallel_agents)] + + # Wait until all initialization tasks are done + print("Waiting for initialization tasks to complete...") + await init_queue.join() + for task in init_tasks: + task.cancel() + + print("Initialization tasks completed. Waiting for run tasks to complete...") + # Wait until all running tasks are done + await run_queue.join() + for task in run_tasks: + task.cancel() + + print("Run tasks completed. Waiting for evaluation tasks to complete...") + # Wait until all evaluation tasks are done + await eval_queue.join() + for task in eval_tasks: + task.cancel() + + await pipeline() + + +# Async Batch Dispatcher +@register_dispatcher("async_batch") +async def async_batch_dispatcher(cfg, trajectories: Dict[int, Dict[int, Any]], init_fn: str, run_fn: str, eval_fn: str): + async def run_all(): + tasks = [] + total_instances = cfg["num_instances"] + num_trajectories = cfg["num_trajectories"] + + async def one_traj(instance_id, trajectory_id): + traj = trajectories[instance_id][trajectory_id] + if init_fn is not None: + await getattr(traj, init_fn)() + await getattr(traj, run_fn)() + await getattr(traj, eval_fn)() + + for instance_id in trajectories.keys(): + for trajectory_id in range(num_trajectories): + tasks.append(asyncio.create_task(one_traj(instance_id, trajectory_id))) + + await asyncio.gather(*tasks) + + await run_all() + + +# Async FixedEnv Pool Dispatcher (Env Pool Reuse) +@register_dispatcher("async_fix_pool") +async def async_fix_pool_dispatcher(cfg, init_fn, run_fn, eval_fn): + """ + Dispatcher for pre-initialized environments. Each trajectory is assigned + to a free env. When finished, the env is returned to the pool. + """ + + async def dispatcher(): + envs = cfg["envs"] # List of pre-initialized environments + num_envs = len(envs) + num_instances = cfg["num_instances"] + num_trajectories = cfg["num_trajectories"] + total_trajectories = num_instances * num_trajectories + + # Queue to keep track of available env_ids + env_queue = asyncio.Queue() + for env_id in range(num_envs): + await env_queue.put(env_id) + + # Queue of all pending (batch_idx, trajectory_id) + work_queue = asyncio.Queue() + for trajectory_id in range(num_trajectories): + for batch_idx in range(num_instances): + await work_queue.put((batch_idx, trajectory_id)) + + logger.info(f"FixedEnv dispatcher with {num_envs} envs for {total_trajectories} total trajectories.") + + async def worker(): + while True: + try: + batch_idx, trajectory_id = await work_queue.get() + env_id = await env_queue.get() + + # Reset and assign env + await init_fn(batch_idx, trajectory_id, env_id) + + # Run and eval + await run_fn(batch_idx, trajectory_id, env_id) + await eval_fn(batch_idx, trajectory_id, env_id) + + # Mark trajectory and env as done + work_queue.task_done() + await env_queue.put(env_id) + except Exception as e: + logger.exception(f"Worker failed for ({batch_idx}, {trajectory_id}): {e}") + work_queue.task_done() + await env_queue.put(env_id) + + # Launch one worker per environment + workers = [asyncio.create_task(worker()) for _ in range(num_envs)] + + print("Waiting for all work to complete...") + await work_queue.join() + + # Cancel all workers + for w in workers: + w.cancel() + + await dispatcher() diff --git a/skyrl-agent/skyrl_agent/functional/__init__.py b/skyrl-agent/skyrl_agent/functional/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-agent/skyrl_agent/functional/chat_template.py b/skyrl-agent/skyrl_agent/functional/chat_template.py new file mode 100644 index 0000000000..4d41486b0e --- /dev/null +++ b/skyrl-agent/skyrl_agent/functional/chat_template.py @@ -0,0 +1,50 @@ +# this is for the tokenizer.apply_chat_template to be able to generate assistant masks directly +# todo: this is a hack, we should find a better way to do this +chat_template = ( + "{% for message in messages %}" + "{% if (message['role'] != 'assistant') %}" + "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}" + "{% elif (message['role'] == 'assistant')%}" + "{{'<|im_start|>' + message['role'] + '\n'}}" + "{% generation %}" + "{{message['content'] + '<|im_end|>'}}" + "{% endgeneration %}" + "{{'\n'}}" + "{% endif %}" + "{% endfor %}" +) + +# chat template for qwen3 thinking mode to remove think tokens similar to generation phase +chat_template_qwen3_thinking = ( + "{% for message in messages %}" + "{% if (message['role'] != 'assistant') %}" + "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}" + "{% elif (message['role'] == 'assistant')%}" + "{{'<|im_start|>' + message['role'] + '\n'}}" + "{% generation %}" + "{% set full_content = message['content'] %}" + "{% set mycontent = message['content'] %}" + "{% set is_last_message = loop.last and messages[-1]['role'] == 'assistant' %}" + "{% if '' in full_content and not is_last_message %}" + "{% set mycontent = full_content.split('')[-1].lstrip('\n') %}" + "{% endif %}" + "{{mycontent + '<|im_end|>'}}" + "{% endgeneration %}" + "{{'\n'}}" + "{% endif %}" + "{% endfor %}" +) + + +def get_templates_path(): + from pathlib import Path + + # Get the current file's directory + current_file_path = Path(__file__).resolve() + templates_path = current_file_path.parent / "templates" + + # Ensure the templates directory exists + if not templates_path.exists(): + raise FileNotFoundError(f"Templates directory does not exist: {templates_path}") + + return templates_path diff --git a/skyrl-agent/skyrl_agent/functional/function_calling.py b/skyrl-agent/skyrl_agent/functional/function_calling.py new file mode 100644 index 0000000000..e443c6908c --- /dev/null +++ b/skyrl-agent/skyrl_agent/functional/function_calling.py @@ -0,0 +1,475 @@ +"""Convert function calling messages to non-function calling messages and vice versa. + +This will inject prompts so that models that doesn't support function calling +can still be used with function calling agents. + +We follow format from: https://docs.litellm.ai/docs/completion/function_call +""" + +import copy +import json +import re +from typing import Iterable + +from litellm import ChatCompletionToolParam + + +class FunctionCallConversionError(Exception): + """Exception raised when FunctionCallingConverter failed to convert a non-function call message to a function call message. + + This typically happens when there's a malformed message (e.g., missing tags). But not due to LLM output. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +class FunctionCallValidationError(Exception): + """Exception raised when FunctionCallingConverter failed to validate a function call message. + + This typically happens when the LLM outputs unrecognized function call / parameter names / values. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +# Conversion utility function +def convert_str_to_completion_format(fn_call_messages): + # from types import SimpleNamespace + from litellm import ModelResponse + + role = fn_call_messages[0]["role"] + response_str = fn_call_messages[0]["content"] + tool_calls = fn_call_messages[0].get("tool_calls", None) + + return ModelResponse( + choices=[ + { + "index": 0, + "message": {"content": response_str, "role": role, "tool_calls": tool_calls, "function_calling": None}, + } + ] + ) + + +# Inspired by: https://docs.together.ai/docs/llama-3-function-calling#function-calling-w-llama-31-70b +SYSTEM_PROMPT_SUFFIX_TEMPLATE = """ +You have access to the following functions: + +{description} + +If you choose to call a function ONLY reply in the following format with NO suffix: + + +value_1 + +This is the value for the second parameter +that can span +multiple lines + + + + +Reminder: +- Function calls MUST follow the specified format, start with +- Required parameters MUST be specified +- Only call one function at a time +- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after. +- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls + +""" + +STOP_WORDS = ["]+)>\n(.*?)" +FN_PARAM_REGEX_PATTERN = r"]+)>(.*?)" + +# Add new regex pattern for tool execution results +TOOL_RESULT_REGEX_PATTERN = r"EXECUTION RESULT of \[(.*?)\]:\n(.*)" + + +def convert_tool_call_to_string(tool_call: dict) -> str: + """Convert tool call to content in string format.""" + if "function" not in tool_call: + raise FunctionCallConversionError("Tool call must contain 'function' key.") + if "id" not in tool_call: + raise FunctionCallConversionError("Tool call must contain 'id' key.") + if "type" not in tool_call: + raise FunctionCallConversionError("Tool call must contain 'type' key.") + if tool_call["type"] != "function": + raise FunctionCallConversionError("Tool call type must be 'function'.") + + ret = f'\n' + try: + args = json.loads(tool_call["function"]["arguments"]) + except json.JSONDecodeError as e: + raise FunctionCallConversionError( + f'Failed to parse arguments as JSON. Arguments: {tool_call["function"]["arguments"]}' + ) from e + for param_name, param_value in args.items(): + is_multiline = isinstance(param_value, str) and "\n" in param_value + ret += f"" + if is_multiline: + ret += "\n" + ret += f"{param_value}" + if is_multiline: + ret += "\n" + ret += "\n" + ret += "" + return ret + + +def convert_tools_to_description(tools: list[dict]) -> str: + ret = "" + for i, tool in enumerate(tools): + assert tool["type"] == "function" + fn = tool["function"] + if i > 0: + ret += "\n" + ret += f'---- BEGIN FUNCTION #{i + 1}: {fn["name"]} ----\n' + ret += f'Description: {fn["description"]}\n' + + if "parameters" in fn: + ret += "Parameters:\n" + properties = fn["parameters"].get("properties", {}) + required_params = set(fn["parameters"].get("required", [])) + + for j, (param_name, param_info) in enumerate(properties.items()): + # Indicate required/optional in parentheses with type + is_required = param_name in required_params + param_status = "required" if is_required else "optional" + param_type = param_info.get("type", "string") + + # Get parameter description + desc = param_info.get("description", "No description provided") + + # Handle enum values if present + if "enum" in param_info: + enum_values = ", ".join(f"`{v}`" for v in param_info["enum"]) + desc += f"\nAllowed values: [{enum_values}]" + + ret += f" ({j + 1}) {param_name} ({param_type}, {param_status}): {desc}\n" + else: + ret += "No parameters are required for this function.\n" + + ret += f"---- END FUNCTION #{i + 1} ----\n" + return ret + + +def convert_fncall_messages_to_non_fncall_messages( + messages: list[dict], + tools: list[ChatCompletionToolParam], + add_in_context_learning_example: bool = True, +) -> list[dict]: + """Convert function calling messages to non-function calling messages.""" + messages = copy.deepcopy(messages) + + formatted_tools = convert_tools_to_description(tools) + system_prompt_suffix = SYSTEM_PROMPT_SUFFIX_TEMPLATE.format(description=formatted_tools) + + converted_messages = [] + first_user_message_encountered = False + for message in messages: + role = message["role"] + content = message["content"] + + # 1. SYSTEM MESSAGES + # append system prompt suffix to content + if role == "system": + if isinstance(content, str): + content += system_prompt_suffix + elif isinstance(content, list): + if content and content[-1]["type"] == "text": + content[-1]["text"] += system_prompt_suffix + else: + content.append({"type": "text", "text": system_prompt_suffix}) + else: + raise FunctionCallConversionError( + f"Unexpected content type {type(content)}. Expected str or list. Content: {content}" + ) + converted_messages.append({"role": "system", "content": content}) + + # 2. USER MESSAGES (no change) + elif role == "user": + converted_messages.append( + { + "role": "user", + "content": content, + } + ) + + # 3. ASSISTANT MESSAGES + # - 3.1 no change if no function call + # - 3.2 change if function call + elif role == "assistant": + if "tool_calls" in message and message["tool_calls"] is not None: + if len(message["tool_calls"]) != 1: + raise FunctionCallConversionError( + f'Expected exactly one tool call in the message. More than one tool call is not supported. But got {len(message["tool_calls"])} tool calls. Content: {content}' + ) + try: + tool_content = convert_tool_call_to_string(message["tool_calls"][0]) + except FunctionCallConversionError as e: + raise FunctionCallConversionError( + f'Failed to convert tool call to string.\nCurrent tool call: {message["tool_calls"][0]}.\nRaw messages: {json.dumps(messages, indent=2)}' + ) from e + if isinstance(content, str): + content += "\n\n" + tool_content + content = content.lstrip() + elif isinstance(content, list): + if content and content[-1]["type"] == "text": + content[-1]["text"] += "\n\n" + tool_content + content[-1]["text"] = content[-1]["text"].lstrip() + else: + content.append({"type": "text", "text": tool_content}) + else: + raise FunctionCallConversionError( + f"Unexpected content type {type(content)}. Expected str or list. Content: {content}" + ) + converted_messages.append({"role": "assistant", "content": content}) + + # 4. TOOL MESSAGES (tool outputs) + elif role == "tool": + # Convert tool result as user message + tool_name = message.get("name", "function") + prefix = f"EXECUTION RESULT of [{tool_name}]:\n" + # and omit "tool_call_id" AND "name" + if isinstance(content, str): + content = prefix + content + elif isinstance(content, list): + if content and content[-1]["type"] == "text": + content[-1]["text"] = prefix + content[-1]["text"] + else: + content = [{"type": "text", "text": prefix}] + content + else: + raise FunctionCallConversionError( + f"Unexpected content type {type(content)}. Expected str or list. Content: {content}" + ) + converted_messages.append({"role": "user", "content": content}) + else: + raise FunctionCallConversionError(f"Unexpected role {role}. Expected system, user, assistant or tool.") + return converted_messages + + +def _extract_and_validate_params(matching_tool: dict, param_matches: Iterable[re.Match], fn_name: str) -> dict: + params = {} + # Parse and validate parameters + required_params = set() + if "parameters" in matching_tool and "required" in matching_tool["parameters"]: + required_params = set(matching_tool["parameters"].get("required", [])) + + allowed_params = set() + if "parameters" in matching_tool and "properties" in matching_tool["parameters"]: + allowed_params = set(matching_tool["parameters"]["properties"].keys()) + + param_name_to_type = {} + if "parameters" in matching_tool and "properties" in matching_tool["parameters"]: + param_name_to_type = { + name: val.get("type", "string") for name, val in matching_tool["parameters"]["properties"].items() + } + + # Collect parameters + found_params = set() + for param_match in param_matches: + param_name = param_match.group(1) + param_value = param_match.group(2).strip() + + # Validate parameter is allowed + if allowed_params and param_name not in allowed_params: + raise FunctionCallValidationError( + f"Parameter '{param_name}' is not allowed for function '{fn_name}'. " + f"Allowed parameters: {allowed_params}" + ) + + # Validate and convert parameter type + # supported: string, integer, array + if param_name in param_name_to_type: + if param_name_to_type[param_name] == "integer": + try: + param_value = int(param_value) + except ValueError: + raise FunctionCallValidationError(f"Parameter '{param_name}' is expected to be an integer.") + elif param_name_to_type[param_name] == "array": + try: + param_value = json.loads(param_value) + except json.JSONDecodeError: + raise FunctionCallValidationError(f"Parameter '{param_name}' is expected to be an array.") + else: + # string + pass + + # Enum check + if "enum" in matching_tool["parameters"]["properties"][param_name]: + if param_value not in matching_tool["parameters"]["properties"][param_name]["enum"]: + raise FunctionCallValidationError( + f"Parameter '{param_name}' is expected to be one of {matching_tool['parameters']['properties'][param_name]['enum']}." + ) + + params[param_name] = param_value + found_params.add(param_name) + + # Check all required parameters are present + missing_params = required_params - found_params + if missing_params: + raise FunctionCallValidationError(f"Missing required parameters for function '{fn_name}': {missing_params}") + return params + + +def _fix_stopword(content: str) -> str: + """Fix the issue when some LLM would NOT return the stopword.""" + # if '' + # else: + # content = content + '\n' + return content + + +def convert_non_fncall_messages_to_fncall_messages( + messages: list[dict], + tools: list[ChatCompletionToolParam], +) -> list[dict]: + """Convert non-function calling messages back to function calling messages.""" + messages = copy.deepcopy(messages) + formatted_tools = convert_tools_to_description(tools) + system_prompt_suffix = SYSTEM_PROMPT_SUFFIX_TEMPLATE.format(description=formatted_tools) + + converted_messages = [] + tool_call_counter = 1 # Counter for tool calls + + for message in messages: + role, content = message["role"], message["content"] + content = content or "" # handle cases where content is None + # For system messages, remove the added suffix + if role == "system": + if isinstance(content, str): + # Remove the suffix if present + content = content.split(system_prompt_suffix)[0] + elif isinstance(content, list): + if content and content[-1]["type"] == "text": + # Remove the suffix from the last text item + content[-1]["text"] = content[-1]["text"].split(system_prompt_suffix)[0] + converted_messages.append({"role": "system", "content": content}) + # Skip user messages (no conversion needed) + elif role == "user": + # Check for tool execution result pattern + if isinstance(content, str): + tool_result_match = re.search(TOOL_RESULT_REGEX_PATTERN, content, re.DOTALL) + elif isinstance(content, list): + tool_result_match = next( + ( + _match + for item in content + if item.get("type") == "text" + and (_match := re.search(TOOL_RESULT_REGEX_PATTERN, item["text"], re.DOTALL)) + ), + None, + ) + else: + raise FunctionCallConversionError( + f"Unexpected content type {type(content)}. Expected str or list. Content: {content}" + ) + + if tool_result_match: + if not ( + isinstance(content, str) + or (isinstance(content, list) and len(content) == 1 and content[0].get("type") == "text") + ): + raise FunctionCallConversionError( + f"Expected str or list with one text item when tool result is present in the message. Content: {content}" + ) + tool_name = tool_result_match.group(1) + tool_result = tool_result_match.group(2).strip() + + # Convert to tool message format + converted_messages.append( + { + "role": "tool", + "name": tool_name, + "content": ( + [{"type": "text", "text": tool_result}] if isinstance(content, list) else tool_result + ), + "tool_call_id": f"toolu_{tool_call_counter - 1:02d}", # Use last generated ID + } + ) + else: + converted_messages.append({"role": "user", "content": content}) + + # Handle assistant messages + elif role == "assistant": + if isinstance(content, str): + content = _fix_stopword(content) + fn_match = re.search(FN_REGEX_PATTERN, content, re.DOTALL) + elif isinstance(content, list): + if content and content[-1]["type"] == "text": + content[-1]["text"] = _fix_stopword(content[-1]["text"]) + fn_match = re.search(FN_REGEX_PATTERN, content[-1]["text"], re.DOTALL) + else: + fn_match = None + fn_match_exists = any( + item.get("type") == "text" and re.search(FN_REGEX_PATTERN, item["text"], re.DOTALL) + for item in content + ) + if fn_match_exists and not fn_match: + raise FunctionCallConversionError( + f"Expecting function call in the LAST index of content list. But got content={content}" + ) + else: + raise FunctionCallConversionError( + f"Unexpected content type {type(content)}. Expected str or list. Content: {content}" + ) + + if fn_match: + fn_name = fn_match.group(1) + fn_body = fn_match.group(2) + matching_tool = next( + ( + tool["function"] + for tool in tools + if tool["type"] == "function" and tool["function"]["name"] == fn_name + ), + None, + ) + # Validate function exists in tools + if not matching_tool: + raise FunctionCallValidationError( + f"Function '{fn_name}' not found in available tools: {[tool['function']['name'] for tool in tools if tool['type'] == 'function']}" + ) + + # Parse parameters + param_matches = re.finditer(FN_PARAM_REGEX_PATTERN, fn_body, re.DOTALL) + params = _extract_and_validate_params(matching_tool, param_matches, fn_name) + + # Create tool call with unique ID + tool_call_id = f"toolu_{tool_call_counter:02d}" + tool_call = { + "index": 1, # always 1 because we only support **one tool call per message** + "id": tool_call_id, + "type": "function", + "function": {"name": fn_name, "arguments": json.dumps(params)}, + } + tool_call_counter += 1 # Increment counter + + # Remove the function call part from content + if isinstance(content, list): + assert content and content[-1]["type"] == "text" + content[-1]["text"] = content[-1]["text"].split(" None: + """Add assistant message.""" + self.messages.append({"role": "assistant", "content": content}) + + def add_tool_error(self, error: str, tool_call_id: Optional[str] = None) -> None: + """Add tool error message.""" + msg = { + "role": "tool", + "content": json.dumps({"error": error}), + } + if tool_call_id: + msg["tool_call_id"] = tool_call_id + self.messages.append(msg) + + def add_tool_response(self, output: Any, tool_call_id: str) -> None: + """Add tool response message.""" + self.messages.append( + { + "role": "tool", + "content": json.dumps(output), + "tool_call_id": tool_call_id, + } + ) + + def add_user_guidance(self, guidance: str) -> None: + """Add user guidance message.""" + self.messages.append({"role": "user", "content": guidance}) + + def append_to_last_message(self, content: str) -> None: + """Append content to the last message in history. + + Args: + content: Content to append to the last message + """ + if not self.messages: + raise ValueError("Cannot append to last message: history is empty") + self.messages[-1]["content"] += content + + def add_turn_reminder(self, reminder_text: str) -> None: + """Add a turn reminder to the last message in history. + + This is a convenience method that appends reminder text to the last message. + + Args: + reminder_text: Reminder text to append to the last message + """ + self.append_to_last_message(reminder_text) + + def initialize(self, messages: List[Dict]) -> None: + """Initialize history with messages. + + Sets the _was_reset flag to True to indicate history was reset. + This flag can be checked to determine if state like prompt_token_len + needs to be reset. + """ + # deep copy the messages + self.messages = copy.deepcopy(messages) + self._was_reset = True + + def clear_reset_flag(self) -> None: + """Clear the reset flag after it has been handled.""" + self._was_reset = False + + def was_reset(self) -> bool: + """Check if history was recently reset.""" + return self._was_reset + + def get_messages(self) -> List[Dict]: + """Get all messages.""" + return self.messages + + def __len__(self) -> int: + """Get number of messages.""" + return len(self.messages) + + +class MessageEncoder: + """Handles encoding messages to input_ids for LLM.""" + + def __init__(self, tokenizer, qwen3_enable_thinking: bool = False, qwen3_acc_thinking: bool = False): + self.tokenizer = tokenizer + self.qwen3_enable_thinking = qwen3_enable_thinking + self.chat_template = get_templates_path() / "qwen3_acc_thinking.jinja2" if qwen3_acc_thinking else None + + def encode_messages( + self, + messages: List[Dict], + tool_params: List[Dict], + is_first_message: bool = False, + add_generation: bool = True, + ) -> List[int]: + """Convert messages to input_ids. + + Args: + messages: List of conversation messages + tool_params: List of tool parameter definitions + is_first_message: Whether this is the first message in the conversation + add_generation: Whether to add generation prompt + + Returns: + List of token IDs + + Examples: + Case 1 - First message (is_first_message=True): + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ] + # Encodes the full conversation from scratch with generation prompt + # Formatted as: + # system\nYou are a helpful assistant.\nuser\nHello!\nassistant\n + input_ids = encoder.encode_messages(messages, tool_params, is_first_message=True) + + Case 2 - Not first message, starting from assistant (is_first_message=False): + messages = [ + {"role": "assistant", "content": "Hi! How can I help you?"} + ] + # Performs incremental encoding for assistant response + # Base conversation: system\nYou are a helpful assistant.\nuser\nI am a user.\nassistant\n + # assistant response: Hi! How can I help you?\n + # Returns only the new tokens after base conversation + + Case 3 - Not first message, starting from user (is_first_message=False): + messages = [ + {"role": "user", "content": "Tell me a joke"} + ] + # Base conversation: system\nYou are a helpful assistant.\nuser\nI am a user.\n + # (Note: For user messages, tokens after last EOS are removed from base) + # user message: \nuser\nTell me a joke\nassistant\n + """ + formatted_messages = convert_fncall_messages_to_non_fncall_messages(messages, tool_params) + kwargs = { + "add_generation_prompt": add_generation, + "tokenize": True, + } + if self.chat_template: + kwargs["chat_template"] = self.chat_template.read_text() + kwargs["enable_thinking"] = self.qwen3_enable_thinking + if is_first_message: + input_ids = self.tokenizer.apply_chat_template(formatted_messages, **kwargs) + else: + # do incremental encoding, + # for assistant messages, we assume the generation prompt is already added in the previous message + base_conversation = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "I am a user."}, + ] + is_assistant_message = formatted_messages[0]["role"] == "assistant" + kwargs["add_generation_prompt"] = True if is_assistant_message else False + base_conversation_token_ids = self.tokenizer.apply_chat_template(base_conversation, **kwargs) + + if not is_assistant_message: + # remove tokens after the last EOS + last_eos_token_index = ( + len(base_conversation_token_ids) + - 1 + - base_conversation_token_ids[::-1].index(self.tokenizer.eos_token_id) + ) + base_conversation_token_ids = base_conversation_token_ids[: last_eos_token_index + 1] + kwargs["add_generation_prompt"] = add_generation + full_conversation = base_conversation + formatted_messages + full_conversation_token_ids = self.tokenizer.apply_chat_template(full_conversation, **kwargs) + input_ids = full_conversation_token_ids[len(base_conversation_token_ids) :] + + return input_ids + + +# Tool call parsing and extraction + + +def parse_tool_call( + response_str: str, + tool_params: List[Dict], +) -> Tuple[Optional[Dict], Optional[str]]: + """Parse tool call from LLM response. + + Args: + response_str: Raw LLM response text + tool_params: List of tool parameter definitions + + Returns: + Tuple of (tool_call_dict, error_message) + - If successful: (tool_call, None) + - If failed: (None, error_message) + """ + assistant_msg = {"role": "assistant", "content": response_str} + + try: + fncall_messages = convert_non_fncall_messages_to_fncall_messages([assistant_msg], tool_params) + tool_call = fncall_messages[0].get("tool_calls", [None])[0] + return tool_call, None + + except (FunctionCallValidationError, FunctionCallConversionError) as e: + return None, str(e) + + +def extract_tool_info(tool_call: Optional[Dict]) -> Tuple[Optional[str], Optional[Dict]]: + """Extract tool name and arguments from tool call. + + Args: + tool_call: Parsed tool call dictionary + + Returns: + Tuple of (tool_name, tool_args) + """ + if not tool_call: + return None, None + + tool_name = tool_call.get("function", {}).get("name") + tool_args = tool_call.get("function", {}).get("arguments") + return tool_name, tool_args + + +def check_truncated_tool_call(response_str: str) -> bool: + """Check if response contains a truncated tool call. + + Args: + response_str: Raw LLM response text + + Returns: + True if tool call appears truncated, False otherwise + """ + return "") + + +def format_output_preview(output: Any, max_length: int = 400) -> str: + """Format tool output for logging/preview. + + Args: + output: Tool output to format + max_length: Maximum length of preview + + Returns: + Formatted string preview + """ + try: + out_str = json.dumps(output) + except Exception: + out_str = str(output) + + out_str = out_str.replace("\n", " ") + if len(out_str) > max_length: + out_str = out_str[:max_length] + "..." + + return out_str diff --git a/skyrl-agent/skyrl_agent/functional/templates/qwen3_acc_thinking.jinja2 b/skyrl-agent/skyrl_agent/functional/templates/qwen3_acc_thinking.jinja2 new file mode 100644 index 0000000000..15691e71b6 --- /dev/null +++ b/skyrl-agent/skyrl_agent/functional/templates/qwen3_acc_thinking.jinja2 @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/skyrl-agent/skyrl_agent/functional/utils.py b/skyrl-agent/skyrl_agent/functional/utils.py new file mode 100644 index 0000000000..a5436bb65d --- /dev/null +++ b/skyrl-agent/skyrl_agent/functional/utils.py @@ -0,0 +1,272 @@ +"""Utility functions and dataclasses for agent operations.""" + +from typing import Any, Dict, List, Optional +from dataclasses import dataclass, field +from functools import wraps + +# Type aliases for Transition +Observation = Dict[str, Any] +Metrics = Dict[str, Any] + + +@dataclass +class TokensWithLogprobs: + """Tokens generated by the LLM with their log probabilities.""" + + token_ids: List[int] + logprobs: Optional[List[float]] = None + text: Optional[str] = None + + +@dataclass +class Observation: + """Tokens corresponding to an observation message from the environment""" + + input_ids: List[int] + + +@dataclass +class Transition: + """Records a single LLM call transition for RL training.""" + + ob: Observation + ac: TokensWithLogprobs + reward: float + episode_done: bool + metrics: Metrics = field(default_factory=dict) + + +@dataclass +class StepResult: + """Result of a single agent step.""" + + done: bool + finish_reason: Optional[str] + result: Optional[Any] + + @classmethod + def continuing(cls, result: Optional[Any] = None): + """Create a result indicating the agent should continue.""" + return cls(done=False, finish_reason=None, result=result) + + @classmethod + def finished(cls, reason: str, result: Optional[Any] = None): + """Create a result indicating the agent should stop.""" + return cls(done=True, finish_reason=reason, result=result) + + def to_tuple(self): + """Convert to tuple format for backwards compatibility.""" + return self.done, self.finish_reason, self.result + + +def record_transition(func): + """Decorator to record transitions for each LLM call. + + Captures: + - Observation: input_ids fed to LLM + - Action: output tokens and logprobs from LLM + - Records transition with metadata + """ + + @wraps(func) + async def wrapper(self, *args, **kwargs): + # Capture observation (input_ids) before LLM call + observation = Observation(input_ids=kwargs.get("input_ids", [])) + + # Call the original function (LLM generation) + result = await func(self, *args, **kwargs) + response_str, meta_info = result + print(f"response_str: {response_str}") + + # Extract action information + output_tokens = meta_info.get("output_tokens", []) + logprobs = meta_info.get("logprobs", None) + + action = TokensWithLogprobs( + token_ids=output_tokens, + logprobs=logprobs, + text=response_str, + ) + + # Create transition with default values + # Reward and episode_done will be updated based on step outcome + transition = Transition( + ob=observation, + ac=action, + reward=0.0, # Default, can be updated later + episode_done=False, # Default, can be updated later + metrics={ + "finish_reason": meta_info.get("finish_reason"), + }, + ) + + # TODO(@csy): Add agent_id tracking to transitions + # When tools spawn subagents that also call LLM generation, transitions from + # different agents get mixed in self.transitions. For TITOReActAgent incremental + # encoding, we need to identify which transitions belong to the current agent. + # Proposed solution: Add `agent_id: Optional[str] = None` field to Transition + # dataclass, and set it here: `transition.agent_id = getattr(self, 'agent_id', None)` + + # Store transition + self.transitions.append(transition) + + return result + + return wrapper + + +# from tinker cookbook +@dataclass +class TrainingDatum: + """A single training datum created from one or more transitions.""" + + input_tokens: List[int] + response_tokens: List[int] + response_logprobs: List[float] + response_mask: List[float] # 0 for observation tokens, 1 for action tokens + + +def _is_prefix(sequence: List[int], candidate: List[int]) -> bool: + """Check if sequence is a prefix of candidate.""" + if len(sequence) > len(candidate): + return False + return sequence == candidate[: len(sequence)] + + +def transitions_to_training_data( + transitions: List[Transition], +) -> List[TrainingDatum]: + """ + Convert a list of transitions into training data. + + If observations grow by appending (each successive observation contains + the previous observation+action as a prefix), then we merge them into + a single datum. Otherwise, we create separate data. + + For example, if ob_ac_pairs is: + (O1, A1) + (O1+A1+O2, A2) + (O3, A3) + + We merge the first two into a single datum, and the last into a separate datum. + + Args: + transitions: List of Transition objects + + Returns: + List of TrainingDatum objects + """ + + # Accumulator state for building sequences + full_sequence: List[int] = [] + sampled_logprobs: List[float] = [] + mask: List[float] = [] + + data: List[TrainingDatum] = [] + + def make_datum(): + """Create a TrainingDatum from current accumulated state.""" + if not full_sequence: + return None + + first_nonzero = mask.index(1) if 1 in mask else len(mask) + # till the first non-zero mask + input_tokens = full_sequence[:first_nonzero] + response_tokens = full_sequence[first_nonzero:] + response_logprobs = sampled_logprobs[first_nonzero:] + response_mask = mask[first_nonzero:] + + return TrainingDatum( + input_tokens=input_tokens, + response_tokens=response_tokens, + response_logprobs=response_logprobs, + response_mask=response_mask, + ) + + def clear_accumulator(): + """Clear the accumulator state.""" + nonlocal full_sequence, sampled_logprobs, mask + full_sequence = [] + sampled_logprobs = [] + mask = [] + + # Process each transition + for transition in transitions: + # Get observation tokens + ob_tokens = transition.ob.input_ids + + # Get action tokens and logprobs + ac_tokens = transition.ac.token_ids + ac_logprobs = transition.ac.logprobs or [0.0] * len(ac_tokens) + + # Determine delta observation (new tokens not in accumulated sequence) + if len(full_sequence) == 0: + # First transition, use all observation tokens + delta_ob_tokens = ob_tokens + elif _is_prefix(full_sequence, ob_tokens): + # Current observation extends previous sequence + # Only add the delta (new tokens) + delta_ob_tokens = ob_tokens[len(full_sequence) :] + else: + # Current observation doesn't extend previous sequence + # Save current accumulated datum and start fresh + datum = make_datum() + if datum: + data.append(datum) + clear_accumulator() + delta_ob_tokens = ob_tokens + + # Add delta observation tokens to sequence + full_sequence.extend(delta_ob_tokens) + sampled_logprobs.extend([0.0] * len(delta_ob_tokens)) + mask.extend([0.0] * len(delta_ob_tokens)) + + # Add action tokens to sequence + full_sequence.extend(ac_tokens) + sampled_logprobs.extend(ac_logprobs) + mask.extend([1.0] * len(ac_tokens)) + + # Create final datum from remaining accumulated state + if full_sequence: + datum = make_datum() + if datum: + data.append(datum) + + return data + + +# Custom exceptions for agent step control flow +class StepException(Exception): + """Base exception for step control flow.""" + + def __init__(self, step_result: StepResult): + self.step_result = step_result + super().__init__() + + +class ContextWindowExceeded(StepException): + """Raised when context window is exceeded.""" + + def __init__(self): + super().__init__(StepResult.finished("CONTEXT_WINDOW_EXCEEDED", None)) + + +class ParseError(StepException): + """Raised when tool call parsing fails.""" + + def __init__(self): + super().__init__(StepResult.continuing(None)) + + +class NoToolCall(StepException): + """Raised when no tool call is detected.""" + + def __init__(self): + super().__init__(StepResult.continuing(None)) + + +class ToolExecutionFailed(StepException): + """Raised when tool execution fails.""" + + def __init__(self): + super().__init__(StepResult.continuing(None)) diff --git a/skyrl-agent/skyrl_agent/integrations/__init__.py b/skyrl-agent/skyrl_agent/integrations/__init__.py new file mode 100644 index 0000000000..9fe1ee2c64 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/__init__.py @@ -0,0 +1,3 @@ +from . import skyrl_train +from . import verl +from . import openai diff --git a/skyrl-agent/skyrl_agent/integrations/base.py b/skyrl-agent/skyrl_agent/integrations/base.py new file mode 100644 index 0000000000..cf0023108b --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/base.py @@ -0,0 +1,73 @@ +from abc import ABC, abstractmethod +from typing import Any, List, Type, Dict +import importlib + + +class AsyncInferBackend(ABC): + @abstractmethod + async def async_generate_prompts(self, prompts: Any, sampling_params: Any, **kwargs) -> List[str]: + """Generate outputs for a list of prompts.""" + pass + + @abstractmethod + async def async_generate_ids(self, input_ids: Any, sampling_params: Any, **kwargs) -> List[str]: + """Generate outputs for a list of input_ids.""" + pass + + +class GeneratorOutput(ABC): + def __init__(self, result: Any): + pass + + +class GeneratorInput(ABC): + def __init__(self, input_batch: Any): + pass + + +class BackendSpec: + def __init__( + self, + infer_backend_cls: Type[AsyncInferBackend], + generator_output_cls: Type[GeneratorOutput], + generator_input_cls: Type[GeneratorInput], + ): + self.infer_backend_cls = infer_backend_cls + self.generator_output_cls = generator_output_cls + self.generator_input_cls = generator_input_cls + + +def build_backend(name: str, **kwargs): + spec = BACKEND_REGISTRY.get(name) + if not spec: + raise ValueError(f"Unknown backend: {name}") + infer_backend = spec.infer_backend_cls(**kwargs) + return infer_backend + + +def build_generator_output(name: str, result: Any, **kwargs): + spec = BACKEND_REGISTRY.get(name) + if not spec: + raise ValueError(f"Unknown backend: {name}") + return spec.generator_output_cls(result=result, **kwargs) + + +def build_generator_input(name: str, input_batch: Any, **kwargs): + spec = BACKEND_REGISTRY.get(name) + if not spec: + raise ValueError(f"Unknown backend: {name}") + return spec.generator_input_cls(input_batch=input_batch, **kwargs) + + +def _import_object(path: str): + """Dynamically import a class or function from a module path.""" + module_path, class_name = path.rsplit(".", 1) + module = importlib.import_module(module_path) + return getattr(module, class_name) + + +BACKEND_REGISTRY: Dict[str, BackendSpec] = {} + + +def register_backend(name: str, spec: BackendSpec): + BACKEND_REGISTRY[name] = spec diff --git a/skyrl-agent/skyrl_agent/integrations/openai.py b/skyrl-agent/skyrl_agent/integrations/openai.py new file mode 100644 index 0000000000..294d82bd89 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/openai.py @@ -0,0 +1,96 @@ +from typing import Any, List, TypedDict +from skyrl_agent.integrations.base import ( + AsyncInferBackend, + GeneratorOutput, + GeneratorInput, + register_backend, + BackendSpec, +) +import omegaconf +from loguru import logger +import os +import aiohttp + + +class OpenAIBackendConfig(TypedDict): + model_name: str + api_url: str + + +class OpenAIBackend(AsyncInferBackend): + def __init__(self, infer_engine: Any, cfg: OpenAIBackendConfig): + assert os.environ.get("OPENAI_API_KEY") is not None, "OPENAI_API_KEY is not set" + self.model_name = cfg["model_name"] + self.api_url = cfg["api_url"] + try: + self.model_max_len = cfg["model_max_len"] + except omegaconf.errors.ConfigKeyError: + logger.info("`model_max_len` not provided, using `max_position_embeddings` from the model config") + from transformers import AutoConfig + + config = AutoConfig.from_pretrained(self.model_name) + self.model_max_len = config.max_position_embeddings + logger.info(f"Using `model_max_len` {self.model_max_len}") + + async def async_generate_prompts(self, prompts: str, sampling_params: dict, **kwargs) -> str: + # NOTE: In some agents like OpenHands, the generate calls are from a different thread, so the session needs to be created in the same thread + # TODO: support long lived session depending on the task + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session: + + headers = {"Content-Type": "application/json"} + if not isinstance(sampling_params, dict): + sampling_params = dict(sampling_params) + payload = sampling_params.copy() + payload["model"] = self.model_name + + # TODO: user template might be applied twice here. need to double check. + payload["messages"] = {"role": "user", "content": prompts} + output = await session.post(f"{self.api_url}/v1/chat/completions", json=payload, headers=headers) + output = await output.json() + + try: + return output["choices"][0]["message"]["content"] + except Exception as e: + logger.info(f"Errored out while extracting first response from output {output} with exception {str(e)}") + + async def async_generate_ids(self, input_ids: List[int], sampling_params: dict, **kwargs) -> str: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session: + headers = {"Content-Type": "application/json"} + if not isinstance(sampling_params, dict): + sampling_params = dict(sampling_params) + payload = sampling_params.copy() + payload["model"] = self.model_name + # payload["max_tokens"] = self.model_max_len - len(input_ids) - 1 + print(f"max tokens: {payload['max_tokens']}") + + payload["prompt"] = input_ids + output = await session.post(f"{self.api_url}/v1/completions", json=payload, headers=headers) + output = await output.json() + + meta_info = { + "output_tokens": None, + "finish_reason": output["choices"][0]["finish_reason"], + "logprobs": None, + } + + return output["choices"][0]["text"], meta_info + + +class OpenAIGeneratorOutput(GeneratorOutput): + def __init__(self, result: Any): + self.result = result + + +class OpenAIGeneratorInput(GeneratorInput): + def __init__(self, input_batch: Any): + self.input_batch = input_batch + + +register_backend( + "openai_server", + BackendSpec( + infer_backend_cls=OpenAIBackend, + generator_output_cls=OpenAIGeneratorOutput, + generator_input_cls=OpenAIGeneratorInput, + ), +) diff --git a/skyrl-agent/skyrl_agent/integrations/skyrl_train/__init__.py b/skyrl-agent/skyrl_agent/integrations/skyrl_train/__init__.py new file mode 100644 index 0000000000..59f48a6c78 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/skyrl_train/__init__.py @@ -0,0 +1,11 @@ +from .skyrl_train_backend import SkyRLBackend, SkyRLGeneratorInput, SkyRLGeneratorOutput +from ..base import register_backend, BackendSpec + +register_backend( + "skyrl-train", + BackendSpec( + infer_backend_cls=SkyRLBackend, + generator_output_cls=SkyRLGeneratorOutput, + generator_input_cls=SkyRLGeneratorInput, + ), +) diff --git a/skyrl-agent/skyrl_agent/integrations/skyrl_train/skyrl_train_backend.py b/skyrl-agent/skyrl_agent/integrations/skyrl_train/skyrl_train_backend.py new file mode 100644 index 0000000000..f620c28028 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/skyrl_train/skyrl_train_backend.py @@ -0,0 +1,48 @@ +from typing import Any, List +from ..base import AsyncInferBackend, GeneratorOutput, GeneratorInput + + +class SkyRLBackend(AsyncInferBackend): + def __init__(self, infer_engine, tokenizer: Any = None, cfg: Any = None): + self.client = infer_engine + + async def async_generate_prompts(self, prompts: Any, sampling_params: Any, **kwargs) -> List[str]: + input_obj = { + "prompts": [prompts], + "session_ids": [kwargs.get("request_id", None)], + "sampling_params": sampling_params, + } + output = await self.client.generate(input_obj) + return output["responses"][0], output["stop_reasons"][0] + + async def async_generate_ids(self, input_ids: List[int], sampling_params: Any, **kwargs) -> List[str]: + input_obj = { + "prompt_token_ids": [input_ids], + "session_ids": [kwargs.get("request_id", None)], + "sampling_params": sampling_params, + } + output = await self.client.generate(input_obj) + # todo(@csy) probably need to be finish_reason + # https://github.com/vllm-project/vllm/blob/a0f8a7964694a6077689b242b5eca95de392d4bb/vllm/v1/engine/__init__.py#L22 + meta_info = { + "output_tokens": output["response_ids"][0], + "finish_reason": output["stop_reasons"][0], + "logprobs": None, + } + return output["responses"][0], meta_info + + +class SkyRLGeneratorOutput(GeneratorOutput): + def __init__(self, result: Any): + from skyrl_train.generators.utils import get_rollout_metrics + + # Add more skyrl-specific rollout metrics. + assert "rollout_metrics" in result, "rollout_metrics should be in the result" + skyrl_rollout_metrics = get_rollout_metrics(result["response_ids"], result["rewards"]) + result["rollout_metrics"].update(skyrl_rollout_metrics) + self.result = result + + +class SkyRLGeneratorInput(GeneratorInput): + def __init__(self, input_batch: Any): + self.input_batch = input_batch["env_extras"] diff --git a/skyrl-agent/skyrl_agent/integrations/skyrl_train/skyrl_train_main.py b/skyrl-agent/skyrl_agent/integrations/skyrl_train/skyrl_train_main.py new file mode 100644 index 0000000000..0e32cda71d --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/skyrl_train/skyrl_train_main.py @@ -0,0 +1,89 @@ +import hydra +import os +from omegaconf import DictConfig +from skyrl_train.generators.base import GeneratorInterface, GeneratorInput, GeneratorOutput +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils import initialize_ray +import ray + +from skyrl_agent import AutoAgentRunner + +from .trainer import SkyRLAgentPPOTrainer + + +class SkyRLAgentGenerator(GeneratorInterface): + def __init__(self, generator_cfg: DictConfig, llm_endpoint_client: InferenceEngineClient, tokenizer): + # read the skyagent task yaml + skyagent_task_yaml_path = generator_cfg.task + if not os.path.exists(skyagent_task_yaml_path): + raise FileNotFoundError(f"Task YAML not found: {skyagent_task_yaml_path}") + + self.agent_generator = AutoAgentRunner.from_task( + skyagent_task_yaml_path, infer_engine=llm_endpoint_client, tokenizer=tokenizer + ) + + async def generate(self, input_batch: GeneratorInput) -> GeneratorOutput: + val_mode = input_batch["batch_metadata"].training_phase == "eval" + return await self.agent_generator.run(input_batch, val_mode=val_mode) + + +class SkyRLAgentPPOExp(BasePPOExp): + def get_generator(self, cfg, tokenizer, llm_endpoint_client): + generator = SkyRLAgentGenerator( + generator_cfg=cfg.generator, llm_endpoint_client=llm_endpoint_client, tokenizer=tokenizer + ) + return generator + + def get_trainer( + self, + cfg, + tracker, + tokenizer, + train_dataset, + eval_dataset, + inference_engine_client, + generator: GeneratorInterface, + colocate_pg, + ): + """Initializes the trainer. + + Returns: + RayPPOTrainer: The trainer. + """ + return SkyRLAgentPPOTrainer( + cfg=cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=colocate_pg, + ) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = SkyRLAgentPPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + task = skyrl_entrypoint.remote(cfg) + try: + ray.get(task) + except KeyboardInterrupt: + print("KeyboardInterrupt received, shutting down...") + ray.cancel(task) + raise + + +if __name__ == "__main__": + main() diff --git a/skyrl-agent/skyrl_agent/integrations/skyrl_train/trainer.py b/skyrl-agent/skyrl_agent/integrations/skyrl_train/trainer.py new file mode 100644 index 0000000000..bbbb17786e --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/skyrl_train/trainer.py @@ -0,0 +1,543 @@ +""" +SkyRL Trainer overrides needed for SkyRLAgent integration. + +NOTE(Charlie): we need to get rid of this ASAP, hacky. Either change SkyAgent code or SkyRL code. + +Two changes: +- Do not check `input_batch["prompts"]` length in `validate_generator_output()`, since it is not + repeated n_samples_per_prompt times. +- Do not repeat the prompts in `prepare_generator_input()` since SkyAgent would repeat it. +""" + +from typing import List, Dict, Any, Tuple + +from loguru import logger +import numpy as np +from omegaconf import DictConfig +from torch.utils.data import RandomSampler, SequentialSampler +from torchdata.stateful_dataloader import StatefulDataLoader +import torch + +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.generators.base import ( + GeneratorInput, + GeneratorOutput, + TrajectoryID, + BatchMetadata, + TrainingPhase, +) +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.dataset import PromptDataset + +import asyncio +from pathlib import Path +import ray +from tqdm import tqdm + +from skyrl_train.training_batch import TrainingInputBatch +from skyrl_train.generators.utils import prepare_generator_input, get_metrics_from_generator_output +from skyrl_train.utils import Timer +from skyrl_train.utils.ppo_utils import ( + get_kl_controller, + normalize_advantages_dict, +) +from skyrl_train.utils.trainer_utils import ( + validate_generator_output, + ResumeMode, + build_dataloader, + calculate_per_dataset_metrics, + dump_per_dataset_eval_results, + concatenate_generator_outputs, +) + +from skyrl_train.utils.ppo_utils import register_advantage_estimator + + +@register_advantage_estimator("loop") +def compute_advantages_and_returns_loop( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + values: torch.Tensor, + config, + gamma, + lambd, + grpo_norm_by_std, + **kwargs, +): + from collections import defaultdict + + scores = token_level_rewards.sum(dim=-1) + + id2score = defaultdict(list) + id2mean = {} + + id2samples = defaultdict(list) + + with torch.no_grad(): + bsz = scores.shape[0] + for i in range(bsz): + id2samples[index[i]].append((i, scores[i])) + for group in id2samples.values(): + group_size = len(group) + total_score = sum(score for _, score in group) + for i, score in group: # i is original index + loo_baseline = 0 + if group_size == 1: + print("Cannot compute LOO advantage using 1 sample. 0 baseline is used") + else: + loo_baseline = (total_score - score) / (group_size - 1) + scores[i] = score - loo_baseline + scores = scores.unsqueeze(-1) * response_mask + return scores, scores + + +def validate_generator_output(input_batch: GeneratorInput, generator_output: GeneratorOutput): + """ + Validate the generator output. + + This is exactly the same as the one in SkyRL, except that we do not use the len(input_batch["prompts"]) + since it is not repeated n_samples_per_prompt times. + """ + if len(generator_output["response_ids"]) <= 0: + raise RuntimeError("No outputs generated") + + # check that input prompts, response ids, and prompt token ids are all the same length + # num_prompts = len(input_batch["prompts"]) + num_responses = len(generator_output["response_ids"]) + num_prompt_tokens = len(generator_output["prompt_token_ids"]) + # assert num_prompts == num_responses, f"Mismatch between prompts ({num_prompts}) and responses ({num_responses})" + assert ( + num_responses == num_prompt_tokens + ), f"Mismatch between responses ({num_responses}) and prompt_token_ids ({num_prompt_tokens})" + + # make sure all batch elements have the same length as response_ids (which should be non-zero) + for key in generator_output: + if isinstance(generator_output[key], list) and key in [ + "response_ids", + "loss_masks", + "rewards", + "rollout_logprobs", + ]: + assert len(generator_output[key]) == len( + generator_output["response_ids"] + ), f"Generator output {key} length must be equal to response_ids length, got {len(generator_output[key])} and {len(generator_output['response_ids'])}" + + # make sure that each element of response ids and loss masks are all the same length (and token level rewards if used) + for i, (response_ids, loss_masks, rewards) in enumerate( + zip(generator_output["response_ids"], generator_output["loss_masks"], generator_output["rewards"]) + ): + assert len(response_ids) == len( + loss_masks + ), f"Response ids and loss masks must have the same length, for sample {i} got {len(response_ids)} and {len(loss_masks)}" + if isinstance(rewards, list): + assert len(rewards) == len( + response_ids + ), f"Token rewards and response ids must have the same length, for sample {i} got {len(rewards)} and {len(response_ids)}" + + if generator_output["rollout_logprobs"]: + assert len(response_ids) == len( + generator_output["rollout_logprobs"][i] + ), f"Response ids and rollout logprobs must have the same length, for sample {i} got {len(response_ids)} and {len(generator_output['rollout_logprobs'][i])}" + + # loss masks should be non-zero for at least one element for trainer + if np.concatenate(generator_output["loss_masks"]).sum() == 0: + logger.warning("All outputs are loss masked, which may lead to NaN loss, please check your generation logic!!") + + +def build_dataloader(cfg: DictConfig, dataset: PromptDataset, is_train=True) -> StatefulDataLoader: + """ + Build the dataloader for the training or evaluation dataset + """ + # prepare dataloader + batch_size = cfg.trainer.train_batch_size if is_train else cfg.trainer.eval_batch_size + + # Seed the dataloader for reproducibility. + # NOTE: We seed the dataloader in the same fashion as VERL for exact reproducibility + seeded_generator = torch.Generator() + seeded_generator.manual_seed(cfg.trainer.seed) + if is_train: + sampler = RandomSampler(dataset, generator=seeded_generator) + else: + sampler = SequentialSampler(dataset) + + dataloader = StatefulDataLoader( + dataset, + batch_size=batch_size, + collate_fn=dataset.collate_fn, + # TODO(Charlie): debug why inference http endpoint is slow when num_workers is 8 + num_workers=0 if cfg.generator.enable_http_endpoint else 8, + drop_last=True if is_train else False, + sampler=sampler, + ) + if is_train: + logger.info(f"Total steps: {len(dataloader) * cfg.trainer.epochs}") + else: + logger.info(f"Validation set size: {len(dataloader)}") + + return dataloader + + +def prepare_generator_input( + prompts: List[Any], + n_samples_per_prompt: int, + sampling_params: Dict[str, Any], + default_env_class: str, + training_phase: TrainingPhase, + global_step: int, +) -> Tuple[GeneratorInput, List[str]]: + """Prepares the generator input for training and eval + + Args: + prompts (List[Any]): list of prompts + n_samples_per_prompt (int): how many samples to create per prompt + sampling_params (Dict[str, Any]): sampling parameters + default_env_class (str): env class to use if env class missing from prompts + training_phase (TrainingPhase): training or eval + global_step (int): current global step + + Returns: + Tuple[GeneratorInput, List[str]]: generator input and list of uuids + """ + # skyagent's AgentRunner will repeat trajectories internally + all_prompts = [prompt["prompt"] for prompt in prompts] + + # all the other columns are env_extras + env_extras = [prompt["env_extras"] for prompt in prompts] + + # But for other items like uuids or env classes - repeat by `n_samples_per_prompt` because it is used by + # the trainer and for metrics + all_envs = [ + prompt["env_class"] if prompt["env_class"] is not None else default_env_class + for prompt in prompts + for _ in range(n_samples_per_prompt) + ] + # Create TrajectoryID objects - one UID per row, repetition_id for multiple samples + trajectory_ids = [] + uids = [] + for _, prompt in enumerate(prompts): + uid: str = prompt["uid"] + + # Create TrajectoryID for each repetition + for repetition_id in range(n_samples_per_prompt): + trajectory_ids.append(TrajectoryID(instance_id=uid, repetition_id=repetition_id)) + uids.append(uid) + + generator_input: GeneratorInput = { + "prompts": all_prompts, + "env_classes": all_envs, + "env_extras": env_extras, + "sampling_params": sampling_params, + "trajectory_ids": trajectory_ids, + "batch_metadata": BatchMetadata(global_step=global_step, training_phase=training_phase), + } + + return generator_input, uids + + +class SkyRLAgentPPOTrainer(RayPPOTrainer): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # reinitialize with new dataloader function for exact reproducibility across backends + self.train_dataloader = build_dataloader(self.cfg, self.train_dataset, is_train=True) + self.total_training_steps = len(self.train_dataloader) * self.cfg.trainer.epochs + self.eval_dataloader = ( + build_dataloader(self.cfg, self.eval_dataset, is_train=False) if self.eval_dataset is not None else None + ) + + @torch.no_grad() + async def generate( + self, + input_batch: GeneratorInput, + ) -> GeneratorOutput: + """ + Generate rollouts. + + If colocate_all is enabled: + - before calling this method, the policy model should be on CPU and inference engine should + be awake (i.e. on GPU). + - after calling this method, the same model placement still holds. + """ + # NOTE: we assume that .generate returns samples in the same order as passed in + # Here SkyAgent would return a repeated output (n_samples_per_prompt times) + generator_output: GeneratorOutput = await self.generator.generate(input_batch) + + # add rollout metrics to self.all_metrics + if generator_output["rollout_metrics"] is not None: + self.all_metrics.update(generator_output["rollout_metrics"]) + + validate_generator_output(input_batch, generator_output) + + return generator_output + + def train(self): + """ + Main training loop for PPO + """ + # Initialize weight sync state between policy model and inference engines. + with Timer("init_weight_sync_state"): + self.init_weight_sync_state() + + # Load policy model to GPU before loading checkpoint. + if self.colocate_all: + self.policy_model.backload_to_gpu() + + # Load checkpoint state if resumption is enabled. + if self.resume_mode != ResumeMode.NONE: + with Timer("load_checkpoints"): + self.global_step = self.load_checkpoints() + + if self.colocate_all: + self.policy_model.offload_to_cpu(offload_optimizer=True, offload_model=False) + asyncio.run(self.inference_engine_client.wake_up(tags=["weights"])) + with Timer("sync_weights"): + ray.get(self.sync_policy_weights_to_inference_engines()) + if self.colocate_all: + with Timer("offload_policy_model_to_cpu"): + self.policy_model.offload_to_cpu(offload_optimizer=False, offload_model=True) + asyncio.run(self.inference_engine_client.wake_up(tags=["kv_cache"])) + + # Eval before training + if self.cfg.trainer.eval_interval > 0 and self.cfg.trainer.eval_before_train: + with Timer("eval", self.all_timings): + eval_metrics = asyncio.run(self.eval()) + self.tracker.log(eval_metrics, step=self.global_step, commit=True) + + # initialize kl controller + if self.cfg.trainer.algorithm.use_kl_in_reward: + self.reward_kl_controller = get_kl_controller(self.cfg.trainer.algorithm) + + # main training loop + pbar = tqdm(total=self.total_training_steps, initial=self.global_step, desc="Training Batches Processed") + self.global_step += 1 # start training at global_step 1 + for epoch in range(self.cfg.trainer.epochs): + for iter, rand_prompts in enumerate(self.train_dataloader): + with Timer("step", self.all_timings): + # for colocate_all=true, inference engine is always on GPU when starting the training step + + # 0. truncate data to have even shards + rand_prompts = self._remove_tail_data(rand_prompts) + generator_input, uids = prepare_generator_input( + rand_prompts, + self.cfg.generator.n_samples_per_prompt, + get_sampling_params_for_backend(self.cfg.generator.backend, self.cfg.generator.sampling_params), + self.cfg.environment.env_class, + "train", + self.global_step, + ) + + # 1.1 generation phase + with Timer("generate", self.all_timings): + generator_output: GeneratorOutput = asyncio.run(self.generate(generator_input)) + + # dynamic sampling + if self.cfg.trainer.algorithm.dynamic_sampling.type is not None: + generator_output, uids, keep_sampling = self.handle_dynamic_sampling(generator_output, uids) + if keep_sampling: # continue sampling + # update progress bar for current batch (but not global step) + pbar.update(1) + continue + + # if we are not continuing sampling, we sleep the inference engine + asyncio.run(self.inference_engine_client.sleep()) + + # 1.2 postprocess rewards + with Timer("postprocess_generator_output", self.all_timings): + generator_output = self.postprocess_generator_output(generator_output, uids) + + # 2. print example just for debugging + vis = self.tokenizer.decode(generator_output["response_ids"][0]) + logger.info(f"Example:\n" f" Input: {generator_input['prompts'][0]}\n" f" Output:\n{vis}") + + with Timer("convert_to_training_input", self.all_timings): + training_input: TrainingInputBatch = self.convert_to_training_input(generator_output, uids) + logger.info(f"Number of sequences: {len(training_input['sequences'])}") + + # 1.4 inference and calculate values, log probs, rewards, kl divergence + with Timer("fwd_logprobs_values_reward", self.all_timings): + training_input = self.fwd_logprobs_values_reward(training_input) + + # 1.5 apply kl divergence penalty to rewards + if self.cfg.trainer.algorithm.use_kl_in_reward: + with Timer("apply_reward_kl_penalty", self.all_timings): + training_input = self.apply_reward_kl_penalty(training_input) + + # 3. calculate advantages and returns + with Timer("compute_advantages_and_returns", self.all_timings): + training_input = self.compute_advantages_and_returns(training_input) + # remove some unwanted keys + for key in ["rewards"]: + training_input.pop(key) + training_input.metadata.pop("uids") + + if self.cfg.trainer.algorithm.advantage_batch_normalize: + training_input = normalize_advantages_dict(training_input) + + if self.cfg.trainer.dump_data_batch: + # dump data to file + with Timer("dump_data_batch"): + self.dump_data(training_input, file_name=f"global_step_{self.global_step}_training_input") + + # 4. train policy/critic model + # Policy model is backloaded to GPU during training + with Timer("train_critic_and_policy", self.all_timings): + status = self.train_critic_and_policy(training_input) + + # 5. conditionally save checkpoints and hf model + if self.cfg.trainer.ckpt_interval > 0 and self.global_step % self.cfg.trainer.ckpt_interval == 0: + with Timer("save_checkpoints", self.all_timings): + self.save_checkpoints() + if ( + self.cfg.trainer.hf_save_interval > 0 + and self.global_step % self.cfg.trainer.hf_save_interval == 0 + ): + with Timer("save_hf_model", self.all_timings): + self.save_models() + + # 6. conditionally sync policy and ref at the end of the epoch + if ( + self.cfg.trainer.update_ref_every_epoch + and self.ref_model is not None + and iter == len(self.train_dataloader) - 1 + and epoch != self.cfg.trainer.epochs - 1 # skip updating ref at the end of the last epoch + ): + with Timer("update_ref_with_policy", self.all_timings): + self.update_ref_with_policy() + + # 7. sync weights to inference engines + if self.colocate_all: + self.policy_model.offload_to_cpu(offload_optimizer=True, offload_model=False) + asyncio.run(self.inference_engine_client.wake_up(tags=["weights"])) + with Timer("sync_weights", self.all_timings): + ray.get(self.sync_policy_weights_to_inference_engines()) + if self.colocate_all: + with Timer("offload_policy_model_to_cpu"): + self.policy_model.offload_to_cpu(offload_optimizer=False, offload_model=True) + asyncio.run(self.inference_engine_client.wake_up(tags=["kv_cache"])) + + # 8. set logs + logger.info(status) + # log epoch info + self.all_metrics.update({"trainer/epoch": epoch, "trainer/global_step": self.global_step}) + if self.cfg.trainer.eval_interval > 0 and ( + self.global_step % self.cfg.trainer.eval_interval == 0 + or self.global_step == self.total_training_steps + ): + with Timer("eval", self.all_timings): + eval_metrics = asyncio.run(self.eval()) + self.all_metrics.update(eval_metrics) + + log_payload = { + **self.all_metrics, + **{f"timing/{k}": v for k, v in self.all_timings.items()}, + } + self.tracker.log(log_payload, step=self.global_step, commit=True) + self.all_metrics = {} + self.all_timings = {} + + # update progress bar after logging + pbar.update(1) + + self.global_step += 1 + + del training_input, generator_output + + pbar.close() + if self.colocate_all: + asyncio.run(self.inference_engine_client.sleep()) + self.policy_model.backload_to_gpu() + if self.cfg.trainer.ckpt_interval > 0: + with Timer("save_checkpoints", self.all_timings): + self.save_checkpoints() + logger.info("Saved final checkpoint.") + if self.cfg.trainer.hf_save_interval > 0: + with Timer("save_hf_model", self.all_timings): + self.save_models() + logger.info("Saved final model.") + logger.info("Training done!") + + @torch.no_grad() + async def eval(self): + """ + Run generation and scoring on the evaluation dataset. + + The eval metrics are recorded after having finished training `self.global_step` steps. + Metrics recorded in global_step 0 corresponds to evaluations before training. + + Returns: + A dictionary of evaluation metrics. + """ + # NOTE: We've only injected the custom `prepare_generator_input` function here + + eval_dataloader = self.eval_dataloader + generator = self.generator + cfg = self.cfg + global_step = self.global_step + tokenizer = self.tokenizer + + # 1. Get all generator outputs + generator_outputs: List[GeneratorOutput] = [] + concat_all_envs: List[str] = [] + concat_env_extras: List[Dict[str, Any]] = [] + concat_uids: List[str] = [] + sampling_params = cfg.generator.eval_sampling_params + pbar = tqdm(total=len(eval_dataloader), initial=0, desc="Evaluation Progress") + for _, prompts in enumerate(eval_dataloader): + pbar.update(1) + generator_input, uids = prepare_generator_input( + prompts, + cfg.generator.eval_n_samples_per_prompt, + get_sampling_params_for_backend(cfg.generator.backend, sampling_params), + cfg.environment.env_class, + "eval", + global_step, + ) + generator_output: GeneratorOutput = await generator.generate(generator_input) + validate_generator_output(generator_input, generator_output) + generator_outputs.append(generator_output) + concat_all_envs.extend(generator_input["env_classes"]) + concat_env_extras.extend(generator_input["env_extras"]) + concat_uids.extend(uids) + concat_generator_outputs: GeneratorOutput = concatenate_generator_outputs(generator_outputs) + + # Extract data_sources from env_extras + concat_data_sources = [env_extra.get("data_source") for env_extra in concat_env_extras] + vis = tokenizer.decode(generator_output["response_ids"][0]) + logger.info(f"Eval output example: {vis}") + + # 2. Group data by data source and calculate per-dataset metrics + eval_metrics = calculate_per_dataset_metrics( + concat_generator_outputs, concat_uids, concat_data_sources, cfg.generator.eval_n_samples_per_prompt + ) + + # 3. Calculate overall metrics across all datasets + overall_metrics = get_metrics_from_generator_output(concat_generator_outputs, concat_uids) + eval_metrics.update( + { + "eval/all/avg_score": overall_metrics["avg_score"], + f"eval/all/pass_at_{cfg.generator.eval_n_samples_per_prompt}": overall_metrics["pass_at_n"], + "eval/all/mean_positive_reward": overall_metrics["mean_positive_reward"], + } + ) + + # 4. Prepare dumping data + # TODO[Ben] update this to be cloud-compatible + if cfg.trainer.dump_eval_results: + with Timer("dump_eval_results"): + data_save_dir = ( + Path(cfg.trainer.export_path) + / "dumped_evals" + / ("eval_only" if global_step is None else f"global_step_{global_step}_evals") + ) + data_save_dir.mkdir(parents=True, exist_ok=True) + dump_per_dataset_eval_results( + data_save_dir, + tokenizer, + concat_generator_outputs, + concat_data_sources, + concat_all_envs, + concat_env_extras, + eval_metrics, + ) + + return eval_metrics diff --git a/skyrl-agent/skyrl_agent/integrations/tinker/README.md b/skyrl-agent/skyrl_agent/integrations/tinker/README.md new file mode 100644 index 0000000000..e389777877 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/tinker/README.md @@ -0,0 +1,48 @@ +# Tinker Integration for SkyRL Agent + +This directory contains the integration between SkyRL Agent and Tinker's RL training framework. + +## Overview + +The Tinker integration enables you to: +1. Use **AgentRunner** to collect trajectories from your agents +2. Train models using **Tinker's LoRA training** with importance sampling or PPO +3. Track metrics with **Weights & Biases** +4. Save and resume from checkpoints + +## Architecture + +### Components + +1. **`tinker_backend.py`**: Backend interface for Tinker's sampling client + - Handles async generation from token IDs + - Decodes outputs with proper special token handling + - Detects finish reasons (EOS vs length limit) + +2. **`tinker_train.py`**: Main training script + - Loads data from Parquet files + - Runs AgentRunner for trajectory collection + - Converts AgentRunner output to Tinker's format + - Performs RL training with advantage normalization + - Saves checkpoints periodically + +3. **`__init__.py`**: Backend registration + - Registers Tinker backend in the SkyRL Agent registry + +## Data Flow + +``` +Input Batch (from Parquet) + ↓ +AgentRunner.run() → Trajectories + ↓ +Extract: {prompt_token_ids, response_ids, logprobs, rewards, loss_masks} + ↓ +Compute: Advantages (with optional normalization) + ↓ +Prepare: Tinker Datums (with masks and advantages) + ↓ +Tinker Training: forward_backward() + optim_step() + ↓ +Metrics & Checkpointing +``` diff --git a/skyrl-agent/skyrl_agent/integrations/tinker/__init__.py b/skyrl-agent/skyrl_agent/integrations/tinker/__init__.py new file mode 100644 index 0000000000..2900e8610d --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/tinker/__init__.py @@ -0,0 +1,11 @@ +from .tinker_backend import TinkerBackend, TinkerGeneratorOutput, TinkerGeneratorInput +from ..base import register_backend, BackendSpec + +register_backend( + "tinker", + BackendSpec( + infer_backend_cls=TinkerBackend, + generator_output_cls=TinkerGeneratorOutput, + generator_input_cls=TinkerGeneratorInput, + ), +) diff --git a/skyrl-agent/skyrl_agent/integrations/tinker/tinker_backend.py b/skyrl-agent/skyrl_agent/integrations/tinker/tinker_backend.py new file mode 100644 index 0000000000..e32e8a4aea --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/tinker/tinker_backend.py @@ -0,0 +1,47 @@ +from typing import Any, List +from ..base import AsyncInferBackend, GeneratorOutput, GeneratorInput + + +class TinkerBackend(AsyncInferBackend): + def __init__(self, infer_engine, tokenizer: Any = None, cfg: Any = None): + self.client = infer_engine + self.tokenizer = tokenizer + + async def async_generate_prompts(self, prompts: Any, sampling_params: Any, **kwargs) -> List[str]: + raise NotImplementedError + + async def async_generate_ids(self, input_ids: List[int], sampling_params: Any, **kwargs) -> List[str]: + from tinker.types import ModelInput, SamplingParams + + prompt_ids = ModelInput.from_ints(tokens=input_ids) + tinker_sampling_params = SamplingParams(**sampling_params) + output = await self.client.sample_async( + prompt=prompt_ids, num_samples=1, sampling_params=tinker_sampling_params + ) + output_tokens = output.sequences[0].tokens + # get raw content and skip special tokens like + message = self.tokenizer.decode(output_tokens, skip_special_tokens=True) + # check if finish_reason is stop + # currently check if last token is tokenizer.eos_token + if output_tokens[-1] == self.tokenizer.eos_token_id: + finish_reason = "stop" + else: + finish_reason = "length" + + meta_info = { + "output_tokens": output_tokens, + "finish_reason": finish_reason, + "logprobs": output.sequences[0].logprobs, + } + + return message, meta_info + + +class TinkerGeneratorOutput(GeneratorOutput): + def __init__(self, result: Any): + self.result = result + + +class TinkerGeneratorInput(GeneratorInput): + def __init__(self, input_batch: Any): + self.input_batch = input_batch diff --git a/skyrl-agent/skyrl_agent/integrations/tinker/tinker_train.py b/skyrl-agent/skyrl_agent/integrations/tinker/tinker_train.py new file mode 100644 index 0000000000..45a772baf3 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/tinker/tinker_train.py @@ -0,0 +1,484 @@ +import asyncio +import json +import logging +import os +import pprint +import random +import time +from datetime import datetime +from typing import Any, Literal, List, Dict, cast +from contextlib import contextmanager + +import chz +import numpy as np +import tinker +import torch +import wandb +from tinker import types +from tinker.types.tensor_data import TensorData +from transformers.models.auto.tokenization_auto import AutoTokenizer +from datasets import load_dataset +from torch.utils.data import DataLoader + +from skyrl_agent import AutoAgentRunner + +logger = logging.getLogger(__name__) +logging.getLogger("httpx").setLevel(logging.WARN) + + +def set_seed(seed: int): + """Set random seed for reproducibility across all libraries.""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + logger.info(f"Set random seed to {seed}") + + +@contextmanager +def timed(key: str, metrics: dict[str, Any]): + logger.info(f"Starting {key}") + tstart = time.time() + yield + logger.info(f"{key} took {time.time() - tstart:.2f} seconds") + metrics[f"time/{key}"] = time.time() - tstart + + +safezip = cast(type[zip], lambda *args, **kwargs: zip(*args, **kwargs, strict=True)) + + +def normalize_advantages(advantages: List[float]) -> List[float]: + """Normalize advantages to have mean 0 and std 1 (standard normalization).""" + if not advantages or len(advantages) == 1: + return advantages + mean = np.mean(advantages) + std = np.std(advantages) + if std < 1e-8: + return [0.0] * len(advantages) + return [(a - mean) / (std + 1e-8) for a in advantages] + + +def compute_advantages_grpo( + rewards: List[float], + group_size: int = None, + normalize: bool = True, +) -> List[float]: + """ + GRPO (Group Relative Policy Optimization) advantage estimation. + + For each group of trajectories from the same prompt, compute advantages + as deviations from the group mean. This is particularly useful for + best-of-N sampling scenarios. + + Reference: https://github.com/volcengine/verl/blob/main/verl/trainer/ppo/core_algos.py + + Args: + rewards: List of rewards for all trajectories + group_size: Number of trajectories per prompt group. + If None, treats all as one group. + normalize: Whether to apply additional global normalization + + Returns: + List of advantages, same length as rewards + """ + rewards = np.array(rewards) + + if group_size is None: + # Treat all trajectories as one group (equivalent to simple baseline) + group_size = len(rewards) + + n_groups = len(rewards) // group_size + advantages = [] + + for i in range(n_groups): + start_idx = i * group_size + end_idx = start_idx + group_size + group_rewards = rewards[start_idx:end_idx] + + # GRPO: advantage = reward - mean(group_rewards) + group_mean = group_rewards.mean() + group_advantages = group_rewards - group_mean + advantages.extend(group_advantages.tolist()) + + # Handle remaining trajectories if not evenly divisible + remaining = len(rewards) % group_size + assert remaining == 0, f"Remaining trajectories: {remaining} is not divisible by group_size: {group_size}" + + # Optional: Apply global normalization for extra stability + if normalize: + advantages = normalize_advantages(advantages) + + return advantages + + +def compute_kl_sample_train(data_D: List[tinker.Datum], training_logprobs_D: List[torch.Tensor]) -> Dict[str, float]: + """Compute KL divergence metrics between sampling and training logprobs.""" + all_diffs: list[torch.Tensor] = [] + all_sampling_logprobs: list[torch.Tensor] = [] + + for datum, training_logprobs in safezip(data_D, training_logprobs_D): + # Get logprobs from sampling + sampling_logprobs = datum.loss_fn_inputs["logprobs"].to_torch() + action_mask = datum.loss_fn_inputs["mask"].to_torch() > 0 + # Extract only action token logprobs + sampling_logprobs_actions = sampling_logprobs[action_mask] + training_logprobs_actions = training_logprobs[action_mask] + + if len(sampling_logprobs_actions) > 0: + logprob_diff = sampling_logprobs_actions - training_logprobs_actions + all_diffs.append(logprob_diff) + all_sampling_logprobs.append(sampling_logprobs_actions) + + assert all_diffs + flat_diffs = torch.cat(all_diffs) + kl_sample_train_v1 = flat_diffs.mean().item() + kl_sample_train_v2 = 0.5 * (flat_diffs**2).mean().item() + + flat_sampling_logprobs = torch.cat(all_sampling_logprobs) + entropy_sample = -flat_sampling_logprobs.mean().item() + return { + "optim/kl_sample_train_v1": kl_sample_train_v1, + "optim/kl_sample_train_v2": kl_sample_train_v2, + "optim/entropy": entropy_sample, + } + + +@chz.chz +class Config: + model_name: str = "Qwen/Qwen3-32B" + batch_size: int = 64 + eval_batch_size: int = 1024 + learning_rate: float = 4e-5 + lora_rank: int = 16 + seed: int = 0 + max_steps: int = 200 + save_every: int = 2 + eval_every: int = 10 + resume_exp_name: str = None + + skyrl_agent_task_yaml: str = None + dataset_file: str = None # Path to the training dataset parquet file + eval_dataset_file: str = None # Path to the evaluation dataset parquet file + + # Loss function configuration + loss_fn: Literal["importance_sampling", "ppo", "custom_ppo"] = "ppo" + # Options: + # "ppo" or "importance_sampling": Use Tinker's built-in loss (forward_backward) + + # GRPO (Group Relative Policy Optimization) settings + group_size: int = 8 # Trajectories per prompt group (None = auto-infer from task yaml) + normalize_advantages: bool = True # Apply global normalization after group-relative computation + + wandb_project: str | None = None + wandb_name: str | None = None + log_dir: str | None = None + + +async def save_checkpoint_async( + training_client: tinker.TrainingClient, + name: str, + log_path: str, + loop_state: dict[str, Any], + kind: Literal["state", "sampler", "both"] = "state", +) -> dict[str, str]: + """Save model checkpoint. + Args: + training_client: Training client to save from + name: Name for the checkpoint + log_path: Path to the log directory, where we can find checkpoints.jsonl file + Returns: + Path to the saved checkpoint + """ + futures = {} + if kind in ["state", "both"]: + futures["state"] = await training_client.save_state_async(name) + if kind in ["sampler", "both"]: + futures["sampler"] = await training_client.save_weights_for_sampler_async(name) + + results = {k: await v.result_async() for k, v in futures.items()} + paths = {k + "_path": v.path for k, v in results.items()} + logger.info(f"Saved checkpoints: {paths}") + full_dict = {"name": name, **loop_state, **paths} + with open(os.path.join(log_path, "checkpoints.jsonl"), "a") as f: + f.write(json.dumps(full_dict) + "\n") + + return paths + + +def collate_fn(batch): + """Custom collate function that returns batch as-is without tensor collation. + + This is needed because the agent runner expects to handle the raw batch data + through build_generator_input, rather than having PyTorch stack tensors. + """ + return batch + + +async def main(config: Config): + # Set random seed for reproducibility + set_seed(config.seed) + + # Setup logging + if config.resume_exp_name: + wandb_name = config.resume_exp_name + else: + wandb_name = config.wandb_name or config.model_name.split("/")[-1] + wandb_name += "_" + datetime.now().strftime("%m%dT%H:%M:%S") + save_path = os.path.join("./tinker_output", wandb_name) + os.makedirs(save_path, exist_ok=True) + + # read the most recent checkpoint + checkpoint_path = os.path.join(save_path, "checkpoints.jsonl") + load_state_path = None # Initialize to None + if os.path.exists(checkpoint_path): + with open(checkpoint_path, "r") as f: + checkpoints = [json.loads(line) for line in f] + most_recent_checkpoint = max(checkpoints, key=lambda x: x["policy_iteration_step"]) + resume_from_step = most_recent_checkpoint["policy_iteration_step"] + load_state_path = most_recent_checkpoint["state_path"] + print(f"Resuming training from step {resume_from_step}") + else: + resume_from_step = 0 + print("Starting training from scratch") + + wandb.init( + project=config.wandb_project, + config=chz.asdict(config), + dir=str(config.log_dir) if config.log_dir else None, + name=wandb_name, + ) + + # dataset and dataloader + train_dataset = load_dataset("parquet", data_files=config.dataset_file)["train"] + eval_dataset = load_dataset("parquet", data_files=config.eval_dataset_file)["train"] + + # Calculate steps per epoch for tracking + steps_per_epoch = (len(train_dataset) + config.batch_size - 1) // config.batch_size + logger.info(f"Dataset size: {len(train_dataset)}, Steps per epoch: {steps_per_epoch}") + + # Create function to get dataloader for a specific epoch + def create_train_dataloader(epoch: int): + """Create dataloader with epoch-specific seed for different shuffle orders.""" + return DataLoader( + train_dataset, + batch_size=config.batch_size, + shuffle=True, + collate_fn=collate_fn, + generator=torch.Generator().manual_seed(config.seed + epoch), # Different shuffle per epoch + ) + + # Initialize iterator state for resuming + current_epoch = resume_from_step // steps_per_epoch + batch_offset_in_epoch = resume_from_step % steps_per_epoch + + train_dataloader = create_train_dataloader(current_epoch) + train_iterator = iter(train_dataloader) + + # Skip batches within the current epoch if resuming mid-epoch + if batch_offset_in_epoch > 0: + logger.info(f"Resuming from epoch {current_epoch}, batch {batch_offset_in_epoch}/{steps_per_epoch}") + for _ in range(batch_offset_in_epoch): + next(train_iterator) + + # Setup agent (tinker training client) + service_client = tinker.ServiceClient() + training_client = await service_client.create_lora_training_client_async( + base_model=config.model_name, rank=config.lora_rank + ) + if load_state_path: + future = await training_client.load_state_async(load_state_path) + _ = await future.result_async() + logger.info(f"Loaded state from {load_state_path}") + + adam_params = types.AdamParams(learning_rate=config.learning_rate, beta1=0.9, beta2=0.95, eps=1e-8) + skyrl_agent_task_yaml_path = config.skyrl_agent_task_yaml + tokenizer = AutoTokenizer.from_pretrained(config.model_name) + + # training loop + for policy_iteration_step in range(resume_from_step, config.max_steps): + print("=" * 10 + f" Step {policy_iteration_step} " + "=" * 10) + metrics = { + "step": policy_iteration_step, + "epoch": policy_iteration_step // steps_per_epoch, + "batch_in_epoch": policy_iteration_step % steps_per_epoch, + } + + # save model + if config.save_every > 0 and policy_iteration_step > 0 and policy_iteration_step % config.save_every == 0: + await save_checkpoint_async( + training_client, + f"{policy_iteration_step:06d}", + log_path=save_path, + kind="state", + loop_state={"policy_iteration_step": policy_iteration_step}, + ) + + sampling_path = training_client.save_weights_for_sampler(name=f"{policy_iteration_step:06d}").result().path + sampling_client = service_client.create_sampling_client(model_path=sampling_path) + + agent_generator = AutoAgentRunner.from_task( + skyrl_agent_task_yaml_path, infer_engine=sampling_client, tokenizer=tokenizer + ) + + if policy_iteration_step % config.eval_every == 0: + eval_dataloader = DataLoader( + eval_dataset, batch_size=config.eval_batch_size, shuffle=False, collate_fn=collate_fn + ) + data_source_rewards = {} + for batch in eval_dataloader: + input_batch = batch + rollouts = await agent_generator.run(input_batch, val_mode=True) + traj_rewards_list = rollouts["traj_rewards"] + for data, reward in zip(input_batch, traj_rewards_list): + data_source = data["data_source"] + if data_source not in data_source_rewards: + data_source_rewards[data_source] = [] + data_source_rewards[data_source].append(reward) + # get avg reward per data source + for data_source, rewards in data_source_rewards.items(): + metrics[f"eval/reward/mean/{data_source}"] = np.mean(rewards) + + # Collect rollouts using AgentRunner + print(f"🎲 Start collecting episodes at step {policy_iteration_step}") + st = time.time() + + # Get next batch, handling epoch transitions + try: + input_batch = next(train_iterator) + except StopIteration: + # Start new epoch with different shuffle order + current_epoch += 1 + logger.info(f"Starting epoch {current_epoch} with new shuffle order") + train_dataloader = create_train_dataloader(current_epoch) + train_iterator = iter(train_dataloader) + input_batch = next(train_iterator) + + rollouts = await agent_generator.run(input_batch, val_mode=False) + metrics["time/sample"] = time.time() - st + # rollout time + print(f"Rollout time: {metrics['time/sample']}") + + # Write rollout_metrics to wandb + rollout_metrics = rollouts.get("rollout_metrics", {}) + wandb.log({f"rollout/{k}": v for k, v in rollout_metrics.items()}, step=policy_iteration_step) + + # Extract rollout data + prompt_token_ids = rollouts["prompt_token_ids"] # List of prompt token IDs + response_ids = rollouts["response_ids"] # List of response token IDs + traj_rewards_list = rollouts["traj_rewards"] # List of rewards (binary: 0 or 1) + loss_masks = rollouts["loss_masks"] # List of loss masks + sampled_logprobs = rollouts["rollout_logprobs"] # List of sampled logprobs + num_steps_per_trajectory = rollouts["episode_nums"] # List of number of steps per trajectory + + actual_batch_size = len(response_ids) + logger.info(f"Processing {actual_batch_size} rollouts for training") + + # Compute advantages using GRPO (Group Relative Policy Optimization) + all_returns = [float(r) for r in traj_rewards_list] + + # Determine group size for GRPO + group_size = config.group_size + if group_size is None: + # Try to infer from task config + from omegaconf import OmegaConf + + task_config = OmegaConf.load(skyrl_agent_task_yaml_path) + group_size = task_config.generator.get("num_trajectories", 1) + logger.info(f"Auto-inferred group_size={group_size} from task config") + + # Compute GRPO advantages + logger.info(f"Computing GRPO advantages: group_size={group_size}, normalize={config.normalize_advantages}") + all_advantages = compute_advantages_grpo( + all_returns, group_size=group_size, normalize=config.normalize_advantages + ) + # broadcast advantages to num_steps per trajectory + step_advantages = [] + for idx, num_steps in enumerate(num_steps_per_trajectory): + step_advantages.extend([all_advantages[idx]] * num_steps) + + metrics["reward/mean"] = np.mean(all_returns) + metrics["reward/max"] = np.max(all_returns) + metrics["reward/min"] = np.min(all_returns) + metrics["advantage/mean"] = np.mean(all_advantages) + metrics["advantage/std"] = np.std(all_advantages) + + # Prepare training datums compatible with Tinker API + # For each trajectory, we need to provide: + # - model_input: the full sequence (prompt + response) + # - loss_fn_inputs: target_tokens, advantages, logprobs (if available), mask + training_datums = [] + for idx in range(actual_batch_size): + # Concatenate prompt and response to get full sequence + full_sequence = prompt_token_ids[idx] + response_ids[idx] + prompt_len = len(prompt_token_ids[idx]) + + # Target tokens are same as input (autoregressive training) + target_tokens = full_sequence[1:] + logprobs = ([0] * prompt_len + sampled_logprobs[idx])[1:] + + # Base mask: 0 for prompt, loss_mask value for response + mask = [0] * prompt_len + loss_masks[idx] + + # Advantages: broadcast the single advantage value across all response tokens + advantage_value = step_advantages[idx] + advantages = torch.zeros(len(full_sequence)) + # Only apply advantage to response tokens that are not masked + assert len(mask) == len(full_sequence), f"Mask length mismatch: {len(mask)} vs {len(full_sequence)}" + for i in range(prompt_len, len(full_sequence)): + if mask[i] > 0: + advantages[i] = advantage_value + advantages = advantages[1:] + mask = mask[1:] + + datum = types.Datum( + model_input=types.ModelInput.from_ints(tokens=full_sequence[:-1]), + loss_fn_inputs={ + "target_tokens": TensorData.from_torch(torch.tensor(target_tokens)), + "logprobs": TensorData.from_torch(torch.tensor(logprobs)), + "advantages": TensorData.from_torch(advantages), + }, + ) + training_datums.append(datum) + + # Training step + print(f"🎈 Start training at step {policy_iteration_step}") + st = time.time() + + # Use Tinker's built-in loss function ("ppo" or "importance_sampling") + fwd_bwd_future = training_client.forward_backward(training_datums, loss_fn=config.loss_fn) + # Optimize + optim_step_future = training_client.optim_step(adam_params) + fwd_bwd_result = fwd_bwd_future.result() + + # Extract training logprobs from loss_fn_outputs + training_logprobs_D: list[torch.Tensor] = [] + for output in fwd_bwd_result.loss_fn_outputs: + training_logprobs = output["logprobs"].to_torch() + training_logprobs_D.append(training_logprobs) + # with timed("compute_kl_sample_train", metrics): + # kl_sample_train_metrics = compute_kl_sample_train(training_datums, training_logprobs_D) + # metrics.update(kl_sample_train_metrics) + + _ = optim_step_future.result() + metrics["time/train"] = time.time() - st + + pprint.pprint(metrics) + wandb.log(metrics, step=policy_iteration_step) + + # Save final checkpoint + if config.save_every > 0: + await save_checkpoint_async( + training_client, + "final", + log_path=save_path, + kind="both", + loop_state={"policy_iteration_step": config.max_steps}, + ) + + wandb.finish() + logger.info("Training completed successfully") + + +if __name__ == "__main__": + asyncio.run(main(chz.entrypoint(Config))) diff --git a/skyrl-agent/skyrl_agent/integrations/verl/__init__.py b/skyrl-agent/skyrl_agent/integrations/verl/__init__.py new file mode 100644 index 0000000000..c596204525 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/verl/__init__.py @@ -0,0 +1,13 @@ +from .verl_backend import VeRLGeneratorInput, VeRLBackend, VeRLGeneratorOutput +from ..base import register_backend, BackendSpec + +register_backend( + "verl", + BackendSpec( + infer_backend_cls=VeRLBackend, + generator_output_cls=VeRLGeneratorOutput, + generator_input_cls=VeRLGeneratorInput, + ), +) + +__all__ = ["VeRLGeneratorInput", "VeRLBackend", "VeRLGeneratorOutput"] diff --git a/skyrl-agent/skyrl_agent/integrations/verl/skyagent_async_vllm_server.py b/skyrl-agent/skyrl_agent/integrations/verl/skyagent_async_vllm_server.py new file mode 100644 index 0000000000..f710d46301 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/verl/skyagent_async_vllm_server.py @@ -0,0 +1,59 @@ +from typing import Any, Tuple +from typing import Optional + +import ray +from vllm import SamplingParams +from typing import Dict + +try: + from omegaconf import DictConfig, ListConfig, OmegaConf +except Exception: + DictConfig = None # type: ignore + ListConfig = None # type: ignore + OmegaConf = None # type: ignore + +from vllm.inputs import TokensPrompt +from vllm.outputs import RequestOutput +from verl.workers.rollout.vllm_rollout.vllm_async_server import AsyncvLLMServerRegular + + +@ray.remote(num_cpus=1) +class SkyAgentAsyncvLLMServer(AsyncvLLMServerRegular): + async def generate( + self, prompt_ids: list[int], sampling_params: dict[str, Any], request_id: str + ) -> Tuple[str, str]: + # max_tokens = self.max_model_len - len(prompt_ids) + # sampling_params.pop("max_tokens", None) + # sampling_params = SamplingParams(max_tokens=max_tokens, **sampling_params) + # Defensive sanitize of sampling params for vLLM + sp: Dict[str, Any] = dict(sampling_params) if sampling_params is not None else {} + # Ensure max_tokens exists and is an int within model limit + if "max_tokens" not in sp or sp["max_tokens"] is None: + max_tokens = self.max_model_len - len(prompt_ids) + sp["max_tokens"] = int(max_tokens) + else: + try: + sp["max_tokens"] = int(sp["max_tokens"]) + except Exception: + sp["max_tokens"] = int(self.max_model_len - len(prompt_ids)) + # vLLM expects stop to be a list; rely on caller to provide correct shape + sampling_params = SamplingParams(**sp) + prompt = TokensPrompt(prompt_token_ids=prompt_ids) + generator = self.engine.generate(prompt=prompt, sampling_params=sampling_params, request_id=request_id) + + # Get final response + final_res: Optional[RequestOutput] = None + async for output in generator: + final_res = output + assert final_res is not None + + response_str = final_res.outputs[0].text + stop_reason = final_res.outputs[0].finish_reason + output_tokens = final_res.outputs[0].token_ids + meta_info = { + "output_tokens": output_tokens, + "finish_reason": stop_reason, + "logprobs": None, + } + + return response_str, meta_info diff --git a/skyrl-agent/skyrl_agent/integrations/verl/upload_utils.py b/skyrl-agent/skyrl_agent/integrations/verl/upload_utils.py new file mode 100644 index 0000000000..9d6062ad9b --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/verl/upload_utils.py @@ -0,0 +1,166 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ray +import os +from urllib.parse import urlparse + +from enum import Enum +import torch +import torch.distributed + +import io + + +class Cloud(Enum): + AWS = "aws" + GCP = "gcp" + + +def uploadDirectoryToS3(path, bucketname, prefix): + import boto3 + + s3 = boto3.client("s3") + for root, dirs, files in os.walk(path): + for file in files: + relative_path = os.path.relpath(os.path.join(root, file), path) + s3_key = os.path.join(prefix, relative_path) + s3.upload_file(os.path.join(root, file), bucketname, s3_key) + + +def upload_file_to_s3(path, bucketname, prefix): + import boto3 + + s3 = boto3.client("s3") + s3.upload_file(path, bucketname, prefix) + + +def upload_dir_to_anyscale(local_path, remote_key): + save_bucket, remote_prefix, cloud = _get_anyscale_bucket_and_file_key(remote_key) + if cloud == Cloud.AWS: + uploadDirectoryToS3(local_path, save_bucket, remote_prefix) + elif cloud == Cloud.GCP: + upload_directory_to_gcs(local_path, save_bucket, remote_prefix) + else: + raise NotImplementedError + + +def upload_file_to_anyscale(local_path, remote_key): + save_bucket, remote_prefix, cloud = _get_anyscale_bucket_and_file_key(remote_key) + if cloud == Cloud.AWS: + upload_file_to_s3(local_path, save_bucket, remote_prefix) + elif cloud == Cloud.GCP: + upload_file_to_gcs(local_path, save_bucket, remote_prefix) + else: + raise NotImplementedError + + +def _get_anyscale_bucket_and_file_key(path): + parsed_url = urlparse(os.environ["ANYSCALE_ARTIFACT_STORAGE"]) + if parsed_url.scheme == "s3": + cloud = Cloud.AWS + else: + cloud = Cloud.GCP + save_bucket, prefix = parsed_url.netloc, parsed_url.path + prefix = prefix.lstrip("/") + save_bucket = save_bucket.rstrip("/") + path = os.path.join(prefix, path) + return save_bucket, path, cloud + + +def write_to_s3(obj, path: str): + import boto3 + + save_bucket, path, _ = _get_anyscale_bucket_and_file_key(path) + s3 = boto3.client("s3") + cpu_buffer = io.BytesIO() + torch.save(obj, cpu_buffer) # save to cpu + cpu_buffer.seek(0) + s3.upload_fileobj(cpu_buffer, save_bucket, path) + cpu_buffer.close() + + +ONEGB = 1024 * 1024 * 1024 + + +# Upload a single file to Google Cloud Storage +def upload_file_to_gcs(local_file_path, bucket_name, destination_blob_path): + import os + + os.environ["GOOGLE_RESUMABLE_MEDIA_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD"] = str( + 100 * 1024 * 1024 + ) # 100 MiB threshold + os.environ["GOOGLE_RESUMABLE_MEDIA_PARALLEL_COMPOSITE_PARTS"] = "10" # 10 parts in parallel + from google.cloud import storage + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_path, chunk_size=ONEGB) + + blob.upload_from_filename(local_file_path) + + print(f"File {local_file_path} uploaded to gs://{bucket_name}/{destination_blob_path}") + + +# Upload an entire directory to Google Cloud Storage +def upload_directory_to_gcs(local_directory, bucket_name, destination_prefix=""): + import os + + os.environ["GOOGLE_RESUMABLE_MEDIA_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD"] = str( + 100 * 1024 * 1024 + ) # 100 MiB threshold + os.environ["GOOGLE_RESUMABLE_MEDIA_PARALLEL_COMPOSITE_PARTS"] = "10" # 10 parts in parallel + from google.cloud import storage + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + for root, dirs, files in os.walk(local_directory): + for file in files: + local_path = os.path.join(root, file) + + # Determine the blob path in GCS + relative_path = os.path.relpath(local_path, local_directory) + blob_path = os.path.join(destination_prefix, relative_path).replace( + "\\", "/" + ) # Ensure proper path separators + + # Upload the file + blob = bucket.blob(blob_path, chunk_size=ONEGB) + blob.upload_from_filename(local_path) + + print(f"File {local_path} uploaded to gs://{bucket_name}/{blob_path}") + + print("Directory upload complete") + + +def upload_to_remote_background(config, global_step, local_global_step_folder, main_rank_latest_checkpointed_iteration): + import time + + def _upload_to_remote_background(config, global_step, local_global_step_folder): + dir_path = os.path.join(config.trainer.remote_upload_dir, f"global_step_{global_step}") + print(f"Uploading checkpoint to path: {dir_path}") + s = time.time() + upload_dir_to_anyscale(local_global_step_folder, dir_path) + e = time.time() + print(f"took {e - s} to upload") + + # only upload on main rank/ caller + file_path = os.path.join(config.trainer.remote_upload_dir, "latest_checkpointed_iteration.txt") + upload_file_to_anyscale(main_rank_latest_checkpointed_iteration, file_path) + + # use num_cpus > 0 to schedule only on worker nodes + remote_func = ray.remote(num_cpus=0.01, scheduling_strategy="SPREAD")(_upload_to_remote_background) + + return [remote_func.remote(config, global_step, local_global_step_folder) for _ in range(config.trainer.nnodes)] diff --git a/skyrl-agent/skyrl_agent/integrations/verl/verl_async_manager.py b/skyrl-agent/skyrl_agent/integrations/verl/verl_async_manager.py new file mode 100644 index 0000000000..fa776cca09 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/verl/verl_async_manager.py @@ -0,0 +1,235 @@ +import asyncio +from typing import Any, Dict, Optional + +import numpy as np +import ray +import torch +from omegaconf import DictConfig +from tensordict import TensorDict + +from verl.protocol import DataProto +from verl.single_controller.ray.base import RayWorkerGroup +from verl.utils import hf_tokenizer +from verl.utils.fs import copy_to_local +from verl.experimental.agent_loop.agent_loop import AsyncLLMServerManager +from verl.workers.rollout.async_server import AsyncServerBase + + +from verl.experimental.agent_loop.agent_loop import AgentLoopManager + +from skyrl_agent import AutoAgentRunner + + +def async_server_class( + rollout_backend: str, rollout_backend_module: Optional[str] = None, rollout_backend_class: Optional[str] = None +) -> type[AsyncServerBase]: + """Get async server class. + + Args: + rollout_backend: str, rollout backend type (alias), should be "vllm" or "sglang". + rollout_backend_module: Optional[str], import path of the rollout backend. + rollout_backend_class: Optional[str], class name of the rollout backend. + + Returns: + Type[AsyncServerBase]: async server class. + """ + if rollout_backend_class is None and rollout_backend_module is None: + # If both are None, use the default backend class + # Do not change the original import behavior + # importlib.import_module and from ... import ... have subtle differences in ray + + if rollout_backend == "vllm": + from .skyagent_async_vllm_server import SkyAgentAsyncvLLMServer + + return SkyAgentAsyncvLLMServer + elif rollout_backend == "sglang": + + raise NotImplementedError("Sglang backend for verl with skyagent is not implemented right now") + + else: + raise NotImplementedError(f"rollout backend {rollout_backend} is not supported") + + if rollout_backend_module is None or rollout_backend_class is None: + raise ValueError("rollout_backend_module and rollout_backend_class must be both provided for customization") + + from verl.utils.import_utils import load_extern_type + + return load_extern_type(rollout_backend_module, rollout_backend_class) + + +class SkyAgentLoopManager(AgentLoopManager): + """Agent loop manager that manages a group of agent loop workers.""" + + def __init__(self, config: DictConfig, worker_group: RayWorkerGroup): + """Initialize agent loop manager. + + Args: + config (DictConfig): trainer config. + worker_group (RayWorkerGroup): ActorRolloutRef worker group. + """ + self.config = config + self.worker_group = worker_group + + self._initialize_llm_servers() + # self._init_agent_loop_workers() + + # init tokenizer + model_path = config.actor_rollout_ref.model.path + self.model_name = "/".join(model_path.split("/")[-2:]) + local_path = copy_to_local(config.actor_rollout_ref.model.path) + self.tokenizer = hf_tokenizer(local_path, trust_remote_code=True) + + # init generator + self.server_manager = AsyncLLMServerManager(config, self.async_llm_servers) + self.skyagent_generator = AutoAgentRunner.from_task( + task_yaml=config.skyrl_agent.task_yaml, infer_engine=self.server_manager, tokenizer=self.tokenizer + ) + + # Initially we're in sleep mode. + self.sleep() + + # initialize here with the custom `async_server_class` implementation + def _initialize_llm_servers(self): + self.rollout_tp_size = self.config.actor_rollout_ref.rollout.tensor_model_parallel_size + self.rollout_dp_size = self.worker_group.world_size // self.rollout_tp_size + + register_center = ray.get_actor(f"{self.worker_group.name_prefix}_register_center") + workers_info = ray.get(register_center.get_worker_info.remote()) + assert len(workers_info) == self.worker_group.world_size + + self.async_llm_servers = [None] * self.rollout_dp_size + self.server_addresses = [None] * self.rollout_dp_size + + if self.config.actor_rollout_ref.rollout.agent.custom_async_server: + server_class = async_server_class( + rollout_backend=self.config.actor_rollout_ref.rollout.name, + rollout_backend_module=self.config.actor_rollout_ref.rollout.agent.custom_async_server.path, + rollout_backend_class=self.config.actor_rollout_ref.rollout.agent.custom_async_server.name, + ) + else: + server_class = async_server_class(rollout_backend=self.config.actor_rollout_ref.rollout.name) + + # Start all server instances, restart if address already in use. + unready_dp_ranks = set(range(self.rollout_dp_size)) + while len(unready_dp_ranks) > 0: + servers = { + rollout_dp_rank: server_class.options( + # make sure AsyncvLLMServer colocates with its corresponding workers + scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=workers_info[rollout_dp_rank * self.rollout_tp_size], + soft=False, + ), + name=f"async_llm_server_{rollout_dp_rank}", + ).remote(self.config, self.rollout_dp_size, rollout_dp_rank, self.worker_group.name_prefix) + for rollout_dp_rank in unready_dp_ranks + } + + for rollout_dp_rank, server in servers.items(): + try: + address = ray.get(server.get_server_address.remote()) + self.server_addresses[rollout_dp_rank] = address + self.async_llm_servers[rollout_dp_rank] = server + unready_dp_ranks.remove(rollout_dp_rank) + except Exception: + ray.kill(server) + print(f"rollout server {rollout_dp_rank} failed, maybe address already in use, restarting...") + + # All server instances are ready, init AsyncLLM engine. + ray.get([server.init_engine.remote() for server in self.async_llm_servers]) + + def _postprocess(self, inputs: Dict[str, list[Any]]) -> DataProto: + # NOTE: consistent with batch version of generate_sequences in vllm_rollout_spmd.py + # prompts: left pad + # responses: right pad + # input_ids: prompt + response + # attention_mask: [0,0,0,0,1,1,1,1, | 1,1,1,0,0,0,0,0] + # position_ids: [0,0,0,0,0,1,2,3, | 4,5,6,7,8,9,10,11] + + # inputs + self.tokenizer.padding_side = "left" + max_prompt_length = max( + max([len(input_ids) for input_ids in inputs["prompt_token_ids"]]), + self.config.actor_rollout_ref.rollout.prompt_length, + ) + outputs = self.tokenizer.pad( + [{"input_ids": input_ids} for input_ids in inputs["prompt_token_ids"]], + padding="max_length", + max_length=max_prompt_length, + return_tensors="pt", + return_attention_mask=True, + ) + prompt_ids, prompt_attention_mask = outputs["input_ids"], outputs["attention_mask"] + + # responses + self.tokenizer.padding_side = "right" + max_response_length = max( + max([len(response) for response in inputs["response_ids"]]), + self.config.actor_rollout_ref.rollout.response_length, + ) + outputs = self.tokenizer.pad( + [{"input_ids": response_ids} for response_ids in inputs["response_ids"]], + padding="max_length", + max_length=max_response_length, + return_tensors="pt", + return_attention_mask=True, + ) + response_ids, response_attention_mask = outputs["input_ids"], outputs["attention_mask"] + + # response_mask + response_length = response_ids.shape[1] + loss_masks = [loss_mask + [0] * (response_length - len(loss_mask)) for loss_mask in inputs["loss_masks"]] + response_mask = torch.tensor(loss_masks, dtype=torch.long) + assert ( + response_ids.shape == response_mask.shape + ), f"mismatch in response_ids and response_mask shape: {response_ids.shape} vs {response_mask.shape}" + response_mask = response_mask * response_attention_mask + + input_ids = torch.cat([prompt_ids, response_ids], dim=1) + attention_mask = torch.cat([prompt_attention_mask, response_attention_mask], dim=1) + position_ids = (attention_mask.cumsum(dim=1) - 1) * attention_mask + + batch = TensorDict( + { + "prompts": prompt_ids, # [bsz, prompt_length] + "responses": response_ids, # [bsz, response_length] + "response_mask": response_mask, # [bsz, response_length] + "input_ids": input_ids, # [bsz, prompt_length + response_length] + "attention_mask": attention_mask, # [bsz, prompt_length + response_length] + "position_ids": position_ids, # [bsz, prompt_length + response_length] + }, + batch_size=len(input_ids), + ) + + return DataProto( + batch=batch, + non_tensor_batch={"rewards": np.array(inputs["rewards"])}, + meta_info={"rollout_metrics": inputs["rollout_metrics"], "timing": {}}, + ) + + def generate_sequences(self, prompts: DataProto) -> DataProto: + """Split input batch and dispatch to agent loop workers. + + Args: + prompts (DataProto): Input batch. + + Returns: + DataProto: Output batch. + """ + if self.config.actor_rollout_ref.rollout.free_cache_engine: + self.wake_up() + skyagent_output = asyncio.run( + self.skyagent_generator.run(prompts, val_mode=prompts.meta_info.get("val_mode", False)) + ) + output = self._postprocess(skyagent_output) + if self.config.actor_rollout_ref.rollout.free_cache_engine: + self.sleep() + + return output + + def wake_up(self): + """Wake up all rollout server instances.""" + ray.get([server.wake_up.remote() for server in self.async_llm_servers]) + + def sleep(self): + """Sleep all rollout server instances.""" + ray.get([server.sleep.remote() for server in self.async_llm_servers]) diff --git a/skyrl-agent/skyrl_agent/integrations/verl/verl_backend.py b/skyrl-agent/skyrl_agent/integrations/verl/verl_backend.py new file mode 100644 index 0000000000..cdc3c8fb24 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/verl/verl_backend.py @@ -0,0 +1,43 @@ +from skyrl_agent.integrations.base import AsyncInferBackend, GeneratorOutput, GeneratorInput +from typing import Any, List, Dict +from loguru import logger + + +class VeRLBackend(AsyncInferBackend): + def __init__(self, infer_engine, tokenizer: Any = None, cfg: Dict[str, Any] = None): + self.infer_engine = infer_engine + + async def async_generate_ids( + self, + input_ids: List[int], + sampling_params: Dict[str, Any], + request_id: str, + **kwargs, + ): + response_str, meta_info = await self.infer_engine.generate( + request_id=request_id, + prompt_ids=input_ids, + sampling_params=sampling_params, + ) + return response_str, meta_info + + async def async_generate_prompts(self, prompts: Any, sampling_params: Any) -> List[str]: + raise NotImplementedError + + +class VeRLGeneratorOutput(GeneratorOutput): + def __init__(self, result: Dict[str, Any]): + self.result = result + + +class VeRLGeneratorInput(GeneratorInput): + def __init__(self, input_batch: Any): + self.input_batch: List[Dict[str, Any]] = [] + non_tensor_batch = input_batch.non_tensor_batch + first_key = next(iter(non_tensor_batch.keys())) + num_entries = len(non_tensor_batch[first_key]) + for i in range(num_entries): + data_item: dict = {key: non_tensor_batch[key][i] for key in non_tensor_batch.keys()} + self.input_batch.append(data_item) + logger.info(f"input batch of size: {len(self.input_batch)}") + logger.info(f"keys: {self.input_batch[0].keys()}") diff --git a/skyrl-agent/skyrl_agent/integrations/verl/verl_main_ppo.py b/skyrl-agent/skyrl_agent/integrations/verl/verl_main_ppo.py new file mode 100644 index 0000000000..6aac70604e --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/verl/verl_main_ppo.py @@ -0,0 +1,350 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. +""" + +import os +import socket + +import hydra +import ray +from omegaconf import OmegaConf + +from verl.trainer.constants_ppo import PPO_RAY_RUNTIME_ENV +from verl.utils.device import is_cuda_available + +from .verl_trainer import SkyAgentPPOTrainer +import torch +from collections import defaultdict + + +from verl.workers.reward_manager import register +from verl.protocol import DataProto +from verl.trainer.main_ppo import create_rl_dataset, create_rl_sampler, config_dir + + +@register("skyagent") +class SkyAgentRewardManager: + """The reward manager.""" + + def __init__(self, tokenizer, num_examine, compute_score=None, reward_fn_key="data_source") -> None: + """ + Initialize the NaiveRewardManager instance. + + Args: + tokenizer: The tokenizer used to decode token IDs into text. + num_examine: The number of batches of decoded responses to print to the console for debugging purpose. + compute_score: A function to compute the reward score. If None, `default_compute_score` will be used. + reward_fn_key: The key used to access the data source in the non-tensor batch data. Defaults to + "data_source". + """ + self.tokenizer = tokenizer # Store the tokenizer for decoding token IDs + self.num_examine = num_examine # the number of batches of decoded responses to print to the console + self.reward_fn_key = reward_fn_key # Store the key for accessing the data source + + def __call__(self, data: DataProto, return_dict=False): + """We will expand this function gradually based on the available datasets""" + + # If there is rm score, we directly return rm score. Otherwise, we compute via rm_score_fn + if "rm_scores" in data.batch.keys(): + if return_dict: + return {"reward_tensor": data.batch["rm_scores"]} + else: + return data.batch["rm_scores"] + + reward_tensor = torch.zeros_like(data.batch["responses"], dtype=torch.float32) + reward_extra_info = defaultdict(list) + + already_print_data_sources = {} + print( + f"In Reward manager: data proto batch keys: {data.batch.keys()}, non tensor batch keys: {data.non_tensor_batch.keys()} meta info keys: {data.meta_info.keys()}" + ) + + for i in range(len(data)): + data_item = data[i] # DataProtoItem + + prompt_ids = data_item.batch["prompts"] + + prompt_length = prompt_ids.shape[-1] + + valid_prompt_length = data_item.batch["attention_mask"][:prompt_length].sum() + valid_prompt_ids = prompt_ids[-valid_prompt_length:] + + response_ids = data_item.batch["responses"] + valid_response_length = data_item.batch["attention_mask"][prompt_length:].sum() + valid_response_ids = response_ids[:valid_response_length] + + # decode + prompt_str = self.tokenizer.decode(valid_prompt_ids, skip_special_tokens=True) + response_str = self.tokenizer.decode(valid_response_ids, skip_special_tokens=True) + + reward_tensor[i, valid_response_length - 1] = float(data_item.non_tensor_batch["rewards"]) + + # data_source = data_item.non_tensor_batch["data_source"] + + if return_dict: + return { + "reward_tensor": reward_tensor, + "reward_extra_info": reward_extra_info, + } + else: + return reward_tensor + + +@hydra.main(config_path=config_dir, config_name="ppo_trainer", version_base=None) +def main(config): + """Main entry point for PPO training with Hydra configuration management. + + Args: + config_dict: Hydra configuration dictionary containing training parameters. + """ + run_ppo(config) + + +# Define a function to run the PPO-like training process +def run_ppo(config) -> None: + """Initialize Ray cluster and run distributed PPO training process. + + Args: + config: Training configuration object containing all necessary parameters + for distributed PPO training including Ray initialization settings, + model paths, and training hyperparameters. + """ + # Check if Ray is not initialized + if not ray.is_initialized(): + # Initialize Ray with a local cluster configuration + # Set environment variables in the runtime environment to control tokenizer parallelism, + # NCCL debug level, VLLM logging level, and allow runtime LoRA updating + # `num_cpus` specifies the number of CPU cores Ray can use, obtained from the configuration + + # use V1 for async engine + PPO_RAY_RUNTIME_ENV["env_vars"].update({"VLLM_USE_V1": "1"}) + if os.environ.get("LD_LIBRARY_PATH", None): + # export LD_LIBRARY_PATH path from driver process, needed for EFA on Anyscale rn + print("Exporting `LD_LIBRARY_PATH` from driver") + PPO_RAY_RUNTIME_ENV["env_vars"].update({"LD_LIBRARY_PATH": os.environ["LD_LIBRARY_PATH"]}) + + ray.init( + runtime_env=PPO_RAY_RUNTIME_ENV, + num_cpus=config.ray_init.num_cpus, + ) + + # Create a remote instance of the TaskRunner class, and + # Execute the `run` method of the TaskRunner instance remotely and wait for it to complete + if ( + is_cuda_available + and config.trainer.get("profile_steps") is not None + and len(config.trainer.get("profile_steps", [])) > 0 + ): + nsight_options = OmegaConf.to_container(config.trainer.controller_nsight_options) + runner = TaskRunner.options(runtime_env={"nsight": nsight_options}).remote() + else: + runner = TaskRunner.remote() + + task = runner.run.remote(config) + try: + ray.get(task) + except KeyboardInterrupt: + print("KeyboardInterrupt received, shutting down...") + ray.cancel(task) + raise + + # [Optional] get the path of the timeline trace file from the configuration, default to None + # This file is used for performance analysis + timeline_json_file = config.ray_init.get("timeline_json_file", None) + if timeline_json_file: + ray.timeline(filename=timeline_json_file) + + +@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head +class TaskRunner: + """Ray remote class for executing distributed PPO training tasks. + + This class encapsulates the main training logic and runs as a Ray remote actor + to enable distributed execution across multiple nodes and GPUs. + """ + + def run(self, config): + """Execute the main PPO training workflow. + + This method sets up the distributed training environment, initializes + workers, datasets, and reward functions, then starts the training process. + + Args: + config: Training configuration object containing all parameters needed + for setting up and running the PPO training process. + """ + # Print the initial configuration. `resolve=True` will evaluate symbolic values. + from pprint import pprint + + from omegaconf import OmegaConf + + from verl.utils.fs import copy_to_local + + print(f"SkyAgent TaskRunner hostname: {socket.gethostname()}, PID: {os.getpid()}") + + pprint(OmegaConf.to_container(config, resolve=True)) + + OmegaConf.resolve(config) + + # skyagent validation + if getattr(config, "skyrl_agent", None) is not None: + task_yaml = getattr(config.skyrl_agent, "task_yaml") + import yaml + + with open(task_yaml, "r") as f: + print("Using Skyrlagent now") + skyagent_config = OmegaConf.create(yaml.safe_load(f)) + assert ( + config.skyrl_agent.num_trajectories == skyagent_config.generator.num_trajectories + ), f"Verl configuration received num trajectories per instance `num_trajectories` of {config.skyrl_agent.num_trajectories} but the provided value in the `skyagent_task_yaml` is {skyagent_config.generator.num_trajectories}. Please ensure that both are consistent" + + # Download the checkpoint from HDFS to the local machine. + # `use_shm` determines whether to use shared memory, which could lead to faster model loading if turned on + print(f"[{socket.gethostname()}] Starting model loading from: {config.actor_rollout_ref.model.path}") + import time + + start_time = time.time() + local_path = copy_to_local( + config.actor_rollout_ref.model.path, + use_shm=config.actor_rollout_ref.model.get("use_shm", False), + verbose=True, + ) + print(f"[{socket.gethostname()}] Model loaded in {time.time() - start_time:.2f} seconds from: {local_path}") + + # Instantiate the tokenizer and processor. + from verl.utils import hf_processor, hf_tokenizer + + trust_remote_code = config.data.get("trust_remote_code", False) + tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) + # Used for multimodal LLM, could be None + processor = hf_processor(local_path, trust_remote_code=trust_remote_code, use_fast=True) + + # Version validation for vllm. + if config.actor_rollout_ref.rollout.name in ["vllm"]: + from verl.utils.vllm_utils import is_version_ge + + if config.actor_rollout_ref.model.get("lora_rank", 0) > 0: + if not is_version_ge(pkg="vllm", minver="0.7.3"): + raise NotImplementedError("PPO LoRA is not supported before vllm 0.7.3") + + # Define worker classes based on the actor strategy. + if config.actor_rollout_ref.actor.strategy in {"fsdp", "fsdp2"}: + assert config.critic.strategy in {"fsdp", "fsdp2"} + from verl.single_controller.ray import RayWorkerGroup + from verl.workers.fsdp_workers import ActorRolloutRefWorker, AsyncActorRolloutRefWorker, CriticWorker + + actor_rollout_cls = ( + AsyncActorRolloutRefWorker + if config.actor_rollout_ref.rollout.mode == "async" + else ActorRolloutRefWorker + ) + ray_worker_group_cls = RayWorkerGroup + + elif config.actor_rollout_ref.actor.strategy == "megatron": + assert config.actor_rollout_ref.actor.strategy == config.critic.strategy + from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup + from verl.workers.megatron_workers import ActorRolloutRefWorker, AsyncActorRolloutRefWorker, CriticWorker + + actor_rollout_cls = ( + AsyncActorRolloutRefWorker + if config.actor_rollout_ref.rollout.mode == "async" + else ActorRolloutRefWorker + ) + ray_worker_group_cls = NVMegatronRayWorkerGroup + + else: + raise NotImplementedError + + from .verl_trainer import ResourcePoolManager, Role + + # Map roles to their corresponding remote worker classes. + role_worker_mapping = { + Role.ActorRollout: ray.remote(actor_rollout_cls), + Role.Critic: ray.remote(CriticWorker), + } + + # Define the resource pool specification. + # Map roles to the resource pool. + global_pool_id = "global_pool" + resource_pool_spec = { + global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, + } + mapping = { + Role.ActorRollout: global_pool_id, + Role.Critic: global_pool_id, + } + + # We should adopt a multi-source reward function here: + # - for rule-based rm, we directly call a reward score + # - for model-based rm, we call a model + # - for code related prompt, we send to a sandbox if there are test cases + # finally, we combine all the rewards together + # The reward type depends on the tag of the data + if config.reward_model.enable: + if config.reward_model.strategy in {"fsdp", "fsdp2"}: + from verl.workers.fsdp_workers import RewardModelWorker + elif config.reward_model.strategy == "megatron": + from verl.workers.megatron_workers import RewardModelWorker + else: + raise NotImplementedError + role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) + mapping[Role.RewardModel] = global_pool_id + + # Add a reference policy worker if KL loss or KL reward is used. + if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: + role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) + mapping[Role.RefPolicy] = global_pool_id + + # Load the reward manager for training and validation. + reward_fn = SkyAgentRewardManager(tokenizer, num_examine=0, **config.reward_model.get("reward_kwargs", {})) + val_reward_fn = SkyAgentRewardManager(tokenizer, num_examine=1, **config.reward_model.get("reward_kwargs", {})) + resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) + + from verl.utils.dataset.rl_dataset import collate_fn + + # Create training and validation datasets. + train_dataset = create_rl_dataset(config.data.train_files, config.data, tokenizer, processor, is_train=True) + val_dataset = create_rl_dataset(config.data.val_files, config.data, tokenizer, processor, is_train=False) + # hack for using only swe bench hard + # Dacheng: Delete this hack now, we need to train for other tasks than swe + # mask = val_dataset.dataframe.apply(lambda x: x['instance']["difficulty"] == "1-4 hours", axis=1) + # val_dataset.dataframe = val_dataset.dataframe[mask].reset_index(drop=True) + train_sampler = create_rl_sampler(config.data, train_dataset) + + # Initialize the PPO trainer. + trainer = SkyAgentPPOTrainer( + config=config, + tokenizer=tokenizer, + processor=processor, + role_worker_mapping=role_worker_mapping, + resource_pool_manager=resource_pool_manager, + ray_worker_group_cls=ray_worker_group_cls, + reward_fn=reward_fn, + val_reward_fn=val_reward_fn, + train_dataset=train_dataset, + val_dataset=val_dataset, + collate_fn=collate_fn, + train_sampler=train_sampler, + device_name=config.trainer.device, + ) + # Initialize the workers of the trainer. + trainer.init_workers() + # Start the training process. + trainer.fit() + + +if __name__ == "__main__": + main() diff --git a/skyrl-agent/skyrl_agent/integrations/verl/verl_trainer.py b/skyrl-agent/skyrl_agent/integrations/verl/verl_trainer.py new file mode 100644 index 0000000000..d696d12b20 --- /dev/null +++ b/skyrl-agent/skyrl_agent/integrations/verl/verl_trainer.py @@ -0,0 +1,753 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +FSDP PPO Trainer with Ray-based single controller. +This trainer supports model-agonistic model initialization with huggingface +""" + +import os +import uuid +from collections import defaultdict +from copy import deepcopy +from pprint import pprint +import pickle + +import numpy as np +import ray +import torch +from omegaconf import OmegaConf +from tqdm import tqdm + +from verl import DataProto +from verl.experimental.dataset.sampler import AbstractCurriculumSampler +from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto +from verl.single_controller.base import Worker +from verl.single_controller.ray import RayClassWithInitArgs +from verl.single_controller.ray.base import create_colocated_worker_cls +from verl.trainer.ppo.core_algos import AdvantageEstimator, agg_loss +from verl.trainer.ppo.metric_utils import ( + compute_data_metrics, + compute_throughout_metrics, + compute_timing_metrics, + process_validation_metrics, +) +from verl.trainer.ppo.reward import compute_reward, compute_reward_async +from verl.utils.checkpoint.checkpoint_manager import should_save_ckpt_esi +from verl.utils.debug import marked_timer +from verl.utils.metric import ( + reduce_metrics, +) +from verl.trainer.ppo.ray_trainer import Role, compute_advantage, RayPPOTrainer + +from .upload_utils import upload_to_remote_background + + +WorkerType = type[Worker] + + +class SkyAgentPPOTrainer(RayPPOTrainer): + """Distributed PPO trainer using Ray for scalable reinforcement learning. + + This trainer orchestrates distributed PPO training across multiple nodes and GPUs, + managing actor rollouts, critic training, and reward computation with Ray backend. + Supports various model architectures including FSDP, Megatron, and vLLM integration. + """ + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._upload_refs = None + + def init_workers(self): + """Initialize distributed training workers using Ray backend. + + Creates: + 1. Ray resource pools from configuration + 2. Worker groups for each role (actor, critic, etc.) + """ + self.resource_pool_manager.create_resource_pool() + + self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} + + # create actor and rollout + if self.hybrid_engine: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.ActorRollout) + actor_rollout_cls = RayClassWithInitArgs( + cls=self.role_worker_mapping[Role.ActorRollout], + config=self.config.actor_rollout_ref, + role="actor_rollout", + profile_option=self.config.trainer.npu_profile.options, + ) + self.resource_pool_to_cls[resource_pool]["actor_rollout"] = actor_rollout_cls + else: + raise NotImplementedError + + # create critic + if self.use_critic: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.Critic) + critic_cls = RayClassWithInitArgs(cls=self.role_worker_mapping[Role.Critic], config=self.config.critic) + self.resource_pool_to_cls[resource_pool]["critic"] = critic_cls + + # create reference policy if needed + if self.use_reference_policy: + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) + ref_policy_cls = RayClassWithInitArgs( + self.role_worker_mapping[Role.RefPolicy], + config=self.config.actor_rollout_ref, + role="ref", + profile_option=self.config.trainer.npu_profile.options, + ) + self.resource_pool_to_cls[resource_pool]["ref"] = ref_policy_cls + + # create a reward model if reward_fn is None + if self.use_rm: + # we create a RM here + resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) + rm_cls = RayClassWithInitArgs(self.role_worker_mapping[Role.RewardModel], config=self.config.reward_model) + self.resource_pool_to_cls[resource_pool]["rm"] = rm_cls + + # initialize WorkerGroup + # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, + # you should not use `create_colocated_worker_cls`. + # Instead, directly pass different resource pool to different worker groups. + # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. + all_wg = {} + wg_kwargs = {} # Setting up kwargs for RayWorkerGroup + if OmegaConf.select(self.config.trainer, "ray_wait_register_center_timeout") is not None: + wg_kwargs["ray_wait_register_center_timeout"] = self.config.trainer.ray_wait_register_center_timeout + if OmegaConf.select(self.config.trainer, "profile_steps") is not None: + wg_kwargs["profile_steps"] = OmegaConf.select(self.config.trainer, "profile_steps") + assert ( + OmegaConf.select(self.config.trainer, "worker_nsight_options") is not None + ), "worker_nsight_options must be set when profile_steps is set" + wg_kwargs["worker_nsight_options"] = OmegaConf.to_container( + OmegaConf.select(self.config.trainer, "worker_nsight_options") + ) + wg_kwargs["device_name"] = self.device_name + + for resource_pool, class_dict in self.resource_pool_to_cls.items(): + worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) + wg_dict = self.ray_worker_group_cls( + resource_pool=resource_pool, + ray_cls_with_init=worker_dict_cls, + **wg_kwargs, + ) + spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) + all_wg.update(spawn_wg) + + if self.use_critic: + self.critic_wg = all_wg["critic"] + self.critic_wg.init_model() + + if self.use_reference_policy and not self.ref_in_actor: + self.ref_policy_wg = all_wg["ref"] + self.ref_policy_wg.init_model() + + if self.use_rm: + self.rm_wg = all_wg["rm"] + self.rm_wg.init_model() + + # we should create rollout at the end so that vllm can have a better estimation of kv cache memory + self.actor_rollout_wg = all_wg["actor_rollout"] + self.actor_rollout_wg.init_model() + + # create async rollout manager and request scheduler + self.async_rollout_mode = False + if self.config.actor_rollout_ref.rollout.mode == "async": + from .verl_async_manager import SkyAgentLoopManager + + self.async_rollout_mode = True + self.async_rollout_manager = SkyAgentLoopManager( + config=self.config, + worker_group=self.actor_rollout_wg, + ) + + def _validate(self): + data_source_lst = [] + reward_extra_infos_dict: dict[str, list] = defaultdict(list) + + # Lists to collect samples for the table + sample_inputs = [] + sample_outputs = [] + sample_scores = [] + sample_turns = [] + + val_num_trajectories = ( + self.async_rollout_manager.skyagent_generator.cfg.generator.val_config.num_trajectories + if self.async_rollout_mode + else 1 + ) + + for test_data in self.val_dataloader: + test_batch = DataProto.from_single_dict(test_data) + + # repeat test batch + # test_batch = test_batch.repeat( + # repeat_times=self.config.actor_rollout_ref.rollout.val_kwargs.n, interleave=True + # ) + + # we only do validation on rule-based rm + if self.config.reward_model.enable and test_batch[0].non_tensor_batch["reward_model"]["style"] == "model": + return {} + + # Store original inputs + input_ids = test_batch.batch["input_ids"] + # TODO: Can we keep special tokens except for padding tokens? + input_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in input_ids] + + input_texts = sum([[input_text] * val_num_trajectories for input_text in input_texts], []) + sample_inputs.extend(input_texts) + + batch_keys_to_pop = ["input_ids", "attention_mask", "position_ids"] + # non_tensor_batch_keys_to_pop = ["raw_prompt_ids", "instance", "data_source"] + # Dacheng: a hack now to pop 'instance' if it exists (for swe); TODO: make this a config option + # TODO: A unified pr on data format + non_tensor_batch_keys_to_pop = ["raw_prompt_ids", "data_source"] + if "instance" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("instance") + else: + non_tensor_batch_keys_to_pop.append("reward_model") + non_tensor_batch_keys_to_pop.append("extra_info") + if "multi_modal_data" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("multi_modal_data") + if "raw_prompt" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("raw_prompt") + if "tools_kwargs" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("tools_kwargs") + if "interaction_kwargs" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("interaction_kwargs") + if "agent_name" in test_batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("agent_name") + test_gen_batch = test_batch.pop( + batch_keys=batch_keys_to_pop, + non_tensor_batch_keys=non_tensor_batch_keys_to_pop, + ) + + test_gen_batch.meta_info = { + "eos_token_id": self.tokenizer.eos_token_id, + "pad_token_id": self.tokenizer.pad_token_id, + "recompute_log_prob": False, + "do_sample": self.config.actor_rollout_ref.rollout.val_kwargs.do_sample, + "validate": True, + } + + # pad to be divisible by dp_size + size_divisor = ( + self.actor_rollout_wg.world_size + if not self.async_rollout_mode + else self.config.actor_rollout_ref.rollout.agent.num_workers + ) + test_gen_batch_padded, pad_size = pad_dataproto_to_divisor(test_gen_batch, size_divisor) # [B+ P, S] + # pass val mode for skyrl-agent + test_gen_batch_padded.meta_info["val_mode"] = True + + if not self.async_rollout_mode: + test_output_gen_batch_padded = self.actor_rollout_wg.generate_sequences(test_gen_batch_padded) + else: + test_output_gen_batch_padded = self.async_rollout_manager.generate_sequences( + test_gen_batch_padded + ) # [(B+P)*NT, S] + + # unpad + # SkyRL-Agent will repeat trajectories by `val_num_trajectories` internally + test_output_gen_batch = unpad_dataproto( + test_output_gen_batch_padded, pad_size=pad_size * val_num_trajectories + ) # [B*NT, S] + num_val_samples = test_output_gen_batch.batch["responses"].shape[0] + + print("validation generation end") + + # Store generated outputs + output_ids = test_output_gen_batch.batch["responses"] + output_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in output_ids] + sample_outputs.extend(output_texts) + + # repeat + test_batch = test_batch.repeat(repeat_times=val_num_trajectories, interleave=True) # [B*NT, S] + # union + test_batch = test_batch.union(test_output_gen_batch) # [B*NT, S] + + test_batch.meta_info["validate"] = True + + # evaluate using reward_function + result = self.val_reward_fn(test_batch, return_dict=True) # reward_tensor -> [B*NT,S] + reward_tensor = result["reward_tensor"] # [B*NT, S] + scores = reward_tensor.sum(-1).cpu().tolist() + sample_scores.extend(scores) + + reward_extra_infos_dict["reward"].extend(scores) + if "reward_extra_info" in result: + for key, lst in result["reward_extra_info"].items(): + reward_extra_infos_dict[key].extend(lst) + print(f"len reward_extra_infos_dict['{key}']: {len(reward_extra_infos_dict[key])}") + + # collect num_turns of each prompt + if "__num_turns__" in test_batch.non_tensor_batch: + sample_turns.append(test_batch.non_tensor_batch["__num_turns__"]) + + if "data_source" in test_gen_batch.non_tensor_batch: + data_source_lst.append(test_gen_batch.non_tensor_batch["data_source"].copy()) + + self._maybe_log_val_generations(inputs=sample_inputs, outputs=sample_outputs, scores=sample_scores) + + # dump generations + val_data_dir = self.config.trainer.get("validation_data_dir", None) + if val_data_dir: + self._dump_generations( + inputs=sample_inputs, + outputs=sample_outputs, + scores=sample_scores, + reward_extra_infos_dict=reward_extra_infos_dict, + dump_path=val_data_dir, + ) + + for key_info, lst in reward_extra_infos_dict.items(): + assert len(lst) == 0 or len(lst) == len(sample_scores), f"{key_info}: {len(lst)=}, {len(sample_scores)=}" + + data_sources = np.concatenate(data_source_lst, axis=0) if data_source_lst else np.array([], dtype=object) + if data_sources.size == 0: + return {} + + data_src2var2metric2val = process_validation_metrics(data_sources, sample_inputs, reward_extra_infos_dict) + metric_dict = {} + for data_source, var2metric2val in data_src2var2metric2val.items(): + core_var = "acc" if "acc" in var2metric2val else "reward" + for var_name, metric2val in var2metric2val.items(): + n_max = max([int(name.split("@")[-1].split("/")[0]) for name in metric2val.keys()]) + for metric_name, metric_val in metric2val.items(): + if ( + (var_name == core_var) + and any(metric_name.startswith(pfx) for pfx in ["mean", "maj", "best"]) + and (f"@{n_max}" in metric_name) + ): + metric_sec = "val-core" + else: + metric_sec = "val-aux" + pfx = f"{metric_sec}/{data_source}/{var_name}/{metric_name}" + metric_dict[pfx] = metric_val + + if len(sample_turns) > 0: + sample_turns = np.concatenate(sample_turns) + metric_dict["val-aux/num_turns/min"] = sample_turns.min() + metric_dict["val-aux/num_turns/max"] = sample_turns.max() + metric_dict["val-aux/num_turns/mean"] = sample_turns.mean() + + return metric_dict + + def _save_checkpoint(self): + if self._upload_refs is not None: + ray.get(self._upload_refs) + + super()._save_checkpoint() + + # upload to anyscale (guard missing config key) + remote_upload = False + try: + remote_upload = bool(OmegaConf.select(self.config.trainer, "remote_anyscale_upload")) + except Exception: + remote_upload = False + + if remote_upload: + print(f"Starting background upload for ckpt after step {self.global_steps}") + local_global_step_folder = os.path.join( + self.config.trainer.default_local_dir, f"global_step_{self.global_steps}" + ) + local_latest_checkpointed_iteration = os.path.join( + self.config.trainer.default_local_dir, "latest_checkpointed_iteration.txt" + ) + self._upload_refs = upload_to_remote_background( + self.config, + global_step=self.global_steps, + local_global_step_folder=local_global_step_folder, + main_rank_latest_checkpointed_iteration=local_latest_checkpointed_iteration, + ) + + def fit(self): + """ + The training loop of PPO. + The driver process only need to call the compute functions of the worker group through RPC + to construct the PPO dataflow. + The light-weight advantage computation is done on the driver process. + """ + from omegaconf import OmegaConf + + from verl.utils.tracking import Tracking + + logger = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + config=OmegaConf.to_container(self.config, resolve=True), + ) + + self.global_steps = 0 + + # load checkpoint before doing anything + self._load_checkpoint() + + # perform validation before training + # currently, we only support validation using the reward_function. + if self.val_reward_fn is not None and self.config.trainer.get("val_before_train", True): + val_metrics = self._validate() + assert val_metrics, f"{val_metrics=}" + pprint(f"Initial validation metrics: {val_metrics}") + logger.log(data=val_metrics, step=self.global_steps) + if self.config.trainer.get("val_only", False): + return + + # add tqdm + progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") + + # we start from step 1 + self.global_steps += 1 + last_val_metrics = None + self.max_steps_duration = 0 + + for epoch in range(self.config.trainer.total_epochs): + for batch_dict in self.train_dataloader: + metrics = {} + timing_raw = {} + + do_profile = ( + self.global_steps in self.config.trainer.profile_steps + if self.config.trainer.profile_steps is not None + else False + ) + with marked_timer("start_profile", timing_raw): + if do_profile: + self.actor_rollout_wg.start_profile(role="e2e", profile_step=self.global_steps) + if self.use_reference_policy: + self.ref_policy_wg.start_profile() + if self.use_critic: + self.critic_wg.start_profile() + if self.use_rm: + self.rm_wg.start_profile() + + batch: DataProto = DataProto.from_single_dict(batch_dict) + if self.global_steps == 1: + prompts = self.tokenizer.batch_decode(batch.batch["input_ids"], skip_special_tokens=True) + print("prompts: ", prompts[:2]) + + # pop those keys for generation + batch_keys_to_pop = ["input_ids", "attention_mask", "position_ids"] + # Dacheng: a hack now to pop 'instance' if it exists (for swe); TODO: make this a config option + # TODO: A unified pr on data format + non_tensor_batch_keys_to_pop = ["raw_prompt_ids", "data_source"] + if "instance" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("instance") + else: + non_tensor_batch_keys_to_pop.append("reward_model") + non_tensor_batch_keys_to_pop.append("extra_info") + if "multi_modal_data" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("multi_modal_data") + if "raw_prompt" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("raw_prompt") + if "tools_kwargs" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("tools_kwargs") + if "interaction_kwargs" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("interaction_kwargs") + if "index" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("index") + if "agent_name" in batch.non_tensor_batch: + non_tensor_batch_keys_to_pop.append("agent_name") + + gen_batch = batch.pop( + batch_keys=batch_keys_to_pop, + non_tensor_batch_keys=non_tensor_batch_keys_to_pop, + ) + + # pass global_steps to trace + gen_batch.meta_info["global_steps"] = self.global_steps + gen_batch = gen_batch.repeat(repeat_times=1, interleave=True) + + is_last_step = self.global_steps >= self.total_training_steps + + with marked_timer("step", timing_raw): + # generate a batch + with marked_timer("gen", timing_raw, color="red"): + if not self.async_rollout_mode: + gen_batch_output = self.actor_rollout_wg.generate_sequences(gen_batch) + else: + gen_batch_output = self.async_rollout_manager.generate_sequences(gen_batch) + timing_raw.update(gen_batch_output.meta_info["timing"]) + gen_batch_output.meta_info.pop("timing", None) + if "rollout_metrics" in gen_batch_output.meta_info.keys(): + rollout_metrics = reduce_metrics(gen_batch_output.meta_info["rollout_metrics"]) + metrics.update(rollout_metrics) + print("rollout metrics: ", rollout_metrics) + + if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: + with marked_timer("gen_max", timing_raw, color="purple"): + gen_baseline_batch = deepcopy(gen_batch) + gen_baseline_batch.meta_info["do_sample"] = False + gen_baseline_output = self.actor_rollout_wg.generate_sequences(gen_baseline_batch) + + batch = batch.union(gen_baseline_output) + reward_baseline_tensor = self.reward_fn(batch) + reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) + + batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) + + batch.batch["reward_baselines"] = reward_baseline_tensor + + del gen_baseline_batch, gen_baseline_output + + batch.non_tensor_batch["uid"] = np.array( + [str(uuid.uuid4()) for _ in range(len(batch.batch))], dtype=object + ) + # repeat to align with repeated responses in rollout + batch = batch.repeat(repeat_times=self.config.skyrl_agent.num_trajectories, interleave=True) + batch = batch.union(gen_batch_output) + + rollout_data_dir = self.config.trainer.get("rollout_data_dir", None) + if rollout_data_dir: + # Also dump the full training batch for debugging + with open(os.path.join(rollout_data_dir, "current_batch_after_gen.pkl"), "wb") as f: + pickle.dump(batch, f) + + print( + "Batch size: ", + len(batch.batch), + " Num trajectories: ", + self.config.skyrl_agent.num_trajectories, + ) + + if "response_mask" not in batch.batch.keys(): + batch.batch["response_mask"] = compute_response_mask(batch) + # Balance the number of valid tokens across DP ranks. + # NOTE: This usually changes the order of data in the `batch`, + # which won't affect the advantage calculation (since it's based on uid), + # but might affect the loss calculation (due to the change of mini-batching). + # TODO: Decouple the DP balancing and mini-batching. + if self.config.trainer.balance_batch: + self._balance_batch(batch, metrics=metrics) + + # compute global_valid tokens + batch.meta_info["global_token_num"] = torch.sum(batch.batch["attention_mask"], dim=-1).tolist() + + with marked_timer("reward", timing_raw, color="yellow"): + # compute reward model score + if self.use_rm: + reward_tensor = self.rm_wg.compute_rm_score(batch) + batch = batch.union(reward_tensor) + + if self.config.reward_model.launch_reward_fn_async: + future_reward = compute_reward_async.remote(batch, self.config, self.tokenizer) + else: + reward_tensor, reward_extra_infos_dict = compute_reward(batch, self.reward_fn) + + # recompute old_log_probs + with marked_timer("old_log_prob", timing_raw, color="blue"): + old_log_prob = self.actor_rollout_wg.compute_log_prob(batch) + entropys = old_log_prob.batch["entropys"] + response_masks = batch.batch["response_mask"] + loss_agg_mode = self.config.actor_rollout_ref.actor.loss_agg_mode + entropy_agg = agg_loss(loss_mat=entropys, loss_mask=response_masks, loss_agg_mode=loss_agg_mode) + old_log_prob_metrics = {"actor/entropy": entropy_agg.detach().item()} + metrics.update(old_log_prob_metrics) + old_log_prob.batch.pop("entropys") + batch = batch.union(old_log_prob) + + if "rollout_log_probs" in batch.batch.keys(): + # TODO: we may want to add diff of probs too. + rollout_old_log_probs = batch.batch["rollout_log_probs"] + actor_old_log_probs = batch.batch["old_log_probs"] + attention_mask = batch.batch["attention_mask"] + responses = batch.batch["responses"] + response_length = responses.size(1) + response_mask = attention_mask[:, -response_length:] + + rollout_probs = torch.exp(rollout_old_log_probs) + actor_probs = torch.exp(actor_old_log_probs) + rollout_probs_diff = torch.abs(rollout_probs - actor_probs) + rollout_probs_diff = torch.masked_select(rollout_probs_diff, response_mask.bool()) + rollout_probs_diff_max = torch.max(rollout_probs_diff) + rollout_probs_diff_mean = torch.mean(rollout_probs_diff) + rollout_probs_diff_std = torch.std(rollout_probs_diff) + metrics.update( + { + "training/rollout_probs_diff_max": rollout_probs_diff_max.detach().item(), + "training/rollout_probs_diff_mean": rollout_probs_diff_mean.detach().item(), + "training/rollout_probs_diff_std": rollout_probs_diff_std.detach().item(), + } + ) + + if self.use_reference_policy: + # compute reference log_prob + with marked_timer("ref", timing_raw, color="olive"): + if not self.ref_in_actor: + ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) + else: + ref_log_prob = self.actor_rollout_wg.compute_ref_log_prob(batch) + batch = batch.union(ref_log_prob) + + # compute values + if self.use_critic: + with marked_timer("values", timing_raw, color="cyan"): + values = self.critic_wg.compute_values(batch) + batch = batch.union(values) + + with marked_timer("adv", timing_raw, color="brown"): + # we combine with rule-based rm + reward_extra_infos_dict: dict[str, list] + if self.config.reward_model.launch_reward_fn_async: + reward_tensor, reward_extra_infos_dict = ray.get(future_reward) + batch.batch["token_level_scores"] = reward_tensor + + if reward_extra_infos_dict: + batch.non_tensor_batch.update({k: np.array(v) for k, v in reward_extra_infos_dict.items()}) + + # compute rewards. apply_kl_penalty if available + if self.config.algorithm.use_kl_in_reward: + batch, kl_metrics = apply_kl_penalty( + batch, kl_ctrl=self.kl_ctrl_in_reward, kl_penalty=self.config.algorithm.kl_penalty + ) + metrics.update(kl_metrics) + else: + batch.batch["token_level_rewards"] = batch.batch["token_level_scores"] + + # compute advantages, executed on the driver process + + norm_adv_by_std_in_grpo = self.config.algorithm.get( + "norm_adv_by_std_in_grpo", True + ) # GRPO adv normalization factor + + batch = compute_advantage( + batch, + adv_estimator=self.config.algorithm.adv_estimator, + gamma=self.config.algorithm.gamma, + lam=self.config.algorithm.lam, + num_repeat=1, + norm_adv_by_std_in_grpo=norm_adv_by_std_in_grpo, + config=self.config.algorithm, + ) + + # update critic + if self.use_critic: + with marked_timer("update_critic", timing_raw, color="pink"): + critic_output = self.critic_wg.update_critic(batch) + critic_output_metrics = reduce_metrics(critic_output.meta_info["metrics"]) + metrics.update(critic_output_metrics) + + # implement critic warmup + if self.config.trainer.critic_warmup <= self.global_steps: + # update actor + with marked_timer("update_actor", timing_raw, color="red"): + batch.meta_info["multi_turn"] = self.config.actor_rollout_ref.rollout.multi_turn.enable + actor_output = self.actor_rollout_wg.update_actor(batch) + actor_output_metrics = reduce_metrics(actor_output.meta_info["metrics"]) + metrics.update(actor_output_metrics) + + # Log rollout generations if enabled + rollout_data_dir = self.config.trainer.get("rollout_data_dir", None) + if rollout_data_dir: + with marked_timer("dump_rollout_generations", timing_raw, color="green"): + print(batch.batch.keys()) + inputs = self.tokenizer.batch_decode(batch.batch["prompts"], skip_special_tokens=True) + outputs = self.tokenizer.batch_decode(batch.batch["responses"], skip_special_tokens=True) + scores = batch.batch["token_level_scores"].sum(-1).cpu().tolist() + self._dump_generations( + inputs=inputs, + outputs=outputs, + scores=scores, + reward_extra_infos_dict=reward_extra_infos_dict, + dump_path=rollout_data_dir, + ) + + # validate + if ( + self.val_reward_fn is not None + and self.config.trainer.test_freq > 0 + and (is_last_step or self.global_steps % self.config.trainer.test_freq == 0) + ): + with marked_timer("testing", timing_raw, color="green"): + val_metrics: dict = self._validate() + if is_last_step: + last_val_metrics = val_metrics + metrics.update(val_metrics) + + # Check if the ESI (Elastic Server Instance)/training plan is close to expiration. + esi_close_to_expiration = should_save_ckpt_esi( + max_steps_duration=self.max_steps_duration, + redundant_time=self.config.trainer.esi_redundant_time, + ) + # Check if the conditions for saving a checkpoint are met. + # The conditions include a mandatory condition (1) and + # one of the following optional conditions (2/3/4): + # 1. The save frequency is set to a positive value. + # 2. It's the last training step. + # 3. The current step number is a multiple of the save frequency. + # 4. The ESI(Elastic Server Instance)/training plan is close to expiration. + if self.config.trainer.save_freq > 0 and ( + is_last_step + or self.global_steps % self.config.trainer.save_freq == 0 + or esi_close_to_expiration + ): + if esi_close_to_expiration: + print("Force saving checkpoint: ESI instance expiration approaching.") + with marked_timer("save_checkpoint", timing_raw, color="green"): + self._save_checkpoint() + + with marked_timer("stop_profile", timing_raw): + if do_profile: + self.actor_rollout_wg.stop_profile() + if self.use_reference_policy: + self.ref_policy_wg.stop_profile() + if self.use_critic: + self.critic_wg.stop_profile() + if self.use_rm: + self.rm_wg.stop_profile() + + steps_duration = timing_raw["step"] + self.max_steps_duration = max(self.max_steps_duration, steps_duration) + + # training metrics + metrics.update( + { + "training/global_step": self.global_steps, + "training/epoch": epoch, + } + ) + rollout_data_dir = self.config.trainer.get("rollout_data_dir", None) + if rollout_data_dir: + with open(os.path.join(rollout_data_dir, "current_batch_after_step.pkl"), "wb") as f: + pickle.dump(batch, f) + # collect metrics + metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) + metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) + # TODO: implement actual tflpo and theoretical tflpo + n_gpus = self.resource_pool_manager.get_n_gpus() + metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) + + # this is experimental and may be changed/removed in the future in favor of a general-purpose one + if isinstance(self.train_dataloader.sampler, AbstractCurriculumSampler): + self.train_dataloader.sampler.update(batch=batch) + + # TODO: make a canonical logger that supports various backend + logger.log(data=metrics, step=self.global_steps, commit=True) + + progress_bar.update(1) + self.global_steps += 1 + + if is_last_step: + pprint(f"Final validation metrics: {last_val_metrics}") + progress_bar.close() + return + + # this is experimental and may be changed/removed in the future + # in favor of a general-purpose data buffer pool + if hasattr(self.train_dataset, "on_batch_end"): + # The dataset may be changed after each training batch + self.train_dataset.on_batch_end(batch=batch) diff --git a/skyrl-agent/skyrl_agent/tasks/__init__.py b/skyrl-agent/skyrl_agent/tasks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-agent/skyrl_agent/tasks/base.py b/skyrl-agent/skyrl_agent/tasks/base.py new file mode 100644 index 0000000000..e50f4169bf --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/base.py @@ -0,0 +1,33 @@ +from abc import abstractmethod, ABC +from typing import Dict, Any, List + + +# TODO: See if it makes sense to have an async version +class BaseTask(ABC): + + @classmethod + @abstractmethod + async def initialize_runtime(cls, *args, **kwargs) -> Any: + """Initialize the runtime for the task in an asyncio-compatible way""" + pass + + @classmethod + @abstractmethod + def get_instruction(cls, *args, **kwargs) -> List[Dict[str, str]]: + """Get the initial instruction for the agent in the OpenAI messages format""" + pass + + @classmethod + @abstractmethod + def complete_runtime(cls, *args, **kwargs) -> Dict[str, Any]: + """Complete or finalize the runtime for the task + + For example, this can involve extracting the git patch from the runtime for SWEBench. + """ + pass + + @classmethod + @abstractmethod + async def evaluate_result(cls, *args, **kwargs) -> bool: + """Evaluate model result for the task in an asyncio-compatible way""" + pass diff --git a/skyrl-agent/skyrl_agent/tasks/general_react/__init__.py b/skyrl-agent/skyrl_agent/tasks/general_react/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-agent/skyrl_agent/tasks/general_react/utils.py b/skyrl-agent/skyrl_agent/tasks/general_react/utils.py new file mode 100644 index 0000000000..4fa4e7a9f5 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/general_react/utils.py @@ -0,0 +1,193 @@ +from typing import Dict, Any +import asyncio + +from skyrl_agent.tasks.base import BaseTask +from skyrl_agent.dispatcher.async_utils import call_sync_from_async + + +class GeneralReactTask(BaseTask): + @classmethod + async def initialize_runtime(cls): + pass + + @classmethod + def get_instruction(cls, instance: Dict[str, Any]) -> str: + print(instance) + # (TODO) Dacheng: A hack to make inference only compatible. + # During inference, the key is "prompt" + # During training, the key is "raw_prompt" + if "raw_prompt" in instance: + import numpy + + assert isinstance( + instance.get("raw_prompt"), numpy.ndarray + ), f"Raw prompt must be a numpy array, but got {type(instance.get('raw_prompt'))}" + assert ( + len(instance.get("raw_prompt")) == 1 + ), f"Raw prompt must have only one element, but got {len(instance.get('raw_prompt'))}" + prompt = list(instance.get("raw_prompt")) + else: + prompt = instance.get("prompt") + + print(f"Prompt: {prompt}") + + # assume prompt is a list of messages + assert isinstance(prompt, list), f"Prompt must be a list, but got {type(prompt)}" + + # Check if there's already a system message + has_system = any(msg.get("role") == "system" for msg in prompt) + data_source = instance.get("data_source", "") + + if not has_system: + # Prepend a dummy system prompt so that the agent can use tools by the convert_fncall_messages_to_non_fncall_messages function. + # From Retool prompt: https://arxiv.org/pdf/2504.11536 + if data_source.startswith("math"): + system_prompt = { + "role": "system", + "content": "Please solve the problem with the following tools and return the final answer inside the finish tool. \ + If there are additional requirments such as the answer should be included inside \\boxed{}, please return the answer in the format of \ + \ + \\boxed{'The final answer goes here.'} \ + ", + } + elif data_source.startswith("codegen"): + system_prompt = { + "role": "system", + "content": "Please solve the problem with the following tools and return the final answer inside the finish tool. \ + Given a coding task, you can use the code interpreter tool to run code to test your solution. \ + Please return the final code solution in the format of \ + \ + ```python\n# The final answer goes here.\n``` \ + ", + } + elif data_source in ["2wikimultihopqa", "bamboogle", "hotpotqa", "musique", "nq", "popqa", "triviaqa"]: + system_prompt = { + "role": "system", + "content": "Please solve the problem with the following tools AS MUCH AS POSSIBLE and return the final answer inside the finish tool. \ + Please return the final answer in the format of \ + \ + The final answer goes here. \ + ", + } + elif data_source.startswith("browsecomp"): + system_prompt = { + "role": "system", + "content": "Please solve the problem with the following tools and return the final answer inside the finish tool. You need to follow these steps: \ + 1. Analyze the problem carefully and break it down into smaller parts. \ + 2. Use the available tools to gather information and answer sub-questions as needed. \ + 3. Synthesize the information you have gathered to arrive at the final answer. For each conclusion you reach, make sure to provide evidence from the gathered information. \ + 4. Return the final answer in the format of \ + \ + The final answer goes here. \ + ", + } + elif data_source.startswith("ruler"): + system_prompt = { + "role": "system", + "content": "When you find the answer (or finish reading all chunks), provide your final answer using:\ + \n\nThe final answer goes here.\n", + } + else: + assert False, f"Data source {data_source} is not supported for ReAct agent." + prompt = [system_prompt] + prompt + + print(f"Prompt after system prompt: {prompt}") + return prompt + + @classmethod + def complete_runtime(cls): + pass + + @classmethod + async def evaluate_result( + cls, result: any, instance: any, data_source: str, instance_id: int, trajectory_id: int + ) -> float: + # print(f"Evaluating result: {result=} {instance=} {data_source=} {instance_id=} {trajectory_id=}") + ground_truth = instance["reward_model"]["ground_truth"] + extra_info = instance["extra_info"] + print(f"Evaluating result: {result=}") + if not result: + return 0.0 + if data_source == "ToRL": + from skyrl_agent.tasks.verifiers import torl + + return torl.compute_score(result, ground_truth) + elif data_source.startswith("math"): + from skyrl_agent.tasks.verifiers import naive_dapo + + print(f"Evaluating math task with data_source: {data_source}, got {result=} {ground_truth=} {extra_info=}") + res = naive_dapo.compute_score(result, ground_truth, extra_info=extra_info) + print(f"Evaluated math task with data_source: {data_source}, got {res=}") + return res["score"] + # code generation + elif data_source.startswith("codegen"): + from skyrl_agent.tasks.verifiers import coder1 + + # print(f"Getting result {result}") + print(f"Evaluating codegen task with data_source: {data_source}, got {result=}") + res = coder1.compute_score(result, ground_truth, extra_info=extra_info) + print(f"Evaluated codegen task with data_source: {data_source}, got {res=}") + # print(f"Evaluating codegen task with data_source: {data_source}, got {score=} {extracted_model_output=}") + print(f"Evaluating codegen task with data_source: {data_source}, got {res['score']=}") + return res["score"] + elif data_source in ["2wikimultihopqa", "bamboogle", "hotpotqa", "musique", "nq", "popqa", "triviaqa"]: + ground_truth = instance["reward_model"]["ground_truth"] + from skyrl_agent.tasks.verifiers import qa + + print(f"Evaluating nq / hotpotqa like task with data_source: {data_source}, got {result=}") + res = qa.compute_score_em(result, ground_truth) + print(f"Evaluated nq / hotpotqa like task with data_source: {data_source}, got {res=}") + return res["score"] + elif data_source.startswith("browsecomp"): + ground_truth = instance["reward_model"]["ground_truth"] + from skyrl_agent.tasks.verifiers import qa + + print(f"Evaluating {data_source} task with data_source: {data_source}, got {result=}") + # FIXME: This is a hack to get the question from the prompt. Now inference only supports prompt, not raw_prompt. + if "raw_prompt" in instance: + question = instance["raw_prompt"][0]["content"].replace("Answer the given question:", "") + else: + question = instance["prompt"][0]["content"].replace("Answer the given question:", "") + print(f"during evaluation, Question: {question}") + res = await call_sync_from_async(qa.compute_score_browsecomp, result, ground_truth, question) + print(f"Evaluated {data_source} task with data_source: {data_source}, got {res=}") + return res["score"] + elif data_source.startswith("ruler"): + from skyrl_agent.tasks.verifiers import qa + + ground_truth = instance["reward_model"]["ground_truth"] + print(f"Evaluating ruler task with data_source: {data_source}, got {result=}") + if "raw_prompt" in instance: + question = instance["raw_prompt"][0]["content"] + else: + question = instance["prompt"][0]["content"] + print(f"Question: {question}") + res = await call_sync_from_async(qa.compute_score_ruler, result, ground_truth, question) + print(f"Evaluated ruler task with data_source: {data_source}, got {res=}") + return res["score"] + else: + raise NotImplementedError(f"Reward function is not implemented for {data_source=}") + + if isinstance(res, dict): + return res + elif isinstance(res, (int, float, bool)): + return float(res) + else: + return float(res[0]) + + +class DummyReactTask(GeneralReactTask): + # For Dummy ReAct Agent that does not use tools + @classmethod + def get_instruction(cls, instance: Dict[str, Any]) -> str: + prompt = instance.get("prompt") + return prompt + + +if __name__ == "__main__": + # Simple test + # ToRL + solution_str = "\\boxed{1}" + ground_truth = "1" + instance = {"reward_model": {"ground_truth": ground_truth}, "extra_info": {}} + print(asyncio.run(GeneralReactTask.evaluate_result(solution_str, instance, "math", 0, 0))) diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/__init__.py b/skyrl-agent/skyrl_agent/tasks/swebench/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/r2e_utils.py b/skyrl-agent/skyrl_agent/tasks/swebench/r2e_utils.py new file mode 100644 index 0000000000..cd346a9eba --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/swebench/r2e_utils.py @@ -0,0 +1,70 @@ +import re +import json + + +def parse_log_pytest(log: str | None) -> dict[str, str]: + """ + Parser for test logs generated with Sympy framework + + Args: + log (str): log content + Returns: + dict: test case to test status mapping + """ + if log is None: + return {} + test_status_map = {} + if "short test summary info" not in log: + return test_status_map + log = log.split("short test summary info")[1] + log = log.strip() + log = log.split("\n") + for line in log: + if "PASSED" in line: + test_name = ".".join(line.split("::")[1:]) + test_status_map[test_name] = "PASSED" + elif "FAILED" in line: + test_name = ".".join(line.split("::")[1:]).split(" - ")[0] + test_status_map[test_name] = "FAILED" + elif "ERROR" in line: + try: + test_name = ".".join(line.split("::")[1:]) + except IndexError: + test_name = line + test_name = test_name.split(" - ")[0] + test_status_map[test_name] = "ERROR" + return test_status_map + + +def decolor_dict_keys(key): + decolor = lambda key: re.sub(r"\u001b\[\d+m", "", key) + return {decolor(k): v for k, v in key.items()} + + +def get_reward(parse, instance): + parse = decolor_dict_keys(parse) + expected_json = instance["expected_output_json"] + + expected: dict = json.loads(expected_json) + expected = decolor_dict_keys(expected) + parse = {k.split(" - ")[0]: parse[k] for k in sorted(parse.keys())} + expected = {k.split(" - ")[0]: expected[k] for k in sorted(expected.keys())} + + # Compare + if len(parse) != len(expected): + reward = 0.0 + else: + # If ANY mismatch, reward = 0.0, else = 1.0 + match = True + for k in parse.keys(): + if not k: + continue + if k not in expected: + match = False + break + if parse[k] != expected[k]: + match = False + break + reward = 1.0 if match else 0.0 + + return reward diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/instance_r2e_entry.sh b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/instance_r2e_entry.sh new file mode 100644 index 0000000000..4be225f4b4 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/instance_r2e_entry.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +source ~/.bashrc + +/testbed/.venv/bin/python -m ensurepip --default-pip + +ln -s /testbed/.venv /root/.venv + +ln -s /testbed/.venv/bin/python /root/.local/bin/python + +ln -s /testbed/.venv/bin/python /root/.local/bin/python3 + +find "/testbed/.venv/bin" -type f -executable -exec ln -sf {} "/root/.local/bin/" \; + +export PATH=/root/.local/bin:$PATH +export PATH=/testbed/.venv/bin:$PATH + +# Install chardet +# uv pip install chardet +# /testbed/.venv/bin/python -m pip install chardet + +# for the new search tool +# Install custom BM25 packages for Python 3.6 compatibility +/testbed/.venv/bin/python -m pip install chardet networkx +/testbed/.venv/bin/python -m pip install 'rank-bm25>=0.2.0,<1.0.0' +echo "Custom BM25 components installed successfully" + +# Delete all *.pyc files and __pycache__ dirs in current repo +find . -name '*.pyc' -delete +find . -name '__pycache__' -exec rm -rf {} + + +# Delete *.pyc files and __pycache__ in /r2e_tests +find /r2e_tests -name '*.pyc' -delete +find /r2e_tests -name '__pycache__' -exec rm -rf {} + + +export REPO_PATH="/testbed" +REPO_PATH="/testbed" +ALT_PATH="/root" +SKIP_FILES_NEW=("run_tests.sh" "r2e_tests") +for skip_file in "${SKIP_FILES_NEW[@]}"; do + if [ -e "$REPO_PATH/$skip_file" ]; then + mv "$REPO_PATH/$skip_file" "$ALT_PATH/$skip_file" + fi +done + +# Move /r2e_tests to ALT_PATH +mv /r2e_tests "$ALT_PATH/r2e_tests" + +# Create symlink back to repo +ln -s "$ALT_PATH/r2e_tests" "$REPO_PATH/r2e_tests" diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/instance_swe_entry.sh b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/instance_swe_entry.sh new file mode 100644 index 0000000000..9dfb726b00 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/instance_swe_entry.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +source ~/.bashrc +SWEUTIL_DIR=/swe_util + +# FIXME: Cannot read SWE_INSTANCE_ID from the environment variable +# SWE_INSTANCE_ID=django__django-11099 +if [ -z "$SWE_INSTANCE_ID" ]; then + echo "Error: SWE_INSTANCE_ID is not set." >&2 + exit 1 +fi + +# Read the swe-bench-test-lite.json file and extract the required item based on instance_id +item=$(jq --arg INSTANCE_ID "$SWE_INSTANCE_ID" '.[] | select(.instance_id == $INSTANCE_ID)' $SWEUTIL_DIR/eval_data/instances/swe-bench-instance.json) + +if [[ -z "$item" ]]; then + echo "No item found for the provided instance ID." + exit 1 +fi + +WORKSPACE_NAME=$(echo "$item" | jq -r '(.repo | tostring) + "__" + (.version | tostring) | gsub("/"; "__")') + +echo "WORKSPACE_NAME: $WORKSPACE_NAME" + +# Clear the workspace +if [ -d /workspace ]; then + rm -rf /workspace/* +else + mkdir /workspace +fi +# Copy repo to workspace +if [ -d /workspace/$WORKSPACE_NAME ]; then + rm -rf /workspace/$WORKSPACE_NAME +fi +mkdir -p /workspace +cp -r /testbed /workspace/$WORKSPACE_NAME + +export REPO_PATH="/workspace/$WORKSPACE_NAME" +# Activate instance-specific environment +. /opt/miniconda3/etc/profile.d/conda.sh +conda activate testbed +mkdir -p /root/.venv/bin +# pip install chardet +# for the new search tool +# Install custom BM25 packages for Python 3.6 compatibility +pip install chardet networkx +pip install 'rank-bm25>=0.2.0,<1.0.0' +echo "Custom BM25 components installed successfully" + + +ln -s /opt/miniconda3/envs/testbed/bin/python /root/.venv/bin/python +ln -s /opt/miniconda3/envs/testbed/bin/python /root/.venv/bin/python3 diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/swe_entry.sh b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/swe_entry.sh new file mode 100644 index 0000000000..23fc3a3c50 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/setup/swe_entry.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +set -e + +# assert user name is `root` +if [ "$USER" != "root" ]; then + echo "Error: This script is intended to be run by the 'root' user only." >&2 + exit 1 +fi + +source ~/.bashrc + +SWEUTIL_DIR=/swe_util + +# Create logs directory +LOG_DIR=/openhands/logs +mkdir -p $LOG_DIR && chmod 777 $LOG_DIR + +# FIXME: Cannot read SWE_INSTANCE_ID from the environment variable +# SWE_INSTANCE_ID=django__django-11099 +if [ -z "$SWE_INSTANCE_ID" ]; then + echo "Error: SWE_INSTANCE_ID is not set." >&2 + exit 1 +fi + +# Read the swe-bench-test-lite.json file and extract the required item based on instance_id +item=$(jq --arg INSTANCE_ID "$SWE_INSTANCE_ID" '.[] | select(.instance_id == $INSTANCE_ID)' $SWEUTIL_DIR/eval_data/instances/swe-bench-test-lite.json) + +if [[ -z "$item" ]]; then + echo "No item found for the provided instance ID." + exit 1 +fi + +CONDA_ENV_NAME=$(echo "$item" | jq -r '.repo + "__" + .version | gsub("/"; "__")') + +echo "CONDA_ENV_NAME: $CONDA_ENV_NAME" + +SWE_TASK_DIR=/openhands/swe_tasks +mkdir -p $SWE_TASK_DIR +# Dump test_patch to /workspace/test.patch +echo "$item" | jq -r '.test_patch' > $SWE_TASK_DIR/test.patch +# Dump patch to /workspace/gold.patch +echo "$item" | jq -r '.patch' > $SWE_TASK_DIR/gold.patch +# Dump the item to /workspace/instance.json except for the "test_patch" and "patch" fields +echo "$item" | jq 'del(.test_patch, .patch)' > $SWE_TASK_DIR/instance.json + +# Clear the workspace +rm -rf /workspace/* +# Copy repo to workspace +if [ -d /workspace/$CONDA_ENV_NAME ]; then + rm -rf /workspace/$CONDA_ENV_NAME +fi +cp -r $SWEUTIL_DIR/eval_data/testbeds/$CONDA_ENV_NAME /workspace + +# Reset swe-bench testbed and install the repo +. $SWEUTIL_DIR/miniforge3/etc/profile.d/conda.sh +conda config --set changeps1 False +conda config --append channels conda-forge +conda activate swe-bench-eval + +mkdir -p $SWE_TASK_DIR/reset_testbed_temp +mkdir -p $SWE_TASK_DIR/reset_testbed_log_dir +SWE_BENCH_DIR=/swe_util/OH-SWE-bench +output=$( + export PYTHONPATH=$SWE_BENCH_DIR && \ + cd $SWE_BENCH_DIR && \ + python swebench/harness/reset_swe_env.py \ + --swe_bench_tasks $SWEUTIL_DIR/eval_data/instances/swe-bench-test.json \ + --temp_dir $SWE_TASK_DIR/reset_testbed_temp \ + --testbed /workspace \ + --conda_path $SWEUTIL_DIR/miniforge3 \ + --instance_id $SWE_INSTANCE_ID \ + --log_dir $SWE_TASK_DIR/reset_testbed_log_dir \ + --timeout 900 \ + --verbose +) + +REPO_PATH=$(echo "$output" | awk -F': ' '/repo_path:/ {print $2}') +TEST_CMD=$(echo "$output" | awk -F': ' '/test_cmd:/ {print $2}') +echo "Repo Path: $REPO_PATH" +echo "Test Command: $TEST_CMD" + +echo "export SWE_BENCH_DIR=\"$SWE_BENCH_DIR\"" >> ~/.bashrc +echo "export REPO_PATH=\"$REPO_PATH\"" >> ~/.bashrc +echo "export TEST_CMD=\"$TEST_CMD\"" >> ~/.bashrc + +if [[ "$REPO_PATH" == "None" ]]; then + echo "Error: Failed to retrieve repository path. Tests may not have passed or output was not as expected." >&2 + exit 1 +fi + +# Activate instance-specific environment +. $SWEUTIL_DIR/miniforge3/etc/profile.d/conda.sh +conda activate $CONDA_ENV_NAME + +set +e diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/scripts/tools/search.py b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/tools/search.py new file mode 100644 index 0000000000..f003c140ae --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/tools/search.py @@ -0,0 +1,1667 @@ +#!/root/.venv/bin/python + +""" +Description: + Advanced code search tool that searches for code snippets, entities, and functions using graph-based indexing. + • Searches for code entities (classes, functions, files) by name or keywords using pre-built code indices. + • Can search for specific entities using qualified names (e.g., 'file.py:ClassName.method_name'). + • Supports searching by line numbers within specific files to extract code context. + • Uses graph-based entity search combined with exact substring matching and BM25 semantic search. + • Filters results by file patterns or specific file paths. + • Returns structured results with code snippets, entity definitions, and relevant context. + + Search Strategy (in order): + 1. Entity search: Exact node ID or fuzzy entity name matching + - Matches: 'file.py:ClassName.method' or just 'ClassName' + - Only searches non-test files (from graph index) + 2. Exact substring match: ALWAYS performed (even if entity found) + - Phase A: Searches in specified files (file_path_or_pattern) + - Phase B: Searches all repo files if not found + 3. BM25 semantic search: Only if no entity AND no exact substring + - Token-based semantic matching for related concepts + + Usage combinations: + - search_terms only: searches across all Python files (or pattern-matched files) + - search_terms + file_path_or_pattern: limits search to specific file(s) matching pattern + + Parameters: + 1. search_terms (list of strings, optional) + One or more search terms to find in the codebase. Can be entity names, keywords, qualified names like 'file.py:ClassName.method_name', or exact code substrings. + 2. line_nums (list of integers, optional) + Specific line numbers to extract context from when used with a specific file path. + 3. file_path_or_pattern (string, optional) + File pattern or specific file path to filter search scope. Defaults to '**/*.py'. If a specific file with line_nums, extracts context around those lines. +""" + +import argparse +import collections +import fnmatch +import os +import pickle +import re +from collections import defaultdict +from typing import List, Optional +import ast + +import networkx as nx + +# Lightweight BM25 setup without NLTK to avoid tokenizer/punkt issues +try: + from rank_bm25 import BM25Okapi + + CUSTOM_BM25_AVAILABLE = True +except Exception as e: + CUSTOM_BM25_AVAILABLE = False + print(f"Warning: BM25 backend not available: {e}") + +# Simple, dependency-free tokenizer for code and text +_token_re = re.compile(r"[A-Za-z0-9_]+") + + +def _simple_tokenize(text: str): + try: + return _token_re.findall(text.lower()) + except Exception: + return text.lower().split() + + +def is_skip_dir(dirname): + """Check if directory should be skipped during graph building.""" + skip_dirs = [".github", ".git", "__pycache__", ".pytest_cache", "node_modules", ".venv", "tests"] + for skip_dir in skip_dirs: + if skip_dir in dirname: + return True + return False + + +class CodeAnalyzer(ast.NodeVisitor): + def __init__(self, filename, source_code=None): + self.filename = filename + self.source_code = source_code # Cache source code + self.nodes = [] + self.node_name_stack = [] + self.node_type_stack = [] + + def _get_end_lineno(self, node): + """Get end line number with Python 3.7 fallback.""" + if hasattr(node, "end_lineno") and node.end_lineno is not None: + return node.end_lineno + + # Fallback for Python 3.7: find the last line in the node's body + # This is approximate but better than nothing + try: + last_lineno = node.lineno + # Check if node has a body (classes and functions do) + if hasattr(node, "body") and node.body: + # Recursively find the last line number in the body + for child in node.body: + if hasattr(child, "lineno"): + last_lineno = max(last_lineno, child.lineno) + if hasattr(child, "end_lineno"): + last_lineno = max(last_lineno, child.end_lineno) + return last_lineno + except: + return node.lineno + + def visit_ClassDef(self, node): + class_name = node.name + full_class_name = ".".join(self.node_name_stack + [class_name]) + self.nodes.append( + { + "name": full_class_name, + "type": NODE_TYPE_CLASS, + "code": self._get_source_segment(node), + "start_line": node.lineno, + "end_line": self._get_end_lineno(node), + } + ) + + self.node_name_stack.append(class_name) + self.node_type_stack.append(NODE_TYPE_CLASS) + self.generic_visit(node) + self.node_name_stack.pop() + self.node_type_stack.pop() + + def visit_FunctionDef(self, node): + # if ( + # self.node_type_stack + # and self.node_type_stack[-1] == NODE_TYPE_CLASS + # and node.name == '__init__' + # ): + # return + self._visit_func(node) + + def visit_AsyncFunctionDef(self, node): + self._visit_func(node) + + def _visit_func(self, node): + function_name = node.name + full_function_name = ".".join(self.node_name_stack + [function_name]) + self.nodes.append( + { + "name": full_function_name, + "parent_type": self.node_type_stack[-1] if self.node_type_stack else None, + "type": NODE_TYPE_FUNCTION, + "code": self._get_source_segment(node), + "start_line": node.lineno, + "end_line": self._get_end_lineno(node), + } + ) + + self.node_name_stack.append(function_name) + self.node_type_stack.append(NODE_TYPE_FUNCTION) + self.generic_visit(node) + self.node_name_stack.pop() + self.node_type_stack.pop() + + def _get_source_segment(self, node): + # Use cached source code instead of re-reading file + if self.source_code is None: + with open(self.filename, "r") as file: + self.source_code = file.read() + + # ALWAYS extract lines directly from source to preserve exact formatting + # This ensures exact match for later string-based file modifications + lines = self.source_code.split("\n") + if hasattr(node, "lineno"): + start = node.lineno - 1 # Convert to 0-indexed + end_lineno = self._get_end_lineno(node) + end = end_lineno + if start < len(lines) and end <= len(lines): + return "\n".join(lines[start:end]) + return "" + + +def analyze_file(filepath): + with open(filepath, "r") as file: + code = file.read() + # code = handle_edge_cases(code) + try: + tree = ast.parse(code, filename=filepath) + except: + raise SyntaxError + analyzer = CodeAnalyzer(filepath, source_code=code) # Pass source code to avoid re-reading + try: + analyzer.visit(tree) + except RecursionError: + pass + return analyzer.nodes + + +def resolve_symlink(file_path): + """Resolve the absolute path of a symbolic link.""" + if os.path.islink(file_path): + relative_target = os.readlink(file_path) + symlink_dir = os.path.dirname(os.path.dirname(file_path)) + absolute_target = os.path.abspath(os.path.join(symlink_dir, relative_target)) + if not os.path.exists(absolute_target): + print(f"The target file does not exist: {absolute_target}") + return None + return absolute_target + return None + + +def get_file_content(file_path): + """Get file content with robust error handling and fallbacks.""" + try: + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + return f.read() + except Exception: + try: + with open(file_path, "rb") as f: + return f.read().decode("utf-8", errors="ignore") + except Exception: + return None + + +def build_graph(repo_path, fuzzy_search=True, global_import=False): + import time + + start_time = time.time() + print(f"Building graph for {repo_path}...") + + graph = nx.MultiDiGraph() + file_nodes = {} + file_count = 0 + + ## add nodes + graph.add_node("/", type=NODE_TYPE_DIRECTORY) + dir_stack: List[str] = [] + dir_include_stack: List[bool] = [] + for root, _, files in os.walk(repo_path): + # add directory nodes and edges + dirname = os.path.relpath(root, repo_path) + if dirname == ".": + dirname = "/" + elif is_skip_dir(dirname): + continue + else: + graph.add_node(dirname, type=NODE_TYPE_DIRECTORY) + parent_dirname = os.path.dirname(dirname) + if parent_dirname == "": + parent_dirname = "/" + graph.add_edge(parent_dirname, dirname, type=EDGE_TYPE_CONTAINS) + + # in reverse step, remove directories that do not contain .py file + while len(dir_stack) > 0 and not dirname.startswith(dir_stack[-1]): + if not dir_include_stack[-1]: + # print('remove', dir_stack[-1]) + graph.remove_node(dir_stack[-1]) + dir_stack.pop() + dir_include_stack.pop() + if dirname != "/": + dir_stack.append(dirname) + dir_include_stack.append(False) + + dir_has_py = False + for file in files: + if file.endswith(".py"): + dir_has_py = True + + # add file nodes + try: + file_path = os.path.join(root, file) + filename = os.path.relpath(file_path, repo_path) + if os.path.islink(file_path): + continue + else: + with open(file_path, "r") as f: + file_content = f.read() + + graph.add_node(filename, type=NODE_TYPE_FILE, code=file_content) + file_nodes[filename] = file_path + + nodes = analyze_file(file_path) + file_count += 1 + if file_count % 10 == 0: + print(f" Processed {file_count} files... (current: {filename})") + # break + except (UnicodeDecodeError, SyntaxError): + # Skip the file that cannot decode or parse + continue + + # add function/class nodes + for node in nodes: + full_name = f'{filename}:{node["name"]}' + graph.add_node( + full_name, + type=node["type"], + code=node["code"], + start_line=node["start_line"], + end_line=node["end_line"], + ) + + # add edges with type=contains + graph.add_edge(dirname, filename, type=EDGE_TYPE_CONTAINS) + for node in nodes: + full_name = f'{filename}:{node["name"]}' + name_list = node["name"].split(".") + if len(name_list) == 1: + graph.add_edge(filename, full_name, type=EDGE_TYPE_CONTAINS) + else: + parent_name = ".".join(name_list[:-1]) + full_parent_name = f"{filename}:{parent_name}" + graph.add_edge(full_parent_name, full_name, type=EDGE_TYPE_CONTAINS) + + # keep all parent directories + if dir_has_py: + for i in range(len(dir_include_stack)): + dir_include_stack[i] = True + + # check last traversed directory + while len(dir_stack) > 0: + if not dir_include_stack[-1]: + graph.remove_node(dir_stack[-1]) + dir_stack.pop() + dir_include_stack.pop() + + global_name_dict = defaultdict(list) + # if global_import: + for node in graph.nodes(): + node_name = node.split(":")[-1].split(".")[-1] + global_name_dict[node_name].append(node) + graph.graph["global_name_dict"] = global_name_dict + + elapsed = time.time() - start_time + print("Graph building complete!") + print(f" Files processed: {file_count}") + print(f" Total nodes: {graph.number_of_nodes()}") + print(f" Total edges: {graph.number_of_edges()}") + print(f" Time elapsed: {elapsed:.2f}s") + return graph + + +def find_matching_files_from_list(file_list, file_pattern): + """ + Find and return a list of file paths from the given list that match the given keyword or pattern. + """ + # strip the double/single quotes if any + if (file_pattern.startswith('"') and file_pattern.endswith('"')) or ( + file_pattern.startswith("'") and file_pattern.endswith("'") + ): + file_pattern = file_pattern[1:-1] + + repo_path = REPO_PATH if REPO_PATH else os.environ.get("REPO_PATH", "") + if os.path.isabs(file_pattern): + file_pattern = os.path.relpath(file_pattern, repo_path) + else: + file_pattern = os.path.normpath(file_pattern) + if "*" in file_pattern or "?" in file_pattern or "[" in file_pattern: + matching_files = fnmatch.filter(file_list, file_pattern) + else: + matching_files = [file for file in file_list if file_pattern in file] + return matching_files + + +def merge_intervals(intervals): + if not intervals: + return [] + intervals.sort(key=lambda interval: interval[0]) + merged_intervals = [intervals[0]] + for current in intervals[1:]: + last = merged_intervals[-1] + if current[0] <= last[1]: + merged_intervals[-1] = (last[0], max(last[1], current[1])) + else: + merged_intervals.append(current) + return merged_intervals + + +# Entity and dependency types used by searcher +NODE_TYPE_CLASS = "class" +NODE_TYPE_DIRECTORY = "directory" +NODE_TYPE_FILE = "file" +NODE_TYPE_FUNCTION = "function" + +EDGE_TYPE_CONTAINS = "contains" + +VALID_NODE_TYPES = [NODE_TYPE_FILE, NODE_TYPE_CLASS, NODE_TYPE_FUNCTION, NODE_TYPE_DIRECTORY] +VALID_EDGE_TYPES = [EDGE_TYPE_CONTAINS, "imports", "invokes", "inherits"] + + +class RepoEntitySearcher: + """Retrieve Entity IDs and Code Snippets from Repository Graph""" + + def __init__(self, graph): + self.G = graph + self._global_name_dict = None + self._global_name_dict_lowercase = None + self._etypes_dict = {etype: i for i, etype in enumerate(VALID_EDGE_TYPES)} + + @property + def global_name_dict(self): + if self._global_name_dict is None: # Compute only once + _global_name_dict = defaultdict(list) + for nid in self.G.nodes(): + if is_test_file(nid): + continue + + if nid.endswith(".py"): + fname = nid.split("/")[-1] + _global_name_dict[fname].append(nid) + + name = nid[: -(len(".py"))].split("/")[-1] + _global_name_dict[name].append(nid) + + elif ":" in nid: + name = nid.split(":")[-1].split(".")[-1] + _global_name_dict[name].append(nid) + + self._global_name_dict = _global_name_dict + return self._global_name_dict + + @property + def global_name_dict_lowercase(self): + if self._global_name_dict_lowercase is None: # Compute only once + _global_name_dict_lowercase = defaultdict(list) + for nid in self.G.nodes(): + if is_test_file(nid): + continue + + if nid.endswith(".py"): + fname = nid.split("/")[-1].lower() + _global_name_dict_lowercase[fname].append(nid) + + name = nid[: -(len(".py"))].split("/")[-1].lower() + _global_name_dict_lowercase[name].append(nid) + + elif ":" in nid: + name = nid.split(":")[-1].split(".")[-1].lower() + _global_name_dict_lowercase[name].append(nid) + + self._global_name_dict_lowercase = _global_name_dict_lowercase + return self._global_name_dict_lowercase + + def has_node(self, nid, include_test=False): + if not include_test and is_test_file(nid): + return False + return nid in self.G + + def get_node_data(self, nids, return_code_content=False, wrap_with_ln=True): + rtn = [] + for nid in nids: + node_data = self.G.nodes[nid] + formatted_data = { + "node_id": nid, + "type": node_data["type"], + } + if node_data.get("code", ""): + if "start_line" in node_data: + formatted_data["start_line"] = node_data["start_line"] + start_line = node_data["start_line"] + elif formatted_data["type"] == NODE_TYPE_FILE: + start_line = 1 + formatted_data["start_line"] = start_line + else: + start_line = 1 + + if "end_line" in node_data: + formatted_data["end_line"] = node_data["end_line"] + end_line = node_data["end_line"] + elif formatted_data["type"] == NODE_TYPE_FILE: + end_line = len(node_data["code"].split("\n")) # - 1 + formatted_data["end_line"] = end_line + else: + end_line = 1 + # load formatted code data + if return_code_content and wrap_with_ln: + formatted_data["code_content"] = wrap_code_snippet(node_data["code"], start_line, end_line) + elif return_code_content: + formatted_data["code_content"] = node_data["code"] + rtn.append(formatted_data) + return rtn + + def get_all_nodes_by_type(self, type): + assert type in VALID_NODE_TYPES + nodes = [] + for nid in self.G.nodes(): + if is_test_file(nid): + continue + if self.G.nodes[nid]["type"] == type: + node_data = self.G.nodes[nid] + if type == NODE_TYPE_FILE: + formatted_data = { + "name": nid, + "type": node_data["type"], + "content": node_data.get("code", "").split("\n"), + } + elif type == NODE_TYPE_FUNCTION: + formatted_data = { + "name": nid.split(":")[-1], + "file": nid.split(":")[0], + "type": node_data["type"], + "content": node_data.get("code", "").split("\n"), + "start_line": node_data.get("start_line", 0), + "end_line": node_data.get("end_line", 0), + } + elif type == NODE_TYPE_CLASS: + formatted_data = { + "name": nid.split(":")[-1], + "file": nid.split(":")[0], + "type": node_data["type"], + "content": node_data.get("code", "").split("\n"), + "start_line": node_data.get("start_line", 0), + "end_line": node_data.get("end_line", 0), + "methods": [], + } + dp_searcher = RepoDependencySearcher(self.G) + methods = dp_searcher.get_neighbors( + nid, + "forward", + ntype_filter=[NODE_TYPE_FUNCTION], + etype_filter=[EDGE_TYPE_CONTAINS], + )[0] + formatted_methods = [] + for mid in methods: + mnode = self.G.nodes[mid] + formatted_methods.append( + { + "name": mid.split(".")[-1], + "start_line": mnode.get("start_line", 0), + "end_line": mnode.get("end_line", 0), + } + ) + formatted_data["methods"] = formatted_methods + nodes.append(formatted_data) + return nodes + + +class RepoDependencySearcher: + """Traverse Repository Graph""" + + def __init__(self, graph): + self.G = graph + self._etypes_dict = {etype: i for i, etype in enumerate(VALID_EDGE_TYPES)} + + def subgraph(self, nids): + return self.G.subgraph(nids) + + def get_neighbors( + self, + nid, + direction="forward", + ntype_filter=None, + etype_filter=None, + ignore_test_file=True, + ): + nodes, edges = [], [] + if direction == "forward": + for sn in self.G.successors(nid): + if ntype_filter and self.G.nodes[sn]["type"] not in ntype_filter: + continue + if ignore_test_file and is_test_file(sn): + continue + for key, edge_data in self.G.get_edge_data(nid, sn).items(): + etype = edge_data["type"] + if etype_filter and etype not in etype_filter: + continue + edges.append((nid, sn, self._etypes_dict[etype], {"type": etype})) + nodes.append(sn) + + elif direction == "backward": + for pn in self.G.predecessors(nid): + if ntype_filter and self.G.nodes[pn]["type"] not in ntype_filter: + continue + if ignore_test_file and is_test_file(pn): + continue + for key, edge_data in self.G.get_edge_data(pn, nid).items(): + etype = edge_data["type"] + if etype_filter and etype not in etype_filter: + continue + edges.append((pn, nid, self._etypes_dict[etype], {"type": etype})) + nodes.append(pn) + + return nodes, edges + + +def traverse_tree_structure( + G: nx.MultiDiGraph, + start_node: str, + direction: str, + traversal_depth: int, + entity_type_filter: Optional[List[str]], + dependency_type_filter: Optional[List[str]], +): + # Minimal placeholder for formatting outputs; not used by search_code_snippets + return f"Traversal from {start_node} (depth={traversal_depth}, dir={direction})" + + +def is_test_file(path: str) -> bool: + lowered = path.lower() + return any(token in lowered for token in ["test/", "/tests/", "_test.py", "/test_"]) + + +def wrap_code_snippet(code_content: str, start_line: int, end_line: int) -> str: + """Wrap code snippet with line numbers.""" + lines = code_content.split("\n") + wrapped_lines = [] + for i, line in enumerate(lines, start_line): + wrapped_lines.append(f"{i:4d}|{line}") + return "\n".join(wrapped_lines) + + +REPO_PATH: Optional[str] = None +GRAPH_INDEX_DIR: Optional[str] = None +ALL_FILE: Optional[list] = None +ALL_CLASS: Optional[list] = None +ALL_FUNC: Optional[list] = None + +DP_GRAPH_ENTITY_SEARCHER: Optional[RepoEntitySearcher] = None +DP_GRAPH_DEPENDENCY_SEARCHER: Optional[RepoDependencySearcher] = None +DP_GRAPH: Optional[nx.MultiDiGraph] = None + +# Lightweight BM25 cache (built lazily and reset when graph changes) +BM25_DOCS: Optional[List[List[str]]] = None +BM25_META: Optional[List[dict]] = None +BM25_MODEL: Optional["BM25Okapi"] = None + + +def parse_repo_index(): + global REPO_PATH, GRAPH_INDEX_DIR + global DP_GRAPH_ENTITY_SEARCHER, DP_GRAPH_DEPENDENCY_SEARCHER, DP_GRAPH + + if not REPO_PATH: + REPO_PATH = os.environ.get("REPO_PATH", "") + # print('='*5, REPO_PATH, '='*5) + + if not GRAPH_INDEX_DIR: + BASE_INDEX_DIR = os.path.join(REPO_PATH, "_index_data") + GRAPH_INDEX_DIR = os.path.join(BASE_INDEX_DIR, "graph_index_v2.3") + # print('='*5, BASE_INDEX_DIR, '='*5) + + if not DP_GRAPH: + # setup graph traverser + graph_index_file = os.path.join(GRAPH_INDEX_DIR, "code_graph.pkl") + if not os.path.exists(graph_index_file): + try: + # Try to create the directory, but don't fail if we can't + try: + os.makedirs(GRAPH_INDEX_DIR, exist_ok=True) + except PermissionError: + print(f"Warning: Cannot create index directory {GRAPH_INDEX_DIR}, using in-memory graph only") + + G = build_graph(REPO_PATH, global_import=True) + + # Try to save graph + try: + with open(graph_index_file, "wb") as f: + pickle.dump(G, f) + except (PermissionError, OSError): + pass + except Exception: + import traceback + + traceback.print_exc() + # Create an empty graph as fallback + G = nx.MultiDiGraph() + else: + G = pickle.load(open(graph_index_file, "rb")) + + DP_GRAPH_ENTITY_SEARCHER = RepoEntitySearcher(G) + DP_GRAPH_DEPENDENCY_SEARCHER = RepoDependencySearcher(G) + DP_GRAPH = G + + # Reset BM25 cache whenever graph is (re)loaded + global BM25_DOCS, BM25_META, BM25_MODEL + BM25_DOCS = None + BM25_META = None + BM25_MODEL = None + + global ALL_FILE, ALL_CLASS, ALL_FUNC + ALL_FILE = DP_GRAPH_ENTITY_SEARCHER.get_all_nodes_by_type(NODE_TYPE_FILE) + ALL_CLASS = DP_GRAPH_ENTITY_SEARCHER.get_all_nodes_by_type(NODE_TYPE_CLASS) + ALL_FUNC = DP_GRAPH_ENTITY_SEARCHER.get_all_nodes_by_type(NODE_TYPE_FUNCTION) + + # Custom BM25 implementation doesn't require pre-built indices + # BM25Okapi builds index on-the-fly from documents + + +def get_current_repo_modules(): + return ALL_FILE, ALL_CLASS, ALL_FUNC + + +def get_graph_entity_searcher() -> RepoEntitySearcher: + return DP_GRAPH_ENTITY_SEARCHER + + +def get_graph_dependency_searcher() -> RepoDependencySearcher: + return DP_GRAPH_DEPENDENCY_SEARCHER + + +def get_graph(): + assert DP_GRAPH is not None + return DP_GRAPH + + +def get_module_name_by_line_num(file_path: str, line_num: int): + entity_searcher = get_graph_entity_searcher() + dp_searcher = get_graph_dependency_searcher() + + cur_module = None + if entity_searcher.has_node(file_path): + module_nids, _ = dp_searcher.get_neighbors(file_path, etype_filter=[EDGE_TYPE_CONTAINS]) + module_ndatas = entity_searcher.get_node_data(module_nids) + for module in module_ndatas: + if module["start_line"] <= line_num <= module["end_line"]: + cur_module = module + break + if cur_module and cur_module["type"] == NODE_TYPE_CLASS: + func_nids, _ = dp_searcher.get_neighbors(cur_module["node_id"], etype_filter=[EDGE_TYPE_CONTAINS]) + func_ndatas = entity_searcher.get_node_data(func_nids, return_code_content=True) + for func in func_ndatas: + if func["start_line"] <= line_num <= func["end_line"]: + cur_module = func + break + + if cur_module: + return cur_module + return None + + +class QueryInfo: + query_type: str = "keyword" + term: Optional[str] = None + line_nums: Optional[List] = None + file_path_or_pattern: Optional[str] = None + + def __init__( + self, + query_type: str = "keyword", + term: Optional[str] = None, + line_nums: Optional[List] = None, + file_path_or_pattern: Optional[str] = None, + ): + self.query_type = query_type + if term is not None: + self.term = term + if line_nums is not None: + self.line_nums = line_nums + if file_path_or_pattern is not None: + self.file_path_or_pattern = file_path_or_pattern + + def __str__(self): + parts = [] + if self.term is not None: + parts.append(f"term: {self.term}") + if self.line_nums is not None: + parts.append(f"line_nums: {self.line_nums}") + if self.file_path_or_pattern is not None: + parts.append(f"file_path_or_pattern: {self.file_path_or_pattern}") + return ", ".join(parts) + + def __repr__(self): + return self.__str__() + + +class QueryResult: + file_path: Optional[str] = None + format_mode: Optional[str] = "complete" + nid: Optional[str] = None + ntype: Optional[str] = None + start_line: Optional[int] = None + end_line: Optional[int] = None + query_info_list: Optional[List[QueryInfo]] = None + desc: Optional[str] = None + message: Optional[str] = None + warning: Optional[str] = None + retrieve_src: Optional[str] = None + + def __init__( + self, + query_info: QueryInfo, + format_mode: str, + nid: Optional[str] = None, + ntype: Optional[str] = None, + file_path: Optional[str] = None, + start_line: Optional[int] = None, + end_line: Optional[int] = None, + desc: Optional[str] = None, + message: Optional[str] = None, + warning: Optional[str] = None, + retrieve_src: Optional[str] = None, + ): + self.format_mode = format_mode + self.query_info_list = [] + self.insert_query_info(query_info) + + if nid is not None: + self.nid = nid + + if ntype is not None: + self.ntype = ntype + if ntype in [NODE_TYPE_FILE, NODE_TYPE_CLASS, NODE_TYPE_FUNCTION]: + self.file_path = nid.split(":")[0] + + if file_path is not None: + self.file_path = file_path + if start_line is not None and end_line is not None: + self.start_line = start_line + self.end_line = end_line + + if retrieve_src is not None: + self.retrieve_src = retrieve_src + + if desc is not None: + self.desc = desc + if message is not None: + self.message = message + if warning is not None: + self.warning = warning + + def insert_query_info(self, query_info: QueryInfo): + self.query_info_list.append(query_info) + + def format_output(self, searcher): + cur_result = "" + if self.format_mode == "complete": + node_data = searcher.get_node_data([self.nid], return_code_content=True)[0] + ntype = node_data["type"] + cur_result += f"Found {ntype} `{self.nid}`.\n" + cur_result += "Source: " + self.retrieve_src + "\n" + if "code_content" in node_data: + if ntype in [NODE_TYPE_FUNCTION]: + cur_result += node_data["code_content"] + "\n" + else: + content_size = node_data["end_line"] - node_data["start_line"] + if content_size <= 200: + cur_result += node_data["code_content"] + "\n" + else: + if ntype == NODE_TYPE_CLASS: + dp_searcher = RepoDependencySearcher(searcher.G) + method_nids, _ = dp_searcher.get_neighbors( + self.nid, + "forward", + ntype_filter=[NODE_TYPE_FUNCTION], + etype_filter=[EDGE_TYPE_CONTAINS], + ) + if method_nids: + cur_result += "\nMethods:\n" + for mid in method_nids: + method_name = mid.split(".")[-1] + mnode = searcher.G.nodes[mid] + cur_result += f' - {method_name} (lines {mnode.get("start_line", "?")}-{mnode.get("end_line", "?")})\n' + cur_result += f"\nHint: Search for specific method like `{self.nid}.method_name` to see its implementation.\n" + elif ntype == NODE_TYPE_FILE: + # return class names + dp_searcher = RepoDependencySearcher(searcher.G) + class_nids, _ = dp_searcher.get_neighbors( + self.nid, + "forward", + ntype_filter=[NODE_TYPE_CLASS], + etype_filter=[EDGE_TYPE_CONTAINS], + ) + if class_nids: + cur_result += "\nClasses:\n" + for cid in class_nids: + class_name = cid.split(".")[-1] + cnode = searcher.G.nodes[cid] + cur_result += f' - {class_name} (lines {cnode.get("start_line", "?")}-{cnode.get("end_line", "?")})\n' + cur_result += f"\nHint: Search for specific class like `{self.nid}.class_name` to see its implementation.\n" + + elif self.format_mode == "preview": + node_data = searcher.get_node_data([self.nid], return_code_content=True)[0] + ntype = node_data["type"] + cur_result += f"Found {ntype} `{self.nid}`.\n" + # show line range if available + if "start_line" in node_data and "end_line" in node_data: + cur_result += f'(lines {node_data["start_line"]}-{node_data["end_line"]})\n' + cur_result += "Source: " + self.retrieve_src + "\n" + if ntype == NODE_TYPE_FUNCTION: + # hint + cur_result += f"Hint: Search `{self.nid}` to see the full function code.\n" + elif ntype in [NODE_TYPE_CLASS, NODE_TYPE_FILE]: + content_size = node_data["end_line"] - node_data["start_line"] + if content_size <= 50: + cur_result += node_data["code_content"] + "\n" + else: + if ntype == NODE_TYPE_CLASS: + dp_searcher = RepoDependencySearcher(searcher.G) + method_nids, _ = dp_searcher.get_neighbors( + self.nid, + "forward", + ntype_filter=[NODE_TYPE_FUNCTION], + etype_filter=[EDGE_TYPE_CONTAINS], + ) + if method_nids: + cur_result += "\nMethods:\n" + for mid in method_nids: + method_name = mid.split(".")[-1] + mnode = searcher.G.nodes[mid] + cur_result += f' - {method_name} (lines {mnode.get("start_line", "?")}-{mnode.get("end_line", "?")})\n' + cur_result += f"\nHint: Search for specific method like `{self.nid}.method_name` to see its implementation.\n" + elif ntype == NODE_TYPE_FILE: + cur_result += f"Just show the structure of this {ntype} due to response length limitations:\n" + code_content = searcher.G.nodes[self.nid].get("code", "") + structure = get_skeleton(code_content) + cur_result += "```\n" + structure + "\n```\n" + cur_result += f"Hint: Search `{self.nid}.class_name` to get more information if needed.\n" + elif ntype == NODE_TYPE_DIRECTORY: + pass + elif self.format_mode == "code_snippet": + # If message exists, it's a summary (show message only, no code) + if self.message and self.message.strip(): + cur_result += self.message + "\n" + cur_result += "Source: " + self.retrieve_src + "\n" + else: + # Normal code snippet display + if self.desc: + cur_result += self.desc + "\n" + else: + cur_result += f"Found code snippet in file `{self.file_path}`.\n" + cur_result += "Source: " + self.retrieve_src + "\n" + node_data = searcher.get_node_data([self.file_path], return_code_content=True)[0] + content = node_data["code_content"].split("\n")[1:-1] + code_snippet = content[(self.start_line - 1) : self.end_line] + code_snippet = "```\n" + "\n".join(code_snippet) + "\n```" + cur_result += code_snippet + "\n" + elif self.format_mode == "fold": + node_data = searcher.get_node_data([self.nid], return_code_content=False)[0] + self.ntype = node_data["type"] + cur_result += f"Found {self.ntype} `{self.nid}`.\n" + return cur_result + + +# get_skeleton is referenced by QueryResult.preview; provide a simple fallback +def get_skeleton(code_content: str) -> str: + lines = code_content.splitlines() + preview = [] + for line in lines: + stripped = line.strip() + if stripped.startswith("class ") or stripped.startswith("def "): + preview.append(stripped) + return "\n".join(preview) if preview else "\n".join(lines[:50]) + + +def get_code_block_by_line_nums(query_info, context_window=20): + searcher = get_graph_entity_searcher() + file_path = query_info.file_path_or_pattern + line_nums = query_info.line_nums + cur_query_results = [] + + file_data = searcher.get_node_data([file_path], return_code_content=False)[0] + line_intervals = [] + res_modules = [] + for line in line_nums: + module_data = get_module_name_by_line_num(file_path, line) + if not module_data: + min_line_num = max(1, line - context_window) + max_line_num = min(file_data["end_line"], line + context_window) + line_intervals.append((min_line_num, max_line_num)) + elif module_data["node_id"] not in res_modules: + query_result = QueryResult( + query_info=query_info, + format_mode="preview", + nid=module_data["node_id"], + ntype=module_data["type"], + start_line=module_data["start_line"], + end_line=module_data["end_line"], + retrieve_src=f"Retrieved code context including {query_info.term}.", + ) + cur_query_results.append(query_result) + res_modules.append(module_data["node_id"]) + + if line_intervals: + line_intervals = merge_intervals(line_intervals) + for interval in line_intervals: + start_line, end_line = interval + query_result = QueryResult( + query_info=query_info, + format_mode="code_snippet", + nid=file_path, + file_path=file_path, + start_line=start_line, + end_line=end_line, + retrieve_src=f"Retrieved code context including {query_info.term}.", + ) + cur_query_results.append(query_result) + return cur_query_results + + +def parse_node_id(nid: str): + nfile = nid.split(":")[0] + nname = nid.split(":")[-1] + return nfile, nname + + +def search_entity_in_global_dict(term: str, include_files: Optional[List[str]] = None, prefix_term=None): + searcher = get_graph_entity_searcher() + if term.startswith(("class ", "Class")): + term = term[len("class ") :].strip() + elif term.startswith(("function ", "Function ")): + term = term[len("function ") :].strip() + elif term.startswith(("method ", "Method ")): + term = term[len("method ") :].strip() + elif term.startswith("def "): + term = term[len("def ") :].strip() + + if term in searcher.global_name_dict: + global_name_dict = searcher.global_name_dict + nids = global_name_dict[term] + elif term.lower() in searcher.global_name_dict_lowercase: + term = term.lower() + global_name_dict = searcher.global_name_dict_lowercase + nids = global_name_dict[term] + else: + return None + + node_datas = searcher.get_node_data(nids, return_code_content=False) + found_entities_filter_dict = collections.defaultdict(list) + for ndata in node_datas: + nfile, _ = parse_node_id(ndata["node_id"]) + if not include_files or nfile in include_files: + prefix_terms = [] + candidite_prefixes = re.split(r"[./:]", ndata["node_id"].lower().replace(".py", ""))[:-1] + if prefix_term: + prefix_terms = prefix_term.lower().split(".") + if not prefix_term or all([prefix in candidite_prefixes for prefix in prefix_terms]): + found_entities_filter_dict[ndata["type"]].append(ndata["node_id"]) + return found_entities_filter_dict + + +def bm25_module_retrieve( + query: str, include_files: Optional[List[str]] = None, search_scope: str = "all", similarity_top_k: int = 10 +): + # Simple entity-level retrieval using fuzzy matching on global names. + searcher = get_graph_entity_searcher() + candidates = [] + for nid in searcher.G.nodes: + if query.lower() in nid.lower(): + candidates.append(nid) + if include_files: + candidates = [nid for nid in candidates if nid.split(":")[0] in include_files] + return candidates[:similarity_top_k] + + +def search_entity(query_info, include_files: List[str] = None): + term = query_info.term + searcher = get_graph_entity_searcher() + continue_search = True + cur_query_results = [] + + if searcher.has_node(term): + continue_search = False + query_result = QueryResult( + query_info=query_info, + format_mode="complete", + nid=term, + retrieve_src=f"Exact match found for entity name `{term}`.", + ) + cur_query_results.append(query_result) + + if continue_search: + found_entities_dict = search_entity_in_global_dict(term, include_files) + if not found_entities_dict: + found_entities_dict = search_entity_in_global_dict(term) + + use_sub_term = False + used_term = term + if not found_entities_dict and "." in term: + try: + prefix_term = ".".join(term.split(".")[:-1]).split()[-1] + except IndexError: + prefix_term = None + split_term = term.split(".")[-1].strip() + used_term = split_term + found_entities_dict = search_entity_in_global_dict(split_term, include_files, prefix_term) + if not found_entities_dict: + found_entities_dict = search_entity_in_global_dict(split_term, prefix_term) + if not found_entities_dict: + use_sub_term = True + found_entities_dict = search_entity_in_global_dict(split_term) + + if found_entities_dict: + for ntype, nids in found_entities_dict.items(): + if not nids: + continue + if ntype in [NODE_TYPE_FUNCTION, NODE_TYPE_CLASS, NODE_TYPE_FILE]: + if len(nids) <= 3: + node_datas = searcher.get_node_data(nids, return_code_content=True) + for ndata in node_datas: + query_result = QueryResult( + query_info=query_info, + format_mode="preview", + nid=ndata["node_id"], + ntype=ndata["type"], + start_line=ndata["start_line"], + end_line=ndata["end_line"], + retrieve_src=f"Match found for entity name `{used_term}`.", + ) + cur_query_results.append(query_result) + else: + node_datas = searcher.get_node_data(nids, return_code_content=False) + for ndata in node_datas: + query_result = QueryResult( + query_info=query_info, + format_mode="fold", + nid=ndata["node_id"], + ntype=ndata["type"], + retrieve_src=f"Match found for entity name `{used_term}`.", + ) + cur_query_results.append(query_result) + if not use_sub_term: + continue_search = False + else: + continue_search = True + + if continue_search: + module_nids = bm25_module_retrieve(query=term, include_files=include_files) + if not module_nids: + module_nids = bm25_module_retrieve(query=term) + module_datas = searcher.get_node_data(module_nids, return_code_content=True) + showed_module_num = 0 + for module in module_datas[:5]: + if module["type"] in [NODE_TYPE_FILE, NODE_TYPE_DIRECTORY]: + query_result = QueryResult( + query_info=query_info, + format_mode="fold", + nid=module["node_id"], + ntype=module["type"], + retrieve_src="Retrieved entity using keyword search (approx).", + ) + cur_query_results.append(query_result) + elif showed_module_num < 3: + showed_module_num += 1 + query_result = QueryResult( + query_info=query_info, + format_mode="preview", + nid=module["node_id"], + ntype=module["type"], + start_line=module["start_line"], + end_line=module["end_line"], + retrieve_src="Retrieved entity using keyword search (approx).", + ) + cur_query_results.append(query_result) + + return (cur_query_results, continue_search) + + +def merge_query_results(query_results): + priority = ["complete", "code_snippet", "preview", "fold"] + merged_results = {} + all_query_results: List[QueryResult] = [] + for qr in query_results: + if qr.format_mode == "code_snippet": + all_query_results.append(qr) + elif qr.nid and qr.nid in merged_results: + if qr.query_info_list[0] not in merged_results[qr.nid].query_info_list: + merged_results[qr.nid].query_info_list.extend(qr.query_info_list) + existing_format_mode = merged_results[qr.nid].format_mode + if priority.index(qr.format_mode) < priority.index(existing_format_mode): + merged_results[qr.nid].format_mode = qr.format_mode + merged_results[qr.nid].start_line = qr.start_line + merged_results[qr.nid].end_line = qr.end_line + merged_results[qr.nid].retrieve_src = qr.retrieve_src + elif qr.nid: + merged_results[qr.nid] = qr + all_query_results += list(merged_results.values()) + return all_query_results + + +def rank_and_aggr_query_results(query_results, fixed_query_info_list): + query_info_list_dict = {} + for qr in query_results: + key = tuple(qr.query_info_list) + if key in query_info_list_dict: + query_info_list_dict[key].append(qr) + else: + query_info_list_dict[key] = [qr] + + def sorting_key(key): + for i, fixed_query in enumerate(fixed_query_info_list): + if fixed_query in key: + return i + return len(fixed_query_info_list) + + sorted_keys = sorted(query_info_list_dict.keys(), key=sorting_key) + sorted_query_info_list_dict = {key: query_info_list_dict[key] for key in sorted_keys} + + priority = {"complete": 1, "preview": 2, "fold": 3, "code_snippet": 4} + organized_dict = {} + for key, values in sorted_query_info_list_dict.items(): + nested_dict = {priority_key: [] for priority_key in priority.keys()} + for qr in values: + if qr.format_mode in nested_dict: + nested_dict[qr.format_mode].append(qr) + organized_dict[key] = {k: v for k, v in nested_dict.items() if v} + return organized_dict + + +def exact_substring_search(search_term: str, file_pattern: Optional[str] = None, max_files: int = 22): + """ + Searches for exact substring match by scanning filesystem directly. + Searches specified pattern first, then all .py files if not found. + Includes test files. + Returns dict: {file_path: [line_nums]} + """ + repo_path = os.environ.get("REPO_PATH", "") or REPO_PATH + if not repo_path or not os.path.exists(repo_path): + return {} + + file_matches = {} # {file_path: [line_numbers]} + qlower = search_term.lower() + + # Normalize file_pattern (handle quotes, absolute paths, etc.) + if file_pattern: + # Strip quotes if any + if (file_pattern.startswith('"') and file_pattern.endswith('"')) or ( + file_pattern.startswith("'") and file_pattern.endswith("'") + ): + file_pattern = file_pattern[1:-1] + + # Convert absolute path to relative + if os.path.isabs(file_pattern): + file_pattern = os.path.relpath(file_pattern, repo_path) + else: + file_pattern = os.path.normpath(file_pattern) + + # Build list of files to search + target_files = [] + + for root, dirs, files in os.walk(repo_path): + # Skip hidden and irrelevant directories + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") and d not in [".git", "__pycache__", ".pytest_cache", "node_modules", ".venv"] + ] + + for file in files: + if not file.endswith(".py") or file.startswith("."): + continue + + filepath = os.path.join(root, file) + relative_path = os.path.relpath(filepath, repo_path) + target_files.append((filepath, relative_path)) + + # If file_pattern specified, filter and search those first + if file_pattern and file_pattern != "**/*.py": + pattern_files = [] + + # Check if it's a glob pattern or substring match + is_glob = "*" in file_pattern or "?" in file_pattern or "[" in file_pattern + + for filepath, relative_path in target_files: + # Match against pattern + if is_glob: + # Use fnmatch for glob patterns + if fnmatch.fnmatch(relative_path, file_pattern): + pattern_files.append((filepath, relative_path)) + else: + # Use substring matching for non-glob patterns + if file_pattern in relative_path: + pattern_files.append((filepath, relative_path)) + + # Search pattern-matched files first + for filepath, relative_path in pattern_files: + try: + with open(filepath, "r", encoding="utf-8", errors="ignore") as f: + file_content = f.read() + lines = file_content.split("\n") + + line_nums = [] + for i, line in enumerate(lines, 1): + if qlower in line.lower(): + line_nums.append(i) + + if line_nums: + file_matches[relative_path] = line_nums + + if len(file_matches) >= max_files: + return file_matches + except (UnicodeDecodeError, PermissionError, OSError): + continue + + # If found in pattern files, return immediately + if file_matches: + return file_matches + + # Search all files if not found or no pattern specified + for filepath, relative_path in target_files: + # Skip files already searched in pattern phase + if relative_path in file_matches: + continue + + try: + with open(filepath, "r", encoding="utf-8", errors="ignore") as f: + file_content = f.read() + lines = file_content.split("\n") + + line_nums = [] + for i, line in enumerate(lines, 1): + if qlower in line.lower(): + line_nums.append(i) + + if line_nums: + file_matches[relative_path] = line_nums + + if len(file_matches) >= max_files: + return file_matches + except (UnicodeDecodeError, PermissionError, OSError): + continue + + return file_matches + + +def bm25_semantic_only(query_info: QueryInfo, include_files: Optional[List[str]] = None, similarity_top_k: int = 10): + """ + BM25 semantic search only (no exact substring matching). + Uses token-based matching for semantic similarity. + """ + if not CUSTOM_BM25_AVAILABLE: + return [] # Fallback not available + + query = query_info.term or "" + searcher = get_graph_entity_searcher() + cur_query_results = [] + global BM25_DOCS, BM25_META, BM25_MODEL + + try: + # Build BM25 index if needed + if BM25_DOCS is None or BM25_META is None or BM25_MODEL is None: + documents = [] + metadata = [] + for nid, data in searcher.G.nodes(data=True): + if data.get("type") == "file" and "code" in data: + documents.append(_simple_tokenize(data["code"])) + metadata.append({"nid": nid, "file_path": nid, "type": "file"}) + if not documents: + return [] + BM25_DOCS = documents + BM25_META = metadata + BM25_MODEL = BM25Okapi(BM25_DOCS) + + tokenized_query = _simple_tokenize(query) + if not tokenized_query: + return [] + + # Get BM25 scores + scores = BM25_MODEL.get_scores(tokenized_query) + top_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[: similarity_top_k * 2] + + # Process top-ranked files + for idx in top_indices: + if scores[idx] <= 0: + continue + + meta = BM25_META[idx] + file_path = meta["file_path"] + if include_files and file_path not in include_files: + continue + + node = searcher.G.nodes[file_path] + code_content = node.get("code", "") + if not code_content: + continue + + lines = code_content.split("\n") + q_tokens = [t for t in tokenized_query if len(t) >= 3] + hit_line = None + + # Find line with most token matches + for i, line in enumerate(lines, 1): + low = line.lower() + if any(t in low for t in q_tokens): + hit_line = i + break + + if hit_line is None: + continue # Skip files with no token matches + + start_line = max(1, hit_line - 2) + end_line = min(len(lines), hit_line + 2) + + query_result = QueryResult( + query_info=query_info, + format_mode="code_snippet", + nid=file_path, + file_path=file_path, + start_line=start_line, + end_line=end_line, + retrieve_src="Retrieved using BM25 semantic search.", + ) + cur_query_results.append(query_result) + + return cur_query_results[:3] + + except Exception as e: + print(f"Error in BM25 semantic search: {e}") + return [] + + +def search_code_snippets( + search_terms: Optional[List[str]] = None, + line_nums: Optional[List] = None, + file_path_or_pattern: Optional[str] = "**/*.py", +) -> str: + try: + parse_repo_index() + files, _, _ = get_current_repo_modules() + all_file_paths = [file["name"] for file in files] + + result = "" + if file_path_or_pattern: + include_files = find_matching_files_from_list(all_file_paths, file_path_or_pattern) + if not include_files: + include_files = all_file_paths + result += f"No files found for file pattern '{file_path_or_pattern}'. Will search all files.\n...\n" + else: + include_files = all_file_paths + except Exception as e: + print(f"Error in search_code_snippets initialization: {e}") + import traceback + + traceback.print_exc() + return f"Error: Failed to initialize search. {str(e)}" + + try: + query_info_list = [] + all_query_results = [] + + if search_terms: + filter_terms = [] + for term in search_terms: + if is_test_file(term): + result += f"Do not support searching for test files: `{term}`.\n\n" + else: + filter_terms.append(term) + + # Search each term independently + for term in filter_terms: + term = term.strip().strip(".") + if not term: + continue + + query_info = QueryInfo(term=term) + query_info_list.append(query_info) + cur_query_results = [] + entity_found = False + exact_substring_found = False + + # Step 1: Try entity search (exact node ID or fuzzy entity name) + query_results, continue_search = search_entity(query_info=query_info, include_files=include_files) + if query_results: + entity_found = True + cur_query_results.extend(query_results) + + # Step 2: ALWAYS try exact substring search (even if entity found) + # User might want usages, not just definitions + file_matches = exact_substring_search(term, file_pattern=file_path_or_pattern) + if file_matches: + exact_substring_found = True + num_files = len(file_matches) + + if num_files == 1: + # Single file: show line numbers and suggest str_replace_editor + file_path = list(file_matches.keys())[0] + line_nums = file_matches[file_path] + + message = f"Found {len(line_nums)} match(es) in `{file_path}`:\n" + message += f'Lines: {", ".join(map(str, line_nums[:20]))}' + if len(line_nums) > 20: + message += f" ... and {len(line_nums) - 20} more." + message += "\n**Tip**: Use `str_replace_editor view` with view_range to see the code content." + + query_result = QueryResult( + query_info=query_info, + format_mode="code_snippet", + nid=file_path, + file_path=file_path, + start_line=line_nums[0], + end_line=line_nums[0], + message=message, + retrieve_src="Exact substring match (single file).", + ) + cur_query_results.append(query_result) + + elif num_files <= 20: + # Multiple files (≤20): show file names and match counts + message = f"Found matches in {num_files} files:\n\n" + for file_path, line_nums in sorted(file_matches.items()): + message += f"- `{file_path}`: {len(line_nums)} match(es)" + message += "\n" + message += "\n**Tip**: Please narrow your search using `file_path_or_pattern` parameter." + + # Create a single result summarizing all files + first_file = list(file_matches.keys())[0] + query_result = QueryResult( + query_info=query_info, + format_mode="code_snippet", + nid=first_file, + file_path=first_file, + start_line=1, + end_line=1, + message=message, + retrieve_src="Exact substring match (multiple files).", + ) + cur_query_results.append(query_result) + + else: + # Too many files (>20): suggest narrowing search + message = f"Found matches in {num_files} files (showing first 20):\n\n" + for file_path, line_nums in list(sorted(file_matches.items()))[:20]: + message += f"- `{file_path}`: {len(line_nums)} match(es)\n" + message += "\n**Warning**: More than 20 files matched. Please narrow your search using `file_path_or_pattern` parameter." + + first_file = list(file_matches.keys())[0] + query_result = QueryResult( + query_info=query_info, + format_mode="code_snippet", + nid=first_file, + file_path=first_file, + start_line=1, + end_line=1, + message=message, + retrieve_src="Exact substring match (too many files).", + ) + cur_query_results.append(query_result) + + # Step 3: BM25 semantic search ONLY if no entity AND no exact substring + if not entity_found and not exact_substring_found: + query_results = bm25_semantic_only(query_info=query_info, include_files=include_files) + cur_query_results.extend(query_results) + + all_query_results.extend(cur_query_results) + + if file_path_or_pattern in all_file_paths and line_nums: + if isinstance(line_nums, int): + line_nums = [line_nums] + file_path = file_path_or_pattern + term = file_path + ":line " + ", ".join([str(line) for line in line_nums]) + query_info = QueryInfo(term=term, line_nums=line_nums, file_path_or_pattern=file_path) + query_results = get_code_block_by_line_nums(query_info) + all_query_results.extend(query_results) + + merged_results = merge_query_results(all_query_results) + ranked_query_to_results = rank_and_aggr_query_results(merged_results, query_info_list) + + searcher = get_graph_entity_searcher() + for query_infos, format_to_results in ranked_query_to_results.items(): + term_desc = ", ".join([f'"{query.term}"' for query in query_infos]) + result += f"##Searching for term {term_desc}...\n" + result += "### Search Result:\n" + cur_result = "" + for format_mode, query_results in format_to_results.items(): + if format_mode == "fold": + cur_retrieve_src = "" + for qr in query_results: + if not cur_retrieve_src: + cur_retrieve_src = qr.retrieve_src + if cur_retrieve_src != qr.retrieve_src: + cur_result += "Source: " + cur_retrieve_src + "\n\n" + cur_retrieve_src = qr.retrieve_src + cur_result += qr.format_output(searcher) + cur_result += "Source: " + cur_retrieve_src + "\n" + if len(query_results) > 1: + cur_result += "Hint: Use more detailed query to get the full content of some if needed.\n" + else: + cur_result += f"Hint: Search `{query_results[0].nid}` for the full content if needed.\n" + cur_result += "\n" + elif format_mode == "complete": + for qr in query_results: + cur_result += qr.format_output(searcher) + cur_result += "\n" + elif format_mode == "preview": + filtered_results = [] + grouped_by_file = defaultdict(list) + for qr in query_results: + if (qr.end_line - qr.start_line) < 100: + grouped_by_file[qr.file_path].append(qr) + else: + filtered_results.append(qr) + for file_path, results in grouped_by_file.items(): + sorted_results = sorted(results, key=lambda qr: (qr.start_line, -qr.end_line)) + max_end_line = -1 + for qr in sorted_results: + if qr.end_line > max_end_line: + filtered_results.append(qr) + max_end_line = max(max_end_line, qr.end_line) + for qr in filtered_results: + cur_result += qr.format_output(searcher) + cur_result += "\n" + elif format_mode == "code_snippet": + for qr in query_results: + cur_result += qr.format_output(searcher) + cur_result += "\n" + cur_result += "\n\n" + if cur_result.strip(): + result += cur_result + else: + result += "No locations found.\n\n" + + return result.strip() + except Exception as e: + print(f"Error in search_code_snippets execution: {e}") + import traceback + + traceback.print_exc() + return f"Error: Search failed. {str(e)}" + + +def main(): + parser = argparse.ArgumentParser(description="search_code_snippets tool") + + def parse_line_nums(line_str: str): + # Remove brackets if present + line_str = line_str.strip().strip("[]()") + # Split on commas or whitespace + parts = line_str.replace(",", " ").split() + try: + nums = [int(p) for p in parts] + except ValueError: + raise argparse.ArgumentTypeError(f"Could not convert {parts} to integers.") + if not nums: + raise argparse.ArgumentTypeError("At least one line number must be provided.") + return nums + + parser.add_argument( + "--file_path_or_pattern", + type=str, + default="**/*.py", + help="Glob or file path to filter search scope. If specific file with --line_nums, extracts context around lines.", + ) + parser.add_argument( + "--search_terms", + type=str, + action="append", + help="Repeatable search term(s). " + "Example: --search_terms MyClass --search_terms file.py:Func " + "or --search_terms '[MyClass, func_name]'", + ) + + parser.add_argument( + "--line_nums", + type=parse_line_nums, + action="append", + help="Repeatable line numbers when used with a specific file path. " + "Example: --line_nums 42 --line_nums 56 " + "or --line_nums '[42, 56, 78]'", + ) + args = parser.parse_args() + + # Handle JSON-formatted search terms + search_terms = args.search_terms + if search_terms and len(search_terms) == 1: + # Check if the single argument is a JSON array + try: + import json + + if search_terms[0].startswith("[") and search_terms[0].endswith("]"): + search_terms = json.loads(search_terms[0]) + except (json.JSONDecodeError, ValueError): + # Not JSON, use as-is + pass + + try: + output = search_code_snippets( + search_terms=search_terms, line_nums=args.line_nums, file_path_or_pattern=args.file_path_or_pattern + ) + print(output) + except Exception as e: + print(f"Error: {e}") + import traceback + + traceback.print_exc() + + +if __name__ == "__main__": + main() diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/scripts/tools/str_replace_editor.py b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/tools/str_replace_editor.py new file mode 100644 index 0000000000..0cfb31a279 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/swebench/scripts/tools/str_replace_editor.py @@ -0,0 +1,751 @@ +#!/root/.venv/bin/python + +""" +Description: Custom editing tool for viewing, creating and editing files +* State is persistent across command calls and discussions with the user +* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep +* The `create` command cannot be used if the specified `path` already exists as a file +* If a `command` generates a long output, it will be truncated and marked with `` +* The `undo_edit` command will revert the last edit made to the file at `path` + +Notes for using the `str_replace` command: +* The `old_str` parameter should match EXACTLY one or more consecutive lines from the original file. Be mindful of whitespaces! +* If the `old_str` parameter is not unique in the file, the replacement will not be performed. Make sure to include enough context in `old_str` to make it unique +* The `new_str` parameter should contain the edited lines that should replace the `old_str` + +Parameters: + (1) command (string, required): The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`. +Allowed values: [`view`, `create`, `str_replace`, `insert`, `undo_edit`] + (2) path (string, required): Absolute path to file or directory, e.g. `/testbed/file.py` or `/testbed`. + (3) file_text (string, optional): Required parameter of `create` command, with the content of the file to be created. + (4) old_str (string, optional): Required parameter of `str_replace` command containing the string in `path` to replace. + (5) new_str (string, optional): Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert. + (6) insert_line (integer, optional): Required parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`. + (7) view_range (array, optional): Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [11, 12] will show lines 11 and 12. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file. + (8) enable_linting (boolean, optional): Optional parameter to enable Python linting checks before saving changes. Default is `false`. + (9) concise (boolean, optional): Optional parameter to enable a condensed view for Python files. Default is `false`. Super useful for understanding the structure of a Python file without getting bogged down in the details. +""" + +import argparse +import json +import subprocess +from pathlib import Path +from collections import defaultdict +from typing import Dict, List, Tuple, Optional +import warnings + +import sys +import chardet + +# sys.stdout.reconfigure(encoding='utf-8') + +STATE_FILE = "/var/tmp/editor_state.json" +SNIPPET_LINES = 4 + +# We ignore certain warnings from tree_sitter (optional). +warnings.simplefilter("ignore", category=FutureWarning) + +_LINT_ERROR_TEMPLATE = """Your proposed edit has introduced new syntax error(s). +Please read this error message carefully and then retry editing the file. +ERRORS: +""" + +TRUNCATED_MESSAGE = ( + "To save on context only part of this file has been " + "shown to you. You should retry this tool after you have searched inside the file " + "with `grep -n` in order to find the line numbers of what you are looking for." +) +MAX_RESPONSE_LEN = 10000 # 4000 #12000 # 16000 + + +import io + +# sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") +if hasattr(sys.stdout, "buffer"): + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") +else: + # Fallback + sys.stderr.write("sys.stdout does not have a 'buffer' attribute.\n") + + +def safe_print(x): + try: + print(x) + except UnicodeEncodeError: + print(x.encode("utf-8", errors="replace").decode("utf-8", errors="replace")) + + +def maybe_truncate(content: str, truncate_after: Optional[int] = MAX_RESPONSE_LEN): + if not truncate_after or len(content) <= truncate_after: + return content + return content[:truncate_after] + TRUNCATED_MESSAGE + + +class EditorError(Exception): + """Raised for usage or file system errors within the editor tool.""" + + +class EditorResult: + """ + Simple container for output and optional error messages. + """ + + def __init__(self, output: str, error: str = ""): + self.output = output + self.error = error + + def __str__(self): + if self.error: + return "ERROR: {}\n\n{}".format(self.error, self.output) + return self.output + + +def load_history() -> Dict[str, List[str]]: + """ + Load the file edit history from STATE_FILE if it exists. + """ + try: + with open(STATE_FILE, "r", encoding="utf-8") as f: + data = json.load(f) + return {k: v for k, v in data.items()} + except FileNotFoundError: + return {} + except Exception as e: + safe_print("Warning: Could not load editor history from {}: {}".format(STATE_FILE, e)) + return {} + + +def save_history(history: Dict[str, List[str]]): + """ + Save the file edit history to STATE_FILE as JSON. + """ + try: + with open(STATE_FILE, "w", encoding="utf-8") as f: + json.dump(history, f) + except Exception as e: + safe_print("Warning: Could not write editor history to {}: {}".format(STATE_FILE, e)) + + +class StrReplaceEditor: + """ + A file editor that supports the following commands: + - view + - create + - str_replace + - insert + - undo_edit + + The edit history is kept in memory (self.file_history) and also persisted to disk. + + Additionally, a `--concise` option for `view` on Python files: + - Uses tree-sitter to skip large function bodies. + - Preserves original line numbering, printing placeholders for elided lines. + """ + + def __init__(self, file_history: Dict[str, List[str]], enable_linting: bool = False): + self.file_history = defaultdict(list, file_history) + self.enable_linting = enable_linting + + def run( + self, + command: str, + path_str: str, + file_text: str = None, + view_range: List[int] = None, + old_str: str = None, + new_str: str = None, + insert_line: int = None, + concise: bool = False, + python_only: bool = True, + ) -> EditorResult: + path = Path(path_str) + self.validate_path(command, path) + + if command == "view": + return self.view(path, view_range, concise=concise, python_only=python_only) + elif command == "create": + return self.create(path, file_text) + elif command == "str_replace": + return self.str_replace(path, old_str, new_str) + elif command == "insert": + return self.insert(path, insert_line, new_str) + elif command == "undo_edit": + return self.undo_edit(path) + else: + raise EditorError( + "Unrecognized command '{}'. " + "Allowed commands: view, create, str_replace, insert, undo_edit.".format(command) + ) + + def validate_path(self, command: str, path: Path): + if command == "create": + if path.exists(): + raise EditorError("File already exists at: {}. Cannot overwrite with 'create'.".format(path)) + else: + if not path.exists(): + raise EditorError("The path '{}' does not exist.".format(path)) + + if path.is_dir() and command != "view": + raise EditorError("The path '{}' is a directory. Only 'view' can be used on directories.".format(path)) + + @staticmethod + def read_path(path: Path) -> str: + encoding = chardet.detect(path.read_bytes())["encoding"] + if encoding is None: + encoding = "utf-8" + try: + return path.read_text(encoding=encoding) + except UnicodeDecodeError: + # Retry with UTF-8 if the detected encoding fails + return path.read_text(encoding="utf-8") + + def view( + self, + path: Path, + view_range: Optional[List[int]] = None, + concise: bool = False, + python_only: bool = True, + ) -> EditorResult: + """ + If path is a directory, list contents (2 levels deep, excluding hidden). + If path is a file, optionally use the 'concise' approach for Python. + Then apply [start_line, end_line] slicing if provided. + """ + if path.is_dir(): + if not python_only: + cmd = ["find", str(path), "-maxdepth", "2", "-not", "-path", "*/.*"] + else: + # Use `-type d -o -name '*.py'` to only list directories or *.py files + cmd = [ + "find", + str(path), + "-maxdepth", + "2", + "-not", + "-path", + "*/.*", + "(", + "-type", + "d", + "-o", + "-name", + "*.py", + ")", + ] + try: + # Try using the newer parameters first (Python 3.7+) + try: + proc = subprocess.run(cmd, capture_output=True, text=True, check=False) + except TypeError: + # Fallback for Python 3.5 and 3.6 where capture_output and text are not supported + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=False, + ) + + stderr = proc.stderr.strip() + stdout = proc.stdout + if stderr: + return EditorResult(output="", error=stderr) + + msg = ( + "Here's the files and directories up to 2 levels deep in {}, " + "excluding hidden:\n".format(path) + stdout + ) + msg = maybe_truncate(msg) + return EditorResult(output=msg) + except Exception as e: + return EditorResult( + output="", + error="Ran into {} while trying to list directory contents of {}.".format(e, path), + ) + + # ==================== + # NEW RESTRICTION: only .py files are allowed for viewing + if python_only and path.suffix not in {".py", ".diff"}: + error_msg = ( + "ERROR: Viewing non-Python files is disallowed for saving context. " + "File '{}' is not a .py file.".format(path.name) + ) + return EditorResult(output="", error=error_msg) + # ==================== + + # ----------------------------------------------- + # NEW LOGIC for deciding whether to use 'concise' mode + # ----------------------------------------------- + # If no view_range is given, user did NOT explicitly request 'concise', + # and we have a Python file with more than 50 lines => default to concise + if path.suffix == ".py" and not view_range and not concise: + file_text_tmp = self.read_path(path) + if len(file_text_tmp.splitlines()) > 110: + concise = True + + # For a file + if path.suffix == ".py" and concise: + # Use the tree_sitter approach + lines_with_original_numbers = self._get_elided_lines(path) + else: + # Normal reading + file_text = self.read_path(path) + lines_with_original_numbers = [(i, line) for i, line in enumerate(file_text.splitlines())] + + # Optionally slice by [start_line, end_line] + total_lines = len(lines_with_original_numbers) + if view_range and len(view_range) == 2: + start, end = view_range + if not (1 <= start <= total_lines): + return EditorResult( + output="", + error=("Invalid view_range {}: start line must be in [1, {}]".format(view_range, total_lines)), + ) + if end != -1 and (end < start or end > total_lines): + return EditorResult( + output="", + error=( + "Invalid view_range {}: end must be >= start " + "and <= {}, or -1 to view until end.".format(view_range, total_lines) + ), + ) + + # Filter lines by 1-based index + sliced_lines = [] + for i, text in lines_with_original_numbers: + one_based = i + 1 + if one_based < start: + continue + if end != -1 and one_based > end: + continue + sliced_lines.append((i, text)) + else: + # No slicing + sliced_lines = lines_with_original_numbers + + # for .diff files, we do not show line numbers + if path.suffix == ".diff": + final_output = "" + output_str_list = [text for i, text in sliced_lines] + final_output += "\n".join(output_str_list) + return EditorResult(output=final_output) + + # Now produce a cat-like output (line numbering = i+1) + if concise: + final_output = "Here is a condensed view for file: {}; [Note: Useful for understanding file structure in a concise manner. Please use specific view_range without concise cmd if you want to explore further into the relevant parts.]\n".format( + path + ) + else: + final_output = "Here's the result of running `cat -n` on the file: {}:\n".format(path) + # Then maybe truncate + output_str_list = [] + for i, text in sliced_lines: + # i is 0-based + output_str_list.append("{:6d} {}".format(i + 1, text)) + + final_output += "\n".join(output_str_list) + final_output = maybe_truncate(final_output) + return EditorResult(output=final_output) + + def _get_elided_lines(self, path: Path) -> List[Tuple[int, str]]: + """ + Parse the Python file with the built-in 'ast' module to skip + large function bodies (≥ 5 lines). + Return a list of (zero_based_line_idx, text). + """ + import ast + + file_text = self.read_path(path) + try: + tree = ast.parse(file_text, filename=str(path)) + except SyntaxError as e: + # Raise EditorError to make sure we handle it gracefully upstream + raise EditorError("Syntax error for file {}: {}".format(path, e)) + + def max_lineno_in_subtree(n: ast.AST) -> int: + m = getattr(n, "lineno", 0) + for child in ast.iter_child_nodes(n): + m = max(m, max_lineno_in_subtree(child)) + return m + + # We'll gather the line ranges for all large function bodies + elide_line_ranges = [] + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.body: + # Attempt to use end_lineno (Python 3.8+), else fallback + last_stmt = node.body[-1] + if hasattr(last_stmt, "end_lineno") and last_stmt.end_lineno: + # 0-based start of the body (first statement in the function) + body_start = node.body[0].lineno - 1 + + body_end = last_stmt.end_lineno - 1 + else: + body_start = node.body[0].lineno - 1 + + docstring = ast.get_docstring(node) + + if docstring: + num_docstring_lines = docstring.count("\n") + 1 + body_start -= num_docstring_lines + + # Fallback: traverse the subtree to find max lineno + body_end = max_lineno_in_subtree(last_stmt) - 1 + + # Skip the body if spans ≥ 5 lines + if (body_end - body_start) >= 3: + elide_line_ranges.append((body_start, body_end)) + + if isinstance(node, ast.ClassDef) and node.body: + # elide large docstrings for classes + has_docstring1 = ( + isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant) + and isinstance(node.body[0].value.value, str) + ) + has_docstring2 = isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Str) + has_docstring = has_docstring1 or has_docstring2 + if has_docstring: + if hasattr(node.body[0], "end_lineno") and node.body[0].end_lineno: + docstring_start = node.body[0].lineno - 1 + docstring_end = node.body[0].end_lineno - 1 + else: + docstring_start = node.lineno + docstring_end = max_lineno_in_subtree(node.body[0]) - 1 + + if (docstring_end - docstring_start) >= 4: + elide_line_ranges.append((docstring_start + 1, docstring_end - 1)) + + # Build a set of lines to skip + elide_lines = {line for (start, end) in elide_line_ranges for line in range(start, end + 1)} + + # Add an "elision notice" at the beginning of each range + elide_messages = [ + (start, "... eliding lines {}-{} ...".format(start + 1, end + 1)) for (start, end) in elide_line_ranges + ] + + # Lines we do keep + all_lines = file_text.splitlines() + keep_lines = [(i, line) for i, line in enumerate(all_lines) if i not in elide_lines] + + # Combine and sort by line index + combined = elide_messages + keep_lines + combined.sort(key=lambda x: x[0]) + + return combined + + def create(self, path: Path, file_text: str) -> EditorResult: + if file_text is None: + raise EditorError("Cannot create file without 'file_text' parameter.") + + if self.enable_linting and path.suffix == ".py": + lint_error = self._lint_check(file_text, str(path)) + if lint_error: + return EditorResult(output="", error=_LINT_ERROR_TEMPLATE + lint_error) + + try: + path.write_text(file_text, encoding="utf-8") + self.file_history[str(path)].append("") + except Exception as e: + raise EditorError("Error creating file at {}: {}".format(path, e)) + + success_msg = "File created at {}. ".format(path) + success_msg += self._make_output(file_text, str(path)) + success_msg += "Review the file and make sure that it is as expected. Edit the file if necessary." + + return EditorResult(output="{}".format(success_msg)) + + def str_replace(self, path: Path, old_str: str, new_str: str) -> EditorResult: + if old_str is None: + raise EditorError("Missing required parameter 'old_str' for 'str_replace'.") + + file_content = self.read_file(path).expandtabs() + old_str = old_str.expandtabs() + new_str = new_str.expandtabs() if new_str else "" + occurrences = file_content.count(old_str) + if occurrences == 0: + # Provide guidance with a few close matches to help the agent craft a unique old_str + try: + import difflib + + query = old_str.strip().splitlines()[0] if old_str.strip() else old_str.strip() + candidates = [] + if query: + candidates = difflib.get_close_matches(query, file_content.splitlines(), n=3, cutoff=0.6) + suggestion = "" + if candidates: + suggestion = "\nPossible similar lines:\n" + "\n".join("- {}".format(c[:120]) for c in candidates) + except Exception: + suggestion = "" + raise EditorError("No occurrences of '{}' found in {} for replacement.{}".format(old_str, path, suggestion)) + if occurrences > 1: + # List the first few match locations (line numbers and line text) to aid disambiguation + positions = [] + start_idx = 0 + while True: + idx = file_content.find(old_str, start_idx) + if idx == -1: + break + line_no = file_content[:idx].count("\n") + 1 + positions.append(line_no) + start_idx = idx + 1 + # previews = [] + # for i, pos in enumerate(positions[:5]): + # line_no = file_content[:pos].count("\n") + 1 + # ctx_start = file_content.rfind("\n", 0, pos) + # ctx_start = 0 if ctx_start == -1 else ctx_start + 1 + # ctx_end = file_content.find("\n", pos + len(old_str)) + # if ctx_end == -1: + # ctx_end = len(file_content) + # context = file_content[ctx_start:ctx_end].expandtabs() + # previews.append(f"- match {i+1}: line {line_no}: {context[:160]}") + # hints = "\nHere are the first matches to help you disambiguate:\n" + "\n".join(previews) if previews else "" + raise EditorError( + "Multiple occurrences of '{}' found in lines {}. Please ensure it is unique before using str_replace.".format( + old_str, positions + ) + ) + + old_text = file_content + updated_text = file_content.replace(old_str, new_str if new_str else "") + + if self.enable_linting and path.suffix == ".py": + lint_error = self._lint_check(updated_text, str(path)) + if lint_error: + return EditorResult(output="", error=_LINT_ERROR_TEMPLATE + lint_error) + + self.file_history[str(path)].append(old_text) + self.write_file(path, updated_text) + + # Original snippet logic + replacement_line = file_content.split(old_str)[0].count("\n") + start_line = max(0, replacement_line - SNIPPET_LINES) + end_line = replacement_line + SNIPPET_LINES + (new_str or "").count("\n") + snippet = "\n".join(updated_text.split("\n")[start_line : end_line + 1]) + + success_msg = "The file {} has been edited. ".format(path) + success_msg += self._make_output(snippet, "a snippet of {}".format(path), start_line + 1) + success_msg += "Review the changes and make sure they are as expected. Edit the file again if necessary." + + return EditorResult(output=success_msg) + + def insert(self, path: Path, insert_line: int, new_str: str) -> EditorResult: + if new_str is None: + raise EditorError("Missing required parameter 'new_str' for 'insert'.") + + old_text = self.read_file(path).expandtabs() + new_str = new_str.expandtabs() + file_text_lines = old_text.split("\n") + + if insert_line < 0 or insert_line > len(file_text_lines): + raise EditorError("Invalid insert_line {}. Must be in [0, {}].".format(insert_line, len(file_text_lines))) + + new_str_lines = new_str.split("\n") + new_file_text_lines = file_text_lines[:insert_line] + new_str_lines + file_text_lines[insert_line:] + updated_text = "\n".join(new_file_text_lines) + + if self.enable_linting and path.suffix == ".py": + lint_error = self._lint_check(updated_text, str(path)) + if lint_error: + return EditorResult(output="", error=_LINT_ERROR_TEMPLATE + lint_error) + + self.file_history[str(path)].append(old_text) + self.write_file(path, updated_text) + + # Original snippet logic + snippet_lines = ( + file_text_lines[max(0, insert_line - SNIPPET_LINES) : insert_line] + + new_str_lines + + file_text_lines[insert_line : insert_line + SNIPPET_LINES] + ) + snippet = "\n".join(snippet_lines) + + success_msg = "The file {} has been edited. ".format(path) + success_msg += self._make_output( + snippet, + "a snippet of the edited file", + max(1, insert_line - SNIPPET_LINES + 1), + ) + success_msg += ( + "Review the changes and make sure they are as expected " + "(correct indentation, no duplicate lines, etc). Edit the file again if necessary." + ) + + return EditorResult(output=success_msg) + + def undo_edit(self, path: Path) -> EditorResult: + path_str = str(path) + if not self.file_history[path_str]: + raise EditorError("No previous edits found for {} to undo.".format(path)) + + old_text = self.file_history[path_str].pop() + self.write_file(path, old_text) + + return EditorResult( + output=("Last edit to {} undone successfully. ".format(path) + self._make_output(old_text, str(path))) + ) + + def read_file(self, path: Path) -> str: + try: + return self.read_path(path) + except Exception as e: + raise EditorError("Failed to read file {}: {}".format(path, e)) + + def write_file(self, path: Path, content: str): + try: + path.write_text(content, encoding="utf-8") + except Exception as e: + raise EditorError("Failed to write file {}: {}".format(path, e)) + + def _make_output( + self, + file_content: str, + file_descriptor: str, + init_line: int = 1, + expand_tabs: bool = True, + ) -> str: + """ + Mimics cat -n style numbering, plus maybe_truncate to avoid huge outputs. + """ + file_content = maybe_truncate(file_content) + if expand_tabs: + file_content = file_content.expandtabs() + + lines = file_content.split("\n") + numbered = "\n".join("{:6}\t{}".format(i + init_line, line) for i, line in enumerate(lines)) + return "Here's the result of running `cat -n` on {}:\n".format(file_descriptor) + numbered + "\n" + + def _lint_check(self, new_content: str, file_path: str) -> str: + import ast + + try: + ast.parse(new_content, filename=file_path) + return "" + except SyntaxError as e: + return str(e) + + +def main(): + def parse_view_range(range_str: str): + # Remove surrounding brackets if present + range_str = range_str.strip().strip("[]()") + + # Split on commas or whitespace + parts = range_str.replace(",", " ").split() + + if len(parts) != 2: + raise argparse.ArgumentTypeError("Expected two numbers, got: {}".format(range_str)) + try: + start_line = int(parts[0]) + end_line = int(parts[1]) + except ValueError: + raise argparse.ArgumentTypeError("Could not convert {} to integers.".format(parts)) + return [start_line, end_line] + + def parse_insert_line(line_str: str) -> int: + try: + return int(line_str) + except ValueError: + raise argparse.ArgumentTypeError("Invalid line number: {}".format(line_str)) + + parser = argparse.ArgumentParser( + description=( + "A disk-backed file editing tool (view, create, str_replace, insert, undo_edit) " + "with optional linting. Also supports a 'concise' view for Python files." + ) + ) + parser.add_argument( + "command", + type=str, + help="One of: view, create, str_replace, insert, undo_edit", + ) + parser.add_argument( + "--path", + type=str, + help="Path to the target file or directory (absolute path recommended)", + ) + parser.add_argument( + "--file_text", + type=str, + default=None, + help="File content (for 'create')", + ) + parser.add_argument( + "--view_range", + type=parse_view_range, + default=None, + help="Line range to view [start_line, end_line], use -1 for end.", + ) + parser.add_argument( + "--old_str", + type=str, + default=None, + help="Old string (for 'str_replace')", + ) + parser.add_argument( + "--new_str", + type=str, + default=None, + help="New string (for 'str_replace' or 'insert')", + ) + parser.add_argument( + "--insert_line", + type=parse_insert_line, + default=None, + help="Line number to insert text at (for 'insert')", + ) + parser.add_argument( + "--enable_linting", + type=bool, + default=False, + help="Enable linting checks for Python files before saving changes.", + ) + parser.add_argument( + "--concise", + type=str, + default="False", + help="If True, attempts to produce a condensed view for Python files, eliding large function bodies.", + ) + parser.add_argument( + "--python_only", + type=bool, + default=True, + help="If True, attempts to limit view (for both dir and file level) to Python files only.", + ) + + args = parser.parse_args() + + file_history = load_history() + editor = StrReplaceEditor(file_history, enable_linting=args.enable_linting) + if args.concise.lower() == "true": + args.concise = True + else: + args.concise = False + + try: + result = editor.run( + command=args.command, + path_str=args.path, + file_text=args.file_text, + view_range=args.view_range, + old_str=args.old_str, + new_str=args.new_str, + insert_line=args.insert_line, + concise=args.concise, + ) + safe_print(result.output) + if result.error: + safe_print("ERROR: {}".format(result.error)) + + except EditorError as e: + safe_print("ERROR: {}".format(e)) + except Exception as e: + safe_print("ERROR: Unhandled exception: {}".format(e)) + import traceback + + traceback.print_exc() + + save_history(dict(editor.file_history)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-agent/skyrl_agent/tasks/swebench/utils.py b/skyrl-agent/skyrl_agent/tasks/swebench/utils.py new file mode 100644 index 0000000000..363f7fbf20 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/swebench/utils.py @@ -0,0 +1,1044 @@ +import pandas as pd +import numpy as np +from typing import Any +import os +import json +import tempfile +import time +import re +import asyncio + +from openhands.core.main import create_runtime +from openhands.utils.shutdown_listener import sleep_if_should_continue +from openhands.core.logger import openhands_logger as logger +from openhands.core.config import SandboxConfig, AppConfig +from openhands.events.action import CmdRunAction, MessageAction, FileReadAction +from openhands.events.observation import CmdOutputObservation, ErrorObservation, FileReadObservation +from openhands.runtime.base import Runtime +from openhands.core.exceptions import ( + AgentRuntimeBuildError, + AgentRuntimeDisconnectedError, + AgentRuntimeError, + AgentRuntimeNotFoundError, + AgentRuntimeNotReadyError, + AgentRuntimeTimeoutError, + AgentRuntimeUnavailableError, +) +from skyrl_agent.tasks.base import BaseTask +from skyrl_agent.dispatcher.async_utils import call_sync_from_async + +DOCKER_IMAGE_PREFIX = os.environ.get("EVAL_DOCKER_IMAGE_PREFIX", "docker.io/xingyaoww/") +logger.info(f"Using docker image prefix: {DOCKER_IMAGE_PREFIX}") + + +def ensure_serializable(obj): + """Recursively convert non-serializable objects to JSON serializable formats.""" + if isinstance(obj, np.ndarray): # Convert numpy arrays to lists + return obj.tolist() + elif isinstance(obj, (np.integer, int)): # Convert numpy int to Python int + return int(obj) + elif isinstance(obj, (np.floating, float)): # Convert numpy float to Python float + return float(obj) + elif isinstance(obj, dict): # Recursively process dictionaries + return {key: ensure_serializable(value) for key, value in obj.items()} + elif isinstance(obj, list): # Recursively process lists + return [ensure_serializable(item) for item in obj] + elif isinstance(obj, tuple): # Convert tuples to lists + return tuple(ensure_serializable(item) for item in obj) + return obj # Return as is if already serializable + + +class EvalException(Exception): + pass + + +def assert_and_raise(condition: bool, msg: str): + """Raise an EvalException if the condition is not met. + + This will be used in conjunction with _process_instance_wrapper to handle retries. An EvalException should trigger a retry. + """ + if not condition: + raise EvalException(msg) + + +RUN_WITH_BROWSING = os.environ.get("RUN_WITH_BROWSING", "false").lower() == "true" + + +def _get_swebench_workspace_dir_name(instance: pd.Series, dataset: str) -> str: + if "r2e-gym" in dataset: + return "/testbed" + return "/workspace/" + f'{instance.repo}__{getattr(instance, "version", "null")}'.replace("/", "__") + + +# def get_instruction(instance: pd.Series): +# workspace_dir_name = _get_swebench_workspace_dir_name(instance) +# instruction = f""" +# +# {workspace_dir_name} +# + +# I've uploaded a python code repository in the directory {workspace_dir_name}. Consider the following issue description: + +# +# {instance.problem_statement} +# + +# Can you help me implement the necessary changes to the repository so that the requirements specified in the are met? +# I've already taken care of all changes to any of the test files described in the . This means you DON'T have to modify the testing logic or any of the tests in any way! +# Also the development Python environment is already set up for you (i.e., all dependencies already installed), so you don't need to install other packages. +# Your task is to make the minimal changes to non-test files in the {workspace_dir_name} directory to ensure the is satisfied. + +# Follow these phases to resolve the issue: + +# Phase 1. READING: read the problem and reword it in clearer terms +# 1.1 If there are code or config snippets. Express in words any best practices or conventions in them. +# 1.2 Hightlight message errors, method names, variables, file names, stack traces, and technical details. +# 1.3 Explain the problem in clear terms. +# 1.4 Enumerate the steps to reproduce the problem. +# 1.5 Hightlight any best practices to take into account when testing and fixing the issue + +# Phase 2. RUNNING: install and run the tests on the repository +# 2.1 Follow the readme +# 2.2 Install the environment and anything needed +# 2.2 Iterate and figure out how to run the tests + +# Phase 3. EXPLORATION: find the files that are related to the problem and possible solutions +# 3.1 Use `grep` to search for relevant methods, classes, keywords and error messages. +# 3.2 Identify all files related to the problem statement. +# 3.3 Propose the methods and files to fix the issue and explain why. +# 3.4 From the possible file locations, select the most likely location to fix the issue. + +# Phase 4. TEST CREATION: before implementing any fix, create a script to reproduce and verify the issue. +# 4.1 Look at existing test files in the repository to understand the test format/structure. +# 4.2 Create a minimal reproduction script that reproduces the located issue. +# 4.3 Run the reproduction script to confirm you are reproducing the issue. +# 4.4 Adjust the reproduction script as necessary. + +# Phase 5. FIX ANALYSIS: state clearly the problem and how to fix it +# 5.1 State clearly what the problem is. +# 5.2 State clearly where the problem is located. +# 5.3 State clearly how the test reproduces the issue. +# 5.4 State clearly the best practices to take into account in the fix. +# 5.5 State clearly how to fix the problem. + +# Phase 6. FIX IMPLEMENTATION: Edit the source code to implement your chosen solution. +# 6.1 Make minimal, focused changes to fix the issue. + +# Phase 7. VERIFICATION: Test your implementation thoroughly. +# 7.1 Run your reproduction script to verify the fix works. +# 7.2 Add edge cases to your test script to ensure comprehensive coverage. +# 7.3 Run existing tests related to the modified code to ensure you haven't broken anything. + +# 8. FINAL REVIEW: Carefully re-read the problem description and compare your changes with the base commit {instance['base_commit']}. +# 8.1 Ensure you've fully addressed all requirements. +# 8.2 Run any tests in the repository related to: +# 8.2.1 The issue you are fixing +# 8.2.2 The files you modified +# 8.2.3 The functions you changed +# 8.3 If any tests fail, revise your implementation until all tests pass + +# Be thorough in your exploration, testing, and reasoning. It's fine if your thinking process is lengthy - quality and completeness are more important than brevity. +# """ + +# if RUN_WITH_BROWSING: +# instruction += ( +# '\nYou SHOULD NEVER attempt to browse the web. \n' +# ) + +# if 'image_assets' in instance: +# assets = json.loads(instance['image_assets']) +# assert 'problem_statement' in assets, ( +# 'problem_statement is required in image_assets' +# ) +# image_urls = assets['problem_statement'] +# return MessageAction(content=instruction, image_urls=image_urls) +# return MessageAction(content=instruction) + + +def initialize_runtime(runtime: Runtime, instance: pd.Series, dataset: str): + logger.info("-" * 30) + logger.info("BEGIN Runtime Initialization Fn") + logger.info("-" * 30) + workspace_dir_name = _get_swebench_workspace_dir_name(instance, dataset) + obs: CmdOutputObservation + + # Set instance id and git configuration + action = CmdRunAction( + command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc && git config --global core.pager "" && git config --global diff.binary false""" + ) + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + obs.exit_code == 0, + f"Failed to export SWE_INSTANCE_ID and configure git: {str(obs)}", + ) + + action = CmdRunAction(command="""export USER=$(whoami); echo USER=${USER} """) + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise(obs.exit_code == 0, f"Failed to export USER: {str(obs)}") + + # inject the init script + script_dir = os.path.dirname(__file__) + + # inject the instance info + action = CmdRunAction(command="mkdir -p /swe_util/eval_data/instances") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + obs.exit_code == 0, + f"Failed to create /swe_util/eval_data/instances: {str(obs)}", + ) + + # @(csy)todo: make it go through all the tools + # copy the search tools to the runtime + for tool_name in ["search", "str_replace_editor"]: + runtime.copy_to( + str(os.path.join(script_dir, f"scripts/tools/{tool_name}.py")), + "/usr/local/bin", + ) + # strip the .py suffix + runtime.run_action(CmdRunAction(command=f"mv /usr/local/bin/{tool_name}.py /usr/local/bin/{tool_name}")) + runtime.run_action(CmdRunAction(command=f"chmod +x /usr/local/bin/{tool_name}")) + + swe_instance_json_name = "swe-bench-instance.json" + with tempfile.TemporaryDirectory() as temp_dir: + # Construct the full path for the desired file name within the temporary directory + temp_file_path = os.path.join(temp_dir, swe_instance_json_name) + # Write to the file with the desired name within the temporary directory + with open(temp_file_path, "w") as f: + if not isinstance(instance, dict): + json.dump([instance.to_dict()], f) + else: + json.dump([instance], f) + + # Copy the file to the desired location + runtime.copy_to(temp_file_path, "/swe_util/eval_data/instances/") + + # inject the instance swe entry + if "r2e-gym" in dataset: + runtime.copy_to( + str(os.path.join(script_dir, "scripts/setup/instance_r2e_entry.sh")), + "/swe_util/", + ) + else: + runtime.copy_to( + str(os.path.join(script_dir, "scripts/setup/instance_swe_entry.sh")), + "/swe_util/", + ) + + action = CmdRunAction(command="cat ~/.bashrc") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise(obs.exit_code == 0, f"Failed to cat ~/.bashrc: {str(obs)}") + + action = CmdRunAction(command="source ~/.bashrc") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + if isinstance(obs, ErrorObservation): + logger.error(f"Failed to source ~/.bashrc: {str(obs)}") + assert_and_raise(obs.exit_code == 0, f"Failed to source ~/.bashrc: {str(obs)}") + + if "r2e-gym" in dataset: + action = CmdRunAction(command="source /swe_util/instance_r2e_entry.sh") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + obs.exit_code == 0, + f"Failed to source /swe_util/instance_r2e_entry.sh: {str(obs)}", + ) + else: + action = CmdRunAction(command="source /swe_util/instance_swe_entry.sh") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + obs.exit_code == 0, + f"Failed to source /swe_util/instance_swe_entry.sh: {str(obs)}", + ) + + action = CmdRunAction(command=f"cd {workspace_dir_name}") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + obs.exit_code == 0, + f"Failed to cd to {workspace_dir_name}: {str(obs)}", + ) + + # git fetch and checkout needed by swe-smith + if dataset == "swe-smith": + action = CmdRunAction(command="git fetch") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise(obs.exit_code == 0, f"Failed to git fetch: {str(obs)}") + + action = CmdRunAction(command=f'git checkout {instance["instance_id"]}') + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + obs.exit_code == 0, + f'Failed to git checkout {instance["instance_id"]}: {str(obs)}', + ) + + if "r2e-gym" not in dataset: + action = CmdRunAction(command="git reset --hard") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise(obs.exit_code == 0, f"Failed to git reset --hard: {str(obs)}") + + action = CmdRunAction(command='for remote_name in $(git remote); do git remote remove "${remote_name}"; done') + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise(obs.exit_code == 0, f"Failed to remove git remotes: {str(obs)}") + + if dataset == "swt-ci": + # set up repo + setup_commands = [] + if instance["repo"] in MAP_REPO_TO_INSTALL: + setup_commands.append(MAP_REPO_TO_INSTALL[instance["repo"]]) + + # Run pre-install set up if provided + install = MAP_VERSION_TO_INSTALL.get(instance["repo"], {}).get(instance["version"], []) + if "pre_install" in install: + for pre_install in install["pre_install"]: + setup_commands.append(pre_install) + + if "install" in install: + setup_commands.append(install["install"]) + + for command in setup_commands: + action = CmdRunAction(command=command) + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + + if "multimodal" not in dataset: + # Only for non-multimodal datasets, we need to activate the testbed environment for Python + # SWE-Bench multimodal datasets are not using the testbed environment + action = CmdRunAction(command="which python") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + obs.exit_code == 0 and "testbed" in obs.metadata.py_interpreter_path, + f"Expected to find python interpreter from testbed, but got: {str(obs)}", + ) + + logger.info("-" * 30) + logger.info("END Runtime Initialization Fn") + logger.info("-" * 30) + + +class SWEBenchTask(BaseTask): + @classmethod + def get_instruction(cls, instance: pd.Series, dataset: str): + workspace_dir_name = _get_swebench_workspace_dir_name(instance, dataset) + instruction = f""" + + {workspace_dir_name} + + + I've uploaded a python code repository in the directory {workspace_dir_name}. Consider the following issue description: + + + {instance.problem_statement} + + + Can you help me implement the necessary changes to the repository so that the requirements specified in the are met? + I've already taken care of all changes to any of the test files described in the . This means you DON'T have to modify the testing logic or any of the tests in any way! + Your task is to make the minimal changes to non-test files in the {workspace_dir_name} directory to ensure the is satisfied. + + Follow these steps to resolve the issue: + 1. First, explore the codebase to locate and understand the code relevant to the . + - Use efficient search commands to identify key files and functions. + - You should err on the side of caution and look at various relevant files and build your understanding of + - how the code works + - what are the expected behaviors and edge cases + - what are the potential root causes for the given issue + + 2. Assess whether you can reproduce the issue: + - Create a script at {workspace_dir_name}/reproduce_issue.py that demonstrates the error. + - Execute this script to confirm the error behavior. + - You should reproduce the issue before fixing it. + - Your reproduction script should also assert the expected behavior for the fixed code. + + 3. Analyze the root cause: + - Identify the underlying problem based on your code exploration and reproduction results. + - Critically analyze different potential approaches to fix the issue. + - You NEED to explicitly reason about multiple approaches to fix the issue. Next, find the most elegant and effective solution among them considering the tradeoffs (correctness, generality, side effects, etc.). + - You would need to reason about execution paths, edge cases, and other potential issues. You should look at the unit tests to understand the expected behavior of the relevant code. + + 4. Implement your solution: + - Make targeted changes to the necessary files following idiomatic code patterns once you determine the root cause. + - You should be thorough and methodical. + + 5. Verify your solution: + - Rerun your reproduction script to confirm the error is fixed. + - If verification fails, iterate on your solution until successful. If you identify the reproduction script is buggy, adjust it as needed. + + 6. Run unit tests: + - Find and run the relevant unit tests relevant to the performed fix. + - You should run the unit tests to ensure your solution is correct and does not cause any regressions. + - In cases where the unit tests are do not pass, you should consider whether the unit tests does not reflect the *new* expected behavior of the code. If so, you can test it by writing additional edge test cases. + - Use the existing test runner to run the unit tests you identify as relevant to the changes you made. For example: + - `python -m pytest -xvs sympy/physics/units/tests/test_dimensions_transcendental.py` + - `python -m pytest tests/test_domain_py.py::test_pymethod_options` + - `./tests/runtests.py constraints.tests.CheckConstraintTests -v 2` + - RUN ALL relevant unit tests to ensure your solution is correct and does not cause any regressions. + + 7. Test edge cases: + - Identify potential edge cases that might challenge your solution. + - Create additional test cases in a separate file {workspace_dir_name}/edge_case_tests.py. + - Execute these tests to verify your solution's robustness. + - You should run multiple rounds of edge cases. When creating edge cases: + - Consider complex scenarios beyond the original issue description + - Test for regressions to ensure existing functionality remains intact + + 8. Refine if necessary: + - If edge case testing reveals issues, refine your solution accordingly. + - Ensure your final implementation handles all identified scenarios correctly. + - Document any assumptions or limitations of your solution. + + 9. Submit your solution: + - Once you have verified your solution, submit your solution using the `finish` tool. + + A successful resolution means: + - The specific error/issue described no longer occurs + - Your changes maintain compatibility with existing functionality + - Edge cases are properly handled + + """ + return MessageAction(content=instruction) + + @classmethod + def get_config(cls, instance, data_source, agent_config=None, max_iterations=None) -> AppConfig: + # Configure sandbox + RUN_WITH_BROWSING = os.environ.get("RUN_WITH_BROWSING", "false").lower() == "true" + SWE_BENCH_CONTAINER_IMAGE = "ghcr.io/opendevin/eval-swe-bench:full-v1.2.1" + + if os.environ.get("USE_INSTANCE_IMAGE", "true").lower() == "true": + # Use a different instance image for each instance of swe-bench eval + base_container_image = get_instance_docker_image(instance, data_source) + logger.info( + f"Using instance container image: {base_container_image}. " + f"Please make sure this image exists. " + f"Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues." + ) + else: + base_container_image = SWE_BENCH_CONTAINER_IMAGE + logger.info(f"Using swe-bench container image: {base_container_image}") + + sandbox_config = get_default_sandbox_config_for_eval() + sandbox_config.base_container_image = base_container_image + sandbox_config.enable_auto_lint = True + sandbox_config.use_host_network = False + sandbox_config.platform = "linux/amd64" + extra_kwargs = {} + if max_iterations is not None: + extra_kwargs["max_iterations"] = max_iterations + + app_config = AppConfig( + run_as_openhands=False, + runtime="remote", + sandbox=sandbox_config, + workspace_base=None, + workspace_mount_path=None, + **extra_kwargs, + ) + if agent_config is not None: + app_config.set_agent_config(agent_config) + + return app_config + + @classmethod + async def initialize_runtime(cls, instance: pd.Series, dataset, agent_cfg, max_iterations): + """Initialize the runtime for the agent. + + This function is called before the runtime is used to run the agent. + """ + app_config = cls.get_config(instance, dataset, agent_cfg, max_iterations) + + # Create runtime + runtime = create_runtime(app_config) + + # Connect runtime + # Retry loop for runtime_initializer + max_retries = 3 + retry_delay = 2 # initial delay in seconds + for attempt in range(1, max_retries + 1): + try: + await runtime.connect() + break # success, exit retry loop + except Exception as e_inner: + if attempt < max_retries: + logger.warning( + f"[Retry {attempt}/{max_retries}] runtime_initializer failed for " + f"{str(e_inner)}. " + f"Retrying in {retry_delay} sec..." + ) + await asyncio.sleep(retry_delay) + retry_delay *= 2 # exponential backoff + else: + raise # re-raise if max retries exceeded + + await call_sync_from_async(initialize_runtime, runtime, instance, dataset) + + return runtime + + @classmethod + def complete_runtime( + cls, + runtime: Runtime, + instance: pd.Series, # this argument is not required, but it is used to get the workspace_dir_name + dataset: str, + ) -> dict[str, Any]: + """Complete the runtime for the agent. + + This function is called before the runtime is used to run the agent. + If you need to do something in the sandbox to get the correctness metric after + the agent has run, modify this function. + """ + logger.info("-" * 30) + logger.info("BEGIN Runtime Completion Fn") + logger.info("-" * 30) + obs: CmdOutputObservation + workspace_dir_name = _get_swebench_workspace_dir_name(instance, dataset) + + action = CmdRunAction(command=f"cd {workspace_dir_name}") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + + if obs.exit_code == -1: + # The previous command is still running + # We need to kill previous command + logger.info("The previous command is still running, trying to kill it...") + action = CmdRunAction(command="C-c", is_input=True) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + + # Then run the command again + action = CmdRunAction(command=f"cd {workspace_dir_name}") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + + if obs.exit_code == -1: + # The previous command is still running + # We need to kill previous command + logger.info("The previous command is still running, trying to ctrl+z it...") + action = CmdRunAction(command="C-z", is_input=True) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + + # Then run the command again + action = CmdRunAction(command=f"cd {workspace_dir_name}") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + + assert_and_raise( + isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, + f"Failed to cd to {workspace_dir_name}: {str(obs)}", + ) + + # r2e directly run the tests + if "r2e-gym" in dataset: + action = CmdRunAction(command="bash /root/run_tests.sh") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + if isinstance(obs, CmdOutputObservation): + test_output = obs.content + output = re.sub(r"\x1b\[[0-9;]*m|\r", "", test_output) + assert isinstance(output, str) + from .r2e_utils import parse_log_pytest, get_reward + + results = parse_log_pytest(output) + reward = get_reward(results, instance) + return {"reward": reward} + else: + logger.info(f"Running bash /root/run_tests.sh, but got unexpected observation type: {str(obs)}") + return {"reward": 0, "finish_reason": "error_evaluation"} + + action = CmdRunAction(command='git config --global core.pager ""') + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, + f'Failed to git config --global core.pager "": {str(obs)}', + ) + + # First check for any git repositories in subdirectories + action = CmdRunAction(command='find . -type d -name .git -not -path "./.git"') + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, + f"Failed to find git repositories: {str(obs)}", + ) + + git_dirs = [p for p in obs.content.strip().split("\n") if p] + if git_dirs: + # Remove all .git directories in subdirectories + for git_dir in git_dirs: + action = CmdRunAction(command=f'rm -rf "{git_dir}"') + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, + f"Failed to remove git directory {git_dir}: {str(obs)}", + ) + + # add all files + action = CmdRunAction(command="git add -A") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, + f"Failed to git add -A: {str(obs)}", + ) + + # Remove binary files from git staging + action = CmdRunAction(command=remove_binary_files_from_git()) + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert_and_raise( + isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, + f"Failed to remove binary files: {str(obs)}", + ) + + n_retries = 0 + git_patch = None + while n_retries < 5: + if "swe-smith" in dataset or "r2e-gym" in dataset: + # For swe-smith, directly get the git diff without base commit + action = CmdRunAction(command="git diff --no-color --cached > patch.diff") + else: + action = CmdRunAction(command=f'git diff --no-color --cached {instance["base_commit"]} > patch.diff') + action.set_hard_timeout(max(300 + 100 * n_retries, 600)) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + n_retries += 1 + if isinstance(obs, CmdOutputObservation): + if obs.exit_code == 0: + # Read the patch file + action = FileReadAction(path="patch.diff") + action.set_hard_timeout(max(300 + 100 * n_retries, 600)) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + if isinstance(obs, FileReadObservation): + git_patch = obs.content + break + elif isinstance(obs, CmdOutputObservation): + git_patch = obs.content + break + elif isinstance(obs, ErrorObservation): + # Fall back to cat "patch.diff" to get the patch + assert "File could not be decoded as utf-8" in obs.content + action = CmdRunAction(command="cat patch.diff") + action.set_hard_timeout(max(300 + 100 * n_retries, 600)) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + assert isinstance(obs, CmdOutputObservation) and obs.exit_code == 0 + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + git_patch = obs.content + break + else: + assert_and_raise(False, f"Unexpected observation type: {str(obs)}") + else: + logger.info("Failed to get git diff, retrying...") + sleep_if_should_continue(10) + elif isinstance(obs, ErrorObservation): + logger.error(f"Error occurred: {obs.content}. Retrying...") + sleep_if_should_continue(10) + else: + assert_and_raise(False, f"Unexpected observation type: {str(obs)}") + + assert_and_raise(git_patch is not None, "Failed to get git diff (None)") + + # Remove binary diffs from the patch + git_patch = remove_binary_diffs(git_patch) + + logger.info("-" * 30) + logger.info("END Runtime Completion Fn") + logger.info("-" * 30) + return {"git_patch": git_patch} + + @classmethod + async def evaluate_result(cls, instance, run_results, instance_id, trajectory_id, dataset) -> bool: + app_config = cls.get_config(instance, dataset) + + try: + # Create runtime + # TODO(csy): some tasks may not need a runtime for evaluation + runtime = create_runtime(app_config) + + # Connect runtime + await runtime.connect() + + eval_results: bool = await call_sync_from_async( + evaluate_result, runtime, instance, run_results, instance_id, trajectory_id, dataset + ) + finally: + logger.info(f"Running cleanup for eval agent task for instance {instance_id}, trajectory {trajectory_id}") + if "runtime" in locals() and runtime: + runtime.event_stream.close() + runtime.close() + return eval_results + + +def evaluate_result(runtime, instance, run_results, instance_id, trajectory_id, dataset) -> bool: + """Apply patch and evaluate the solution.""" + if "swe-gym" in dataset: + from swegym.harness.grading import get_eval_report + from swegym.harness.run_evaluation import ( + APPLY_PATCH_FAIL, + APPLY_PATCH_PASS, + ) + from swegym.harness.test_spec import ( + make_test_spec, + ) + elif "swe-smith" in dataset: + from swesmith.harness.grading import get_eval_report + from swebench.harness.constants import ( + APPLY_PATCH_FAIL, + APPLY_PATCH_PASS, + ) + from .swesmith_utils import make_test_spec + else: # Newer version of SWE-Bench have different import paths + from swebench.harness.grading import get_eval_report + from swebench.harness.constants import ( + APPLY_PATCH_FAIL, + APPLY_PATCH_PASS, + ) + from swebench.harness.test_spec import ( + make_test_spec, + ) + + model_patch = run_results.get("git_patch", None) + if not model_patch: + raise Exception(f"No git patch found for instance {instance_id}, trajectory {trajectory_id}") + + test_spec = make_test_spec(instance=instance) + if "sphinx-doc" in instance["instance_id"]: + for item in test_spec.eval_script_list: + if "tox" in item: + # remove --current-env to avoid issues + new_item = item.replace("--current-env", "") + test_spec.eval_script_list[test_spec.eval_script_list.index(item)] = new_item + model_patch = process_git_patch(model_patch) + # Get patch and save it to /tmp/patch.diff + with tempfile.TemporaryDirectory() as temp_dir: + # Patch file + patch_file_path = os.path.join(temp_dir, "patch.diff") + with open(patch_file_path, "w") as f: + f.write(model_patch) + runtime.copy_to(patch_file_path, "/tmp") + # Eval script + eval_script_path = os.path.join(temp_dir, "eval.sh") + with open(eval_script_path, "w") as f: + f.write(test_spec.eval_script) + runtime.copy_to(eval_script_path, "/tmp") + + # Set +x + action = CmdRunAction(command="chmod +x /tmp/eval.sh") + action.set_hard_timeout(600) + logger.info(action, extra={"msg_type": "ACTION"}) + obs = runtime.run_action(action) + logger.info(obs, extra={"msg_type": "OBSERVATION"}) + assert obs.exit_code == 0 + + # Apply patch + if "swe-smith" in dataset: + # need to fetch and checkout the branch first + exec_command = ( + "cd /testbed && " + "git fetch && " + f"git checkout {instance['instance_id']} && " + "(git apply -v /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " + "(echo 'Failed to apply patch with git apply, trying with patch command...' && " + "(patch --batch --fuzz=5 -p1 -i /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " + "echo 'APPLY_PATCH_FAIL')))" + ) + else: + exec_command = ( + "cd /testbed && " + "(git apply -v /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " + "(echo 'Failed to apply patch with git apply, trying with patch command...' && " + "(patch --batch --fuzz=5 -p1 -i /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " + "echo 'APPLY_PATCH_FAIL')))" + ) + action = CmdRunAction(command=exec_command) + action.set_hard_timeout(600) + obs = runtime.run_action(action) + assert isinstance(obs, CmdOutputObservation) + apply_patch_output = obs.content + assert isinstance(apply_patch_output, str) + # instance['test_result']['apply_patch_output'] = apply_patch_output + + if "APPLY_PATCH_FAIL" in apply_patch_output: + raise Exception(f"Instance {instance_id}, trajectory {trajectory_id} {APPLY_PATCH_FAIL}:\n{apply_patch_output}") + elif "APPLY_PATCH_PASS" in apply_patch_output: + logger.info(f"[{instance_id}, {trajectory_id}] {APPLY_PATCH_PASS}:\n{apply_patch_output}") + + # Run eval script in background and save output to log file + log_file = "/tmp/eval_output.log" + action = CmdRunAction(command=f"/tmp/eval.sh > {log_file} 2>&1 & echo $!") + action.set_hard_timeout(300) # Short timeout just to get the process ID + obs = runtime.run_action(action) + + if isinstance(obs, CmdOutputObservation) and obs.exit_code == 0: + pid = obs.content.split()[-1].strip() + logger.info(f"[{instance_id}, {trajectory_id}] Evaluation process started with PID: {pid}") + + # Poll for completion + start_time = time.time() + timeout = 1800 # 30 minutes + while True: + seconds_elapsed = time.time() - start_time + if seconds_elapsed > timeout: + raise Exception(f"[{instance_id}, {trajectory_id}] Evaluation timed out after {timeout} seconds") + check_action = CmdRunAction(command=f"ps -p {pid} > /dev/null; echo $?") + check_action.set_hard_timeout(300) + check_obs = runtime.run_action(check_action) + if isinstance(check_obs, CmdOutputObservation) and check_obs.content.split()[-1].strip() == "1": + logger.info( + f"[{instance_id}, {trajectory_id}] Evaluation process completed after {seconds_elapsed} seconds" + ) + break + logger.info( + f"[{instance_id}, {trajectory_id}] [{seconds_elapsed:.0f}s] Evaluation still running, waiting..." + ) + time.sleep(30) # Wait for 30 seconds before checking again + + # Read the log file + cat_action = CmdRunAction(command=f"cat {log_file}") + cat_action.set_hard_timeout(300) + cat_obs = runtime.run_action(cat_action) + + # Grade answer + if isinstance(cat_obs, CmdOutputObservation) and cat_obs.exit_code == 0: + test_output = cat_obs.content + assert isinstance(test_output, str) + # instance['test_result']['test_output'] = test_output + + # Get report from test output + logger.info(f"[{instance_id}, {trajectory_id}] Grading answer...") + + with tempfile.TemporaryDirectory() as temp_dir: + # Create a directory structure that matches the expected format + # NOTE: this is a hack to make the eval report format consistent + # with the original SWE-Bench eval script + log_dir = os.path.join(temp_dir, "logs", instance_id.lower()) + os.makedirs(log_dir, exist_ok=True) + test_output_path = os.path.join(log_dir, "test_output.txt") + with open(test_output_path, "w") as f: + f.write(test_output) + try: + extra_kwargs = {} + if "swe-smith" in dataset: + # SWE-Gym uses a different version of the package, hence a different eval report argument + extra_kwargs["test_log_path"] = test_output_path + else: + extra_kwargs["log_path"] = test_output_path + + if "swe-smith" in dataset: + extra_kwargs["inst"] = instance + else: + extra_kwargs["test_spec"] = test_spec + extra_kwargs["include_tests_status"] = True + + _report = get_eval_report( + prediction={ + "model_patch": model_patch, + "instance_id": instance_id, + }, + **extra_kwargs, + ) + # in swe-smith, the report is a single dict + # in swe-gym and swe-bench, the report is a dict with instance_id + report = _report if "swe-smith" in dataset else _report[instance_id] + logger.info( + f"[{instance_id}, {trajectory_id}] report: {report}\nResult for [{instance_id}, {trajectory_id}]: resolved: {report['resolved']}" + ) + return report["resolved"] + except Exception as e: + logger.error(f"[{instance_id}, {trajectory_id}] Error when getting eval report: {e}") + return False + else: + raise Exception(f"[{instance_id}, {trajectory_id}] Error when starting eval:\n{obs.content}") + else: + raise Exception(f"[{instance_id}] Unexpected output when applying patch:\n{apply_patch_output}") + + +def get_instance_docker_image(instance, data_source) -> str: + if "swe-smith" in data_source: + image_name = instance["image_name"] + return f"jyangballin/{image_name.replace('__', '_1776_')}" + if "r2e-gym" in data_source: + return instance["instance_id"] + + instance_id = instance["instance_id"] + image_name = "sweb.eval.x86_64." + instance_id + image_name = image_name.replace("__", "_s_") # to comply with docker image naming convention + return (DOCKER_IMAGE_PREFIX.rstrip("/") + "/" + image_name).lower() + + +# Helper function for sandbox config +def get_default_sandbox_config_for_eval(): + return SandboxConfig( + use_host_network=False, + timeout=300, + api_key=os.environ.get("ALLHANDS_API_KEY", None), + remote_runtime_api_url=os.environ.get("SANDBOX_REMOTE_RUNTIME_API_URL"), + keep_runtime_alive=False, + remote_runtime_init_timeout=3600, + remote_runtime_api_timeout=120, + remote_runtime_enable_retries=True, + remote_runtime_class="sysbox", + ) + + +def remove_binary_diffs(patch_text): + """ + Remove binary file diffs from a git patch. + + Args: + patch_text (str): The git patch text + + Returns: + str: The cleaned patch text with binary diffs removed + """ + lines = patch_text.splitlines() + cleaned_lines = [] + block = [] + is_binary_block = False + + for line in lines: + if line.startswith("diff --git "): + if block and not is_binary_block: + cleaned_lines.extend(block) + block = [line] + is_binary_block = False + elif "Binary files" in line: + is_binary_block = True + block.append(line) + else: + block.append(line) + + if block and not is_binary_block: + cleaned_lines.extend(block) + return "\n".join(cleaned_lines) + + +# avoid depend on file cmd +def remove_binary_files_from_git(): + """ + Generate a bash command to remove binary files from git staging. + + Returns: + str: A bash command that removes binary files from git staging + """ + return """ + for file in $(git status --porcelain | grep -E "^(M| M|\\?\\?|A| A)" | cut -c4-); do + if [ -f "$file" ] && (test -x "$file" || git check-attr binary "$file" | grep -q "binary: set"); then + git rm -f "$file" 2>/dev/null || rm -f "$file" + echo "Removed: $file" + fi + done + """.strip() + + +def is_fatal_evaluation_error(error: str | None) -> bool: + if not error: + return False + + FATAL_EXCEPTIONS = [ + AgentRuntimeError, + AgentRuntimeBuildError, + AgentRuntimeTimeoutError, + AgentRuntimeUnavailableError, + AgentRuntimeNotReadyError, + AgentRuntimeDisconnectedError, + AgentRuntimeNotFoundError, + ConnectionError, + ] + + if any(exception.__name__ in error for exception in FATAL_EXCEPTIONS): + logger.error(f"Fatal evaluation error detected: {error}") + return True + + return False + + +def process_git_patch(patch): + if not isinstance(patch, str): + return "" + + if not patch.strip(): + print("Skipping empty patch....") + # skip empty patches + return "" + + patch = patch.replace("\r\n", "\n") + # There might be some weird characters at the beginning of the patch + # due to some OpenHands inference command outputs + + # FOR EXAMPLE: + # git diff --no-color --cached 895f28f9cbed817c00ab68770433170d83132d90 + # 0 + # diff --git a/django/db/models/sql/.backup.query.py b/django/db/models/sql/.backup.query.py + # new file mode 100644 + # index 0000000000..fc13db5948 + + # We "find" the first line that starts with "diff" and then we remove lines before it + lines = patch.split("\n") + for i, line in enumerate(lines): + if line.startswith("diff --git"): + patch = "\n".join(lines[i:]) + break + + patch = patch.rstrip() + "\n" # Make sure the last line ends with a newline + return patch diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/__init__.py b/skyrl-agent/skyrl_agent/tasks/verifiers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/__init__.py b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/__init__.py new file mode 100644 index 0000000000..ca3c9a64e2 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/__init__.py @@ -0,0 +1,273 @@ +import re +from typing import Tuple, Optional +import time +import json +import os +from concurrent.futures import ThreadPoolExecutor, as_completed + +import numpy as np + +from .utils import _ERROR_MSG_PREFIX + +_MAX_CHAR_DISPLAY = 2048 + +# CODER1_EXEC = os.environ.get("CODER1_EXEC", "bwrap") +CODER1_EXEC = os.environ.get("CODER1_EXEC", "unsafe_local") + +if CODER1_EXEC == "unsafe_local": + from .unsafe_local_exec import code_exec_local + + code_exec = code_exec_local +elif CODER1_EXEC == "sandboxfusion": + from .sandboxfusion_exec import code_exec_sandboxfusion + + code_exec = code_exec_sandboxfusion +else: + raise ValueError(f"Unknown CODER1_EXEC: {CODER1_EXEC}") + + +CODE_PATTERN = re.compile(r"```(?:\w+)?\n(.*?)\n```", re.DOTALL) +ANSWER_PATTERN = re.compile(r"(.*)", re.DOTALL) + + +def remote_check_stdio(code, stdin, stdout): + succ, output = code_exec(code=code, stdin=stdin) + return succ, output, stdin, stdout + + +# def validate_response_structure(processed_str: str) -> bool: +# pattern = re.compile(r".*.*", re.DOTALL) +# return bool(pattern.search(processed_str.strip())) + + +def try_extract_solution(solution_str: str) -> str: + match = re.search(ANSWER_PATTERN, solution_str) + + if match: + return match.group(1).strip() + + return solution_str + + +def extract_code_from_string(solution_str): + solution_str = try_extract_solution(solution_str) + code_blocks = CODE_PATTERN.findall(solution_str) + return "\n".join(code_blocks).strip() + + +def fuzzy_equal(actual: str, expected: str, tolerance: float = 1e-6, verbose=True) -> bool: + """ + Compare two outputs line by line and element by element for approximate equality. + Handles: + 1. Integer and floating-point number comparison with tolerance + 2. Case-insensitive comparison for yes/no + + Args: + actual: The actual output from code execution + expected: The expected output + tolerance: Tolerance for floating point number comparison + + Returns: + bool: True if outputs are approximately equal + """ + # Save original values for debugging + original_actual = actual + original_expected = expected + + # Normalize line endings + actual = actual.strip().replace("\r\n", "\n") + expected = expected.strip().replace("\r\n", "\n") + + # If exact match after normalization, return early + if actual == expected: + return True + + # Split into lines + actual_lines = actual.split("\n") + expected_lines = expected.split("\n") + + # If different number of lines, they're definitely not equal + if len(actual_lines) != len(expected_lines): + return False + + # Track fuzzy matches for debugging + fuzzy_match_reasons = [] + + # Compare each line + for i, (actual_line, expected_line) in enumerate(zip(actual_lines, expected_lines)): + # If lines match exactly, continue + if actual_line == expected_line: + continue + + # Split into tokens by whitespace + actual_tokens = actual_line.split() + expected_tokens = expected_line.split() + + # If different number of tokens, they're not equal + if len(actual_tokens) != len(expected_tokens): + return False + + # Compare each token + for j, (actual_token, expected_token) in enumerate(zip(actual_tokens, expected_tokens)): + # If tokens match exactly, continue + if actual_token == expected_token: + continue + + # For yes/no, use case-insensitive comparison + if actual_token.lower() in ["yes", "no"] and expected_token.lower() in ["yes", "no"]: + if actual_token.lower() == expected_token.lower(): + fuzzy_match_reasons.append( + f"Line {i+1}, Token {j+1}: Case-insensitive yes/no match '{actual_token}' ≈ '{expected_token}'" + ) + continue + else: + return False + + # Try numeric comparison + try: + actual_num = float(actual_token) + expected_num = float(expected_token) + diff = abs(actual_num - expected_num) + + if diff <= tolerance: + fuzzy_match_reasons.append( + f"Line {i+1}, Token {j+1}: Numeric match '{actual_token}' ≈ '{expected_token}' (diff: {diff})" + ) + continue + else: + return False + except ValueError: + # Not numeric values + return False + + # Output fuzzy match information if any occurred + if fuzzy_match_reasons and verbose: + print("😅 FUZZY MATCH - Outputs approximately equal:") + print(f" Expected: {repr(original_expected)}") + print(f" Actual: {repr(original_actual)}") + print(" Reasons for fuzzy matching:") + for reason in fuzzy_match_reasons: + print(f" • {reason}") + + # If we made it here, all lines are approximately equal + return True + + +def _compute_score(solution_str, ground_truth, extra_info, format_reward=0.0, answer_reward=1.0): + reward_log = [] + + # ground_truth is not code, but tests + # pass_fmt = validate_response_structure(solution_str) + solution_code = extract_code_from_string(solution_str) + # print(f"Extracted code: {solution_code}") + + # NameError: name 'List' is not defined. Did you mean: 'list'? + # NameError: name 'defaultdict' is not defined + # reference solutions fail due to imports not being present + + if ( + # not pass_fmt or len(solution_code) == 0 + len(solution_code) + == 0 + ): # only print full output when there is an error + reward_log.append("-" * 16 + "Bad format detected!" + "-" * 16) + reward_log.append("-" * 16 + "Original Model Output" + "-" * 16) + reward_log.append(solution_str) + # return -answer_reward - format_reward, "\n".join(reward_log) + return 0.0, "\n".join(reward_log) + + reward_log.append("-" * 16 + "Extracted Code to Execute" + "-" * 16) + ground_truth = json.loads(ground_truth) + + t_start = time.time() + + # log code + if "functional" in ground_truth: + if "prefix" in extra_info and extra_info["prefix"] != None: + solution_code = extra_info["prefix"] + "\n" + solution_code + reward_log.append(solution_code + "\n" + ground_truth["functional"]) + else: + reward_log.append(solution_code) + + if "pytest" in ground_truth or "functional" in ground_truth or "solution_file" in ground_truth: + if "functional" in ground_truth: + if "prefix" in extra_info and extra_info["prefix"] != None: + solution_code = extra_info["prefix"] + "\n" + solution_code + succ, output = code_exec(solution_code + "\n" + ground_truth["functional"]) + elif "solution_file" in ground_truth: + succ, output = code_exec(solution_code, solution=ground_truth["solution_file"]) + else: # pytest + succ, output = code_exec(solution_code, pytest=ground_truth["pytest"]) + if not succ: + # reward_log.append( + # "!" * 16 + # + f"⚠️ Test Execution Failed in {time.time() - t_start:.1f}s" + # + "!" * 16 + # ) + # reward_log.append(output[:_MAX_CHAR_DISPLAY]) + # reward_log.append("-" * 16 + "Failed Prompt" + "-" * 16) + # reward_log.append(extra_info["original_prompt"].replace("\n\n", "\n")) + return format_reward, "\n".join(reward_log) + elif "inputs" in ground_truth and "outputs" in ground_truth: + stdin_list: str = ground_truth["inputs"] + stdout_list: str = ground_truth["outputs"] + + # Add parallelism + # ok i see why they had concurrency issues. + # they call this one by one by one for each of the inputs + # spawns a separate process for each input as opposed to just being a single file + with ThreadPoolExecutor(max_workers=min(8, len(stdin_list))) as executor: + futures = [ + executor.submit(remote_check_stdio, solution_code, stdin, stdout) + for stdin, stdout in zip(stdin_list, stdout_list) + ] + for future in as_completed(futures): + succ, output, stdin, stdout = future.result() + if not succ or not fuzzy_equal(output.strip(), stdout.strip()): + output = output[:_MAX_CHAR_DISPLAY] # truncate output to print + # reward_log.append( + # "!" * 16 + # + f"⚠️ Test Execution Failed in {time.time() - t_start:.1f}s" + # + "!" * 16 + # ) + # reward_log.append(f"🔎Input: {repr(stdin)}") + # reward_log.append(f"✅Expected: {repr(stdout.strip())}") + # reward_log.append( + # f"❌Actual: {output if output.startswith(_ERROR_MSG_PREFIX) else repr(output.strip())}" + # ) + # reward_log.append("-" * 16 + "Failed Prompt" + "-" * 16) + # reward_log.append(extra_info["original_prompt"].replace("\n\n", "\n")) + return format_reward, "\n".join(reward_log) + else: + raise ValueError( + "Current supports for ground-truth are ['pytest', 'functional', 'solution_file', 'inputs/outputs'] -- No idea what's: {ground_truth = }" + ) + + reward_log.append("+" * 16 + "Test Execution Passed! (Output)" + "+" * 16) + reward_log.append(output) + return format_reward + answer_reward, "\n".join(reward_log) + + +def compute_score(solution_str, ground_truth, extra_info, format_reward=0.0, answer_reward=1.0): + if isinstance(extra_info, np.ndarray): + extra_info = extra_info.item() + score, reward_log = _compute_score( + solution_str, + ground_truth, + extra_info=extra_info, + format_reward=format_reward, + answer_reward=answer_reward, + ) + marker = "✅" if score == (format_reward + answer_reward) else "❌" + reward_log = ( + marker * 16 + + "Reward Calculation" + + marker * 16 + + "\n" + + reward_log + + "\n" + + marker * 16 + + f"Final Rward = {score}" + + marker * 16 + ) + return {"score": score, "acc": score} diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/sandboxfusion_exec.py b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/sandboxfusion_exec.py new file mode 100644 index 0000000000..48b23c1882 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/sandboxfusion_exec.py @@ -0,0 +1,123 @@ +import requests +from itertools import cycle +import threading +import os +from .utils import _ERROR_MSG_PREFIX, _DEFAULT_TIMEOUT_SECONDS, RunCodeResponse, RunStatus + +# Default sandbox servers - can be overridden via environment variable or function parameter +DEFAULT_SANDBOX_SERVERS = [ + # "fs-mbz-gpu-044", # Add more servers here +] + +# Thread-safe cycle iterator for round-robin load balancing +server_cycle = None +cycle_lock = threading.Lock() + + +def _parse_sandbox_servers(servers_input): + """Parse sandbox servers from various input formats""" + if not servers_input: + return DEFAULT_SANDBOX_SERVERS + + if isinstance(servers_input, str): + # Single server or comma-separated servers + if "," in servers_input: + return [server.strip() for server in servers_input.split(",")] + else: + return [servers_input.strip()] + elif isinstance(servers_input, list): + return servers_input + else: + raise ValueError(f"Invalid sandbox servers format: {type(servers_input)}. Expected str or list.") + + +def _get_next_server(server_cycle): + """Get the next server in round-robin fashion thread-safely.""" + with cycle_lock: + return next(server_cycle) + + +def code_exec_sandboxfusion(code, stdin: str = None, timeout=_DEFAULT_TIMEOUT_SECONDS, sandbox_servers=None): + """ + Execute Python code using SandboxFusion remote service. + + Args: + code: Python code to execute + stdin: Optional input to pass to the code + timeout: Timeout in seconds (default from utils) + sandbox_servers: Optional server names for sandbox servers. Can be: + - Single server string: "fs-mbz-gpu-044" + - Comma-separated servers: "fs-mbz-gpu-044,fs-mbz-gpu-045" + - List of servers: ["fs-mbz-gpu-044", "fs-mbz-gpu-045"] + - None: Uses SANDBOX_FUSION_SERVERS environment variable or default + + Returns: + tuple: (success: bool, output: str) + """ + try: + # Determine sandbox servers to use + if sandbox_servers is None: + sandbox_servers = os.getenv("SANDBOX_FUSION_SERVERS", "") + + servers = _parse_sandbox_servers(sandbox_servers) + server_cycle = cycle(servers) + + if not servers: + return ( + False, + _ERROR_MSG_PREFIX + + "No sandbox servers configured. Set SANDBOX_FUSION_SERVERS environment variable or pass sandbox_servers parameter.", + ) + + request_data = {"language": "python", "code": code, "stdin": stdin, "run_timeout": timeout} + + # Try each server (for load balancing/failover) + for _ in range(len(servers)): + try: + server = _get_next_server(server_cycle) + url = f"http://{server}:8080/run_code" + response = requests.post(url, json=request_data, timeout=timeout + 2) + + if response.status_code != 200: + continue # Try next server + + result = RunCodeResponse(**response.json()) + if result.status == RunStatus.Success: + return True, result.run_result.stdout + else: + return ( + False, + _ERROR_MSG_PREFIX + + f"STDOUT:\n{result.run_result.stdout}\n\nSTDERR:\n{result.run_result.stderr}", + ) + + except requests.exceptions.RequestException: + continue # Try next server + + # If we get here, all servers failed + return False, _ERROR_MSG_PREFIX + f"All sandbox servers failed to process the request. Servers tried: {servers}" + + except Exception as e: + return False, _ERROR_MSG_PREFIX + f"Execution error: {str(e)}" + + +def code_exec_sandboxfusion_with_pytest(code, pytest_code, timeout=_DEFAULT_TIMEOUT_SECONDS, sandbox_servers=None): + """ + Execute Python code with pytest using SandboxFusion remote service. + + Args: + code: Python solution code + pytest_code: Pytest test code + timeout: Timeout in seconds + sandbox_servers: Optional server names for sandbox servers (same format as code_exec_sandboxfusion) + + Returns: + tuple: (success: bool, output: str) + """ + # Combine the solution code and test code + combined_code = f""" +{code} + +{pytest_code} +""" + return code_exec_sandboxfusion(combined_code, timeout=timeout, sandbox_servers=sandbox_servers) diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/unsafe_local_exec.py b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/unsafe_local_exec.py new file mode 100644 index 0000000000..ce4bce7b04 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/unsafe_local_exec.py @@ -0,0 +1,115 @@ +import os +import subprocess +import shlex +from tempfile import NamedTemporaryFile, TemporaryDirectory + +from .utils import _ERROR_MSG_PREFIX + +CLI_ARG_SIZE_LIMIT = 1024 * 3 +MEMORY_LIMIT_KB = 10 * 1024 * 1024 # 10GB in KB + + +def wrap_command_with_ulimit(command): + cmd_str = " ".join(shlex.quote(c) for c in command) + return ["bash", "-c", f"ulimit -v {MEMORY_LIMIT_KB}; exec {cmd_str}"] + + +def code_exec_local( + code, + stdin: str = None, + timeout=30, + pytest: str = None, + solution: str = None, + python_env: str = os.environ.get("CONDA_BIN_PATH", None), +): + env = os.environ.copy() + env["OPENBLAS_NUM_THREADS"] = "1" + if "PYTHONPATH" in env: + del env["PYTHONPATH"] + + if python_env is None: + python_executable = "/usr/bin/python3" + else: + python_executable = os.path.join(python_env, "python3") + + if solution: + with TemporaryDirectory() as tmpdir: + assert stdin is None, "STDIN is not supported with solution_file" + with open(os.path.join(tmpdir, "solution.py"), "w") as f: + f.write(code) + with open(os.path.join(tmpdir, "test_solution.py"), "w") as f: + f.write(solution) + + command = [ + "timeout", + str(timeout), + python_executable, + os.path.join(tmpdir, "test_solution.py"), + ] + command = wrap_command_with_ulimit(command) + result = subprocess.run( + command, + cwd=tmpdir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + elif pytest: + with TemporaryDirectory() as tmpdir: + assert stdin is None, "STDIN is not supported with pytest" + with open(os.path.join(tmpdir, "solution.py"), "w") as f: + f.write(code) + with open(os.path.join(tmpdir, "test_solution.py"), "w") as f: + f.write(pytest) + + command = [ + "timeout", + str(timeout), + python_executable, + "-m", + "pytest", + tmpdir, + ] + command = wrap_command_with_ulimit(command) + result = subprocess.run( + command, + cwd=tmpdir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + else: + if len(code) < CLI_ARG_SIZE_LIMIT: + command = ["timeout", str(timeout), python_executable, "-c", code] + command = wrap_command_with_ulimit(command) + result = subprocess.run( + command, + input=(stdin.encode() if stdin else None), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + else: + with NamedTemporaryFile() as tmp: + tmp.write(code.encode()) + tmp.flush() + command = ["timeout", str(timeout), python_executable, tmp.name] + command = wrap_command_with_ulimit(command) + result = subprocess.run( + command, + input=(stdin.encode() if stdin else None), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + + stderr = result.stderr.decode().strip() + stdout = result.stdout.decode() + + if result.returncode == 0: + return True, stdout + return False, _ERROR_MSG_PREFIX + f"STDOUT:\n{stdout}\n\nSTDERR:\n{stderr}" diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/utils.py b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/utils.py new file mode 100644 index 0000000000..66b613bfe7 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/coder1/utils.py @@ -0,0 +1,50 @@ +import requests +from pydantic import BaseModel +from typing import Optional, Dict +from enum import Enum + +_ERROR_MSG_PREFIX = "Failed to execute program: " +_DEFAULT_TIMEOUT_SECONDS = 30 # 30 seconds is the default timeout for the executor + + +def check_executor_alive(executor): + try: + return requests.get(executor + "/").status_code in [200, 404] + except Exception: + return False + + +# The below code is from the code_sandbox repo: https://github.com/bytedance/SandboxFusion/blob/main/sandbox/server/sandbox_api.py +# - RunStatus, CommandRunStatus, CommandRunResult, RunCodeResponse are from the code_sandbox repo + + +class RunStatus(str, Enum): + # all command finished successfully + Success = "Success" + # one of the process has non-zero return code + Failed = "Failed" + # error on sandbox side + SandboxError = "SandboxError" + + +class CommandRunStatus(str, Enum): + Finished = "Finished" + Error = "Error" + TimeLimitExceeded = "TimeLimitExceeded" + + +class CommandRunResult(BaseModel): + status: CommandRunStatus + execution_time: Optional[float] = None + return_code: Optional[int] = None + stdout: Optional[str] = None + stderr: Optional[str] = None + + +class RunCodeResponse(BaseModel): + status: RunStatus + message: str + compile_result: Optional[CommandRunResult] = None + run_result: Optional[CommandRunResult] = None + executor_pod_name: Optional[str] = None + files: Dict[str, str] = {} diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/naive_dapo.py b/skyrl-agent/skyrl_agent/tasks/verifiers/naive_dapo.py new file mode 100644 index 0000000000..63bed7724c --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/naive_dapo.py @@ -0,0 +1,528 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/hendrycks_math/utils.py + +import re +import signal + +import sympy +from pylatexenc import latex2text +from sympy.parsing import sympy_parser +import os +from .prime_math import math_normalize +from .prime_math.grader import math_equal + + +class timeout: + + def __init__(self, seconds=1, error_message="Timeout"): + self.seconds = seconds + self.error_message = error_message + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) + + +# Constants for normalization +SUBSTITUTIONS = [ + ("an ", ""), + ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), +] + +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """Normalize a final answer to a quantitative reasoning question. + + Args: + final_answer: The answer string to normalize + + Returns: + Normalized answer string + """ + final_answer = final_answer.split("=")[-1] + + # Apply substitutions and removals + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + + # Extract and normalize LaTeX math + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + + # Normalize numbers + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + + return final_answer.strip() + + +# sympy might hang -- we don't care about trying to be lenient in these cases +BAD_SUBSTRINGS = ["^{", "^("] +BAD_REGEXES = ["\^[0-9]+\^", "\^[0-9][0-9]+"] +TUPLE_CHARS = "()[]" + + +def timeout(timeout_seconds: int = 8): + if os.name == "posix": + import signal + + def decorator(func): + + def handler(signum, frame): + raise TimeoutError("Operation timed out!") + + def wrapper(*args, **kwargs): + old_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, handler) + signal.alarm(timeout_seconds) + + try: + return func(*args, **kwargs) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + return wrapper + + return decorator + else: + raise NotImplementedError(f"Unsupported OS: {os.name}") + + +def _sympy_parse(expr: str): + """Parses an expression with sympy.""" + py_expr = expr.replace("^", "**") + return sympy_parser.parse_expr( + py_expr, + transformations=(sympy_parser.standard_transformations + (sympy_parser.implicit_multiplication_application,)), + ) + + +def _parse_latex(expr: str) -> str: + """Attempts to parse latex to an expression sympy can read.""" + expr = expr.replace("\\tfrac", "\\frac") + expr = expr.replace("\\dfrac", "\\frac") + expr = expr.replace("\\frac", " \\frac") # Play nice with mixed numbers. + expr = latex2text.LatexNodes2Text().latex_to_text(expr) + + # Replace the specific characters that this parser uses. + expr = expr.replace("√", "sqrt") + expr = expr.replace("π", "pi") + expr = expr.replace("∞", "inf") + expr = expr.replace("∪", "U") + expr = expr.replace("·", "*") + expr = expr.replace("×", "*") + + return expr.strip() + + +def _is_float(num: str) -> bool: + try: + float(num) + return True + except ValueError: + return False + + +def _is_int(x: float) -> bool: + try: + return abs(x - int(round(x))) <= 1e-7 + except: + return False + + +def _is_frac(expr: str) -> bool: + return bool(re.search(r"^-?[0-9]+.?/0*[1-9][0-9]*.?$", expr)) + + +def _str_is_int(x: str) -> bool: + try: + x = _strip_properly_formatted_commas(x) + x = float(x) + return abs(x - int(round(x))) <= 1e-7 + except: + return False + + +def _str_to_int(x: str) -> bool: + x = x.replace(",", "") + x = float(x) + return int(x) + + +def _inject_implicit_mixed_number(step: str): + """ + Automatically make a mixed number evalable + e.g. 7 3/4 => 7+3/4 + """ + p1 = re.compile("([0-9]) +([0-9])") + step = p1.sub("\\1+\\2", step) ## implicit mults + return step + + +def _strip_properly_formatted_commas(expr: str): + # We want to be careful because we don't want to strip tuple commas + p1 = re.compile("(\d)(,)(\d\d\d)($|\D)") + while True: + next_expr = p1.sub("\\1\\3\\4", expr) + if next_expr == expr: + break + expr = next_expr + return next_expr + + +def _normalize(expr: str) -> str: + """Normalize answer expressions.""" + if expr is None: + return None + + # Remove enclosing `\text{}`. + m = re.search("^\\\\text\{(?P.+?)\}$", expr) + if m is not None: + expr = m.group("text") + + expr = expr.replace("\\%", "%") + expr = expr.replace("\\$", "$") + expr = expr.replace("$", "") + expr = expr.replace("%", "") + expr = expr.replace(" or ", " , ") + expr = expr.replace(" and ", " , ") + + expr = expr.replace("million", "*10^6") + expr = expr.replace("billion", "*10^9") + expr = expr.replace("trillion", "*10^12") + + for unit in [ + "degree", + "cm", + "centimeter", + "meter", + "mile", + "second", + "minute", + "hour", + "day", + "week", + "month", + "year", + "foot", + "feet", + "inch", + "yard", + "liter", + ]: + expr = re.sub(f"{unit}(es)?(s)? *(\^[0-9]+)?", "", expr) + expr = re.sub("\^ *\\\\circ", "", expr) + + if len(expr) > 0 and expr[0] == "{" and expr[-1] == "}": + expr = expr[1:-1] + + expr = re.sub(",\\\\! *", "", expr) + if _is_float(expr) and _is_int(float(expr)): + expr = str(int(round(float(expr)))) + if "\\" in expr: + try: + expr = _parse_latex(expr) + except: + pass + + # edge case with mixed numbers and negative signs + expr = re.sub("- *", "-", expr) + + expr = _inject_implicit_mixed_number(expr) + + # don't be case sensitive for text answers + expr = expr.lower() + + if _str_is_int(expr): + expr = str(_str_to_int(expr)) + + return expr + + +def count_unknown_letters_in_expr(expr: str): + expr = expr.replace("sqrt", "") + expr = expr.replace("frac", "") + letters_in_expr = set([x for x in expr if x.isalpha()]) + return len(letters_in_expr) + + +def should_allow_eval(expr: str): + # we don't want to try parsing unknown text or functions of more than two variables + if count_unknown_letters_in_expr(expr) > 2: + return False + + for bad_string in BAD_SUBSTRINGS: + if bad_string in expr: + return False + + for bad_regex in BAD_REGEXES: + if re.search(bad_regex, expr) is not None: + return False + + return True + + +@timeout(timeout_seconds=10) +def are_equal_under_sympy(ground_truth_normalized: str, given_normalized: str): + are_equal = False + try: + expr = f"({ground_truth_normalized})-({given_normalized})" + if should_allow_eval(expr): + sympy_diff = _sympy_parse(expr) + simplified = sympy.simplify(sympy_diff) + if simplified == 0: + are_equal = True + except: + pass + return are_equal + + +def split_tuple(expr: str): + """ + Split the elements in a tuple/interval, while handling well-formatted commas in large numbers + """ + expr = _strip_properly_formatted_commas(expr) + if len(expr) == 0: + return [] + if ( + len(expr) > 2 + and expr[0] in TUPLE_CHARS + and expr[-1] in TUPLE_CHARS + and all([ch not in expr[1:-1] for ch in TUPLE_CHARS]) + ): + elems = [elem.strip() for elem in expr[1:-1].split(",")] + else: + elems = [expr] + return elems + + +def grade_answer(given_answer: str, ground_truth: str) -> tuple[bool, str]: + """ + The answer will be considered correct if: + (a) it normalizes to the same string as the ground truth answer + OR + (b) sympy can simplify the difference between the expressions to 0 + """ + if given_answer is None: + return False + + ground_truth_normalized_mathd = math_normalize.normalize_answer(ground_truth) + given_answer_normalized_mathd = math_normalize.normalize_answer(given_answer) + + # be at least as lenient as mathd + if ground_truth_normalized_mathd == given_answer_normalized_mathd: + return True, given_answer_normalized_mathd + + ground_truth_normalized = _normalize(ground_truth) + given_normalized = _normalize(given_answer) + + if ground_truth_normalized is None: + return False, given_normalized + + if ground_truth_normalized == given_normalized: + return True, given_normalized + + if len(given_normalized) == 0: + return False, given_normalized + + ground_truth_elems = split_tuple(ground_truth_normalized) + given_elems = split_tuple(given_normalized) + + if len(ground_truth_elems) > 1 and ( + ground_truth_normalized[0] != given_normalized[0] or ground_truth_normalized[-1] != given_normalized[-1] + ): + is_correct = False + elif len(ground_truth_elems) != len(given_elems): + is_correct = False + else: + for ground_truth_elem, given_elem in zip(ground_truth_elems, given_elems): + if _is_frac(ground_truth_elem) and _is_frac(given_elem): + # if fractions aren't reduced, then shouldn't be marked as correct + # so, we don't want to allow sympy.simplify in this case + is_correct = ground_truth_elem == given_elem + elif _str_is_int(ground_truth_elem) != _str_is_int(given_elem): + # if the ground truth answer is an integer, we require the given answer to be a strict match (no sympy.simplify) + is_correct = False + else: + is_correct = are_equal_under_sympy(ground_truth_elem, given_elem) + if not is_correct: + break + + return is_correct, given_normalized + + +def _last_boxed_only_string(string): + idx = string.rfind("\\boxed") + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + left_brace_idx = None + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if left_brace_idx is None: + left_brace_idx = i + elif string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + + i += 1 + + if left_brace_idx is None or right_brace_idx is None: + return None + + return string[left_brace_idx + 1 : right_brace_idx].strip() + + +def match_answer(response): + is_matched = False + response = response.split("")[-1] + + # Find boxed + ans_boxed = _last_boxed_only_string(response) + if ans_boxed: + is_matched = True + response = ans_boxed + + return is_matched, response + + +import math + + +def compute_score(solution_str: str, ground_truth: str, extra_info: dict) -> float: + """Compute the reward score for a solution. This draws heavily from the LLM-as-judge and PRIME reward functions + + Args: + solution_str: The solution string + ground_truth: The ground truth answer + extra_info: dict with additional info for the score computation + + Returns: + Reward score (1.0 for correct, -1.0 for incorrect) + """ + # First assert intended generation and gt type + model_output = str(solution_str) + ground_truth = str(ground_truth) + + # Extract answer from generated output + is_matched, extracted_model_output = match_answer(model_output) + + # TWK NOTE: WE REMOVED THE RESPONSE TRUNCATION FROM math_dapo.compute_score + + # Verify the solution, first check simple comparisons. + correct, pred = grade_answer(extracted_model_output, ground_truth) + + if not correct: + try: + if "\\pi" in extracted_model_output or "\\pi" in ground_truth: + equivs = [] + for pi in [math.pi, 3.14]: + equivs.append(math_equal(extracted_model_output, ground_truth, tiemout=True, pi=pi)) + correct = any(equivs) + else: + correct = math_equal(extracted_model_output, ground_truth, timeout=True) + except: + correct = False + + # reward = 1.0 if correct else -1.0 + reward = 1.0 if correct else 0.0 + acc = correct + + return { + "score": reward, + "acc": acc, + } diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/__init__.py b/skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/__init__.py new file mode 100644 index 0000000000..1df04f1987 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/__init__.py @@ -0,0 +1,415 @@ +# Copyright 2024 PRIME team and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Answer checker API that uses sympy to simplify expressions and check for equality. + +Call grade_answer(given_answer: str, ground_truth: str). + +FROM: https://github.com/openai/prm800k/blob/main/prm800k/grading/grader.py +""" + +import contextlib +import math +import os +import re + +import sympy +from pylatexenc import latex2text +from sympy.parsing import sympy_parser + +from skyrl_agent.tasks.verifiers.py_functional import timeout_limit + +from . import math_normalize +from .grader import math_equal + +# import math_normalize +# from grader import math_equal + +# sympy might hang -- we don't care about trying to be lenient in these cases +BAD_SUBSTRINGS = ["^{", "^("] +BAD_REGEXES = ["\^[0-9]+\^", "\^[0-9][0-9]+"] +TUPLE_CHARS = "()[]" + + +def _sympy_parse(expr: str): + """Parses an expression with sympy.""" + py_expr = expr.replace("^", "**") + return sympy_parser.parse_expr( + py_expr, + transformations=(sympy_parser.standard_transformations + (sympy_parser.implicit_multiplication_application,)), + ) + + +def _parse_latex(expr: str) -> str: + """Attempts to parse latex to an expression sympy can read.""" + expr = expr.replace("\\tfrac", "\\frac") + expr = expr.replace("\\dfrac", "\\frac") + expr = expr.replace("\\frac", " \\frac") # Play nice with mixed numbers. + expr = latex2text.LatexNodes2Text().latex_to_text(expr) + + # Replace the specific characters that this parser uses. + expr = expr.replace("√", "sqrt") + expr = expr.replace("π", "pi") + expr = expr.replace("∞", "inf") + expr = expr.replace("∪", "U") + expr = expr.replace("·", "*") + expr = expr.replace("×", "*") + + return expr.strip() + + +def _is_float(num: str) -> bool: + try: + float(num) + return True + except ValueError: + return False + + +def _is_int(x: float) -> bool: + try: + return abs(x - int(round(x))) <= 1e-7 + except Exception: + return False + + +def _is_frac(expr: str) -> bool: + return bool(re.search(r"^-?[0-9]+.?/0*[1-9][0-9]*.?$", expr)) + + +def _str_is_int(x: str) -> bool: + try: + x = _strip_properly_formatted_commas(x) + x = float(x) + return abs(x - int(round(x))) <= 1e-7 + except Exception: + return False + + +def _str_to_int(x: str) -> bool: + x = x.replace(",", "") + x = float(x) + return int(x) + + +def _inject_implicit_mixed_number(step: str): + """ + Automatically make a mixed number evalable + e.g. 7 3/4 => 7+3/4 + """ + p1 = re.compile("([0-9]) +([0-9])") + step = p1.sub("\\1+\\2", step) ## implicit mults + return step + + +def _strip_properly_formatted_commas(expr: str): + # We want to be careful because we don't want to strip tuple commas + p1 = re.compile("(\d)(,)(\d\d\d)($|\D)") + while True: + next_expr = p1.sub("\\1\\3\\4", expr) + if next_expr == expr: + break + expr = next_expr + return next_expr + + +def _normalize(expr: str) -> str: + """Normalize answer expressions.""" + if expr is None: + return None + + # Remove enclosing `\text{}`. + m = re.search("^\\\\text\{(?P.+?)\}$", expr) + if m is not None: + expr = m.group("text") + + expr = expr.replace("\\%", "%") + expr = expr.replace("\\$", "$") + expr = expr.replace("$", "") + expr = expr.replace("%", "") + expr = expr.replace(" or ", " , ") + expr = expr.replace(" and ", " , ") + + expr = expr.replace("million", "*10^6") + expr = expr.replace("billion", "*10^9") + expr = expr.replace("trillion", "*10^12") + + for unit in [ + "degree", + "cm", + "centimeter", + "meter", + "mile", + "second", + "minute", + "hour", + "day", + "week", + "month", + "year", + "foot", + "feet", + "inch", + "yard", + "liter", + ]: + expr = re.sub(f"{unit}(es)?(s)? *(\^[0-9]+)?", "", expr) + expr = re.sub("\^ *\\\\circ", "", expr) + + if len(expr) > 0 and expr[0] == "{" and expr[-1] == "}": + expr = expr[1:-1] + + expr = re.sub(",\\\\! *", "", expr) + if _is_float(expr) and _is_int(float(expr)): + expr = str(int(round(float(expr)))) + if "\\" in expr: + with contextlib.suppress(Exception): + expr = _parse_latex(expr) + + # edge case with mixed numbers and negative signs + expr = re.sub("- *", "-", expr) + + expr = _inject_implicit_mixed_number(expr) + + # don't be case sensitive for text answers + expr = expr.lower() + + if _str_is_int(expr): + expr = str(_str_to_int(expr)) + + return expr + + +def count_unknown_letters_in_expr(expr: str): + expr = expr.replace("sqrt", "") + expr = expr.replace("frac", "") + letters_in_expr = set([x for x in expr if x.isalpha()]) + return len(letters_in_expr) + + +def should_allow_eval(expr: str): + # we don't want to try parsing unknown text or functions of more than two variables + if count_unknown_letters_in_expr(expr) > 2: + return False + + for bad_string in BAD_SUBSTRINGS: + if bad_string in expr: + return False + + return all(re.search(bad_regex, expr) is None for bad_regex in BAD_REGEXES) + + +@timeout_limit(seconds=10) +def are_equal_under_sympy(ground_truth_normalized: str, given_normalized: str): + are_equal = False + try: + expr = f"({ground_truth_normalized})-({given_normalized})" + if should_allow_eval(expr): + sympy_diff = _sympy_parse(expr) + simplified = sympy.simplify(sympy_diff) + if simplified == 0: + are_equal = True + except Exception: + pass + return are_equal + + +def split_tuple(expr: str): + """ + Split the elements in a tuple/interval, while handling well-formatted commas in large numbers + """ + expr = _strip_properly_formatted_commas(expr) + if len(expr) == 0: + return [] + if ( + len(expr) > 2 + and expr[0] in TUPLE_CHARS + and expr[-1] in TUPLE_CHARS + and all([ch not in expr[1:-1] for ch in TUPLE_CHARS]) + ): + elems = [elem.strip() for elem in expr[1:-1].split(",")] + else: + elems = [expr] + return elems + + +def grade_answer(given_answer: str, ground_truth: str) -> bool: + """ + The answer will be considered correct if: + (a) it normalizes to the same string as the ground truth answer + OR + (b) sympy can simplify the difference between the expressions to 0 + """ + if given_answer is None: + return False + + ground_truth_normalized_mathd = math_normalize.normalize_answer(ground_truth) + given_answer_normalized_mathd = math_normalize.normalize_answer(given_answer) + + # be at least as lenient as mathd + if ground_truth_normalized_mathd == given_answer_normalized_mathd: + return True + + ground_truth_normalized = _normalize(ground_truth) + given_normalized = _normalize(given_answer) + + if ground_truth_normalized is None: + return False + + if ground_truth_normalized == given_normalized: + return True + + if len(given_normalized) == 0: + return False + + ground_truth_elems = split_tuple(ground_truth_normalized) + given_elems = split_tuple(given_normalized) + + if ( + len(ground_truth_elems) > 1 + and (ground_truth_normalized[0] != given_normalized[0] or ground_truth_normalized[-1] != given_normalized[-1]) + or len(ground_truth_elems) != len(given_elems) + ): + is_correct = False + else: + for ground_truth_elem, given_elem in zip(ground_truth_elems, given_elems): + if _is_frac(ground_truth_elem) and _is_frac(given_elem): + # if fractions aren't reduced, then shouldn't be marked as correct + # so, we don't want to allow sympy.simplify in this case + is_correct = ground_truth_elem == given_elem + elif _str_is_int(ground_truth_elem) != _str_is_int(given_elem): + # if the ground truth answer is an integer, we require the given answer to be a strict match (no sympy.simplify) + is_correct = False + else: + try: + is_correct = are_equal_under_sympy(ground_truth_elem, given_elem) + except Exception as e: + # if there's an error, we'll just say it's not correct + is_correct = False + print(f"Error: {e} from are_equal_under_sympy, {ground_truth_elem}, {given_elem}") + if not is_correct: + break + + return is_correct + + +def remove_boxed(s): + left = "\\boxed{" + try: + assert s[: len(left)] == left + assert s[-1] == "}" + return s[len(left) : -1] + except Exception: + return None + + +def _last_boxed_only_string(string): + idx = string.rfind("\\boxed") + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + left_brace_idx = None + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if left_brace_idx is None: + left_brace_idx = i + elif string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + + i += 1 + + if left_brace_idx is None or right_brace_idx is None: + return None + + return string[left_brace_idx + 1 : right_brace_idx].strip() + + +def match_answer(response): + # NOTE: Reasoning360 used to comment out all response stripping + + is_matched = False + for ans_marker in ["answer:", "answer is", "answers are"]: + ans_idx = response.lower().rfind(ans_marker) + if ans_idx != -1: + is_matched = True + response = response[ans_idx + len(ans_marker) :].strip() + if response.endswith("\n"): + response = response[:-2] + + for ans_marker in ["is answer", "is the answer", "are answers", "are the answers"]: + ans_idx = response.lower().rfind(ans_marker) + if ans_idx != -1: + is_matched = True + response = response[:ans_idx].strip() + if response.endswith("\n"): + response = response[:-2] + + # Find boxed + ans_boxed = _last_boxed_only_string(response) + if ans_boxed: + is_matched = True + response = ans_boxed + + if ". " in response: + dot_idx = response.lower().rfind(". ") + if dot_idx != -1: + response = response[:dot_idx].strip() + + for ans_marker in ["be ", "is ", "are ", "=", ": ", "get ", "be\n", "is\n", "are\n", ":\n", "get\n"]: + ans_idx = response.lower().rfind(ans_marker) + if ans_idx != -1: + is_matched = True + response = response[ans_idx + len(ans_marker) :].strip() + if response.endswith("\n"): + response = response[:-2] + + is_matched = is_matched if any([c.isdigit() for c in response]) else False # answer must have a digit + # Grade + return is_matched, response + + +def compute_score(model_output: str, ground_truth: str) -> bool: + model_output = str(model_output) + ground_truth = str(ground_truth) + + is_matched, extracted_model_output = match_answer(model_output) + format_correctness = "Step 2:" in model_output and "\\box" in model_output + + # grade simple algebra questions. if succeeded, return; otherwise, proceed to more complex grading + if grade_answer(extracted_model_output, ground_truth): + # return True, True, extracted_model_output + return {"score": 1, "acc": 1} + + try: + if "\\pi" in extracted_model_output or "\\pi" in ground_truth: + equivs = [] + for pi in [math.pi, 3.14]: + equivs.append(math_equal(extracted_model_output, ground_truth, timeout=True, pi=pi)) + is_correct = any(equivs) + else: + is_correct = math_equal(extracted_model_output, ground_truth, timeout=True) + except Exception: + is_correct = False + + # return is_correct, format_correctness, extracted_model_output + return {"score": is_correct, "acc": is_correct} diff --git a/verl/utils/reward_score/prime_math/grader.py b/skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/grader.py similarity index 78% rename from verl/utils/reward_score/prime_math/grader.py rename to skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/grader.py index e3e1686f15..8d8158967b 100644 --- a/verl/utils/reward_score/prime_math/grader.py +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/grader.py @@ -93,16 +93,19 @@ """ import contextlib -import re -import signal import math +import re from math import isclose from typing import Union +# sympy related from sympy import N, simplify from sympy.parsing.latex import parse_latex from sympy.parsing.sympy_parser import parse_expr +# verl related +from skyrl_agent.tasks.verifiers.py_functional import timeout_limit + def is_digit(s): try: @@ -118,12 +121,13 @@ def is_digit(s): def normalize(answer, pi) -> str: # checking if answer is $ and removing $ in that case to compare - if isinstance(answer, str) and bool(re.match(r'\$\d+(\.\d+)?', answer)): + if isinstance(answer, str) and bool(re.match(r"\$\d+(\.\d+)?", answer)): return answer[1:] # checking if answer is % or \\% and removing % - if isinstance(answer, str) and (bool(re.match(r'^\d+(\.\d+)?%$', answer)) or - bool(re.match(r'^\d+(\.\d+)?\\%$', answer))): + if isinstance(answer, str) and ( + bool(re.match(r"^\d+(\.\d+)?%$", answer)) or bool(re.match(r"^\d+(\.\d+)?\\%$", answer)) + ): return answer.replace("\\%", "").replace("%", "") # handle base @@ -145,38 +149,37 @@ def handle_base(x) -> str: def handle_pi(string, pi): - if isinstance(string, str) and "\pi" in string: + if isinstance(string, str) and "\\pi" in string: # Find the first occurrence of "\pi" - idx = string.find("\pi") + idx = string.find("\\pi") # Iterate over the string and find all occurrences of "\pi" with a valid previous character while idx != -1: - if idx > 0 and string[idx - 1].isdigit(): # Replace "\pi" with "*math.pi" if the previous character is a digit - string = string[:idx] + f"*{pi}" + string[idx + 3:] + string = string[:idx] + f"*{pi}" + string[idx + 3 :] else: # Replace "\pi" with "1*math.pi" if the previous character is not a digit - string = string[:idx] + f"1*{pi}" + string[idx + 3:] + string = string[:idx] + f"1*{pi}" + string[idx + 3 :] # Find the next occurrence of "\pi" - idx = string.find("\pi", idx + 1) + idx = string.find("\\pi", idx + 1) # Evaluate the expression using eval() function - try: + with contextlib.suppress(Exception): string = eval(string) - except: - pass return string -def math_equal(prediction: Union[bool, float, str], - reference: Union[float, str], - include_percentage: bool = True, - tolerance: float = 1e-4, - timeout: float = 10.0, - pi: float = math.pi) -> bool: +def math_equal( + prediction: Union[bool, float, str], + reference: Union[float, str], + include_percentage: bool = True, + tolerance: float = 1e-4, + timeout: float = 10.0, + pi: float = math.pi, +) -> bool: """ Exact match of math if and only if: 1. numerical equal: both can convert to float and are equal @@ -201,10 +204,7 @@ def math_equal(prediction: Union[bool, float, str], prediction = is_digit(prediction)[1] reference = is_digit(reference)[1] # number questions - if include_percentage: - gt_result = [reference / 100, reference, reference * 100] - else: - gt_result = [reference] + gt_result = [reference / 100, reference, reference * 100] if include_percentage else [reference] for item in gt_result: try: if isclose(item, prediction, rel_tol=tolerance): @@ -226,9 +226,9 @@ def math_equal(prediction: Union[bool, float, str], prediction = format_intervals(prediction) pred_str, ref_str = prediction, reference - if (prediction.startswith("[") and prediction.endswith("]") and - not reference.startswith("(")) or (prediction.startswith("(") and prediction.endswith(")") and - not reference.startswith("[")): + if (prediction.startswith("[") and prediction.endswith("]") and not reference.startswith("(")) or ( + prediction.startswith("(") and prediction.endswith(")") and not reference.startswith("[") + ): pred_str = pred_str.strip("[]()") ref_str = ref_str.strip("[]()") for s in ["{", "}", "(", ")"]: @@ -238,52 +238,62 @@ def math_equal(prediction: Union[bool, float, str], return True ## [a, b] vs. [c, d], return a==c and b==d - if (prediction and reference and prediction[0] in "([" and prediction[-1] in ")]" and - prediction[0] == reference[0] and prediction[-1] == reference[-1]): + if ( + prediction + and reference + and prediction[0] in "([" + and prediction[-1] in ")]" + and prediction[0] == reference[0] + and prediction[-1] == reference[-1] + ): pred_parts = prediction[1:-1].split(",") ref_parts = reference[1:-1].split(",") - if len(pred_parts) == len(ref_parts): - if all([ - math_equal(pred_pt, ref_pt, include_percentage, tolerance) - for pred_pt, ref_pt in zip(pred_parts, ref_parts) - ]): - return True + if len(pred_parts) == len(ref_parts) and all( + [ + math_equal(pred_pt, ref_pt, include_percentage, tolerance) + for pred_pt, ref_pt in zip(pred_parts, ref_parts) + ] + ): + return True if "," in prediction and "," in reference: pred_parts = [item.strip() for item in prediction.split(",")] ref_parts = [item.strip() for item in reference.split(",")] if len(pred_parts) == len(ref_parts): - if all([ - math_equal(pred_parts[i], ref_parts[i], include_percentage, tolerance) - for i in range(len(pred_parts)) - ]): - return True - else: - return False + return bool( + all( + [ + math_equal(pred_parts[i], ref_parts[i], include_percentage, tolerance) + for i in range(len(pred_parts)) + ] + ) + ) # if we have point == tuple of values if prediction.startswith("Point") and reference[0] == "(" and reference[-1] == ")": - pred_parts = prediction[prediction.find("(") + 1:-1].split(",") + pred_parts = prediction[prediction.find("(") + 1 : -1].split(",") ref_parts = reference[1:-1].split(",") - if len(pred_parts) == len(ref_parts): - if all([ - math_equal(pred_pt, ref_pt, include_percentage, tolerance) - for pred_pt, ref_pt in zip(pred_parts, ref_parts) - ]): - return True + if len(pred_parts) == len(ref_parts) and all( + [ + math_equal(pred_pt, ref_pt, include_percentage, tolerance) + for pred_pt, ref_pt in zip(pred_parts, ref_parts) + ] + ): + return True # if reference is a matrix if "\begin{pmatrix}" in reference and prediction.startswith("Matrix"): try: pred_matrix = parse_expr(prediction) ref_matrix_items = reference.split()[1:-1:2] - if len(pred_matrix) == len(ref_matrix_items): - if all([ - math_equal(pred, ref, include_percentage, tolerance) - for ref, pred in zip(ref_matrix_items, pred_matrix) - ]): - return True + if len(pred_matrix) == len(ref_matrix_items) and all( + [ + math_equal(pred, ref, include_percentage, tolerance) + for ref, pred in zip(ref_matrix_items, pred_matrix) + ] + ): + return True except Exception: pass elif "\begin{pmatrix}" in reference and prediction.startswith("[") and prediction.endswith("]"): @@ -291,16 +301,21 @@ def math_equal(prediction: Union[bool, float, str], try: pred_matrix = eval(prediction) # ref_matrix_items = reference.split()[1:-1:2] - ref_matrix_items = reference.lstrip("\\begin{pmatrix}").lstrip("\begin{pmatrix}").rstrip( - "\\end{pmatrix}").rstrip("\end{pmatrix}") + ref_matrix_items = ( + reference.lstrip("\\begin{pmatrix}") + .lstrip("\begin{pmatrix}") + .rstrip("\\end{pmatrix}") + .rstrip("\\end{pmatrix}") + ) # noqa: B005 ref_matrix_items = ref_matrix_items.split("\\") ref_matrix_items = [row.split("&") if "&" in row else row for row in ref_matrix_items] - if len(pred_matrix) == len(ref_matrix_items): - if all([ - math_equal(pred, ref, include_percentage, tolerance) - for ref, pred in zip(ref_matrix_items, pred_matrix) - ]): - return True + if len(pred_matrix) == len(ref_matrix_items) and all( + [ + math_equal(pred, ref, include_percentage, tolerance) + for ref, pred in zip(ref_matrix_items, pred_matrix) + ] + ): + return True except Exception: pass @@ -308,53 +323,43 @@ def math_equal(prediction: Union[bool, float, str], def symbolic_equal(a, b, tolerance, timeout=10.0): - def _parse(s): for f in [parse_expr, parse_latex]: try: - with time_limit(timeout): + with timeout_limit(seconds=timeout): return f(s) + except TimeoutError: + print(f"Parsing timed out for {s}") + continue except Exception: - pass + continue return s a = _parse(a) b = _parse(b) try: - with time_limit(timeout): + with timeout_limit(seconds=timeout): if simplify(a - b) == 0: return True + except TimeoutError: + print(f"Simplification timed out for {a} - {b}") + pass except Exception: pass try: - with time_limit(timeout): + with timeout_limit(seconds=timeout): if isclose(N(a), N(b), rel_tol=tolerance): return True + except TimeoutError: + print(f"Numerical evaluation timed out for {a}, {b}") + pass except Exception: pass return False -class TimeoutException(Exception): - pass - - -@contextlib.contextmanager -def time_limit(seconds: float): - - def signal_handler(signum, frame): - raise TimeoutException("Timed out!") - - signal.setitimer(signal.ITIMER_REAL, seconds) - signal.signal(signal.SIGALRM, signal_handler) - try: - yield - finally: - signal.setitimer(signal.ITIMER_REAL, 0) - - def format_intervals(prediction): patterns = { "Interval(": r"^Interval\((.*)\)$", diff --git a/verl/utils/reward_score/prime_math/math_normalize.py b/skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/math_normalize.py similarity index 94% rename from verl/utils/reward_score/prime_math/math_normalize.py rename to skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/math_normalize.py index e9921a5ad2..2036ad1421 100644 --- a/verl/utils/reward_score/prime_math/math_normalize.py +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/prime_math/math_normalize.py @@ -36,6 +36,7 @@ From: https://github.com/openai/prm800k/blob/main/prm800k/grading/math_normalize.py """ + import re from typing import Optional @@ -46,11 +47,11 @@ def normalize_answer(answer: Optional[str]) -> Optional[str]: answer = answer.strip() try: # Remove enclosing `\text{}`. - m = re.search("^\\\\text\{(?P.+?)\}$", answer) + m = re.search("^\\\\text\\{(?P.+?)\\}$", answer) if m is not None: answer = m.group("text").strip() return _strip_string(answer) - except: + except: # noqa: E722 return answer @@ -66,7 +67,7 @@ def _fix_fracs(string): else: try: assert len(substr) >= 2 - except: + except: # noqa: E722 return string a = substr[0] b = substr[1] @@ -97,7 +98,7 @@ def _fix_a_slash_b(string): assert string == "{}/{}".format(a, b) new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" return new_string - except: + except: # noqa: E722 return string @@ -156,7 +157,7 @@ def _strip_string(string): # remove percentage string = string.replace("\\%", "") - string = string.replace("\%", "") + string = string.replace("\\%", "") # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string string = string.replace(" .", " 0.") @@ -168,9 +169,8 @@ def _strip_string(string): string = "0" + string # to consider: get rid of e.g. "k = " or "q = " at beginning - if len(string.split("=")) == 2: - if len(string.split("=")[0]) <= 2: - string = string.split("=")[1] + if len(string.split("=")) == 2 and len(string.split("=")[0]) <= 2: + string = string.split("=")[1] # fix sqrt3 --> sqrt{3} string = _fix_sqrt(string) @@ -188,4 +188,4 @@ def _strip_string(string): # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y string = _fix_a_slash_b(string) - return string \ No newline at end of file + return string diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/py_functional.py b/skyrl-agent/skyrl_agent/tasks/verifiers/py_functional.py new file mode 100644 index 0000000000..ad3a08d373 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/py_functional.py @@ -0,0 +1,284 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Contain small python utility functions +""" + +import importlib +import multiprocessing +import os +import queue # Import the queue module for exception type hint +import signal +from functools import wraps +from types import SimpleNamespace +from typing import Any, Callable, Dict, Iterator, Optional, Tuple + + +# --- Top-level helper for multiprocessing timeout --- +# This function MUST be defined at the top level to be pickleable +def _mp_target_wrapper(target_func: Callable, mp_queue: multiprocessing.Queue, args: Tuple, kwargs: Dict[str, Any]): + """ + Internal wrapper function executed in the child process. + Calls the original target function and puts the result or exception into the queue. + """ + try: + result = target_func(*args, **kwargs) + mp_queue.put((True, result)) # Indicate success and put result + except Exception as e: + # Ensure the exception is pickleable for the queue + try: + import pickle + + pickle.dumps(e) # Test if the exception is pickleable + mp_queue.put((False, e)) # Indicate failure and put exception + except (pickle.PicklingError, TypeError): + # Fallback if the original exception cannot be pickled + mp_queue.put((False, RuntimeError(f"Original exception type {type(e).__name__} not pickleable: {e}"))) + + +# Renamed the function from timeout to timeout_limit +def timeout_limit(seconds: float, use_signals: bool = False): + """ + Decorator to add a timeout to a function. + + Args: + seconds: The timeout duration in seconds. + use_signals: (Deprecated) This is deprecated because signals only work reliably in the main thread + and can cause issues in multiprocessing or multithreading contexts. + Defaults to False, which uses the more robust multiprocessing approach. + + Returns: + A decorated function with timeout. + + Raises: + TimeoutError: If the function execution exceeds the specified time. + RuntimeError: If the child process exits with an error (multiprocessing mode). + NotImplementedError: If the OS is not POSIX (signals are only supported on POSIX). + """ + + def decorator(func): + if use_signals: + if os.name != "posix": + raise NotImplementedError(f"Unsupported OS: {os.name}") + # Issue deprecation warning if use_signals is explicitly True + print( + "WARN: The 'use_signals=True' option in the timeout decorator is deprecated. \ + Signals are unreliable outside the main thread. \ + Please use the default multiprocessing-based timeout (use_signals=False)." + ) + + @wraps(func) + def wrapper_signal(*args, **kwargs): + def handler(signum, frame): + # Update function name in error message if needed (optional but good practice) + raise TimeoutError(f"Function {func.__name__} timed out after {seconds} seconds (signal)!") + + old_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, handler) + # Use setitimer for float seconds support, alarm only supports integers + signal.setitimer(signal.ITIMER_REAL, seconds) + + try: + result = func(*args, **kwargs) + finally: + # Reset timer and handler + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, old_handler) + return result + + return wrapper_signal + else: + # --- Multiprocessing based timeout (existing logic) --- + @wraps(func) + def wrapper_mp(*args, **kwargs): + q = multiprocessing.Queue(maxsize=1) + process = multiprocessing.Process(target=_mp_target_wrapper, args=(func, q, args, kwargs)) + process.start() + process.join(timeout=seconds) + + if process.is_alive(): + process.terminate() + process.join(timeout=0.5) # Give it a moment to terminate + if process.is_alive(): + print(f"Warning: Process {process.pid} did not terminate gracefully after timeout.") + # Update function name in error message if needed (optional but good practice) + raise TimeoutError(f"Function {func.__name__} timed out after {seconds} seconds (multiprocessing)!") + + try: + success, result_or_exc = q.get(timeout=0.1) # Small timeout for queue read + if success: + return result_or_exc + else: + raise result_or_exc # Reraise exception from child + except queue.Empty as err: + exitcode = process.exitcode + if exitcode is not None and exitcode != 0: + raise RuntimeError( + f"Child process exited with error (exitcode: {exitcode}) before returning result." + ) from err + else: + # Should have timed out if queue is empty after join unless process died unexpectedly + # Update function name in error message if needed (optional but good practice) + raise TimeoutError( + f"Operation timed out or process finished unexpectedly without result (exitcode: {exitcode})." + ) from err + finally: + q.close() + q.join_thread() + + return wrapper_mp + + return decorator + + +def union_two_dict(dict1: Dict, dict2: Dict): + """Union two dict. Will throw an error if there is an item not the same object with the same key. + + Args: + dict1: + dict2: + + Returns: + + """ + for key, val in dict2.items(): + if key in dict1: + assert dict2[key] == dict1[key], f"{key} in meta_dict1 and meta_dict2 are not the same object" + dict1[key] = val + + return dict1 + + +def append_to_dict(data: Dict, new_data: Dict): + """Append values from new_data to lists in data. + + For each key in new_data, this function appends the corresponding value to a list + stored under the same key in data. If the key doesn't exist in data, a new list is created. + + Args: + data (Dict): The target dictionary containing lists as values. + new_data (Dict): The source dictionary with values to append. + + Returns: + None: The function modifies data in-place. + """ + for key, val in new_data.items(): + if key not in data: + data[key] = [] + data[key].append(val) + + +class NestedNamespace(SimpleNamespace): + """A nested version of SimpleNamespace that recursively converts dictionaries to namespaces. + + This class allows for dot notation access to nested dictionary structures by recursively + converting dictionaries to NestedNamespace objects. + + Example: + config_dict = {"a": 1, "b": {"c": 2, "d": 3}} + config = NestedNamespace(config_dict) + # Access with: config.a, config.b.c, config.b.d + + Args: + dictionary: The dictionary to convert to a nested namespace. + **kwargs: Additional attributes to set on the namespace. + """ + + def __init__(self, dictionary, **kwargs): + super().__init__(**kwargs) + for key, value in dictionary.items(): + if isinstance(value, dict): + self.__setattr__(key, NestedNamespace(value)) + else: + self.__setattr__(key, value) + + +class DynamicEnumMeta(type): + def __iter__(cls) -> Iterator[Any]: + return iter(cls._registry.values()) + + def __contains__(cls, item: Any) -> bool: + # allow `name in EnumClass` or `member in EnumClass` + if isinstance(item, str): + return item in cls._registry + return item in cls._registry.values() + + def __getitem__(cls, name: str) -> Any: + return cls._registry[name] + + def __reduce_ex__(cls, protocol): + # Always load the existing module and grab the class + return getattr, (importlib.import_module(cls.__module__), cls.__name__) + + def names(cls): + return list(cls._registry.keys()) + + def values(cls): + return list(cls._registry.values()) + + +class DynamicEnum(metaclass=DynamicEnumMeta): + _registry: Dict[str, "DynamicEnum"] = {} + _next_value: int = 0 + + def __init__(self, name: str, value: int): + self.name = name + self.value = value + + def __repr__(self): + return f"<{self.__class__.__name__}.{self.name}: {self.value}>" + + def __reduce_ex__(self, protocol): + """ + Unpickle via: getattr(import_module(module).Dispatch, 'ONE_TO_ALL') + so the existing class is reused instead of re-executed. + """ + module = importlib.import_module(self.__class__.__module__) + enum_cls = getattr(module, self.__class__.__name__) + return getattr, (enum_cls, self.name) + + @classmethod + def register(cls, name: str) -> "DynamicEnum": + key = name.upper() + if key in cls._registry: + raise ValueError(f"{key} already registered") + member = cls(key, cls._next_value) + cls._registry[key] = member + setattr(cls, key, member) + cls._next_value += 1 + return member + + @classmethod + def remove(cls, name: str): + key = name.upper() + member = cls._registry.pop(key) + delattr(cls, key) + return member + + @classmethod + def from_name(cls, name: str) -> Optional["DynamicEnum"]: + return cls._registry.get(name.upper()) + + +def convert_to_regular_types(obj): + """Convert Hydra configs and other special types to regular Python types.""" + from omegaconf import DictConfig, ListConfig + + if isinstance(obj, (ListConfig, DictConfig)): + return {k: convert_to_regular_types(v) for k, v in obj.items()} if isinstance(obj, DictConfig) else list(obj) + elif isinstance(obj, (list, tuple)): + return [convert_to_regular_types(x) for x in obj] + elif isinstance(obj, dict): + return {k: convert_to_regular_types(v) for k, v in obj.items()} + return obj diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/qa.py b/skyrl-agent/skyrl_agent/tasks/verifiers/qa.py new file mode 100644 index 0000000000..df7fc6f2b8 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/qa.py @@ -0,0 +1,288 @@ +# From https://github.com/PeterGriffinJin/Search-R1/blob/9828adecbbdb184333e4d8ca9d6b8bd10275da44/verl/utils/reward_score/qa_em.py#L36 +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import string +import random +import litellm +import json + +JUDGE_PROMPT_BROWSECOMP_OFFICIAL = """Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below. + +[question]: {question} + +[response]: {response} + +Your judgement must be in the format and criteria specified below: + +extracted_final_answer: The final exact answer extracted from the [response]. Put the extracted answer as 'None' if there is no exact, final answer to extract from the response. + +[correct_answer]: {correct_answer} + +reasoning: Explain why the extracted_final_answer is correct or incorrect based on [correct_answer], focusing only on if there are meaningful differences between [correct_answer] and the extracted_final_answer. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match. + +correct: Answer 'yes' if extracted_final_answer matches the [correct_answer] given above, or is within a small margin of error for numerical problems. Answer 'no' otherwise, i.e. if there if there is any inconsistency, ambiguity, non-equivalency, or if the extracted answer is incorrect. + +confidence: The extracted confidence score between 0|\%| and 100|\%| from [response]. Put 100 if there is no confidence score available. +""".strip() + +extracted_answer_format_for_confidence = { + "type": "json_schema", + "json_schema": { + "name": "extracted_answer", + "schema": { + "type": "object", + "properties": { + "extracted_final_answer": {"type": "string"}, + "reasoning": {"type": "string"}, + "correct": {"type": "string", "enum": ["yes", "no"]}, + "confidence": {"type": "number"}, + "strict": {"type": "boolean"}, + }, + "required": ["extracted_final_answer", "reasoning", "correct", "confidence", "strict"], + "additionalProperties": False, + }, + "strict": True, + }, +} + +JUDGE_PROMPT_RULER_OFFICIAL = """Does the [response] correctly answer the [question] based on [correct_answer]? + +[question]: {question} +[correct_answer]: {correct_answer} +[response]: {response} + +Answer 'yes' if the response matches the correct answer, 'no' otherwise. +""".strip() + +ruler_answer_format = { + "type": "json_schema", + "json_schema": { + "name": "simple_judgment", + "schema": { + "type": "object", + "properties": { + "correct": {"type": "string", "enum": ["yes", "no"]}, + }, + "required": ["correct"], + "additionalProperties": False, + }, + "strict": True, + }, +} + + +def normalize_answer(s): + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + # remove \boxed{} if it exists + s = re.sub(r"\\boxed\{([^}]+)\}", r"\1", s) + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def em_check(prediction, golden_answers): + if isinstance(golden_answers, str): + golden_answers = [golden_answers] + normalized_prediction = normalize_answer(prediction) + score = 0 + for golden_answer in golden_answers: + golden_answer = normalize_answer(golden_answer) + if golden_answer == normalized_prediction: + score = 1 + break + return score + + +def bool_mapping(s): + """Map boolean strings to yes/no format""" + if s == "True": + return "yes" + elif s == "False": + return "no" + else: + return s + + +def contains_chinese(text): + """Check if the given text contains Chinese characters""" + for char in text: + if "\u4e00" <= char <= "\u9fff": + return True + if "\u3400" <= char <= "\u4dbf": + return True + if "\uf900" <= char <= "\ufaff": + return True + return False + + +def normalize_text(text: str) -> str: + """Normalize text for F1 scoring""" + for punct in string.punctuation: + text = text.replace(punct, " ") + text = re.sub(r"\s+", " ", text) + return text.strip().lower() + + +def f1_score(answer_content, gt): + """Compute F1 score between answer and ground truth""" + answer_content = normalize_text(bool_mapping(answer_content)) + gt = normalize_text(bool_mapping(gt)) + + if contains_chinese(gt): + + def parse_chinese_str(s): + numbers = [] + for i, c in enumerate(s): + if c.isdigit(): + if i > 0 and s[i - 1].isdigit(): + numbers[-1] = numbers[-1] + c + else: + numbers.append(c) + for c in "0123456789，。 ,.-": + s = s.replace(c, "") + return set(list(s) + numbers) + + pred_tokens = parse_chinese_str(answer_content) + gt_tokens = parse_chinese_str(gt) + else: + pred_tokens = set(answer_content.split()) + gt_tokens = set(gt.split()) + + if not gt_tokens or not pred_tokens: + return 0 + + common_tokens = pred_tokens & gt_tokens + precision = len(common_tokens) / len(pred_tokens) + recall = len(common_tokens) / len(gt_tokens) + + if precision + recall > 0: + return 2 * (precision * recall) / (precision + recall) + return 0 + + +def compute_score_f1(solution_str, ground_truth, format_score=0.0, score=1.0): + """Compute F1 score - handles both single targets and lists""" + target = ground_truth["target"] + + if solution_str is None: + return {"score": 0} + + # Handle numpy arrays and lists + if hasattr(target, "tolist"): + target = target.tolist() + + # Handle lists by computing F1 against each and taking max + if isinstance(target, list): + scores = [] + for gt in target: + scores.append(f1_score(solution_str, gt)) + return {"score": max(scores) if scores else 0} + + # Single target case + return {"score": f1_score(solution_str, target)} + + +def compute_score_em(solution_str, ground_truth, format_score=0.0, score=1.0): + """The scoring function for exact match (EM). + + Args: + solution_str: the solution text + ground_truth: the ground truth + method: the method to extract the solution, choices are 'strict' and 'flexible' + format_score: the score for the format + score: the score for the correct answer + """ + do_print = random.randint(1, 64) == 1 + + if do_print: + print("--------------------------------") + print(f"Golden answers: {ground_truth['target']}") + print(f"Solution string: {solution_str}") + + if solution_str is None: + return {"score": 0} + else: + if em_check(solution_str, ground_truth["target"]): + return {"score": score} + else: + return {"score": format_score} + + +# use llm as a verifier +def compute_score_browsecomp(solution_str, ground_truth, question): + """The scoring function for LLM""" + + if isinstance(ground_truth["target"], list): + assert len(ground_truth["target"]) == 1, "Only one correct answer is supported for browsecomp" + ground_truth["target"] = ground_truth["target"][0] + + if solution_str is None: + return {"score": 0} + else: + # use llm as a verifier + prompt = JUDGE_PROMPT_BROWSECOMP_OFFICIAL.format( + question=question, response=solution_str, correct_answer=ground_truth["target"] + ) + print(f"Prompt for judge model: {prompt}") + response = litellm.completion( + model="gpt-4o-2024-08-06", + messages=[{"role": "user", "content": prompt}], + num_retries=5, + response_format=extracted_answer_format_for_confidence, + ) + print(f"Response from judge model: {response}") + raw_content = response.choices[0].message["content"] + raw_judge = json.loads(raw_content) + judgement = "Correct" if raw_judge["correct"].lower() == "yes" else "" + + return {"score": 1 if judgement == "Correct" else 0} + + +# use llm as a verifier +def compute_score_ruler(solution_str, ground_truth, question): + """The scoring function for LLM""" + if solution_str is None: + return {"score": 0} + else: + # use llm as a verifier + prompt = JUDGE_PROMPT_RULER_OFFICIAL.format( + question=question, response=solution_str, correct_answer=ground_truth + ) + print(f"Prompt for judge model: {prompt}") + response = litellm.completion( + model="gpt-5-nano", + messages=[{"role": "user", "content": prompt}], + num_retries=1, + response_format=ruler_answer_format, + ) + print(f"Response from judge model: {response}") + raw_content = response.choices[0].message["content"] + print(f"Raw content from judge model: {raw_content}") + raw_judge = json.loads(raw_content) + judgement = "Correct" if raw_judge["correct"].lower() == "yes" else "" + + return {"score": 1 if judgement == "Correct" else 0} diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/sandbox_fusion/utils.py b/skyrl-agent/skyrl_agent/tasks/verifiers/sandbox_fusion/utils.py new file mode 100644 index 0000000000..71879586e4 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/sandbox_fusion/utils.py @@ -0,0 +1,569 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import concurrent.futures # <-- Import concurrent.futures +import json +import logging +import os +import threading +import time +import traceback +import uuid +from typing import Any, Optional + +import requests + +DEFAULT_TIMEOUT = 10 # Default compile and run timeout +MAX_RETRIES = 3 +INITIAL_RETRY_DELAY = 1 +API_TIMEOUT = 10 + +logger = logging.getLogger(__name__) + +# Define supported languages list (optional, for documentation or validation) +SUPPORTED_LANGUAGES = [ + "python", + "cpp", + "nodejs", + "go", + "go_test", + "java", + "php", + "csharp", + "bash", + "typescript", + "sql", + "rust", + "cuda", + "lua", + "R", + "perl", + "D_ut", + "ruby", + "scala", + "julia", + "pytest", + "junit", + "kotlin_script", + "jest", + "verilog", + "python_gpu", + "lean", + "swift", + "racket", +] + + +def call_sandbox_api( + sandbox_fusion_url: str, + code: str, + stdin: Optional[str], + compile_timeout: int, + run_timeout: int, + memory_limit_mb: int, + language: str = "python", +) -> tuple[Optional[dict[str, Any]], Optional[str]]: # <-- Remove request_id parameter + """ + Calls the remote sandbox API to execute code with retry logic for Gateway Timeout, + using increasing delay between retries. Logs internal calls with a unique ID. + + Args: + sandbox_fusion_url: The URL of the sandbox fusion API. + code: The code string to execute. + stdin: The standard input string. + compile_timeout: Compile timeout in seconds. + run_timeout: Run timeout in seconds. + language: The programming language of the code (e.g., "python", "cpp", "java"). Defaults to "python". + + Returns: + A tuple (response_json, error_message). + If successful, response_json is the API's returned JSON object, error_message is None. + If failed after retries, response_json is None, error_message contains the error information. + """ + request_id = str(uuid.uuid4()) # <-- Generate request_id internally + log_prefix = f"[Request ID: {request_id}] " # <-- Create log prefix + + if language not in SUPPORTED_LANGUAGES: + error_msg = f"{log_prefix}Unsupported language: {language}" + logger.error(error_msg) + return None, error_msg + + payload = json.dumps( + { + "compile_timeout": compile_timeout, + "run_timeout": run_timeout, + "code": code, + "stdin": stdin, + "memory_limit_MB": memory_limit_mb, + "language": language, # Use the passed language parameter + "files": {}, + "fetch_files": [], + } + ) + headers = {"Content-Type": "application/json", "Accept": "application/json"} + # Calculate a reasonable request timeout based on compile/run timeouts plus a buffer + request_timeout = compile_timeout + run_timeout + API_TIMEOUT + + last_error = None # Store the last error encountered + + for attempt in range(MAX_RETRIES): + try: + # logger.info( + # f"{log_prefix}Attempt {attempt + 1}/{MAX_RETRIES}: Calling sandbox API at {sandbox_fusion_url}" + # ) # <-- Use internal log_prefix + response = requests.post( + sandbox_fusion_url, + headers=headers, + data=payload, + timeout=request_timeout, # Use the calculated timeout + ) + + # Check for Gateway Timeout (504) specifically for retrying + if response.status_code == 504: + last_error = ( + f"{log_prefix}API Request Error: Gateway Timeout (504) on attempt " f"{attempt + 1}/{MAX_RETRIES}" + ) # <-- Use internal log_prefix + logger.warning(last_error) + if attempt < MAX_RETRIES - 1: # Don't sleep after the last attempt + # Calculate increasing delay (e.g., 1s, 2s, 4s, ...) or (1s, 2s, 3s, ...) + # Simple linear increase: delay = INITIAL_RETRY_DELAY * (attempt + 1) + # Exponential backoff: delay = INITIAL_RETRY_DELAY * (2 ** attempt) + delay = INITIAL_RETRY_DELAY * (attempt + 1) # Using linear increase for simplicity + logger.info(f"{log_prefix}Retrying after {delay} seconds...") # <-- Use internal log_prefix + time.sleep(delay) + continue # Go to the next retry attempt + + # Check for other HTTP errors (e.g., 4xx, other 5xx) + response.raise_for_status() + + # If successful (status code 2xx) + logger.info( + f"{log_prefix}Sandbox API call successful on attempt {attempt + 1}" + ) # <-- Use internal log_prefix + return response.json(), None + + except requests.exceptions.RequestException as e: + last_error = f"{log_prefix}API Request Error: {e}" # <-- Use internal log_prefix + break # Exit retry loop on non-504 request errors + except json.JSONDecodeError as e: + raw_response_text = response.text if "response" in locals() else "N/A" + last_error = f"{log_prefix}API Response JSON Decode Error: {e}" # <-- Use internal log_prefix + break # Exit retry loop on JSON decode errors + except Exception as e: + last_error = f"{log_prefix}Unexpected Error: {e}" # <-- Use internal log_prefix + break # Exit retry loop on other unexpected errors + + # If loop finishes without returning success, return the last recorded error + logger.error(f"{log_prefix}Sandbox API call failed. Last error: {last_error}") # <-- Use internal log_prefix + # Return the error message without the prefix, as the caller doesn't need the internal ID + # Ensure API call failure returns error message, leading to -1 in check_correctness + return None, last_error.replace(log_prefix, "API Call Failed: ") if last_error else "API Call Failed after retries" + + +def _process_single_case( + case_index: int, + stdin_data: Any, + expected_output: Any, + sandbox_fusion_url: str, + generation: str, + timeout: int, + memory_limit_mb: int, + language: str, + concurrent_semaphore: Optional[threading.Semaphore] = None, + fn_name: Optional[str] = None, +) -> tuple[int, dict[str, Any]]: + """Helper function to process a single test case.""" + api_response = None + error_msg = None + logger.info(f"Processing test case {case_index + 1}.") + + current_generation_code = generation + + if fn_name and language == "python": + # Wrapper assumes stdin_data is a JSON string for function arguments. + wrapper_code = f""" +import traceback +from string import * +from re import * +from datetime import * +from collections import * +from heapq import * +from bisect import * +from copy import * +from math import * +from random import * +from statistics import * +from itertools import * +from functools import * +from operator import * +from io import * +from sys import * +from json import * +from builtins import * +from typing import * +import string +import re +import datetime +import collections +import heapq +import bisect +import copy +import math +import random +import statistics +import itertools +import functools +import operator +import io +import sys +import json + +# === User's Original Code START === +{generation} +# === User's Original Code END === + +_SANDBOX_FN_NAME = "{fn_name}" + +def _execute_user_function(): + # --- Input Parsing --- + _raw_input_str = sys.stdin.read() + _args = [] + if _raw_input_str.strip(): # If there's input + try: + _args = [json.loads(line) for line in _raw_input_str.split('\\n')] + except json.JSONDecodeError as _je: + sys.stderr.write(f"WrapperError: Invalid JSON input for '{{_SANDBOX_FN_NAME}}': {{_je}}\\nInput was: " + f"{{_raw_input_str[:200]}}\\n") + return None, True # result, error_occurred + + # --- Function Location and Execution --- + try: + _target_callable = None + # Try global scope first + if _SANDBOX_FN_NAME in globals(): + _target_callable = globals()[_SANDBOX_FN_NAME] + # Else, if 'Solution' class exists, try to get its method + elif 'Solution' in globals(): + _Solution_class = globals()['Solution'] + # Attempt to instantiate and get method. + # Errors (e.g., Solution not a class, instantiation fails, method missing) + # will be caught by the broad except block below. + _solution_instance = _Solution_class() + _target_callable = getattr(_solution_instance, _SANDBOX_FN_NAME) + + if not _target_callable: + sys.stderr.write(f"WrapperError: Function or method '{{_SANDBOX_FN_NAME}}' not found.\\n") + return None, True # result, error_occurred + + _fn_result = _target_callable(*_args) + return _fn_result, False # result, no_error + except Exception: # Catches errors from Solution instantiation, getattr, or function call + sys.stderr.write(f"Error during setup or execution of '{{_SANDBOX_FN_NAME}}':\\n{{traceback.format_exc()}}\\n") + return None, True # result, error_occurred + +if __name__ == '__main__': + _result, _error_occurred = _execute_user_function() + + if not _error_occurred: + # Serialize result to stdout + if isinstance(_result, (dict, list, tuple)) or _result is None or isinstance(_result, bool): + print(json.dumps(_result)) + elif isinstance(_result, (int, float, str)): + print(str(_result)) # Ensure string conversion for print + else: + # For other types, default to string representation. + print(str(_result)) + # Optional: To explicitly exit with an error code if the sandbox relies on it + # else: + # sys.exit(1) +""" + current_generation_code = wrapper_code + + stdin = None if stdin_data is None else str(stdin_data) + try: + if concurrent_semaphore: + # logger.debug(f"Case {case_index + 1}: Attempting to acquire semaphore.") + with concurrent_semaphore: + # logger.debug(f"Case {case_index + 1}: Semaphore acquired. Calling API.") + api_response, error_msg = call_sandbox_api( + sandbox_fusion_url=sandbox_fusion_url, + code=current_generation_code, + stdin=stdin, + compile_timeout=timeout, + run_timeout=timeout, + memory_limit_mb=memory_limit_mb, + language=language, + ) + # logger.debug(f"Case {case_index + 1}: Semaphore released.") + else: + api_response, error_msg = call_sandbox_api( + sandbox_fusion_url=sandbox_fusion_url, + code=current_generation_code, + stdin=stdin, + compile_timeout=timeout, + run_timeout=timeout, + memory_limit_mb=memory_limit_mb, + language=language, + ) + except Exception as e: + error_msg = f"API Request Exception during check_correctness for case {case_index + 1}: {e}" + logger.error(f"Case {case_index + 1}: {error_msg}") + traceback.print_exc() + + metadata = { + "case_index": case_index, + "input": stdin, + "expected_output": str(expected_output), + "api_request_error": error_msg, + "api_response": None, + "status": "unknown", + "stdout": None, + "stderr": None, + "exit_code": None, + "duration": None, + "compile_duration": None, + "compile_stderr": None, + "api_status": None, + "compile_status": None, + "run_status": None, + } + result_status = -1 # Default error: API request error or unknown sandbox error + + if error_msg: + metadata["status"] = "api_error" + result_status = -1 # API request itself failed (includes timeout after retries) + logger.error(f"Case {case_index}: API error occurred: {error_msg}") + # Log code and input only on error for brevity + generation_to_log = generation[:200] + "..." if len(generation) > 200 else generation + logger.error(f"Case {case_index}: code: {generation_to_log}") + logger.error(f"Case {case_index}: input: {stdin}") + elif api_response: + # --- Add debug logging --- + logger.debug(f"Case {case_index}: API Response: {api_response}") + metadata["api_response"] = api_response + metadata["api_status"] = api_response.get("status") + compile_result = api_response.get("compile_result") + run_result = api_response.get("run_result") + + # Extract compile information + if compile_result: + metadata["compile_status"] = compile_result.get("status") + metadata["compile_duration"] = compile_result.get("execution_time") + metadata["compile_stderr"] = compile_result.get("stderr") + + # Extract run information + if run_result: + metadata["run_status"] = run_result.get("status") + metadata["stdout"] = run_result.get("stdout") + metadata["stderr"] = run_result.get("stderr") # stderr during runtime + metadata["exit_code"] = run_result.get("return_code") + metadata["duration"] = run_result.get("execution_time") + + # --- Determine status based on API response --- + api_status = metadata["api_status"] + + if api_status == "SandboxError": + metadata["status"] = "sandbox_error" + result_status = -1 # Internal sandbox error + elif api_status == "Failed": + # --- Add debug logging --- + logger.debug(f"API returned Failed status. Response: {api_response}") + logger.debug(f"Compile Result: {compile_result}") + logger.debug(f"Run Result: {run_result}") + # --- Check the logic here --- + # Compile failed or timed out + is_compile_error = compile_result and ( + metadata["compile_status"] in ["Error", "TimeLimitExceeded"] + or (metadata["compile_status"] == "Finished" and compile_result.get("return_code") != 0) + ) + if is_compile_error: + # Differentiate between compile_error and compile_timeout based on specific status + if metadata["compile_status"] == "TimeLimitExceeded": + metadata["status"] = "compile_timeout" + else: # Includes Error and Finished but return_code != 0 cases + metadata["status"] = "compile_error" + result_status = -4 + # Run failed or timed out + elif run_result: + # Modified condition: Check for TimeLimitExceeded OR (Finished with non-zero exit code) OR Error status + is_runtime_error = ( + metadata["run_status"] == "TimeLimitExceeded" + or metadata["run_status"] == "Error" + or (metadata["run_status"] == "Finished" and run_result.get("return_code") != 0) + ) + if is_runtime_error: + if metadata["run_status"] == "TimeLimitExceeded": + metadata["status"] = "timeout" # Runtime timeout + result_status = -3 + else: # Includes Error and Finished with non-zero return_code + metadata["status"] = "runtime_error" + result_status = -2 + else: + # Other Failed status with run_result, classify as unknown failure + logger.warning(f"Unknown run_status '{metadata['run_status']}' or state within Failed API status.") + metadata["status"] = "unknown_failure" + result_status = -1 # Default to -1 + else: + # Status is Failed but neither a clear compile error nor run_result exists + logger.warning("API status Failed but cannot determine specific error type (compile/run).") + metadata["status"] = "unknown_failure_state" + result_status = -1 # Default to -1 + elif api_status == "Success": + # Run completed successfully, now check the answer + if run_result and metadata["run_status"] == "Finished": + actual_output = metadata["stdout"] if metadata["stdout"] is not None else "" + # Note: Output might contain trailing newlines, need normalization + if str(actual_output).rstrip("\n") == str(expected_output).rstrip("\n"): + result_status = True + metadata["status"] = "success" + else: + result_status = False + metadata["status"] = "wrong_answer" + else: + # Status is Success but run_result status is not Finished, this is unexpected + metadata["status"] = "unexpected_success_state" + result_status = -1 # Classify as unknown error + else: + # API returned an unknown top-level status + logger.warning(f"Unknown API status received: {api_status}") + metadata["status"] = f"unknown_api_status_{api_status}" + result_status = -1 # Default to -1 + else: # api_response is None and no error_msg (Should not happen with current call_sandbox_api logic) + metadata["status"] = "unknown_api_state" + result_status = -1 + logger.error(f"Case {case_index}: Unknown API state (no response and no error message).") + return result_status, metadata + + +def check_correctness( + sandbox_fusion_url: str, + in_outs: Optional[dict], + generation: str, + timeout: int = DEFAULT_TIMEOUT, + memory_limit_mb: int = 1024, + language: str = "python", + concurrent_semaphore: Optional[threading.Semaphore] = None, +) -> tuple[list[Any], list[dict[str, Any]]]: + """ + Checks the correctness of code generation using the remote sandbox API, + processing test cases concurrently. + + Args: + sandbox_fusion_url: The URL of the sandbox fusion API. + in_outs: Dictionary containing "inputs" and "outputs" lists. + generation: The generated code string. + timeout: Timeout for each test case (compile and run share this timeout). + language: The programming language of the code. + + Returns: + A tuple (results, metadata_list). + results: A list containing the test result for each input/output pair + (True/False/-1 api/sandbox err, -2 runtime err, -3 timeout, -4 compile err). + Results are ordered corresponding to the inputs. + metadata_list: A list containing metadata dictionaries for each test case, + ordered corresponding to the inputs. + """ + logger.info("Starting correctness check for generation.") + + if not in_outs or "inputs" not in in_outs or "outputs" not in in_outs: + logger.warning("Invalid in_outs format provided.") + return [-1], [{"error": "Invalid input/output data"}] + + inputs = in_outs["inputs"] + expected_outputs = in_outs["outputs"] + fn_name = in_outs.get("fn_name") + num_cases = len(inputs) + results = [None] * num_cases # Initialize with placeholders + metadata_list = [None] * num_cases # Initialize with placeholders + + if num_cases == 0: + logger.warning("Empty inputs provided.") + return [], [] + + if len(inputs) != len(expected_outputs): + logger.warning(f"Mismatch between number of inputs ({len(inputs)}) and outputs ({len(expected_outputs)}).") + # Return error based on the number of inputs provided + return [-1] * num_cases, [{"error": "Input/output count mismatch", "case_index": i} for i in range(num_cases)] + + first_compile_error_index = -1 + + # max_workers is limited by sandbox_fusion_max_concurrent from concurrent_semaphore + with concurrent.futures.ThreadPoolExecutor(max_workers=max(32, os.cpu_count() * 5)) as executor: + # Submit all tasks, passing the concurrent_semaphore to _process_single_case + future_to_index = { + executor.submit( + _process_single_case, + i, + stdin_data, + expected_outputs[i], + sandbox_fusion_url, + generation, + timeout, + memory_limit_mb, + language, + concurrent_semaphore, + fn_name, + ): i + for i, stdin_data in enumerate(inputs) + } + + # Process results as they complete + for future in concurrent.futures.as_completed(future_to_index): + index = future_to_index[future] + try: + result_status, metadata = future.result() + results[index] = result_status + metadata_list[index] = metadata + + # Check for compile error (-4) + if result_status == -4: + if first_compile_error_index == -1 or index < first_compile_error_index: + first_compile_error_index = index + # Optimization: could potentially cancel futures for index > first_compile_error_index + # However, cancellation is not guaranteed. Post-processing is safer. + + except Exception as exc: + logger.error(f"Test case {index} generated an exception: {exc}") + traceback.print_exc() + results[index] = -1 # Mark as API/internal error + metadata_list[index] = { + "case_index": index, + "input": str(inputs[index]), + "expected_output": str(expected_outputs[index]), + "api_request_error": f"Internal execution error: {exc}", + "status": "internal_error", + } + + # Post-processing for compile errors + if first_compile_error_index != -1: + logger.warning( + f"Compile error detected in case {first_compile_error_index}. Marking subsequent cases as compile errors." + ) + for i in range(first_compile_error_index + 1, num_cases): + # Only update if not already processed (though it should be None or have a result) + if results[i] != -4: # Avoid overwriting if it somehow already got -4 + results[i] = -4 + # Update or create metadata for skipped cases due to compile error + if metadata_list[i] is None: # If future failed before returning metadata + metadata_list[i] = { + "case_index": i, + "input": str(inputs[i]), + "expected_output": str(expected_outputs[i]), + "api_request_error": None, + "status": "compile_error_skipped", # Indicate skipped due to prior compile error + } + else: # If future completed but result is overridden + metadata_list[i]["status"] = "compile_error_skipped" + + logger.info(f"Correctness check finished. Results: {results}") + return results, metadata_list diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/torl/__init__.py b/skyrl-agent/skyrl_agent/tasks/verifiers/torl/__init__.py new file mode 100644 index 0000000000..4dfe29f20e --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/torl/__init__.py @@ -0,0 +1 @@ +from .math_verify import compute_score diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/torl/eval.py b/skyrl-agent/skyrl_agent/tasks/verifiers/torl/eval.py new file mode 100644 index 0000000000..17cd922673 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/torl/eval.py @@ -0,0 +1,537 @@ +import re +import copy + +PATTERNS = [ + r"(?i)Answer\s*:\s*([^\n]+)", + r"\\boxed\{((?:[^{}]|\\{|\\}|(?:\{(?:[^{}]|\\{|\\}|(?:\{(?:[^{}]|\\{|\\}|(?:\{[^{}]*\}))*\}))*\}))*\})", +] + + +def extract_pattern(pred: str, pattern: str): + match = re.findall(pattern, pred) + # 从pred中extract出一个answerlist，代表所有可能的answer + if match: + extracted_answer = match[-1] + if ( + pattern + == r"\\boxed\{((?:[^{}]|\\{|\\}|(?:\{(?:[^{}]|\\{|\\}|(?:\{(?:[^{}]|\\{|\\}|(?:\{[^{}]*\}))*\}))*\}))*\})" + ): + extracted_answer = extracted_answer[:-1] + return extracted_answer.strip("*").strip().strip("*") + else: + return "" + + +SPLIT = [ + "####" "\n", + "Answer:", +] + + +def extract_split(pred: str, split: str): + """ + 最后一个换行符之后的部分 + """ + pred = pred.split(split)[-1] + return pred.strip("*").strip().strip("*") + + +def expansion(answer_list: str): + org_answer_list = copy.deepcopy(answer_list) + for answer in org_answer_list: + if "=" in answer: + answer_list.append(answer.split("=")[-1]) + for choice in ["A", "B", "C", "D", "E", "F"]: + if ( + f"({choice.upper()})" in answer.upper() + or f"{choice.upper()}:" in answer.upper() + or f"{choice.upper()}. " in answer.upper() + ): + answer_list.append(f"{choice.upper()}") + break + for answer in org_answer_list: + pattern = r"^(\d+(\.\d+)?)\s+[a-zA-Z]+(?:\s+[a-zA-Z]+)*$" + if bool(re.match(pattern, answer)): + answer_list.append(answer.split(" ")[0]) + for answer in org_answer_list: + if "\\in" in answer: + answer_list.append(answer.split("\\in")[-1].strip()) + if "\u2208" in answer: + answer_list.append(answer.split("\u2208")[-1].strip()) + return answer_list + + +def extract(pred: str): + answer_list = [] + answer_list.append(pred.split("####")[-1].strip()) + + for split in SPLIT: + answer_list.append(extract_split(copy.deepcopy(pred), split=split)) + for pattern in PATTERNS: + answer_list.append(extract_pattern(copy.deepcopy(pred), pattern=pattern)) + answer_list = expansion(answer_list) + return answer_list + + +SUBSTITUTIONS = [ + ("an ", ""), + # ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), + ("\\left", ""), + ("\\right", ""), + ("∶", ":"), + ("，", ","), + ("$", ""), + ("\\approx", "="), + ("\\simeq", "="), + ("\\sim", "="), + ("^\\prime", "'"), + ("^{\\prime}", "'"), + ("\\dfrac", "\\frac"), + ("\\tfrac", "\\frac"), + ("^\\circ", ""), + ("%", ""), + ("\u221a", "\\sqrt"), + ("\u221e", "\\infty"), + ("\u222a", "\\cup"), +] + +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + # "ft", #this is dangerous, infty, left will be damaged! + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """ + Normalize a final answer to a quantitative reasoning question. + Copied character for character from appendix D of Lewkowycz et al. (2022) + """ + # final_answer = final_answer.split("=")[-1] + final_answer = final_answer.strip() + if final_answer[:2] == "\\(" or final_answer[:2] == "\\[": + final_answer = final_answer[2:] + if final_answer[-2:] == "\\)" or final_answer[-2:] == "\\]": + final_answer = final_answer[:-2] + + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + # Extract answer that is in LaTeX math, is bold, + # is surrounded by a box, etc. + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + # Normalize 100,000 -> 100000 + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + if final_answer[:2] == "\\(" or final_answer[:2] == "\\[": + final_answer = final_answer[2:] + if final_answer[-2:] == "\\)" or final_answer[-2:] == "\\]": + final_answer = final_answer[:-2] + return final_answer.strip() + + +""" +This logic is largely copied from the Hendrycks' MATH release (math_equivalence), and borrowed from: +- https://github.com/microsoft/ProphetNet/tree/master/CRITIC +- https://github.com/openai/prm800k +- https://github.com/microsoft/ToRA/blob/main/src/eval/grader.py +- https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/eval/eval_utils.py +""" + +import regex +import multiprocessing +from math import isclose +from typing import Union + +from sympy import simplify, N +from sympy.parsing.sympy_parser import parse_expr +from sympy.parsing.latex import parse_latex + +# from latex2sympy2 import latex2sympy + +# from .parser import choice_answer_clean, strip_string +# from parser import choice_answer_clean + + +def choice_answer_clean(pred: str): + pred = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":") + # Clean the answer based on the dataset + tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper()) + if tmp: + pred = tmp + else: + pred = [pred.strip().strip(".")] + pred = pred[-1] + # Remove the period at the end, again! + pred = pred.rstrip(".").rstrip("/") + return pred + + +def parse_digits(num): + num = regex.sub(",", "", str(num)) + try: + return float(num) + except: + if num.endswith("%"): + num = num[:-1] + if num.endswith("\\"): + num = num[:-1] + try: + return float(num) / 100 + except: + pass + return None + + +def is_digit(num): + # paired with parse_digits + return parse_digits(num) is not None + + +def str_to_pmatrix(input_str): + input_str = input_str.strip() + matrix_str = re.findall(r"\{.*,.*\}", input_str) + pmatrix_list = [] + + for m in matrix_str: + m = m.strip("{}") + pmatrix = r"\begin{pmatrix}" + m.replace(",", "\\") + r"\end{pmatrix}" + pmatrix_list.append(pmatrix) + + return ", ".join(pmatrix_list) + + +def math_equal( + prediction: Union[bool, float, str], + reference: Union[float, str], + include_percentage: bool = True, + is_close: bool = True, + timeout: bool = False, +) -> bool: + """ + Exact match of math if and only if: + 1. numerical equal: both can convert to float and are equal + 2. symbolic equal: both can convert to sympy expression and are equal + """ + # print("Judge:", prediction, reference) + if prediction is None or reference is None: + return False + if str(prediction.strip().lower()) == str(reference.strip().lower()): + return True + if reference in ["A", "B", "C", "D", "E"] and choice_answer_clean(prediction) == reference: + return True + + try: # 1. numerical equal + if is_digit(prediction) and is_digit(reference): + prediction = parse_digits(prediction) + reference = parse_digits(reference) + # number questions + if include_percentage: + gt_result = [reference / 100, reference, reference * 100] + else: + gt_result = [reference] + for item in gt_result: + try: + if is_close: + if numeric_equal(prediction, item): + return True + else: + if item == prediction: + return True + except Exception: + continue + return False + except: + pass + + if not prediction and prediction not in [0, False]: + return False + + # 2. symbolic equal + reference = str(reference).strip() + prediction = str(prediction).strip() + + ## pmatrix (amps) + if "pmatrix" in prediction and "pmatrix" not in reference: + reference = str_to_pmatrix(reference) + + ## deal with [], (), {} + pred_str, ref_str = prediction, reference + if (prediction.startswith("[") and prediction.endswith("]") and not reference.startswith("(")) or ( + prediction.startswith("(") and prediction.endswith(")") and not reference.startswith("[") + ): + pred_str = pred_str.strip("[]()") + ref_str = ref_str.strip("[]()") + for s in ["{", "}", "(", ")"]: + ref_str = ref_str.replace(s, "") + pred_str = pred_str.replace(s, "") + if pred_str.lower() == ref_str.lower(): + return True + + ## [a, b] vs. [c, d], return a==c and b==d + if ( + regex.match(r"(\(|\[).+(\)|\])", prediction) is not None + and regex.match(r"(\(|\[).+(\)|\])", reference) is not None + ): + pred_parts = prediction[1:-1].split(",") + ref_parts = reference[1:-1].split(",") + if len(pred_parts) == len(ref_parts): + if all( + [math_equal(pred_parts[i], ref_parts[i], include_percentage, is_close) for i in range(len(pred_parts))] + ): + return True + if ( + (prediction.startswith("\\begin{pmatrix}") or prediction.startswith("\\begin{bmatrix}")) + and (prediction.endswith("\\end{pmatrix}") or prediction.endswith("\\end{bmatrix}")) + and (reference.startswith("\\begin{pmatrix}") or reference.startswith("\\begin{bmatrix}")) + and (reference.endswith("\\end{pmatrix}") or reference.endswith("\\end{bmatrix}")) + ): + pred_lines = [ + line.strip() + for line in prediction[len("\\begin{pmatrix}") : -len("\\end{pmatrix}")].split("\\\\") + if line.strip() + ] + ref_lines = [ + line.strip() + for line in reference[len("\\begin{pmatrix}") : -len("\\end{pmatrix}")].split("\\\\") + if line.strip() + ] + matched = True + if len(pred_lines) == len(ref_lines): + for pred_line, ref_line in zip(pred_lines, ref_lines): + pred_parts = pred_line.split("&") + ref_parts = ref_line.split("&") + if len(pred_parts) == len(ref_parts): + if not all( + [ + math_equal( + pred_parts[i], + ref_parts[i], + include_percentage, + is_close, + ) + for i in range(len(pred_parts)) + ] + ): + matched = False + break + else: + matched = False + if not matched: + break + else: + matched = False + if matched: + return True + + if prediction.count("=") == 1 and reference.count("=") == 1: + pred = prediction.split("=") + pred = f"{pred[0].strip()} - ({pred[1].strip()})" + ref = reference.split("=") + ref = f"{ref[0].strip()} - ({ref[1].strip()})" + if symbolic_equal(pred, ref) or symbolic_equal(f"-({pred})", ref): + return True + elif prediction.count("=") == 1 and len(prediction.split("=")[0].strip()) <= 2 and "=" not in reference: + if math_equal(prediction.split("=")[1], reference, include_percentage, is_close): + return True + elif reference.count("=") == 1 and len(reference.split("=")[0].strip()) <= 2 and "=" not in prediction: + if math_equal(prediction, reference.split("=")[1], include_percentage, is_close): + return True + + # symbolic equal with sympy + if timeout: + if call_with_timeout(symbolic_equal_process, prediction, reference): + return True + else: + if symbolic_equal(prediction, reference): + return True + # symbolic == numeric + try: + prediction = float(N(parse_latex(prediction))) + if abs(prediction - float(reference)) <= 1e-8: + True + except: + pass + try: + reference = float(N(parse_latex(reference))) + if abs(prediction - reference) <= 1e-8: + return True + except: + pass + return False + + +def math_equal_process(param): + return math_equal(param[-2], param[-1]) + + +def numeric_equal(prediction: float, reference: float): + # prediction = round(prediction, len(str(reference).split(".")[-1])) + return isclose(reference, prediction, rel_tol=1e-4) + + +def symbolic_equal(a, b): + def _parse(s): + for f in [parse_latex, parse_expr]: + try: + return f(s.replace("\\\\", "\\")) + except: + try: + return f(s) + except: + pass + return s + + a = _parse(a) + b = _parse(b) + # direct equal + try: + if str(a) == str(b) or a == b: + return True + except: + pass + + # simplify equal + try: + if a.equals(b) or simplify(a - b) == 0: + return True + except: + pass + + # equation equal + try: + if (abs(a.lhs - a.rhs)).equals(abs(b.lhs - b.rhs)): + return True + except: + pass + + try: + if numeric_equal(float(N(a)), float(N(b))): + return True + except: + pass + + # matrix + try: + # if a and b are matrix + if a.shape == b.shape: + _a = a.applyfunc(lambda x: round(x, 3)) + _b = b.applyfunc(lambda x: round(x, 3)) + if _a.equals(_b): + return True + except: + pass + + return False + + +def symbolic_equal_process(a, b, output_queue): + result = symbolic_equal(a, b) + output_queue.put(result) + + +def call_with_timeout(func, *args, timeout=1, **kwargs): + output_queue = multiprocessing.Queue() + process_args = args + (output_queue,) + process = multiprocessing.Process(target=func, args=process_args, kwargs=kwargs) + process.start() + process.join(timeout) + + if process.is_alive(): + process.terminate() + process.join() + return False + + return output_queue.get() + + +def process_answer_list(answer_list): + answer_list = list(set(answer_list)) + if "" in answer_list: + answer_list.remove("") + return answer_list + + +# api + + +def is_equal(pred, gt): + pred = normalize_final_answer(pred) + gt = normalize_final_answer(gt) + return math_equal(pred, gt) + + +def exact_match_eval(pred, gt): + gt = normalize_final_answer(gt) + + answer_list = extract(pred) + normalized_answer_list = [] + for answer in copy.deepcopy(answer_list): + normalized_answer_list.append(normalize_final_answer(answer)) + normalized_answer_list = process_answer_list(normalized_answer_list) + + for answer in normalized_answer_list: + if math_equal(gt, answer): + return normalized_answer_list, True + return normalized_answer_list, False diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/torl/math_verify.py b/skyrl-agent/skyrl_agent/tasks/verifiers/torl/math_verify.py new file mode 100644 index 0000000000..d2571acee8 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/torl/math_verify.py @@ -0,0 +1,329 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/hendrycks_math/utils.py +import signal +import re +from contextlib import contextmanager +from .eval import normalize_final_answer +from math_verify import parse, verify + + +class TimeoutException(Exception): + pass + + +@contextmanager +def timeout(seconds): + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +timeout_seconds = 2 +chinese_pattern = re.compile(r"[\u4e00-\u9fff]") +english_pattern = re.compile(r"[a-zA-Z]") +boxed_pattern = re.compile( + r"\\boxed\{((?:[^{}]|\\{|\\}|(?:\{(?:[^{}]|\\{|\\}|(?:\{(?:[^{}]|\\{|\\}|(?:\{[^{}]*\}))*\}))*\}))*\})" +) +valid_char_pattern = re.compile(r'[a-zA-Z0-9\s\.,!?"\'\(\)\{\}\[\]_\-+=<>/@#$%^&*\\|:;~`\u2200-\u22FF]') +repeat_pattern = re.compile(r"(.{5,}?)\1{4,}") + + +def check_mixed_languages(text): + chinese_chars = len(chinese_pattern.findall(text)) + english_chars = len(english_pattern.findall(text)) + return chinese_chars >= 20 and english_chars >= 20 + + +def undesired_format(text): + if "<|endoftext|>" not in text: + return True + else: + return False + + +def check_garbled_characters(text): + valid_chars = valid_char_pattern.sub("", text) + if not text: + return False + invalid_ratio = len(valid_chars) / len(text) + return invalid_ratio > 0.3 + + +def has_repeated_patterns(text): + return bool(repeat_pattern.search(text)) + + +def correctness_score_default(response, gt): + matches = boxed_pattern.findall(response) + if not matches: + return -1.0 + pred = matches[-1][:-1] + return 1.0 if is_equiv(pred, gt) else -1.0 + + +def correctness_score_v2(response, gt): + matches = boxed_pattern.findall(response) + if not matches: + return -1.0 + pred = matches[-1][:-1] + return 1.0 if is_equiv(pred, gt) else -0.5 + + +def compute_score(solution_str, ground_truth, reward_type="default") -> float: + if reward_type == "default": + try: + # if undesired_format(solution_str): return -1.0 + return correctness_score_default(solution_str, ground_truth) + except TimeoutException: + return -1.0 + except Exception: + return -1.0 + elif reward_type == "v2.wformat": + try: + return correctness_score_v2(solution_str, ground_truth) + except TimeoutException: + return -1.0 + except Exception: + return -1.0 + else: + try: + # if undesired_format(solution_str): return -1.0 + return correctness_score_default(solution_str, ground_truth) + except TimeoutException: + return -1.0 + except Exception: + return -1.0 + + +# string normalization from https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/hendrycks_math.py +def is_equiv(str1, str2, verbose=False): + if str1 is None and str2 is None: + print("WARNING: Both None") + return True + if str1 is None or str2 is None: + return False + if str1.strip().lower() == str2.strip().lower(): + return True + try: + str1 = normalize_final_answer(str1) + str2 = normalize_final_answer(str2) + str1 = parse(str1) + str2 = parse(str2) + return verify(str1, str2) + except: + pass + + try: + ss1 = strip_string(str1) + ss2 = strip_string(str2) + if verbose: + print(ss1, ss2) + return ss1 == ss2 + except Exception: + return str1 == str2 + + +def remove_boxed(s): + if "\\boxed " in s: + left = "\\boxed " + assert s[: len(left)] == left + return s[len(left) :] + + left = "\\boxed{" + + assert s[: len(left)] == left + assert s[-1] == "}" + + return s[len(left) : -1] + + +def last_boxed_only_string(string): + idx = string.rfind("\\boxed") + if "\\boxed " in string: + return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx is None: + retval = None + else: + retval = string[idx : right_brace_idx + 1] + + return retval + + +def fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except AssertionError: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + +def fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + a = int(a) + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except AssertionError: + return string + + +def remove_right_units(string): + # "\\text{ " only ever occurs (at least in the val set) when describing units + if "\\text{ " in string: + splits = string.split("\\text{ ") + assert len(splits) == 2 + return splits[0] + else: + return string + + +def fix_sqrt(string): + if "\\sqrt" not in string: + return string + splits = string.split("\\sqrt") + new_string = splits[0] + for split in splits[1:]: + if split[0] != "{": + a = split[0] + new_substr = "\\sqrt{" + a + "}" + split[1:] + else: + new_substr = "\\sqrt" + split + new_string += new_substr + return new_string + + +def strip_string(string): + # linebreaks + string = string.replace("\n", "") + + # remove inverse spaces + string = string.replace("\\!", "") + + # replace \\ with \ + string = string.replace("\\\\", "\\") + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "") + string = string.replace("^\\circ", "") + + # remove dollar signs + string = string.replace("\\$", "") + + # remove units (on the right) + string = remove_right_units(string) + + # remove percentage + string = string.replace("\\%", "") + string = string.replace("\%", "") # noqa: W605 + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split("=")) == 2: + if len(string.split("=")[0]) <= 2: + string = string.split("=")[1] + + # fix sqrt3 --> sqrt{3} + string = fix_sqrt(string) + + # remove spaces + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = fix_fracs(string) + + # manually change 0.5 --> \frac{1}{2} + if string == "0.5": + string = "\\frac{1}{2}" + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = fix_a_slash_b(string) + + return string + + +if __name__ == "__main__": + response = 'To determine which digit appears in the 534th place after the decimal point in the decimal representation of $\\frac{5}{13}$, we need to first find the repeating decimal sequence of $\\frac{5}{13}$. \n\nLet\'s start by calculating the decimal representation of $\\frac{5}{13}$.\n```python\nfrom decimal import Decimal, getcontext\r\n\r\n# Set the precision high enough to see the repeating pattern clearly\r\ngetcontext().prec = 1000\r\n\r\n# Calculate the decimal representation of 5/13\r\ndecimal_rep = Decimal(5) / Decimal(13)\r\nprint(str(decimal_rep))\n```\n```output\n0.3846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846153846\n```\nThe decimal representation of $\\frac{5}{13}$ is $0.\\overline{384615}$. This means the repeating sequence is "384615" and it has a length of 6 digits.\n\nTo find the digit in the 534th place after the decimal point, we need to determine the position within the repeating sequence. Since the sequence repeats every 6 digits, we can find the position by calculating the remainder when 534 is divided by 6.\n\nLet\'s calculate this.\n```python\n# Length of the repeating sequence\r\nrepeating_sequence = "384615"\r\nsequence_length = len(repeating_sequence)\r\n\r\n# Find the position within the repeating sequence\r\nposition = (534 - 1) % sequence_length # -1 because indexing starts from 0\r\n\r\n# Get the digit at that position\r\ndigit_in_534th_place = repeating_sequence[position]\r\nprint(digit_in_534th_place)\n```\n```output\n6\n```\nThe digit in the 534th place after the decimal point in the decimal representation of $\\frac{5}{13}$ is $\\boxed{6}$. <|endoftext|>' + answer = "6" + res = compute_score(response, answer) + print(res) diff --git a/skyrl-agent/skyrl_agent/tasks/verifiers/web_search/stem_llm_judge.py b/skyrl-agent/skyrl_agent/tasks/verifiers/web_search/stem_llm_judge.py new file mode 100644 index 0000000000..e34a6beeac --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/verifiers/web_search/stem_llm_judge.py @@ -0,0 +1,207 @@ +""" +Minimal STEM judge: every answer is graded by an external LLM. + +Prerequisite: + - Set env var STEM_LLM_JUDGE_URL to an OpenAI-compatible base URL. + - Launch the service with your preferred model beforehand, e.g. + + +""" + +import os +import re +import requests +from typing import Tuple + +# # ------------ Prompt template ------------------------------------------------ +# JUDGE_TEMPLATE = """\ +# You are a strict grader for university-level STEM problems. + +# Question: +# {QUESTION} + +# Reference Answer: +# {REFERENCE_ANSWER} + +# Student Answer: +# {STUDENT_ANSWER} + +# Carefully think and check whether the student answer is equivalent to the reference answer. +# You only need to refer to the reference answer to grade the student's answer. Sometimes the student's answer is expressed in a different way from the reference answer, but the meaning is the same, and you should still consider it correct. If they are not equivalent in mathematical sense, you should consider it incorrect. + +# : CORRECT or INCORRECT + +# """ + + +# ------------ Core LLM call -------------------------------------------------- +def _llm_judge(question: str, student: str, reference: str, verbose: bool = False) -> bool: + url_base = os.getenv("STEM_LLM_JUDGE_URL") + if not url_base: + raise EnvironmentError("STEM_LLM_JUDGE_URL not set") + url = url_base.rstrip("/") + "/v1/chat/completions" + + # prompt = JUDGE_TEMPLATE.format( + # QUESTION=question, + # STUDENT_ANSWER=student, + # REFERENCE_ANSWER=reference, + # ) + + prompt = ( + f"User: ### Question: {question}\n\n" + f"### Ground Truth Answer: {reference}\n\n" + f"### Student Answer: {student}\n\n" + "For the above question, please verify if the student's answer is equivalent to the ground truth answer.\n" + "Do not solve the question by yourself; just check if the student's answer is equivalent to the ground truth answer.\n" + 'If the student\'s answer is correct, output "Final Decision: Yes". If the student\'s answer is incorrect, output "Final Decision: No". Assistant:' + ) + + payload = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": prompt}], + "chat_template_kwargs": {"enable_thinking": False}, + "temperature": 0.0, + } + + resp = requests.post(url, json=payload, timeout=120) + resp.raise_for_status() + data = resp.json() + + text = data["choices"][0]["message"]["content"] + + decision = _extract_final_decision(text) + if decision is None: + if verbose: + print("[judge-warning] Unable to parse final decision; treating as incorrect.") + score = 0.0 + else: + score = float(decision) + + marker = "✅" if score == 1.0 else "❌" + + if verbose: + print(marker * 50) + print("student answer: ", student) + print("gt: ", reference) + import json as _json + + print(_json.dumps(data, indent=2, ensure_ascii=False)) + print(marker * 16 + " LLM Judge CONTENT " + marker * 16) + print(text) + print(marker * 16 + "End of LLM Judge Reply \n" + marker * 16) + + return score + + +_FINAL_DECISION_RE = re.compile(r"final\s*decision\s*(?:[:\-=])\s*(yes|no)\b", re.IGNORECASE) + + +def _extract_final_decision(text: str): + """Return True for Yes, False for No, or None if no marker is found.""" + matches = list(_FINAL_DECISION_RE.finditer(text)) + if not matches: + return None + last_decision = matches[-1].group(1).lower() + return last_decision == "yes" + + +def _last_boxed_only_string(string): + idx = string.rfind("\\boxed") + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + left_brace_idx = None + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if left_brace_idx is None: + left_brace_idx = i + elif string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + + i += 1 + + if left_brace_idx is None or right_brace_idx is None: + return None + + return string[left_brace_idx + 1 : right_brace_idx].strip() + + +def match_answer(response): + is_matched = False + + # Find boxed + ans_boxed = _last_boxed_only_string(response) + if ans_boxed: + is_matched = True + response = ans_boxed + + return is_matched, response + + +# ------------ Public API ----------------------------------------------------- +def compute_score(data_source: str, model_output: str, ground_truth: str, extra_info: dict) -> Tuple[bool, float, str]: + """ + Arguments + --------- + model_output : str – agent's raw answer + ground_truth : str – reference answer + extra_info : dict – MUST contain key "question" + + Returns + ------- + (is_correct, score, normalized_student_answer) + score is 1.0 if correct, else 0.0 + """ + model_output = str(model_output) + ground_truth = str(ground_truth) + + # Try to extract boxed answer; if not found, fall back to full output + is_matched, extracted_model_output = match_answer(model_output) + + # Exact-match fast path: if dataset specifies exactMatch, do direct comparison + answer_type = str(extra_info.get("answer_type", "")).lower() + if answer_type == "exactmatch": + student = extracted_model_output.strip() + gt = ground_truth.strip() + return 1.0 if student == gt else 0.0 + + # Otherwise, require boxed for LLM judging (legacy behavior) + question = extra_info["question"] + if is_matched == False: + return 0.0 + else: + try: + is_correct = _llm_judge(question, extracted_model_output, ground_truth, verbose=False) + except Exception as e: + instance_id = extra_info.get("instance_id", "unknown") if isinstance(extra_info, dict) else "unknown" + print(f"[judge-error] instance_id={instance_id} {e}") + return 0.0 + return float(is_correct) + + +# ---------------- Demo ------------------------------------------------------- +if __name__ == "__main__": + demo_item = { + "question": "A cash-strapped young professional offers to buy your car " + "with four equal annual payments of $3,000, beginning 2 " + "years from today. Assuming you can invest at 10% and want " + "to receive $9,000 for the car, should you accept?", + "answer": "$8,645.09", + } + agent_reply = "The answer is\\boxed{$8,645.09}" # pretend this comes from your agent + score = compute_score( + data_source="", + model_output=agent_reply, + ground_truth=demo_item["answer"], + extra_info={"question": demo_item["question"]}, + ) + print("score =", score) diff --git a/skyrl-agent/skyrl_agent/tasks/web_research_task.py b/skyrl-agent/skyrl_agent/tasks/web_research_task.py new file mode 100644 index 0000000000..fe66ec5f98 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tasks/web_research_task.py @@ -0,0 +1,249 @@ +""" +Web Research Task - Uses search_engine and web_browser tools together +""" + +from typing import Dict, Any, List +from skyrl_agent.tasks.base import BaseTask + + +class WebResearchTask(BaseTask): + """ + A task that combines search_engine and web_browser tools for web research. + This task uses the existing tools without modification. + """ + + @classmethod + async def initialize_runtime(cls, *args, **kwargs) -> Any: + """Initialize the runtime for the web research task""" + # No special runtime initialization needed + return {} + + @classmethod + def get_instruction(cls, instance: Dict[str, Any]) -> List[Dict[str, str]]: + """ + Get the initial instruction for the agent in OpenAI messages format. + The agent will use tools sequentially as per ReActAgent's design. + """ + # Handle pandas Series or dict input + import pandas as pd + + # Extract prompt - simplified since our unified dataset has 'prompt' field + if isinstance(instance, pd.Series): + # Use the prompt field from our unified dataset + prompt = instance.get("prompt", "") + + # Debug output (optional) + if prompt: + print(f"[DEBUG] Using prompt from unified dataset (length: {len(prompt)})") + else: + # Fallback to Question field if using old format + prompt = instance.get("Question", "") + if prompt: + print("[DEBUG] Using Question field as fallback") + else: + # Handle dict-like objects + prompt = instance.get("prompt", instance.get("Question", "")) if hasattr(instance, "get") else str(instance) + + # Debug: show raw prompt source when enabled + import os as _os + + _debug = _os.getenv("SKYAGENT_DEBUG_LOG", "0") == "1" + + # Handle different input formats + if isinstance(prompt, str): + prompt = [{"role": "user", "content": prompt}] + elif isinstance(prompt, dict): + prompt = [prompt] + elif not isinstance(prompt, list): + prompt = [{"role": "user", "content": str(prompt)}] + + # Check if there's already a system message + has_system = any(msg.get("role") == "system" for msg in prompt) + + if not has_system: + # Add system prompt that explains how to use the tools + system_prompt = { + "role": "system", + "content": """You are a precise, helpful assistant. You decide whether to use tools (search_engine, web_browser) based on the task. + +Use search_engine to find relevant sources. +Use web_browser to visit URLs for detailed information. +If you DID call search_engine, do NOT answer from snippets—open at least one source with web_browser first. + +Always end by calling the finish tool with exactly one \\boxed{...} as the final answer. + +Typical workflow: +1) search_engine → get URLs +2) web_browser → extract key facts from sources +3) finish → \\boxed{YOUR_ANSWER} +""", + } + prompt = [system_prompt] + prompt + + if _debug: + # Print system + first user message preview + def _preview(msg, w=300): + s = str(msg).replace("\n", " ") + return s[:w] + ("..." if len(s) > w else "") + + if isinstance(prompt, list) and len(prompt) > 0: + sys_msg = next((m for m in prompt if m.get("role") == "system"), prompt[0]) + usr_msg = next((m for m in prompt if m.get("role") == "user"), prompt[min(1, len(prompt) - 1)]) + print("[WebResearchTask] System prompt preview:", _preview(sys_msg.get("content", ""))) + print("[WebResearchTask] User prompt preview:", _preview(usr_msg.get("content", ""))) + return prompt + + @classmethod + def complete_runtime(cls, *args, **kwargs) -> Dict[str, Any]: + """Complete or finalize the runtime for the task""" + # No special cleanup needed + return {} + + @classmethod + async def evaluate_result( + cls, result: Any, instance: Any, data_source: str = None, instance_id: int = None, trajectory_id: int = None + ) -> float: + """ + STEM evaluation using LLM judge for textbook data; otherwise simple pass-through. + """ + import pandas as pd + + # No result => incorrect + if result is None or (isinstance(result, str) and not result.strip()): + return 0.0 + + # Debug logging for instance type and content + print(f"[evaluate_result] instance_id={instance_id} instance type: {type(instance).__name__}") + + # Normalize instance into a dict + try: + if isinstance(instance, pd.Series): + print(f"[evaluate_result] Converting Series to dict for instance_id={instance_id}") + instance = instance.to_dict() + except Exception as e: + print(f"[evaluate_result] Failed to convert Series for instance_id={instance_id}: {e}") + pass + + # Log instance structure after normalization + if isinstance(instance, dict): + print(f"[evaluate_result] instance_id={instance_id} keys: {list(instance.keys())}") + if "reward_model" in instance: + rm_type = type(instance["reward_model"]).__name__ + print(f"[evaluate_result] reward_model type: {rm_type}") + if "extra_info" in instance: + ei_type = type(instance["extra_info"]).__name__ + print(f"[evaluate_result] extra_info type: {ei_type}") + + # Determine data_source + ds = data_source + if not ds and isinstance(instance, dict): + ds = instance.get("data_source") + + # Apply LLM judge for STEM textbook and HLE data + ds_lower = str(ds or "").lower() + if ds_lower in ["stem_textbook", "hle"]: + try: + # Extract fields with robust GT fallback (align with training-side logic) + import json as _json + + rm_raw = instance.get("reward_model") if isinstance(instance, dict) else None + ei_raw = instance.get("extra_info") if isinstance(instance, dict) else None + # Handle JSON-serialized dicts + if isinstance(rm_raw, str): + try: + rm = _json.loads(rm_raw) + print(f"[evaluate_result] Parsed reward_model from JSON for instance_id={instance_id}") + except Exception as e: + print(f"[evaluate_result] Failed to parse reward_model JSON for instance_id={instance_id}: {e}") + rm = {} + else: + rm = rm_raw or {} + if isinstance(ei_raw, str): + try: + ei = _json.loads(ei_raw) + print(f"[evaluate_result] Parsed extra_info from JSON for instance_id={instance_id}") + except Exception as e: + print(f"[evaluate_result] Failed to parse extra_info JSON for instance_id={instance_id}: {e}") + ei = {} + else: + ei = ei_raw or {} + + # Prefer: nested instance.answer -> top-level answer -> extra_info.reference_answer -> reward_model.ground_truth + ground_truth = "" + try: + if isinstance(instance, dict): + nested_inst = instance.get("instance") if isinstance(instance.get("instance"), dict) else {} + if nested_inst and nested_inst.get("answer"): + ground_truth = str(nested_inst.get("answer")) + except Exception: + ground_truth = "" + if not ground_truth and isinstance(instance, dict) and instance.get("answer"): + ground_truth = str(instance.get("answer")) + if not ground_truth and isinstance(ei, dict) and ei.get("reference_answer"): + ground_truth = str(ei.get("reference_answer")) + if not ground_truth and isinstance(rm, dict) and rm.get("ground_truth"): + ground_truth = str(rm.get("ground_truth")) + + if not ground_truth: + # No GT → cannot judge; return 0 and emit a concise trace + try: + print( + f"[rollout-judge-skip] instance_id={instance_id} traj={trajectory_id} ds={ds_lower} reason=missing_ground_truth" + ) + # Debug why ground truth is missing + print(f" [DEBUG] rm keys: {list(rm.keys()) if isinstance(rm, dict) else 'not dict'}") + print(f" [DEBUG] ei keys: {list(ei.keys()) if isinstance(ei, dict) else 'not dict'}") + if isinstance(rm, dict) and "ground_truth" in rm: + print(f" [DEBUG] rm.ground_truth exists but is: {rm['ground_truth']!r}") + if isinstance(ei, dict) and "reference_answer" in ei: + print(f" [DEBUG] ei.reference_answer exists but is: {ei['reference_answer']!r}") + except Exception: + pass + return 0.0 + + # Ensure question/subject available for judge + question = ei.get("question") or (instance.get("raw_prompt", "") if isinstance(instance, dict) else "") + subject = ei.get("subject", "") + judge_extra = dict(ei) + judge_extra.setdefault("question", question) + judge_extra.setdefault("subject", subject) + + # Call synchronous judge safely from async context + from skyrl_agent.dispatcher.async_utils import call_sync_from_async + + # Use the original minimal web-search STEM judge (as before) + from skyrl_agent.tasks.verifiers.web_search.stem_llm_judge import compute_score as _compute_score + + score = await call_sync_from_async( + _compute_score, + ds, + str(result), + ground_truth, + judge_extra, + ) + # Lightweight trace to confirm rollout-side LLM judge usage + try: + import os as _os + + _base = _os.environ.get("STEM_LLM_JUDGE_URL", "") or _os.environ.get("MATH_LLM_JUDGE_URL", "") or "" + print( + f"[rollout-judge-used] instance_id={instance_id} traj={trajectory_id} ds={ds_lower} score={float(score)} base={_base}" + ) + except Exception: + pass + return float(score) + except Exception as e: + print(f"[eval-error] STEM LLM judge failed: {e}") + return 0.0 + else: + # For non-STEM datasets, do not call judge; emit a concise trace once per item when result exists + try: + if result: + print( + f"[rollout-judge-skip] instance_id={instance_id} traj={trajectory_id} ds={ds_lower} reason=non_stem_dataset" + ) + except Exception: + pass + + # Default behavior for other data sources: success if result exists + return 1.0 if result else 0.0 diff --git a/skyrl-agent/skyrl_agent/tools/__init__.py b/skyrl-agent/skyrl_agent/tools/__init__.py new file mode 100644 index 0000000000..75affb948a --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/__init__.py @@ -0,0 +1,9 @@ +from .base import TOOL_REGISTRY, BaseTool +from .finish import FinishTool +from .em_finish import EMFinishTool +from .sandbox_fusion import CodeInterpreter +from .search_engine import SearchEngine +from .web_browser import WebBrowser +from .local_search import LocalSearchTool +from .next_memagent import NextWithSummary +from .search import FaissSearch diff --git a/skyrl-agent/skyrl_agent/tools/base.py b/skyrl-agent/skyrl_agent/tools/base.py new file mode 100644 index 0000000000..eac9d9b468 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/base.py @@ -0,0 +1,211 @@ +# https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/tools/base.py + +import json +from abc import ABC, abstractmethod +from typing import List, Optional, Union +import logging +import json5 +from litellm import ChatCompletionToolParam, ChatCompletionToolParamFunctionChunk + +logger = logging.getLogger(__name__) + +TOOL_REGISTRY = {} + + +def json_loads(text: str) -> dict: + text = text.strip("\n") + if text.startswith("```") and text.endswith("\n```"): + text = "\n".join(text.split("\n")[1:-1]) + try: + return json.loads(text) + except json.decoder.JSONDecodeError as json_err: + try: + return json5.loads(text) + except ValueError: + raise json_err + + +class ToolServiceError(Exception): + + def __init__( + self, + exception: Optional[Exception] = None, + code: Optional[str] = None, + message: Optional[str] = None, + extra: Optional[dict] = None, + ): + if exception is not None: + super().__init__(exception) + else: + super().__init__(f"\nError code: {code}. Error message: {message}") + self.exception = exception + self.code = code + self.message = message + self.extra = extra + + +def register_tool(name, allow_overwrite=False): + + def decorator(cls): + if name in TOOL_REGISTRY: + if allow_overwrite: + logger.warning(f"Tool `{name}` already exists! Overwriting with class {cls}.") + else: + raise ValueError(f"Tool `{name}` already exists! Please ensure that the tool name is unique.") + # if cls.name and (cls.name != name): + # raise ValueError(f'{cls.__name__}.name="{cls.name}" conflicts with @register_tool(name="{name}").') + # cls.name = name + TOOL_REGISTRY[name] = cls + + return cls + + return decorator + + +def is_tool_schema(obj: dict) -> bool: + """ + Check if obj is a valid JSON schema describing a tool compatible with OpenAI's tool calling. + Example valid schema: + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + } + """ + import jsonschema + + try: + assert set(obj.keys()) == {"name", "description", "parameters"} + assert isinstance(obj["name"], str) + assert obj["name"].strip() + assert isinstance(obj["description"], str) + assert isinstance(obj["parameters"], dict) + + assert set(obj["parameters"].keys()) == {"type", "properties", "required"} + assert obj["parameters"]["type"] == "object" + assert isinstance(obj["parameters"]["properties"], dict) + assert isinstance(obj["parameters"]["required"], list) + assert set(obj["parameters"]["required"]).issubset(set(obj["parameters"]["properties"].keys())) + except AssertionError: + return False + try: + jsonschema.validate(instance={}, schema=obj["parameters"]) + except jsonschema.exceptions.SchemaError: + return False + except jsonschema.exceptions.ValidationError: + pass + return True + + +class BaseTool(ABC): + name: str = "" + description: str = "" + parameters: Union[List[dict], dict] = [] + + def __init__(self, cfg: Optional[dict] = None): + self.cfg = cfg or {} + if not self.name: + raise ValueError( + f"You must set {self.__class__.__name__}.name, either by @register_tool(name=...) or explicitly setting {self.__class__.__name__}.name" + ) + if isinstance(self.parameters, dict): + if not is_tool_schema({"name": self.name, "description": self.description, "parameters": self.parameters}): + raise ValueError( + "The parameters, when provided as a dict, must confirm to a valid openai-compatible JSON schema." + ) + + def get_tool_param(self) -> ChatCompletionToolParam: + """Get the tool parameter for OpenAI API.""" + return ChatCompletionToolParam( + type="function", + function=ChatCompletionToolParamFunctionChunk( + name=self.name, + description=self.description, + parameters=self.parameters, + ), + ) + + @abstractmethod + def call(self, params: Union[str, dict], **kwargs) -> Union[str, list, dict]: + """The interface for calling tools. + + Each tool needs to implement this function, which is the workflow of the tool. + + Args: + params: The parameters of func_call. + kwargs: Additional parameters for calling tools. + + Returns: + The result returned by the tool, implemented in the subclass. + """ + raise NotImplementedError + + def _verify_json_format_args(self, params: Union[str, dict], strict_json: bool = False) -> dict: + """Verify the parameters of the function call""" + if isinstance(params, str): + try: + if strict_json: + params_json: dict = json.loads(params) + else: + params_json: dict = json_loads(params) + except json.decoder.JSONDecodeError: + raise ValueError("Parameters must be formatted as a valid JSON!") + else: + params_json: dict = params + if isinstance(self.parameters, list): + for param in self.parameters: + if "required" in param and param["required"]: + if param["name"] not in params_json: + raise ValueError("Parameters %s is required!" % param["name"]) + elif isinstance(self.parameters, dict): + import jsonschema + + jsonschema.validate(instance=params_json, schema=self.parameters) + else: + raise ValueError + return params_json + + @property + def function(self) -> dict: # Bad naming. It should be `function_info`. + return { + # 'name_for_human': self.name_for_human, + "name": self.name, + "description": self.description, + "parameters": self.parameters, + # 'args_format': self.args_format + } + + @property + def name_for_human(self) -> str: + return self.cfg.get("name_for_human", self.name) + + @property + def args_format(self) -> str: + fmt = self.cfg.get("args_format") + if fmt is None: + fmt = "Format the arguments as a JSON object." + return fmt + + def get_system_prompt_prefix(self) -> Optional[str]: + """Return an optional system prompt prefix that should be prepended to the system message. + + Tools can override this method to provide tool-specific system prompt instructions. + The returned prefix will be automatically prepended to the system message when the agent initializes. + + Returns: + Optional[str]: System prompt prefix string, or None if no prefix is needed. + """ + return None diff --git a/skyrl-agent/skyrl_agent/tools/cache.py b/skyrl-agent/skyrl_agent/tools/cache.py new file mode 100644 index 0000000000..b6ea658590 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/cache.py @@ -0,0 +1,170 @@ +import atexit +from collections import OrderedDict +import hashlib +import json +import os +import threading +import time +from typing import Any, Dict, Optional + + +class WebPageCache: + """ + A simple persistent LRU cache for webpage content, modeled after ASearcher. + - Keys are md5(url) + - Values store url, content, timestamp + - Periodically saved to a JSON file preserving LRU order + """ + + def __init__(self, max_size: int = 100000, cache_file: str = "./webpage_cache.json", save_interval: int = 10): + self.max_size = max_size + self.cache_file = cache_file + self.cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + self.lock = threading.Lock() + self.stats = {"hits": 0, "misses": 0, "evictions": 0} + self.save_interval = save_interval + self.operations_since_save = 0 + + self.load_from_file() + atexit.register(self.save_to_file) + + def _generate_cache_key(self, url: str) -> str: + return hashlib.md5(url.encode()).hexdigest() + + def put(self, url: str, content: str): + if not url or not content: + return + + cache_key = self._generate_cache_key(url) + with self.lock: + if cache_key in self.cache: + del self.cache[cache_key] + + while len(self.cache) >= self.max_size: + self.cache.popitem(last=False) + self.stats["evictions"] += 1 + + self.cache[cache_key] = { + "url": url, + "content": content, + "timestamp": time.time(), + } + + self.operations_since_save += 1 + if self.operations_since_save >= self.save_interval: + self.operations_since_save = 0 + threading.Thread(target=self._background_save, daemon=True).start() + + def get(self, url: str) -> Optional[str]: + cache_key = self._generate_cache_key(url) + with self.lock: + if cache_key in self.cache: + entry = self.cache.pop(cache_key) + self.cache[cache_key] = entry # move to end (MRU) + self.stats["hits"] += 1 + return entry.get("content") + else: + self.stats["misses"] += 1 + return None + + def has(self, url: str) -> bool: + cache_key = self._generate_cache_key(url) + with self.lock: + return cache_key in self.cache + + def clear(self): + with self.lock: + self.cache.clear() + self.stats = {"hits": 0, "misses": 0, "evictions": 0} + self.operations_since_save = 0 + + def force_save(self): + self.save_to_file() + self.operations_since_save = 0 + + def get_stats(self) -> Dict[str, Any]: + with self.lock: + total_requests = self.stats["hits"] + self.stats["misses"] + hit_rate = self.stats["hits"] / total_requests if total_requests > 0 else 0 + return { + "cache_size": len(self.cache), + "max_size": self.max_size, + "hits": self.stats["hits"], + "misses": self.stats["misses"], + "evictions": self.stats["evictions"], + "hit_rate": hit_rate, + "total_requests": total_requests, + } + + def _background_save(self): + try: + self.save_to_file() + except Exception as e: + print(f"[ERROR] WebPageCache: Background save failed: {e}") + + def save_to_file(self): + try: + with self.lock: + ordered_cache = [] + for key, value in self.cache.items(): + ordered_cache.append((key, value)) + + cache_data = { + "cache_ordered": ordered_cache, + "stats": self.stats, + "max_size": self.max_size, + "saved_at": time.time(), + } + + # Ensure directory exists for absolute paths like /workspace/webcache/webpage_cache.json + try: + dirpath = os.path.dirname(self.cache_file) + if dirpath: + os.makedirs(dirpath, exist_ok=True) + except Exception: + pass + + with open(self.cache_file, "w", encoding="utf-8") as f: + json.dump(cache_data, f, indent=2, ensure_ascii=False) + + print(f"[DEBUG] WebPageCache: Saved {len(self.cache)} entries to {self.cache_file}") + + except Exception as e: + print(f"[ERROR] WebPageCache: Failed to save cache to {self.cache_file}: {e}") + + def load_from_file(self): + if not os.path.exists(self.cache_file): + print(f"[DEBUG] WebPageCache: No existing cache file {self.cache_file}, starting fresh") + return + + try: + with open(self.cache_file, "r", encoding="utf-8") as f: + cache_data = json.load(f) + + with self.lock: + if "cache_ordered" in cache_data: + ordered_cache = cache_data["cache_ordered"] + self.cache = OrderedDict(ordered_cache) + print("[DEBUG] WebPageCache: Loaded ordered cache format") + else: + loaded_cache = cache_data.get("cache", {}) + self.cache = OrderedDict(loaded_cache) + print("[DEBUG] WebPageCache: Loaded legacy cache format (LRU order may be lost)") + + self.stats = cache_data.get("stats", {"hits": 0, "misses": 0, "evictions": 0}) + + while len(self.cache) > self.max_size: + self.cache.popitem(last=False) + self.stats["evictions"] += 1 + + saved_at = cache_data.get("saved_at", 0) + saved_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(saved_at)) + print( + f"[DEBUG] WebPageCache: Loaded {len(self.cache)} entries from {self.cache_file} (saved at {saved_time})" + ) + + except Exception as e: + print(f"[ERROR] WebPageCache: Failed to load cache from {self.cache_file}: {e}") + with self.lock: + self.cache = OrderedDict() + self.stats = {"hits": 0, "misses": 0, "evictions": 0} diff --git a/skyrl-agent/skyrl_agent/tools/em_finish.py b/skyrl-agent/skyrl_agent/tools/em_finish.py new file mode 100644 index 0000000000..912233944f --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/em_finish.py @@ -0,0 +1,48 @@ +from .base import BaseTool, register_tool +from typing import Union + + +@register_tool("em_finish") +class EMFinishTool(BaseTool): + name = "finish" + description = ( + "Signals the completion of the current task or conversation.\n\n" + "Use this tool when:\n" + "- You have successfully completed the requested task\n" + "- You cannot proceed further due to technical limitations or missing information\n\n" + "Always end by calling the finish tool.\n" + ) + parameters = { + "type": "object", + "required": ["answer"], + "properties": { + "answer": { + "type": "string", + "description": ( + "Provide exactly one \\boxed{...} as the final answer.\n" + "You MUST ALWAYS use \\boxed{YOUR_ANSWER} format in the finish tool's answer parameter.\n" + "This applies to ALL types of answers:\n" + " - Numbers: \\boxed{42}\n" + " - Words: \\boxed{word}\n" + " - Yes/No: \\boxed{Yes} or \\boxed{No}\n" + " - Expressions: \\boxed{x^2 + 1}\n" + " - Multiple choice: \\boxed{A} or \\boxed{B}\n" + "- NEVER use plain text without \\boxed{} in the finish tool\n" + "- This format is MANDATORY for ALL answers" + ), + }, + }, + } + + def call(self, params: Union[str, dict], **kwargs) -> str: + try: + params = self._verify_json_format_args(params) + except ValueError as e: + return {"error": f"Invalid parameters: {str(e)}"} + + # If the parameters are valid, we can proceed to finish the task. + answer = params.get("answer", "") + # Enforce boxed format if missing (universal requirement per instruction) + if isinstance(answer, str) and "\\boxed{" not in answer: + answer = f"\\boxed{{{answer}}}" + return answer diff --git a/skyrl-agent/skyrl_agent/tools/finish.py b/skyrl-agent/skyrl_agent/tools/finish.py new file mode 100644 index 0000000000..137f5863dd --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/finish.py @@ -0,0 +1,31 @@ +from .base import BaseTool, register_tool +from typing import Union + + +@register_tool("finish") +class FinishTool(BaseTool): + name = "finish" + description = ( + "Signals the completion of the current task or conversation.\n\n" + "Use this tool when:\n" + "- You have successfully completed the requested task\n" + "- You cannot proceed further due to technical limitations or missing information\n\n" + "The answer field should include the final answer to the problem (follow the required format) if an answer is required by the problem.\n" + ) + parameters = { + "type": "object", + "required": ["answer"], + "properties": { + "answer": {"type": "string", "description": "Final message summarizing the task or containing the answer."}, + }, + } + + def call(self, params: Union[str, dict], **kwargs) -> str: + try: + params = self._verify_json_format_args(params) + except ValueError as e: + return {"error": f"Invalid parameters: {str(e)}"} + + # If the parameters are valid, we can proceed to finish the task. + answer = params.get("answer", "") + return answer diff --git a/skyrl-agent/skyrl_agent/tools/local_search.py b/skyrl-agent/skyrl_agent/tools/local_search.py new file mode 100644 index 0000000000..7aa15bbad1 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/local_search.py @@ -0,0 +1,246 @@ +# From https://github.com/PeterGriffinJin/Search-R1/blob/main/search_r1/llm_agent/generation.py +# Erro Handling from https://github.com/NovaSky-AI/SkyRL/blob/main/skyrl-gym/skyrl_gym/tools/search.py + +from skyrl_agent.tools.base import BaseTool, register_tool +import requests +import os +import time +import uuid +import json +from typing import Union, List, Tuple, Optional, Dict, Any + +# Constants for retry logic +DEFAULT_TIMEOUT = 30 +MAX_RETRIES = 10 +INITIAL_RETRY_DELAY = 1 + + +@register_tool("local_search") +class LocalSearchTool(BaseTool): + name = "local_search" + description = "Performs a single query with a local search engine, use this if you need more information." + parameters = { + "type": "object", + "properties": {"query": {"type": "string", "description": "The query to search for."}}, + "required": ["query"], + } + + def __init__(self): + super().__init__() + # Check for required environment variable + assert "LOCAL_SEARCH_URL" in os.environ, "Environment variable LOCAL_SEARCH_URL must be set." + self.search_url = os.getenv("LOCAL_SEARCH_URL") + self.topk = int(os.getenv("LOCAL_SEARCH_TOP_K", "3")) + self.timeout = DEFAULT_TIMEOUT + self.log_requests = True + + def _call_search_api(self, query: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """ + Calls the search API with a single query using improved error handling. + + Args: + query: The query to search for. + + Returns: + response: The response from the search API (json if successful, None otherwise) + error_msg: The error message if the request failed. + """ + request_id = str(uuid.uuid4()) + log_prefix = f"[Search Request ID: {request_id}] " + + payload = {"queries": [query], "topk": self.topk, "return_scores": True} + headers = {"Content-Type": "application/json", "Accept": "application/json"} + + last_error = None + for attempt in range(MAX_RETRIES): + try: + if self.log_requests: + print(f"{log_prefix}Attempt {attempt + 1}/{MAX_RETRIES}: Calling search API at {self.search_url}") + + response = requests.post( + self.search_url, + headers=headers, + json=payload, + timeout=self.timeout, + ) + + # Check for server errors that should trigger retries + if response.status_code in [500, 502, 503, 504]: + last_error = f"{log_prefix}API Request Error: Server Error ({response.status_code}) on attempt {attempt + 1}/{MAX_RETRIES}" + print(f"Warning: {last_error}") + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + print(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + + # Check for other HTTP errors + response.raise_for_status() + + # If successful (status code 2xx) + if self.log_requests: + print(f"{log_prefix}Search API call successful on attempt {attempt + 1}") + + return response.json(), None + + except requests.exceptions.ConnectionError as e: + last_error = f"{log_prefix}Connection Error: {e}" + print(f"Warning: {last_error}") + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + print(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + except requests.exceptions.Timeout as e: + last_error = f"{log_prefix}Timeout Error: {e}" + print(f"Warning: {last_error}") + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + print(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + except requests.exceptions.RequestException as e: + last_error = f"{log_prefix}API Request Error: {e}" + break # Exit retry loop on other request errors + except json.JSONDecodeError as e: + raw_response_text = response.text if "response" in locals() else "N/A" + last_error = f"{log_prefix}API Response JSON Decode Error: {e}, Response: {raw_response_text[:200]}" + break # Exit retry loop on JSON decode errors + except Exception as e: + last_error = f"{log_prefix}Unexpected Error: {e}" + break # Exit retry loop on other unexpected errors + + # If we reach here, all attempts failed + print(f"Error: {log_prefix}API Request Failed after {MAX_RETRIES} attempts: {last_error}") + return None, last_error + + def _passages2string(self, retrieval_result: List[dict]) -> str: + """ + Convert search results to formatted string representation. + + Args: + retrieval_result: List of document results from search API + + Returns: + Formatted string with document titles and content + """ + if not retrieval_result: + return "No search results found." + + format_reference = "" + for idx, doc_item in enumerate(retrieval_result): + content = doc_item["document"]["contents"] + title = content.split("\n")[0] + text = "\n".join(content.split("\n")[1:]) + format_reference += f"Doc {idx+1}(Title: {title}) {text}\n" + + return format_reference.strip() + + def call(self, params: dict, **kwargs) -> Union[str, list, dict]: + """ + Main entry point for the search tool with improved error handling. + + Args: + params (dict): Dictionary containing 'query' parameter. + **kwargs: Additional keyword arguments. + + Returns: + Formatted search results as string or error message. + """ + # Verify required parameters + try: + params = self._verify_json_format_args(params) + except ValueError as e: + return {"error": f"Invalid parameters: {str(e)}"} + + query = params.get("query") + + # Execute search + try: + api_response, error_msg = self._call_search_api(query) + + if error_msg: + return f"Search error: {error_msg}" + + if not api_response: + return "No response from search API" + + if "result" not in api_response: + return "Unexpected response format from search API" + + # Convert results to formatted string + assert len(api_response["result"]) == 1, "Expected 1 result, got " + str(len(api_response["result"])) + formatted_result = self._passages2string(api_response["result"][0]) + return formatted_result + + except Exception as e: + error_msg = f"Unexpected error processing query '{query}': {str(e)}" + return error_msg + + +if __name__ == "__main__": + # Example usage for testing + tool = LocalSearchTool() + + # 10 random test parameters + test_queries = [ + "artificial intelligence machine learning", + "quantum computing applications", + "climate change renewable energy", + "blockchain cryptocurrency bitcoin", + "space exploration mars mission", + "genetic engineering CRISPR technology", + "virtual reality augmented reality", + "cybersecurity data protection", + "robotics automation manufacturing", + "neural networks deep learning", + ] + + # Import tokenizer + try: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B") + print("Successfully loaded Qwen2.5-7B tokenizer") + except ImportError: + print("transformers library not available, cannot count tokens") + tokenizer = None + except Exception as e: + print(f"Error loading tokenizer: {e}") + tokenizer = None + + total_tokens = 0 + results = [] + + for i, query in enumerate(test_queries, 1): + print(f"\n--- Test {i}/10 ---") + print(f"Query: {query}") + + test_params = {"query": query} + result = tool.call(test_params) + + if tokenizer: + # Count tokens in the result + tokens = tokenizer.encode(str(result), add_special_tokens=False) + token_count = len(tokens) + total_tokens += token_count + print(f"Result tokens: {token_count}") + else: + print("Cannot count tokens - tokenizer not available") + + results.append({"query": query, "result": result, "tokens": token_count if tokenizer else 0}) + + print(f"Result preview: {str(result)[:200]}...") + + print("\n=== SUMMARY ===") + print(f"Total queries tested: {len(test_queries)}") + if tokenizer: + print(f"Total tokens used: {total_tokens}") + print(f"Average tokens per result: {total_tokens / len(test_queries):.2f}") + + # Show token distribution + print("\nToken distribution per query:") + for i, result in enumerate(results, 1): + print(f"Query {i}: {result['tokens']} tokens") + else: + print("Token counting not available") diff --git a/skyrl-agent/skyrl_agent/tools/next_memagent.py b/skyrl-agent/skyrl_agent/tools/next_memagent.py new file mode 100644 index 0000000000..28e220918d --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/next_memagent.py @@ -0,0 +1,203 @@ +""" +Next With Summary Tool - Handles memory update, chunking, and context reset. + +This tool: +- Chunks the document from instance['context'] on first use +- Updates cumulative memory with provided summary +- Resets conversation history to system + problem + memory + next chunk +- Manages chunk index and provides feedback about progress +""" + +from skyrl_agent.tools.base import BaseTool, register_tool +from skyrl_agent.tools.prompt import SIMPLE_MEMORY_SYSTEM_PROMPT +from typing import Union, List, Optional +import re + + +@register_tool("next_with_summary") +class NextWithSummary(BaseTool): + name = "next_with_summary" + description = "Update your memory with new information from the current section and move to the next chunk. Provide a summary that combines important details from previous memory with new findings from this section. This tool will reset the conversation context, keeping only your updated memory and the next section." + parameters = { + "type": "object", + "properties": { + "summary": { + "type": "string", + "description": "Updated memory that combines: (1) relevant information from previous memory, and (2) new information from the current section that helps answer the problem. Be comprehensive but concise.", + } + }, + "required": ["summary"], + } + + def get_system_prompt_prefix(self) -> Optional[str]: + """Return the system prompt prefix required for this tool.""" + return SIMPLE_MEMORY_SYSTEM_PROMPT + + def _chunk_documents_by_tokens(self, context: str, tokenizer, max_tokens: int = 4000) -> List[str]: + """Chunk documents by tokens. Groups by 'Document X:' and packs up to max_tokens. + + Uses the provided tokenizer to count tokens per doc block. + """ + doc_pattern = r"(Document \d+:)" + parts = re.split(doc_pattern, context) + + documents: List[str] = [] + for i in range(1, len(parts), 2): + if i + 1 < len(parts): + doc = parts[i] + "\n" + parts[i + 1] + documents.append(doc.strip()) + elif i < len(parts): + documents.append(parts[i].strip()) + + chunks: List[str] = [] + current_chunk: List[str] = [] + current_tokens = 0 + + for doc in documents: + doc_tokens = len(tokenizer.encode(doc)) + if current_tokens + doc_tokens > max_tokens and current_chunk: + chunks.append("\n\n".join(current_chunk)) + current_chunk = [doc] + current_tokens = doc_tokens + else: + current_chunk.append(doc) + current_tokens += doc_tokens + + if current_chunk: + chunks.append("\n\n".join(current_chunk)) + + return chunks + + def _ensure_document_chunks(self, agent) -> None: + """Initialize chunking from instance['context'] on first use. + + Also stores system prompt and problem for later context resets. + """ + # Use agent attributes to store chunking state + if hasattr(agent, "document_chunks") and agent.document_chunks: + return + + raw_context = agent.instance.get("context") + if not raw_context or not isinstance(raw_context, str): + raise ValueError("next_with_summary requires instance['context'] (str) to be set for chunking.") + + # Chunk the documents + agent.document_chunks = self._chunk_documents_by_tokens( + raw_context, + agent.tokenizer, + 4000, + ) + agent.current_chunk_index = 0 + agent.cumulative_memory = "" + + # Store system prompt and problem for context resets + # Extract system prompt + for msg in agent.history.messages: + if msg.get("role") == "system": + agent.mem_system_prompt = msg.get("content", "") + break + + # Extract problem from user message + problem_content = "" + for msg in agent.history.messages: + if msg.get("role") == "user": + problem_content = msg.get("content", "") + break + + # Extract just the problem text if wrapped in tags + if "" in problem_content: + problem_match = re.search(r"(.*?)", problem_content, re.DOTALL) + if problem_match: + agent.mem_cur_question = problem_match.group(1).strip() + else: + agent.mem_cur_question = problem_content + else: + agent.mem_cur_question = problem_content + + def _build_flushed_user_message(self, agent, next_chunk: str) -> str: + """Build the user message with problem, memory, next section, and feedback.""" + memory_section = ( + f"\n{agent.cumulative_memory}\n" + if agent.cumulative_memory + else "\nNo information recorded yet.\n" + ) + + section_content = ( + f"
\nChunk {agent.current_chunk_index}/{len(agent.document_chunks)}:\n\n{next_chunk}\n
" + ) + + # Use stored problem question + problem_text = getattr(agent, "mem_cur_question", "") + + # Build feedback message about progress + if agent.current_chunk_index >= len(agent.document_chunks): + feedback = ( + f"Memory updated. Moved to chunk {agent.current_chunk_index}/{len(agent.document_chunks)} " + "(FINAL CHUNK). After reading this chunk, use the finish tool to provide your answer." + ) + else: + feedback = ( + f"Memory updated. Moved to chunk {agent.current_chunk_index}/{len(agent.document_chunks)}. " + "Continue reading and updating your memory." + ) + + return ( + f"\n{problem_text}\n\n\n" + f"{memory_section}\n\n{section_content}\n\n" + f"**Progress:** {feedback}" + ) + + def _flush_history_with(self, agent, user_content: str) -> None: + """Reset history to only system + new user message. + + The history.reset flag will be set by initialize(), which will be + detected in _prepare_llm_input() to reset prompt_token_len. + """ + # Use stored system prompt + system_content = getattr(agent, "mem_system_prompt", "") + + # Replace history with only system + new user message + new_messages = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": user_content}, + ] + agent.history.initialize(new_messages) + + def call(self, params: dict, **kwargs) -> Optional[Union[str, dict]]: + """Handle next_with_summary: update memory, get next chunk, reset context. + + Returns None to indicate that feedback is embedded in the user message + and no tool output should be added to history. + """ + agent = kwargs.get("agent") + if agent is None: + return "Error: Agent context missing for next_with_summary. Use MemAgent which passes agent=self." + + params = self._verify_json_format_args(params) + + # Get summary from params + summary = params.get("summary", "") + if not summary: + return "Error: 'summary' parameter is required." + + # Ensure chunks are initialized + self._ensure_document_chunks(agent) + + # Update cumulative memory with provided summary + agent.cumulative_memory = summary + + # Check if we've run out of chunks + if agent.current_chunk_index >= len(agent.document_chunks): + return "You have read all chunks. Please use the finish tool to provide your final " "answer IMMEDIATELY." + + # Get next chunk and advance index + next_chunk = agent.document_chunks[agent.current_chunk_index] + agent.current_chunk_index += 1 + + # Flush context with memory + next section (feedback is embedded in user message) + new_user_message = self._build_flushed_user_message(agent, next_chunk) + self._flush_history_with(agent, new_user_message) + + # Return None to indicate no tool output should be added to history + # (feedback is already in the new user message) + return None diff --git a/skyrl-agent/skyrl_agent/tools/prompt.py b/skyrl-agent/skyrl_agent/tools/prompt.py new file mode 100644 index 0000000000..fe951b9e2d --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/prompt.py @@ -0,0 +1,32 @@ +EXTRACTOR_PROMPT = """Please process the following webpage content and user goal to extract relevant information: + +## **Webpage Content** +{webpage_content} + +## **User Goal** +{goal} + +## **Task Guidelines** +1. **Content Scanning for Rational**: Locate the **specific sections/data** directly related to the user's goal within the webpage content +2. **Key Extraction for Evidence**: Identify and extract the **most relevant information** from the content, you never miss any important information, output the **full original context** of the content as far as possible, it can be more than three paragraphs. +3. **Summary Output for Summary**: Organize into a concise paragraph with logical flow, prioritizing clarity and judge the contribution of the information to the goal. +4. **Word limits**: "rational" ≤ 50 words; "evidence" ≤ 70 words; "summary" ≤ 90 words. + +**Final Output Format using JSON format has "rational", "evidence", "summary" feilds** +""" + +SIMPLE_MEMORY_SYSTEM_PROMPT = """You are presented with a problem and sections of articles that may contain the answer. + +Your task is to: +1. Read each section carefully +2. Identify information relevant to answering the problem +3. Update your memory to combine previous findings with new information from the current section +4. Keep all relevant details while maintaining a concise, organized memory +5. Once you have enough information, provide your final answer + +The format you'll work with: +- : The question you need to answer +- : Your cumulative findings from previous sections (empty at start) +-
: The current section of text to analyze + +Use the 'next_with_summary' tool to update your memory and move to the next section. When you have sufficient information to answer the problem, use the 'finish' tool with your answer.""" diff --git a/skyrl-agent/skyrl_agent/tools/sandbox_fusion.py b/skyrl-agent/skyrl_agent/tools/sandbox_fusion.py new file mode 100644 index 0000000000..abf35e4a95 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/sandbox_fusion.py @@ -0,0 +1,135 @@ +from skyrl_agent.tools.base import BaseTool, register_tool +import os +import time +from typing import Union + + +@register_tool("code_interpreter") +class CodeInterpreter(BaseTool): + name = "code_interpreter" + description = "Executes code in a sandbox for supported languages. Supports Python, JavaScript, C++, and more." + parameters = { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "The code to be executed. Should be valid in the specified language.", + }, + "language": { + "type": "string", + "description": "Programming language of the code (e.g., python, javascript, cpp). Defaults to python.", + "default": "python", + }, + }, + "required": ["code"], + } + # Don't check at import time, only when tool is actually used + sandbox_url = os.getenv("SANDBOX_FUSION_URL", None) + memory_limit_mb = 1024 # 1 GB memory limit + + # Adapted from https://arxiv.org/pdf/2502.14382. + @staticmethod + def _post_process_code(code: str) -> str: + """ + Remove any markdown formatting from the code. + This is important to ensure the code is executed correctly in the sandbox. + """ + if not code: + return code + + # Remove markdown code block delimiters + # Handle cases like ```python, ```javascript, ```cpp, or just ``` + import re + + # Remove opening markdown code blocks (```language or just ```) + code = re.sub(r"^```\w*\s*\n?", "", code.strip(), flags=re.MULTILINE) + + # Remove closing markdown code blocks + code = re.sub(r"\n?```\s*$", "", code, flags=re.MULTILINE) + + # Remove any remaining ``` that might be in the middle + code = code.replace("```", "") + + code = code.strip() + + # From https://github.com/volcengine/verl/blob/7fc3029a1ec407f6e56f1f1ff02a659071da3b1d/recipe/retool/retool.py#L41C9-L49C32 + # NOTE: some script may not explicitly print result, we need to add a print statement to the end of the script + # Dacheng: Add a " " handle to avoid adding print with meaningful code with indentation. + lines = code.split("\n") + for i, line in reversed(list(enumerate(lines))): + if line == "": + continue + if not lines[i].startswith("print") and not lines[i].startswith(" "): + lines[i] = f"print({line})" + break + code = "\n".join(lines) + + return code + + # From https://github.com/volcengine/verl/blob/main/verl/tools/sandbox_fusion_tools.py + def _execute_code_retool(self, code, timeout=30, language="python"): + # Check sandbox_url when actually using the tool + if not self.sandbox_url: + raise ValueError("SANDBOX_FUSION_URL environment variable must be set to use CodeInterpreter tool") + from skyrl_agent.tasks.verifiers.sandbox_fusion.utils import _process_single_case + + result_status, metadata = _process_single_case( + 0, None, None, self.sandbox_url + "/run_code", code, timeout, self.memory_limit_mb, language + ) + # we should always expect this since we don't have correct answer + if metadata["run_status"] == "Finished": + actual_output = metadata["stdout"] + metadata["stderr"] + # logger.debug(f"actual_output from sandbox fusion: {actual_output}") + return actual_output + else: + return "no stdout here" + + def call(self, params: dict, **kwargs) -> Union[str, list, dict]: + """ + Executes the provided code in a sandbox environment with retry logic. + + Args: + params (dict): Dictionary containing 'code' and optionally 'language'. + **kwargs: Additional keyword arguments. + + Returns: + str: The output of the executed code or an error message. + """ + # verify required parameters + try: + params = self._verify_json_format_args(params) + except ValueError as e: + return {"error": f"Invalid parameters: {str(e)}"} + # extract code and language from params + + code = params.get("code") + # Post-process code to remove markdown formatting + # Dacheng: sometimes model write ```py instead of ```python, so we just extract the code part. + code = self._post_process_code(code) + + language = params.get("language", "python") + if not code: + return {"error": "Code parameter is required."} + + # Retry logic for timeout and connection errors + for attempt in range(5): + output = self._execute_code_retool(code, 30, language) + if "RLIMIT_NPROC" in output: + # This indicates a timeout or resource limit error + print(f"Attempt {attempt + 1}: Timeout or resource limit exceeded.") + time.sleep(5) + continue # Retry the request + return output + + return {"error": "Max retries exceeded"} + + +if __name__ == "__main__": + # Example usage for testing + tool = CodeInterpreter() + test_params = {"code": "print('Hello, World!')", "language": "python"} + import json + + test_params = json.dumps(test_params) # Convert to JSON string if needed + result = tool.call(test_params) + print("Test Result:", result) # Should print: {'output': 'Hello, World!'} or an error message diff --git a/skyrl-agent/skyrl_agent/tools/search.py b/skyrl-agent/skyrl_agent/tools/search.py new file mode 100644 index 0000000000..a93e30ab56 --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/search.py @@ -0,0 +1,313 @@ +""" +Dense retrieval search tool using FAISS. +Ported from BrowseComp-Plus/searcher/searchers/faiss_searcher.py +""" + +import glob +import json +import logging +import os +import pickle +import threading +from itertools import chain +from typing import Any, Dict, List, Optional, Union + +import faiss +import numpy as np +import requests +from datasets import load_dataset +from tevatron.retriever.searcher import FaissFlatSearcher +from tqdm import tqdm + +from skyrl_agent.tools.base import BaseTool, register_tool + +logger = logging.getLogger(__name__) + + +@register_tool("faiss_search") +class FaissSearch(BaseTool): + name = "faiss_search" + description = "Performs dense retrieval search using FAISS: supply a 'query' string; the tool retrieves the top 5 most relevant documents from a local corpus. : You may need to adjust your queries or split them into sub-queries for better results." + parameters = { + "type": "object", + "properties": {"query": {"type": "string", "description": "Search query string for semantic retrieval"}}, + "required": ["query"], + } + + # Class-level cache for shared resources + _retriever = None + _lookup = None + _docid_to_text = None + _snippet_tokenizer = None + _initialized = False + _init_lock = threading.Lock() # Thread-safe initialization + + def __init__(self, cfg: Optional[dict] = None): + super().__init__(cfg) + + # Required configuration from environment variables + self.embedding_api_url = os.getenv("FAISS_EMBEDDING_API_URL") # e.g., "http://remote-server:8000/v1" + self.index_path = os.getenv("FAISS_INDEX_PATH") + + # Use hardcoded defaults (matching BrowseComp-Plus) + self.dataset_name = "Tevatron/browsecomp-plus-corpus" + self.task_prefix = ( + "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:" + ) + self.k = 5 # Fixed k value as in BrowseComp-Plus + self.snippet_max_tokens = 512 + + # Validate required environment variables + if not self.embedding_api_url: + raise ValueError( + "FAISS_EMBEDDING_API_URL environment variable is required (e.g., 'http://remote-server:8000/v1')" + ) + if not self.index_path: + raise ValueError("FAISS_INDEX_PATH environment variable is required") + + logger.info(f"FAISS search tool created (lazy init, remote API: {self.embedding_api_url})") + + def _ensure_initialized(self): + """Lazy initialization - load resources only once on first use (thread-safe).""" + # Fast path: check without lock first (double-checked locking pattern) + if FaissSearch._initialized: + return + + # Slow path: acquire lock and check again + with FaissSearch._init_lock: + # Another thread might have initialized while we were waiting for the lock + if FaissSearch._initialized: + logger.info("FAISS resources already initialized by another thread") + return + + logger.info("Initializing FAISS resources (first use)...") + + # Load FAISS index + self._load_faiss_index() + + # Load dataset + self._load_dataset() + + # Skip tokenizer loading - using character-based truncation instead for speed + FaissSearch._snippet_tokenizer = None + logger.info("Using character-based snippet truncation (faster than tokenizer)") + + FaissSearch._initialized = True + logger.info("FAISS search tool initialized successfully") + + def _load_faiss_index(self) -> None: + """Load FAISS index from pickle files into class-level cache.""" + + def pickle_load(path): + with open(path, "rb") as f: + reps, lookup = pickle.load(f) + return np.array(reps), lookup + + index_files = glob.glob(self.index_path) + logger.info(f"Found {len(index_files)} index files matching pattern: {self.index_path}") + + if not index_files: + raise ValueError(f"No files found matching pattern: {self.index_path}") + + # Load first shard + p_reps_0, p_lookup_0 = pickle_load(index_files[0]) + FaissSearch._retriever = FaissFlatSearcher(p_reps_0) + + # Load remaining shards + shards = chain([(p_reps_0, p_lookup_0)], map(pickle_load, index_files[1:])) + if len(index_files) > 1: + shards = tqdm(shards, desc="Loading index shards", total=len(index_files)) + + FaissSearch._lookup = [] + for p_reps, p_lookup in shards: + FaissSearch._retriever.add(p_reps) + FaissSearch._lookup += p_lookup + + self._setup_gpu() + + def _setup_gpu(self) -> None: + """Setup GPU for FAISS if available.""" + num_gpus = faiss.get_num_gpus() + if num_gpus == 0: + logger.info("No GPU found. Using CPU for FAISS.") + else: + logger.info(f"Using {num_gpus} GPU(s) for FAISS") + if num_gpus == 1: + co = faiss.GpuClonerOptions() + co.useFloat16 = True + res = faiss.StandardGpuResources() + FaissSearch._retriever.index = faiss.index_cpu_to_gpu(res, 0, FaissSearch._retriever.index, co) + else: + co = faiss.GpuMultipleClonerOptions() + co.shard = True + co.useFloat16 = True + FaissSearch._retriever.index = faiss.index_cpu_to_all_gpus( + FaissSearch._retriever.index, co, ngpu=num_gpus + ) + + def _load_dataset(self) -> None: + """Load the document dataset into class-level cache.""" + logger.info(f"Loading dataset: {self.dataset_name}") + + try: + dataset_cache = os.getenv("HF_DATASETS_CACHE") + cache_dir = dataset_cache if dataset_cache else None + + ds = load_dataset(self.dataset_name, split="train", cache_dir=cache_dir) + FaissSearch._docid_to_text = {row["docid"]: row["text"] for row in ds} + logger.info(f"Loaded {len(FaissSearch._docid_to_text)} documents from dataset") + except Exception as e: + if "doesn't exist on the Hub or cannot be accessed" in str(e): + logger.error(f"Dataset '{self.dataset_name}' access failed. This is likely an authentication issue.") + logger.error("Solutions:") + logger.error("1. Run: huggingface-cli login") + logger.error("2. Set: export HF_TOKEN=your_token_here") + logger.error(f"Current HF_TOKEN: {'Set' if os.getenv('HF_TOKEN') else 'Not set'}") + raise RuntimeError(f"Failed to load dataset '{self.dataset_name}': {e}") + + def _get_embedding_from_api(self, text: str) -> np.ndarray: + """Get embedding from remote API using OpenAI-compatible interface.""" + import time + + start_time = time.time() + + url = f"{self.embedding_api_url}/embeddings" + headers = {"Content-Type": "application/json"} + + # Get the actual model name from environment or use the served model + # vLLM requires the actual model name (e.g., "Qwen/Qwen3-Embedding-8B") + model_name = os.getenv("FAISS_EMBEDDING_MODEL_NAME", "Qwen/Qwen3-Embedding-8B") + + payload = {"input": text, "model": model_name} + + try: + response = requests.post(url, headers=headers, json=payload, timeout=120) # Increased to 120s + response.raise_for_status() + + result = response.json() + embedding = result["data"][0]["embedding"] + + elapsed = time.time() - start_time + logger.info(f"[TIMING] Embedding API request took {elapsed:.3f}s (query length: {len(text)} chars)") + + return np.array(embedding, dtype=np.float32) + + except requests.exceptions.Timeout as e: + logger.error(f"Embedding API request timed out after 120s: {e}") + logger.error(f"API URL: {self.embedding_api_url}") + raise RuntimeError("Embedding API timeout (120s) - server may be overloaded or model is slow") + except requests.exceptions.ConnectionError as e: + logger.error(f"Cannot connect to embedding API: {e}") + logger.error(f"API URL: {self.embedding_api_url}") + logger.error("Check if vLLM server is running and accessible") + raise RuntimeError(f"Embedding API connection failed: {e}") + except requests.exceptions.RequestException as e: + logger.error(f"Embedding API request failed: {e}") + logger.error(f"API URL: {self.embedding_api_url}") + raise RuntimeError(f"Embedding API request failed: {e}") + + def _search(self, query: str) -> List[Dict[str, Any]]: + """Perform FAISS search using remote embedding API.""" + import time + + total_start = time.time() + + # Lazy initialization on first use + self._ensure_initialized() + + if not FaissSearch._retriever or not FaissSearch._lookup: + raise RuntimeError("Search tool not properly initialized") + + # Get embedding from remote API + query_with_prefix = self.task_prefix + query + q_reps = self._get_embedding_from_api(query_with_prefix) + # Reshape to match expected format [1, embedding_dim] + q_reps = q_reps.reshape(1, -1) + + # Search with fixed k + faiss_start = time.time() + all_scores, psg_indices = FaissSearch._retriever.search(q_reps, self.k) + faiss_elapsed = time.time() - faiss_start + logger.info(f"[TIMING] FAISS search took {faiss_elapsed:.3f}s") + + # Format results + format_start = time.time() + results = [] + for score, index in zip(all_scores[0], psg_indices[0]): + passage_id = FaissSearch._lookup[index] + passage_text = FaissSearch._docid_to_text.get(passage_id, "Text not found") + # Simple character-based truncation instead of tokenizer (much faster) + snippet = passage_text[:2000] if len(passage_text) > 2000 else passage_text + + results.append( + { + "docid": passage_id, + "score": float(score), + "snippet": snippet, + "text": passage_text, # Include full text as well + } + ) + + format_elapsed = time.time() - format_start + total_elapsed = time.time() - total_start + logger.info(f"[TIMING] Result formatting took {format_elapsed:.3f}s") + logger.info(f"[TIMING] Total search took {total_elapsed:.3f}s") + + return results + + def call(self, params: dict, **kwargs) -> Union[str, dict]: + """ + Execute search query. + + Args: + params (dict): Dictionary containing 'query' (string). + **kwargs: Additional keyword arguments. + + Returns: + dict: Search results or error message. + """ + try: + params = self._verify_json_format_args(params) + except ValueError as e: + return {"error": f"Invalid parameters: {str(e)}"} + + query = params.get("query") + if not query: + return {"error": "Query parameter is required"} + + logger.info( + f"[SEARCH] Starting search for query: {query[:100]}..." + if len(query) > 100 + else f"[SEARCH] Starting search for query: {query}" + ) + + try: + results = self._search(query) + + # Format for display + formatted_results = [] + for i, result in enumerate(results, 1): + formatted_results.append( + {"rank": i, "docid": result["docid"], "score": result["score"], "snippet": result["snippet"]} + ) + + logger.info(f"[SEARCH] Completed search successfully, returned {len(formatted_results)} results") + + return {"query": query, "num_results": len(formatted_results), "results": formatted_results} + + except Exception as e: + logger.error(f"Search failed: {e}", exc_info=True) + return {"error": f"Search failed: {str(e)}"} + + +if __name__ == "__main__": + # Example usage for testing + # Make sure to set environment variables first: + # export FAISS_MODEL_NAME="your-model-name" + # export FAISS_INDEX_PATH="/path/to/index/*.pkl" + + tool = FaissSearch() + test_params = {"query": "What is machine learning?"} + result = tool.call(test_params) + print("Test Result:") + print(json.dumps(result, indent=2)) diff --git a/skyrl-agent/skyrl_agent/tools/search_engine.py b/skyrl-agent/skyrl_agent/tools/search_engine.py new file mode 100644 index 0000000000..479a8cb6eb --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/search_engine.py @@ -0,0 +1,242 @@ +from skyrl_agent.tools.base import BaseTool, register_tool, json_loads +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Union +import requests +import os + + +@register_tool("search_engine") +class SearchEngine(BaseTool): + name = "search_engine" + description = ( + "Performs batched web searches: supply an array 'query'; the tool retrieves the top 10 results for each query in one call.\n\n" + 'For search_engine, query must be JSON array: ["term1", "term2"] NOT [term1, term2] or [["term1", "term2"]]' + ) + parameters = { + "type": "object", + "properties": { + "query": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Array of query strings. Include multiple complementary search queries in a single call.\n\n" + 'For search_engine, query must be JSON array: ["term1", "term2"] NOT [term1, term2] or [["term1", "term2"]]' + ), + }, + }, + "required": ["query"], + } + google_search_key = os.getenv("GOOGLE_SEARCH_KEY") + if not google_search_key: + raise ValueError("GOOGLE_SEARCH_KEY environment variable is required") + + # Optional blocklists to prevent data leakage (e.g., excluding benchmark/dataset sites) + # Configure via env vars: + # SEARCH_BLOCKLIST_DOMAINS: comma-separated domains to exclude (e.g., "huggingface.co,github.com") + # SEARCH_BLOCKLIST_KEYWORDS: comma-separated keywords to filter out (e.g., "gpqa,web_research_hle") + # SEARCH_NEGATIVE_FILTERS: if "true" (default), append -site:domain to the query for blocklisted domains + _default_block_domains = os.getenv( + "SEARCH_BLOCKLIST_DOMAINS", + # Sensible defaults to avoid benchmark leakage and paywalled homework sites + "huggingface.co,github.com,gitlab.com,chegg.com,coursehero.com,studocu.com,brainly.com,quizlet.com", + ).strip() + _default_block_keywords = os.getenv( + "SEARCH_BLOCKLIST_KEYWORDS", + # Common benchmark/dataset keywords observed in HLE/GPQA scenarios + "gpqa,Chemistry-GPQA,gpqa-diamond", + ).strip() + _use_negative_filters = os.getenv("SEARCH_NEGATIVE_FILTERS", "true").lower() == "true" + + # Normalize to sets for faster checks + blocklist_domains = {d.strip().lower() for d in _default_block_domains.split(",") if d.strip()} + blocklist_keywords = {k.strip().lower() for k in _default_block_keywords.split(",") if k.strip()} + + def google_search(self, query: str): + """ + Performs a Google search using the Serper API. + + Args: + query (str): The search query string + + Returns: + str: Formatted search results or error message + """ + url = "https://google.serper.dev/search" + headers = { + "X-API-KEY": self.google_search_key, + "Content-Type": "application/json", + } + # Optionally add negative site filters directly to the query + q = query + if self._use_negative_filters and self.blocklist_domains: + try: + q = query + " " + " ".join([f"-site:{d}" for d in sorted(self.blocklist_domains)]) + except Exception: + q = query + + data = { + "q": q, + "num": 10, + "extendParams": { + "country": "en", + "page": 1, + }, + } + + for i in range(5): + try: + response = requests.post(url, headers=headers, data=json.dumps(data), timeout=10) + results = response.json() + break + except Exception: + if i == 4: + return f"Google search timeout for query '{query}'. Please try again later." + continue + + if response.status_code != 200: + return f"Search API error: {response.status_code} - {response.text}" + + try: + if "organic" not in results: + return f"No results found for query: '{query}'. Use a less specific query." + + # Filter results by blocklists (domains/keywords) to reduce leakage + pages = results.get("organic", []) + filtered_pages = [] + for p in pages: + try: + link = str(p.get("link", "")) + title = str(p.get("title", "")) + snippet = str(p.get("snippet", "")) + link_l = link.lower() + combined_l = f"{title} {snippet} {link}".lower() + # Domain filtering based on URL substring match + blocked_domain = ( + any(d in link_l for d in self.blocklist_domains) if self.blocklist_domains else False + ) + # Keyword filtering anywhere in title/snippet/link + blocked_keyword = ( + any(k in combined_l for k in self.blocklist_keywords) if self.blocklist_keywords else False + ) + if not blocked_domain and not blocked_keyword: + filtered_pages.append(p) + except Exception: + # If anything goes wrong during filtering, conservatively keep the page + filtered_pages.append(p) + + web_snippets = [] + idx = 0 + for page in filtered_pages: + idx += 1 + # Only include title, link, and snippet; omit publish date/source/citations metadata + snippet = "" + if "snippet" in page: + snippet = "\n" + page["snippet"] + + redacted_version = f"{idx}. [{page['title']}]({page['link']})\n{snippet}" + redacted_version = redacted_version.replace("Your browser can't play this video.", "") + web_snippets.append(redacted_version) + + content = ( + f"A Google search for '{query}' found {len(web_snippets)} results:\n\n## Web Results\n" + + "\n\n".join(web_snippets) + ) + return content + except Exception as e: + return f"Error parsing search results for '{query}': {str(e)}" + + def call(self, params: dict, **kwargs) -> Union[str, dict]: + """ + Executes web search queries. + + Args: + params (dict): Dictionary containing 'query' (array of strings or single string). + **kwargs: Additional keyword arguments. + + Returns: + str or dict: The search results or an error message. + """ + # Normalize and validate parameters robustly + # 1) Parse to dict (be tolerant of JSON5/markdowny inputs) + raw: dict + if isinstance(params, dict): + raw = dict(params) + else: + try: + raw = json_loads(params) if isinstance(params, str) else {"query": params} + except Exception: + raw = {"query": params} + + # 2) Normalize "query" into a flat list[str] + def _normalize_query(q): + if q is None: + return None + # If it's a JSON-like string representing an array, try to parse + if isinstance(q, str): + s = q.strip() + if s.startswith("[") and s.endswith("]"): + try: + parsed = json.loads(s) + q = parsed + except Exception: + # fall back to single string query + return [q] + else: + return [q] + # If it's a list, flatten nested lists and stringify items + if isinstance(q, list): + flat = [] + for item in q: + if isinstance(item, list): + flat.extend([str(x) for x in item]) + else: + flat.append(str(item)) + return flat + # Any other type → cast to single string element + return [str(q)] + + normalized_query = _normalize_query(raw.get("query")) + if not normalized_query: + return { + "error": "Query parameter is required.", + "hint": "Provide a JSON array of search strings.", + "example": {"query": ["term1", "term2"]}, + } + + raw["query"] = normalized_query + + # 3) Schema-validate, but catch any schema error and return actionable hint + try: + params = self._verify_json_format_args(raw) + except Exception as e: + return { + "error": f"Invalid parameters: {str(e)}", + "hint": "query must be an array of strings (no nested arrays).", + "example": {"query": ["term1", "term2"]}, + } + + query = params.get("query") + + try: + if isinstance(query, str): + response = self.google_search(query) + elif isinstance(query, list): + with ThreadPoolExecutor(max_workers=3) as executor: + response = list(executor.map(self.google_search, query)) + response = "\n=======\n".join(response) + else: + return {"error": "Query must be a string or array of strings."} + + return {"results": response} + + except Exception as e: + return {"error": f"Search failed: {str(e)}"} + + +if __name__ == "__main__": + # Example usage for testing + tool = SearchEngine() + test_params = {"query": ["python programming", "machine learning"]} + result = tool.call(test_params) + print("Test Result:", result) diff --git a/skyrl-agent/skyrl_agent/tools/web_browser.py b/skyrl-agent/skyrl_agent/tools/web_browser.py new file mode 100644 index 0000000000..0dcbc0cdab --- /dev/null +++ b/skyrl-agent/skyrl_agent/tools/web_browser.py @@ -0,0 +1,752 @@ +from skyrl_agent.tools.base import BaseTool, register_tool +import json +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Union +import requests +from skyrl_agent.tools.prompt import EXTRACTOR_PROMPT +import os +from openai import OpenAI +import random +from skyrl_agent.tools.cache import WebPageCache +import copy + + +@register_tool("web_browser") +class WebBrowser(BaseTool): + name = "web_browser" + description = ( + "Visit webpage(s) and return the summary of the content.\n\n" + "CRITICAL REQUIREMENTS:\n" + "- Search results alone are NOT sufficient - you need full webpage content\n" + "- If search returns papers/PDFs, you MUST visit them with web_browser to extract key findings\n" + "- Skipping web_browser will result in incomplete and unreliable answers" + ) + parameters = { + "type": "object", + "properties": { + "url": { + "type": ["string", "array"], + "items": {"type": "string"}, + "minItems": 1, + "description": "The URL(s) of the webpage(s) to visit. Can be a single URL string or an array of URL strings. Use plain URLs only, never Markdown format.", + }, + "goal": {"type": "string", "description": "The goal of the visit for webpage(s)."}, + }, + "required": ["url", "goal"], + } + + webcontent_maxlength = int(os.getenv("WEBCONTENT_MAXLENGTH", "150000")) + ignore_jina = os.getenv("IGNORE_JINA", "false").lower() == "true" + jina_reader_url_prefix = "https://r.jina.ai/" + jina_api_keys = os.getenv("JINA_API_KEYS", "").split(",") if os.getenv("JINA_API_KEYS") else [] + use_jina = len(jina_api_keys) > 0 # Only use Jina if we have API keys + # Timeouts and retries (tunable via env) + _default_max_attempts = int(os.getenv("BROWSER_MAX_ATTEMPTS", "2")) + _jina_timeout_s = float(os.getenv("JINA_TIMEOUT_SECONDS", "5")) + _jina_retries = int(os.getenv("JINA_MAX_RETRIES", "2")) + # ASearcher-style length controls (only affect content size and chunking) + page_char_cap = int(os.getenv("BROWSER_PAGE_MAX_CHARS", "240000")) + chunk_size = int(os.getenv("BROWSER_CHUNK_SIZE", "16000")) + max_chunks = int(os.getenv("BROWSER_MAX_CHUNKS", "15")) + # Shared persistent page cache (ASearcher-style) + _page_cache = WebPageCache( + max_size=10000, + cache_file=os.getenv( + "WEBPAGE_CACHE_FILE", + os.path.join( + os.getenv("SKYAGENT_WEB_CACHE_DIR", os.path.expanduser("~/.skyagent_web_cache")), + "webpage_cache.json", + ), + ), + ) + + # Optional domain/keyword blocklists to avoid benchmark leakage during eval + _block_domains_env = os.getenv("WEB_BLOCKLIST_DOMAINS", os.getenv("SEARCH_BLOCKLIST_DOMAINS", "")).strip() + _block_keywords_env = os.getenv("WEB_BLOCKLIST_KEYWORDS", os.getenv("SEARCH_BLOCKLIST_KEYWORDS", "")).strip() + block_domains = {d.strip().lower() for d in _block_domains_env.split(",") if d.strip()} + block_keywords = {k.strip().lower() for k in _block_keywords_env.split(",") if k.strip()} + + def _is_blocked_url(self, url: str) -> bool: + try: + lu = (url or "").lower() + if self.block_domains and any(d in lu for d in self.block_domains): + return True + if self.block_keywords and any(k in lu for k in self.block_keywords): + return True + except Exception: + return False + return False + + def _normalize_url(self, url: str) -> str: + """Minimal normalization for anti-bot wrappers and PDF endpoints. + + - Unwrap validate.perfdrive.com links (decode 'ssc' target). + - For iopscience.iop.org article PDFs/EPUBs (path ends with '/pdf' or '/epub'), + prefer the HTML article page by dropping the suffix. + - Drop leading 'www.' when host already has multiple labels (e.g., www.pmc.ncbi.nlm.nih.gov → pmc.ncbi.nlm.nih.gov). + """ + try: + from urllib.parse import urlparse, parse_qs, unquote, urlunparse + + u = (url or "").strip() + if not u: + return url + p = urlparse(u) + + # Unwrap Radware/Perfdrive wrapper + if p.netloc.endswith("validate.perfdrive.com"): + qs = parse_qs(p.query) + ssc = qs.get("ssc", [""])[0] + if ssc: + target = unquote(ssc) + return self._normalize_url(target) + + # Prefer HTML article page over direct PDF/EPUB on iopscience + if p.netloc.endswith("iopscience.iop.org") and (p.path.endswith("/pdf") or p.path.endswith("/epub")): + new_path = p.path.rsplit("/", 1)[0] + return urlunparse((p.scheme, p.netloc, new_path, p.params, "", "")) + + # Drop www. for multi-label hosts (general fix for www.{sub}.{domain}) + host = p.netloc + if host.startswith("www.") and host.count(".") >= 2: + host = host[4:] + return urlunparse((p.scheme, host, p.path, p.params, p.query, p.fragment)) + + return u + except Exception: + return url + + def _url_variants(self, url: str) -> List[str]: + """Generate generic URL variants to handle common fetch failures. + + Variants include toggling scheme (https/http) and adding/removing a + leading 'www.' for simple two-label hosts, or removing it for + multi-label hosts. This is general and not site-specific. + """ + try: + from urllib.parse import urlparse, urlunparse + + base = (url or "").strip() + if not base: + return [] + p = urlparse(base) + # Schemes to try (prefer https first) + scheme0 = p.scheme if p.scheme else "https" + schemes = [scheme0, ("http" if scheme0 == "https" else "https")] + # Host variants + hosts = [] + host = p.netloc + if host: + hosts.append(host) + if host.startswith("www."): + # For multi-label hosts like www.sub.example.com, prefer dropping www. + hosts.append(host[4:]) + else: + # Only for simple two-label hosts like example.com, try adding www. + if host.count(".") == 1: + hosts.append("www." + host) + + variants = [] + for sc in schemes: + for h in hosts: + cand = urlunparse((sc, h, p.path, p.params, p.query, p.fragment)) + variants.append(cand) + # Deduplicate while preserving order + seen = set() + uniq = [] + for v in variants: + if v and v not in seen: + seen.add(v) + uniq.append(v) + return uniq or [base] + except Exception: + return [url] + + def call(self, params: dict, **kwargs) -> Union[str, dict]: + """ + Visits webpage(s) and returns summarized content. + + Args: + params (dict): Dictionary containing 'url' and 'goal'. + **kwargs: Additional keyword arguments. + + Returns: + str or dict: The webpage summary or an error message. + """ + # Verify required parameters (be tolerant of schema/JSON issues) + # Capture trajectory id for sticky routing (optional) + try: + self._trajectory_id = kwargs.get("trajectory_id") + except Exception: + self._trajectory_id = None + + try: + params = self._verify_json_format_args(params) + except Exception as e: + return { + "error": f"Invalid parameters: {str(e)}", + "hint": "Pass plain URL string or an array of URL strings in 'url', and a non-empty 'goal' string.", + } + + try: + url = params["url"] + goal = params["goal"] + except KeyError as e: + return {"error": f"Missing required field: {str(e)}"} + + if not goal: + return { + "error": "Goal parameter is required.", + "hint": "Provide a concise description of what to extract from the page.", + } + + try: + # Handle case where url is a JSON string that looks like a list + if isinstance(url, str): + # Strip extra quotes if present + url = url.strip() + if (url.startswith('"') and url.endswith('"')) or (url.startswith("'") and url.endswith("'")): + url = url[1:-1] + print(f"Removed extra quotes from URL: {url}") + + # First, check if it's a Markdown link format [text](url) + import re + + markdown_pattern = r"\[([^\]]+)\]\(([^)]+)\)" + markdown_match = re.match(markdown_pattern, url.strip()) + if markdown_match: + # Extract the actual URL from Markdown format + url = markdown_match.group(2) + print(f"Extracted URL from Markdown format: {url}") + + # Check if it's a JSON array string + url_stripped = url.strip() + if url_stripped.startswith("[") and url_stripped.endswith("]"): + try: + parsed = json.loads(url_stripped) + if isinstance(parsed, list): + # Clean any Markdown formatted URLs in the list + cleaned_urls = [] + for u in parsed: + if isinstance(u, str): + # Check if this URL is also in Markdown format + md_match = re.match(markdown_pattern, u.strip()) + if md_match: + cleaned_urls.append(md_match.group(2)) + else: + cleaned_urls.append(u) + if cleaned_urls: + url = cleaned_urls + # print(f"Parsed URL list: {url}") + except json.JSONDecodeError as e: + print(f"Failed to parse JSON array: {e}") + # Try to extract URLs from malformed array like [url1, url2] + inner_content = url_stripped[1:-1] # Remove brackets + # Split by comma and clean each URL + potential_urls = inner_content.split(",") + cleaned_urls = [] + for u in potential_urls: + u = u.strip() + # Remove any quotes if present + if (u.startswith('"') and u.endswith('"')) or (u.startswith("'") and u.endswith("'")): + u = u[1:-1] + # Check for Markdown format + md_match = re.match(markdown_pattern, u) + if md_match: + cleaned_urls.append(md_match.group(2)) + elif u.startswith("http://") or u.startswith("https://"): + cleaned_urls.append(u) + if cleaned_urls: + url = cleaned_urls + print(f"Extracted URLs from malformed array: {url}") + # Otherwise keep as string + + # Fallbacks for stray leading '[' or embedded URLs within a string + if isinstance(url, str): + # Extract any http(s) URLs embedded in the string + candidates = re.findall(r'https?://[^\s\]\\)"\'<>]+', url) + if candidates: + # Deduplicate while preserving order + seen = set() + dedup = [] + for c in candidates: + if c not in seen: + seen.add(c) + dedup.append(c) + url = dedup[0] if len(dedup) == 1 else dedup + print(f"Extracted URL(s) from freeform string: {url}") + else: + # If starts with stray '[' but no closing ']', strip and retry extraction once + if url_stripped.startswith("[") and not url_stripped.endswith("]"): + candidate = url_stripped.lstrip("[").rstrip("]") + candidates2 = re.findall(r'https?://[^\s\]\\)"\'<>]+', candidate) + if candidates2: + url = candidates2[0] if len(candidates2) == 1 else candidates2 + print(f"Recovered URL(s) from stray-bracket string: {url}") + + if isinstance(url, str): + url = self._normalize_url(url) + if self._is_blocked_url(url): + response = ( + f"The useful information in {url} for user goal {goal} as follows: \n\n" + "Evidence in page: \nBlocked by evaluation policy (domain/keyword blocklist).\n\n" + "Summary: \nThis URL is excluded to avoid benchmark leakage.\n\n" + ) + else: + response = self.readpage(url, goal) + elif isinstance(url, list): + response = [] + with ThreadPoolExecutor(max_workers=3) as executor: + # Skip blocked URLs + urls_to_fetch = [] + for u in url: + if isinstance(u, str): + u = self._normalize_url(u) + if self._is_blocked_url(u): + response.append( + f"The useful information in {u} for user goal {goal} as follows: \n\n" + "Evidence in page: \nBlocked by evaluation policy (domain/keyword blocklist).\n\n" + "Summary: \nThis URL is excluded to avoid benchmark leakage.\n\n" + ) + else: + urls_to_fetch.append(u) + futures = {executor.submit(self.readpage, u, goal): u for u in urls_to_fetch} + for future in as_completed(futures): + try: + response.append(future.result()) + except Exception as e: + response.append(f"Error fetching {futures[future]}: {str(e)}") + response = "\n=======\n".join(response) + else: + return {"error": "URL must be a string or array of strings."} + + # print(f'Summary Length {len(response)}; Summary Content {response}') + return {"results": response.strip()} + + except Exception as e: + return {"error": f"Web browsing failed: {str(e)}"} + + def call_server(self, msgs, max_tries=10): + """ + Call the OpenAI API server to process webpage content. + + Args: + msgs: Messages to send to the API + max_tries: Maximum number of retry attempts + + Returns: + str: The API response content + """ + openai_api_key = "EMPTY" + openai_api_base = os.getenv("WEB_SUMMARY_API_BASE") + summary_model = os.getenv("WEB_SUMMARY_MODEL") + assert openai_api_base, "WEB_SUMMARY_API_BASE is not set" + assert summary_model, "WEB_SUMMARY_MODEL is not set" + + client = OpenAI( + api_key=openai_api_key, + base_url=openai_api_base, + ) + + def _shrink_extractor_prompt_content(text: str, ratio: float = 0.5) -> str: + """Shrink only the Webpage Content section inside EXTRACTOR_PROMPT. + + Falls back to trimming the whole message if markers are not found. + """ + head_marker = "## **Webpage Content**" + goal_marker = "## **User Goal**" + try: + head_idx = text.find(head_marker) + goal_idx = text.find(goal_marker) + if head_idx != -1 and goal_idx != -1 and goal_idx > head_idx: + # Start of the variable content is the newline after the header + nl_idx = text.find("\n", head_idx + len(head_marker)) + content_start = nl_idx + 1 if nl_idx != -1 else head_idx + len(head_marker) + # Existing content segment to shrink + segment = text[content_start:goal_idx] + keep = max(1, int(len(segment) * ratio)) + return text[:content_start] + segment[:keep] + text[goal_idx:] + except Exception: + pass + # Fallback: trim entire text + return text[: max(1, int(len(text) * ratio))] + + def _shrink_messages(msgs_in: list, ratio: float = 0.5) -> list: + new_msgs = copy.deepcopy(msgs_in) + for m in new_msgs: + if isinstance(m, dict) and m.get("role") == "user" and isinstance(m.get("content"), str): + m["content"] = _shrink_extractor_prompt_content(m["content"], ratio) + return new_msgs + + def _split_extractor_prompt_content(text: str) -> tuple[str, str]: + head_marker = "## **Webpage Content**" + goal_marker = "## **User Goal**" + try: + head_idx = text.find(head_marker) + goal_idx = text.find(goal_marker) + if head_idx != -1 and goal_idx != -1 and goal_idx > head_idx: + nl_idx = text.find("\n", head_idx + len(head_marker)) + content_start = nl_idx + 1 if nl_idx != -1 else head_idx + len(head_marker) + segment = text[content_start:goal_idx] + half = len(segment) // 2 + first = text[:content_start] + segment[:half] + text[goal_idx:] + second = text[:content_start] + segment[half:] + text[goal_idx:] + return first, second + except Exception: + pass + mid = max(1, len(text) // 2) + return text[:mid], text[mid:] + + def _split_messages_into_two(msgs_in: list) -> tuple[list, list]: + a = copy.deepcopy(msgs_in) + b = copy.deepcopy(msgs_in) + # find largest user content + idx = -1 + best = -1 + for k, m in enumerate(a): + if isinstance(m, dict) and m.get("role") == "user" and isinstance(m.get("content"), str): + L = len(m["content"]) + if L > best: + best = L + idx = k + if idx >= 0: + first, second = _split_extractor_prompt_content(a[idx]["content"]) + a[idx]["content"] = first + b[idx]["content"] = second + return a, b + + def _normalize_json_str(s: str) -> str: + if not s: + return s + try: + json.loads(s) + return s + except Exception: + left = s.find("{") + right = s.rfind("}") + if left != -1 and right != -1 and left <= right: + return s[left : right + 1] + return s + + def _combine_json_strs(a: str, b: str) -> str: + a = _normalize_json_str(a) + b = _normalize_json_str(b) + try: + da = json.loads(a) if a else {} + except Exception: + da = {} + try: + db = json.loads(b) if b else {} + except Exception: + db = {} + if not da and not db: + return "" + rational = (da.get("rational") or "").strip() + if not rational: + rational = (db.get("rational") or "").strip() + ev = "\n\n".join( + [x for x in [str(da.get("evidence", "")).strip(), str(db.get("evidence", "")).strip()] if x] + ) + su = "\n\n".join([x for x in [str(da.get("summary", "")).strip(), str(db.get("summary", "")).strip()] if x]) + try: + return json.dumps({"rational": rational, "evidence": ev, "summary": su}, ensure_ascii=False) + except Exception: + return (a or "") + ("\n\n" if a and b else "") + (b or "") + + def _call_once_with_shrink(msgs_try: list) -> str: + local_msgs = copy.deepcopy(msgs_try) + local_shrink = 2 + while True: + try: + resp = client.chat.completions.create( + model=summary_model, + messages=local_msgs, + stop=["\n", ""], + temperature=0, + user=(str(self._trajectory_id) if getattr(self, "_trajectory_id", None) else None), + ) + cont = resp.choices[0].message.content + return _normalize_json_str(cont or "") + except Exception as ie: + ses = str(ie).lower() + ictx = ( + "maximum context length" in ses + or ("context" in ses and "exceed" in ses) + or "reduce the length" in ses + or "too many tokens" in ses + ) + if ictx and local_shrink > 0: + local_msgs = _shrink_messages(local_msgs, 0.5) + local_shrink -= 1 + continue + return "" + + # Work on a local copy we can shrink progressively on context errors + msgs_local = copy.deepcopy(msgs) + shrink_attempts = 3 + + for attempt in range(max_tries): + try: + chat_response = client.chat.completions.create( + model=summary_model, + messages=msgs_local, + stop=["\n", ""], + temperature=0, + user=(str(self._trajectory_id) if getattr(self, "_trajectory_id", None) else None), + ) + content = chat_response.choices[0].message.content + if content: + try: + json.loads(content) + except: + # extract json from string + left = content.find("{") + right = content.rfind("}") + if left != -1 and right != -1 and left <= right: + content = content[left : right + 1] + return content + except Exception as e: + es = str(e).lower() + ctx_err = ( + "maximum context length" in es + or "context length" in es + or "reduce the length" in es + or "too many tokens" in es + or "exceed" in es + and "context" in es + ) + if ctx_err: + a_msgs, b_msgs = _split_messages_into_two(msgs_local) + part_a = _call_once_with_shrink(a_msgs) + part_b = _call_once_with_shrink(b_msgs) + combined = _combine_json_strs(part_a, part_b) + if combined: + return combined + # fallback to shrinking the original message if split failed + if shrink_attempts > 0: + msgs_local = _shrink_messages(msgs_local, 0.5) + shrink_attempts -= 1 + continue + if attempt == (max_tries - 1): + return "" + continue + return "" + + def jina_readpage(self, url: str) -> str: + """Read webpage using Jina (only if API keys available)""" + """ + Read webpage content using Jina service. + + Args: + url: The URL to read + + Returns: + str: The webpage content or error message + """ + if not self.jina_api_keys: + return "[visit] No Jina API keys available." + + headers = { + "Authorization": f"Bearer {random.choice(self.jina_api_keys)}", + } + # Keep retries/timeouts conservative to reduce wall time; tunable via env + max_retries = max(1, int(getattr(self, "_jina_retries", 2))) + timeout = max(1.0, float(getattr(self, "_jina_timeout_s", 5.0))) + + # Try generic URL variants (scheme/www) within each retry attempt + for attempt in range(max_retries): + try: + for target in self._url_variants(url) or [url]: + try: + response = requests.get(f"https://r.jina.ai/{target}", headers=headers, timeout=timeout) + if response.status_code == 200 and response.text: + return response.text + else: + # Print remote error body as before for visibility + try: + print(response.text) + except Exception: + pass + # Raise to align with original retry flow + raise ValueError("jina readpage error") + except Exception: + # Try next variant + continue + except Exception: + # Fall through to next attempt + pass + if attempt == max_retries - 1: + return "[visit] Failed to read page." + + return "[visit] Failed to read page." + + def readpage(self, url: str, goal: str) -> str: + """ + Attempt to read webpage content and extract relevant information. + + Args: + url: The URL to read + goal: The goal/purpose of reading the page + + Returns: + str: The processed webpage content or error message + """ + # Normalize upfront to avoid anti-bot wrappers and blocked PDFs + url = self._normalize_url(url) + + # Avoid repeated hammering of the same failing URL within a short TTL + try: + if not hasattr(self, "_recent_failures"): + self._recent_failures = {} + ttl = int(os.getenv("WEB_BROWSER_FAIL_TTL", "10")) + except Exception: + ttl = 10 + now = None + try: + now = __import__("time").time() + last = self._recent_failures.get(url) + if last and (now - last) < ttl: + # Still print a concise note to signal skip + print(f"[web_browser][skip] recently failed url={url} since={int(now-last)}s ttl={ttl}s") + useful_information = "The useful information in {url} for user goal {goal} as follows: \n\n".format( + url=url, goal=goal + ) + useful_information += ( + "Evidence in page: \n" + + "Recently attempted but unreachable; skipping re-access to avoid consecutive failures." + + "\n\n" + ) + useful_information += ( + "Summary: \n" + + "Prior attempts failed very recently. Try alternative sources or retry later." + + "\n\n" + ) + return useful_information + except Exception: + pass + + # Keep overall attempts small to avoid long waits (env: BROWSER_MAX_ATTEMPTS) + try: + max_attempts = max(1, int(self._default_max_attempts)) + except Exception: + max_attempts = 2 + for attempt in range(max_attempts): + # Try cache first (exactly like ASearcher-style persistent LRU) + content = self._page_cache.get(url) + service = "cache" if content else "jina" + if not content: + content = self.jina_readpage(url) + + # print(service) + # print(content) + if ( + content + and not content.startswith("[visit] Failed to read page.") + and content != "[visit] Empty content." + and not content.startswith("[document_parser]") + ): + # On valid fetch (including first-time), populate cache + if service == "jina": + try: + self._page_cache.put(url, content) + except Exception: + pass + # ASearcher-style length handling: cap page size and split into chunks, summarize per chunk + content = content[: self.page_char_cap] + + # Build chunks: 10k each, up to 10 chunks + chunks = [] + lim = min(len(content), self.chunk_size * self.max_chunks) + i = 0 + while i < lim and len(chunks) < self.max_chunks: + chunks.append(content[i : i + self.chunk_size]) + i += self.chunk_size + + aggregated_evidence = [] + aggregated_summary = [] + + for chunk in chunks: + messages = [{"role": "user", "content": EXTRACTOR_PROMPT.format(webpage_content=chunk, goal=goal)}] + raw = self.call_server(messages) + + # Handle long chunk content by progressive truncation + summary_retries = 3 + current = chunk + while (not raw or len(raw) < 10) and summary_retries >= 0: + truncate_length = int(0.7 * len(current)) if summary_retries > 0 else min(25000, len(current)) + current = current[:truncate_length] + extraction_prompt = EXTRACTOR_PROMPT.format(webpage_content=current, goal=goal) + messages = [{"role": "user", "content": extraction_prompt}] + raw = self.call_server(messages) + summary_retries -= 1 + + # Parse JSON response with a few retries + parse_retry_times = 0 + parsed = None + while parse_retry_times < 3: + try: + parsed = json.loads(raw) + break + except Exception: + raw = self.call_server(messages) + parse_retry_times += 1 + + if parsed and isinstance(parsed, dict): + aggregated_evidence.append(str(parsed.get("evidence", ""))) + aggregated_summary.append(str(parsed.get("summary", ""))) + + # Generate final aggregated response if any chunk succeeded + if aggregated_evidence or aggregated_summary: + useful_information = "The useful information in {url} for user goal {goal} as follows: \n\n".format( + url=url, goal=goal + ) + useful_information += "Evidence in page: \n" + "\n\n".join(aggregated_evidence) + "\n\n" + useful_information += "Summary: \n" + "\n\n".join(aggregated_summary) + "\n\n" + return useful_information + + # If all chunks failed to parse, fall back to original failure message + useful_information = "The useful information in {url} for user goal {goal} as follows: \n\n".format( + url=url, goal=goal + ) + useful_information += ( + "Evidence in page: \n" + + "The provided webpage content could not be accessed. Please check the URL or file format." + + "\n\n" + ) + useful_information += ( + "Summary: \n" + + "The webpage content could not be processed, and therefore, no information is available." + + "\n\n" + ) + return useful_information + + # If we're on the last attempt, record failure and return failure message + if attempt == max_attempts - 1: + try: + if now is None: + now = __import__("time").time() + self._recent_failures[url] = now + except Exception: + pass + useful_information = "The useful information in {url} for user goal {goal} as follows: \n\n".format( + url=url, goal=goal + ) + useful_information += ( + "Evidence in page: \n" + + "The provided webpage content could not be accessed. Please check the URL or file format." + + "\n\n" + ) + useful_information += ( + "Summary: \n" + + "The webpage content could not be processed, and therefore, no information is available." + + "\n\n" + ) + return useful_information + + +if __name__ == "__main__": + # Example usage for testing + tool = WebBrowser() + test_params = {"url": "https://apple.com", "goal": "Find information about the company"} + result = tool.call(test_params) + print("Test Result:", result) diff --git a/skyrl-agent/tests/__init__.py b/skyrl-agent/tests/__init__.py new file mode 100644 index 0000000000..31a60909dc --- /dev/null +++ b/skyrl-agent/tests/__init__.py @@ -0,0 +1 @@ +# SkyRL-Agent Tests diff --git a/skyrl-agent/uv.lock b/skyrl-agent/uv.lock new file mode 100644 index 0000000000..b4864e26f3 --- /dev/null +++ b/skyrl-agent/uv.lock @@ -0,0 +1,8033 @@ +version = 1 +revision = 2 +requires-python = "==3.12.*" +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform == 'darwin' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +conflicts = [[ + { package = "skyagent", extra = "skyrl-train" }, + { package = "skyagent", extra = "verl" }, +]] + +[manifest] +overrides = [ + { name = "opentelemetry-api", specifier = "==1.26.0" }, + { name = "opentelemetry-exporter-otlp", specifier = "==1.26.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = "==1.26.0" }, + { name = "opentelemetry-sdk", specifier = "==1.26.0" }, + { name = "ray", specifier = "==2.48.0" }, + { name = "xgrammar", specifier = "==0.1.17" }, +] + +[[package]] +name = "absl-py" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" }, +] + +[[package]] +name = "accelerate" +version = "1.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/72/ff3961c19ee395c3d30ac630ee77bfb0e1b46b87edc504d4f83bb4a89705/accelerate-1.10.1.tar.gz", hash = "sha256:3dea89e433420e4bfac0369cae7e36dcd6a56adfcfd38cdda145c6225eab5df8", size = 392446, upload-time = "2025-08-25T13:57:06.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/a0/d9ef19f780f319c21ee90ecfef4431cbeeca95bec7f14071785c17b6029b/accelerate-1.10.1-py3-none-any.whl", hash = "sha256:3621cff60b9a27ce798857ece05e2b9f56fcc71631cfb31ccf71f0359c311f11", size = 374909, upload-time = "2025-08-25T13:57:04.55Z" }, +] + +[[package]] +name = "aiodocker" +version = "0.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/d7/30104dfac550ae6570d4dce24c2cbf2ddefd4937c9e861641314abfd8abb/aiodocker-0.24.0.tar.gz", hash = "sha256:661a6f9a479951f11f793031dcd5d55337e232c4ceaee69d51ceb885e5f16fac", size = 135482, upload-time = "2024-11-21T02:23:59.642Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/5b/2bb8b632041e314a0a917ade80382ca6a8f331f12c7eb409e59cd0485cc9/aiodocker-0.24.0-py3-none-any.whl", hash = "sha256:2199b7b01f8ce68f9cabab7910ecb26192f6f3494163f1ccffe527b4c3875689", size = 34849, upload-time = "2024-11-21T02:23:58.053Z" }, +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.12.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/e7/d92a237d8802ca88483906c388f7c201bbe96cd80a165ffd0ac2f6a8d59f/aiohttp-3.12.15.tar.gz", hash = "sha256:4fc61385e9c98d72fcdf47e6dd81833f47b2f77c114c29cd64a361be57a763a2", size = 7823716, upload-time = "2025-07-29T05:52:32.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/97/77cb2450d9b35f517d6cf506256bf4f5bda3f93a66b4ad64ba7fc917899c/aiohttp-3.12.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:802d3868f5776e28f7bf69d349c26fc0efadb81676d0afa88ed00d98a26340b7", size = 702333, upload-time = "2025-07-29T05:50:46.507Z" }, + { url = "https://files.pythonhosted.org/packages/83/6d/0544e6b08b748682c30b9f65640d006e51f90763b41d7c546693bc22900d/aiohttp-3.12.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2800614cd560287be05e33a679638e586a2d7401f4ddf99e304d98878c29444", size = 476948, upload-time = "2025-07-29T05:50:48.067Z" }, + { url = "https://files.pythonhosted.org/packages/3a/1d/c8c40e611e5094330284b1aea8a4b02ca0858f8458614fa35754cab42b9c/aiohttp-3.12.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8466151554b593909d30a0a125d638b4e5f3836e5aecde85b66b80ded1cb5b0d", size = 469787, upload-time = "2025-07-29T05:50:49.669Z" }, + { url = "https://files.pythonhosted.org/packages/38/7d/b76438e70319796bfff717f325d97ce2e9310f752a267bfdf5192ac6082b/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e5a495cb1be69dae4b08f35a6c4579c539e9b5706f606632102c0f855bcba7c", size = 1716590, upload-time = "2025-07-29T05:50:51.368Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/60370d70cdf8b269ee1444b390cbd72ce514f0d1cd1a715821c784d272c9/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6404dfc8cdde35c69aaa489bb3542fb86ef215fc70277c892be8af540e5e21c0", size = 1699241, upload-time = "2025-07-29T05:50:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2b/4968a7b8792437ebc12186db31523f541943e99bda8f30335c482bea6879/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ead1c00f8521a5c9070fcb88f02967b1d8a0544e6d85c253f6968b785e1a2ab", size = 1754335, upload-time = "2025-07-29T05:50:55.394Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/49524ed553f9a0bec1a11fac09e790f49ff669bcd14164f9fab608831c4d/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6990ef617f14450bc6b34941dba4f12d5613cbf4e33805932f853fbd1cf18bfb", size = 1800491, upload-time = "2025-07-29T05:50:57.202Z" }, + { url = "https://files.pythonhosted.org/packages/de/5e/3bf5acea47a96a28c121b167f5ef659cf71208b19e52a88cdfa5c37f1fcc/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd736ed420f4db2b8148b52b46b88ed038d0354255f9a73196b7bbce3ea97545", size = 1719929, upload-time = "2025-07-29T05:50:59.192Z" }, + { url = "https://files.pythonhosted.org/packages/39/94/8ae30b806835bcd1cba799ba35347dee6961a11bd507db634516210e91d8/aiohttp-3.12.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c5092ce14361a73086b90c6efb3948ffa5be2f5b6fbcf52e8d8c8b8848bb97c", size = 1635733, upload-time = "2025-07-29T05:51:01.394Z" }, + { url = "https://files.pythonhosted.org/packages/7a/46/06cdef71dd03acd9da7f51ab3a9107318aee12ad38d273f654e4f981583a/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aaa2234bb60c4dbf82893e934d8ee8dea30446f0647e024074237a56a08c01bd", size = 1696790, upload-time = "2025-07-29T05:51:03.657Z" }, + { url = "https://files.pythonhosted.org/packages/02/90/6b4cfaaf92ed98d0ec4d173e78b99b4b1a7551250be8937d9d67ecb356b4/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6d86a2fbdd14192e2f234a92d3b494dd4457e683ba07e5905a0b3ee25389ac9f", size = 1718245, upload-time = "2025-07-29T05:51:05.911Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e6/2593751670fa06f080a846f37f112cbe6f873ba510d070136a6ed46117c6/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a041e7e2612041a6ddf1c6a33b883be6a421247c7afd47e885969ee4cc58bd8d", size = 1658899, upload-time = "2025-07-29T05:51:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/8f/28/c15bacbdb8b8eb5bf39b10680d129ea7410b859e379b03190f02fa104ffd/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5015082477abeafad7203757ae44299a610e89ee82a1503e3d4184e6bafdd519", size = 1738459, upload-time = "2025-07-29T05:51:09.56Z" }, + { url = "https://files.pythonhosted.org/packages/00/de/c269cbc4faa01fb10f143b1670633a8ddd5b2e1ffd0548f7aa49cb5c70e2/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56822ff5ddfd1b745534e658faba944012346184fbfe732e0d6134b744516eea", size = 1766434, upload-time = "2025-07-29T05:51:11.423Z" }, + { url = "https://files.pythonhosted.org/packages/52/b0/4ff3abd81aa7d929b27d2e1403722a65fc87b763e3a97b3a2a494bfc63bc/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b2acbbfff69019d9014508c4ba0401822e8bae5a5fdc3b6814285b71231b60f3", size = 1726045, upload-time = "2025-07-29T05:51:13.689Z" }, + { url = "https://files.pythonhosted.org/packages/71/16/949225a6a2dd6efcbd855fbd90cf476052e648fb011aa538e3b15b89a57a/aiohttp-3.12.15-cp312-cp312-win32.whl", hash = "sha256:d849b0901b50f2185874b9a232f38e26b9b3d4810095a7572eacea939132d4e1", size = 423591, upload-time = "2025-07-29T05:51:15.452Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d8/fa65d2a349fe938b76d309db1a56a75c4fb8cc7b17a398b698488a939903/aiohttp-3.12.15-cp312-cp312-win_amd64.whl", hash = "sha256:b390ef5f62bb508a9d67cb3bba9b8356e23b3996da7062f1a57ce1a79d2b3d34", size = 450266, upload-time = "2025-07-29T05:51:17.239Z" }, +] + +[[package]] +name = "aiolimiter" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/23/b52debf471f7a1e42e362d959a3982bdcb4fe13a5d46e63d28868807a79c/aiolimiter-1.2.1.tar.gz", hash = "sha256:e02a37ea1a855d9e832252a105420ad4d15011505512a1a1d814647451b5cca9", size = 7185, upload-time = "2024-12-08T15:31:51.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/ba/df6e8e1045aebc4778d19b8a3a9bc1808adb1619ba94ca354d9ba17d86c3/aiolimiter-1.2.1-py3-none-any.whl", hash = "sha256:d3f249e9059a20badcb56b61601a83556133655c11d1eb3dd3e04ff069e5f3c7", size = 6711, upload-time = "2024-12-08T15:31:49.874Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "airportsdata" +version = "20250909" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/c6/17ae8a65f7fa5bbbeee166f8070063eb8b70c89501a65c2e6885db61fc08/airportsdata-20250909.tar.gz", hash = "sha256:f39974fe1101817ced4ccf7c6ed336408469e5e778395d0a3e7a5112ec298f90", size = 907204, upload-time = "2025-09-09T01:07:31.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/87/59b570b9c4b014532777dc3baffc9bea10cf0cc8b232cf3c17e4bd0754a6/airportsdata-20250909-py3-none-any.whl", hash = "sha256:ce7dc6e1485afe3915e708212c7024ad158470c1c934e6a6cb217cf28b798ac7", size = 914391, upload-time = "2025-09-09T01:07:29.364Z" }, +] + +[[package]] +name = "altair" +version = "5.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "narwhals" }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/b1/f2969c7bdb8ad8bbdda031687defdce2c19afba2aa2c8e1d2a17f78376d8/altair-5.5.0.tar.gz", hash = "sha256:d960ebe6178c56de3855a68c47b516be38640b73fb3b5111c2a9ca90546dd73d", size = 705305, upload-time = "2024-11-23T23:39:58.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/f3/0b6ced594e51cc95d8c1fc1640d3623770d01e4969d29c0bd09945fafefa/altair-5.5.0-py3-none-any.whl", hash = "sha256:91a310b926508d560fe0148d02a194f38b824122641ef528113d029fcd129f8c", size = 731200, upload-time = "2024-11-23T23:39:56.4Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anthropic" +version = "0.69.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/9d/9ad1778b95f15c5b04e7d328c1b5f558f1e893857b7c33cd288c19c0057a/anthropic-0.69.0.tar.gz", hash = "sha256:c604d287f4d73640f40bd2c0f3265a2eb6ce034217ead0608f6b07a8bc5ae5f2", size = 480622, upload-time = "2025-09-29T16:53:45.282Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/38/75129688de5637eb5b383e5f2b1570a5cc3aecafa4de422da8eea4b90a6c/anthropic-0.69.0-py3-none-any.whl", hash = "sha256:1f73193040f33f11e27c2cd6ec25f24fe7c3f193dc1c5cde6b7a08b18a16bcc5", size = 337265, upload-time = "2025-09-29T16:53:43.686Z" }, +] + +[package.optional-dependencies] +vertex = [ + { name = "google-auth", extra = ["requests"] }, +] + +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } + +[[package]] +name = "anyio" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, +] + +[[package]] +name = "appnope" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121, upload-time = "2025-07-30T10:01:50.815Z" }, + { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177, upload-time = "2025-07-30T10:01:51.681Z" }, + { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090, upload-time = "2025-07-30T10:01:53.184Z" }, + { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, + { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, + { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, + { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180, upload-time = "2025-07-30T10:01:57.759Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715, upload-time = "2025-07-30T10:01:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149, upload-time = "2025-07-30T10:01:59.329Z" }, +] + +[[package]] +name = "astor" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, +] + +[[package]] +name = "asttokens" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/e7/82da0a03e7ba5141f05cce0d302e6eed121ae055e0456ca228bf693984bc/asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", size = 61978, upload-time = "2024-11-30T04:30:14.439Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918, upload-time = "2024-11-30T04:30:10.946Z" }, +] + +[[package]] +name = "attrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, +] + +[[package]] +name = "bashlex" +version = "0.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/60/aae0bb54f9af5e0128ba90eb83d8d0d506ee8f0475c4fdda3deeda20b1d2/bashlex-0.18.tar.gz", hash = "sha256:5bb03a01c6d5676338c36fd1028009c8ad07e7d61d8a1ce3f513b7fff52796ee", size = 68742, upload-time = "2023-01-18T15:21:26.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/be/6985abb1011fda8a523cfe21ed9629e397d6e06fb5bae99750402b25c95b/bashlex-0.18-py2.py3-none-any.whl", hash = "sha256:91d73a23a3e51711919c1c899083890cdecffc91d8c088942725ac13e9dcfffa", size = 69539, upload-time = "2023-01-18T15:21:24.167Z" }, +] + +[[package]] +name = "beartype" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/78/bf7a20712dbf3a18e37f9ae1c2a1e04169c43b4628871f40f1db5f29c257/beartype-0.12.0.tar.gz", hash = "sha256:3b7545b3f333a6b07042b68b102141554c9add2e979dab7b0f8ed6378f7af7d7", size = 1029204, upload-time = "2023-01-17T08:36:34.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/1f/d066e5689e91e3c8845d275d85d25b23f2c7c78c9523d852b4b73be73875/beartype-0.12.0-py3-none-any.whl", hash = "sha256:3d9d5bec198bcf965c000d7b5120bebdd19a444ef6e39e97d0e93eb8832e10c8", size = 754537, upload-time = "2023-01-17T08:36:27.714Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/e9/df2358efd7659577435e2177bfa69cba6c33216681af51a707193dec162a/beautifulsoup4-4.14.2.tar.gz", hash = "sha256:2a98ab9f944a11acee9cc848508ec28d9228abfd522ef0fad6a02a72e0ded69e", size = 625822, upload-time = "2025-09-29T10:05:42.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/fe/3aed5d0be4d404d12d36ab97e2f1791424d9ca39c2f754a6285d59a3b01d/beautifulsoup4-4.14.2-py3-none-any.whl", hash = "sha256:5ef6fa3a8cbece8488d66985560f97ed091e22bbc4e9c2338508a9d5de6d4515", size = 106392, upload-time = "2025-09-29T10:05:43.771Z" }, +] + +[[package]] +name = "bidict" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" }, +] + +[[package]] +name = "binaryornot" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "chardet" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/fe/7ebfec74d49f97fc55cd38240c7a7d08134002b1e14be8c3897c0dd5e49b/binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061", size = 371054, upload-time = "2017-08-03T15:55:25.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/7e/f7b6f453e6481d1e233540262ccbfcf89adcd43606f44a028d7f5fae5eb2/binaryornot-0.4.4-py2.py3-none-any.whl", hash = "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4", size = 9006, upload-time = "2017-08-03T15:55:31.23Z" }, +] + +[[package]] +name = "blake3" +version = "1.0.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/aa/e767c874392e95c42231a71072e6d47b9411b0c5fc5b221ed1304d63e4c1/blake3-1.0.7.tar.gz", hash = "sha256:228c617d4a92b45ef06c63236348001976245eb8efd561dbe181a0def524173e", size = 114008, upload-time = "2025-09-29T16:41:23.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/f7/fcf1f49bbe650f91570c7b4fe103774bbf5445294083ca127529e6327d45/blake3-1.0.7-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:965b7b4f5f32079b7d974281fd289643197b814dd84e5cdcf153a70e27ad52cd", size = 347519, upload-time = "2025-09-29T16:39:58.669Z" }, + { url = "https://files.pythonhosted.org/packages/9c/3a/5c6ae87ab59026a11d55ba6bf3346e7c71392c57adfda860fc4c6db2b5a0/blake3-1.0.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9676a0a857fbe8052857f0000a44c516eced427fceecba3586deedb0f5253626", size = 324890, upload-time = "2025-09-29T16:40:00.15Z" }, + { url = "https://files.pythonhosted.org/packages/ad/35/1fc595de42168569a83c49aff0aab94feef3c10c4152ad68f8046ad9839a/blake3-1.0.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a7ef1c2c0da34c336e6ee4fa02b22cf142dc1a171e84d1cc46e9948c4e2c76d", size = 370790, upload-time = "2025-09-29T16:40:01.476Z" }, + { url = "https://files.pythonhosted.org/packages/6b/cb/58c5e33dde29b86e8a196d3acb2c1e25436ed58f48cd8ab5d4668cc085ec/blake3-1.0.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46bfeeac655420f7421897a6d4754aafa9cd02bbcda869fc0bc358b19d90d864", size = 375207, upload-time = "2025-09-29T16:40:02.673Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a9/4e59c99d8d4d137424e2175dbd67be48862a6543880dd9d78bc0f42be4ed/blake3-1.0.7-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55095edd17183028c5be4fab204ee48e635ce78bd9f499ca1f95c0ed011fbf12", size = 448151, upload-time = "2025-09-29T16:40:03.888Z" }, + { url = "https://files.pythonhosted.org/packages/b5/55/d2e7d53a654193aad9ac27f3748f903791a9ebb88ba2fbfc1fd3e2629b59/blake3-1.0.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05f8430d31aa11aa9b488b4a6ca07e35cc402b1c104d39ccbc99decdeb895130", size = 504463, upload-time = "2025-09-29T16:40:05.71Z" }, + { url = "https://files.pythonhosted.org/packages/a5/d0/3a2e712202ab85e972a0419caa3465153234adeb8429ba7a0bd4c23b936e/blake3-1.0.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5d19e9a88274c85eb5c9985e58dd3f2cf56b1de0f71a77a4ba7fac2a2c70bf8", size = 393701, upload-time = "2025-09-29T16:40:06.928Z" }, + { url = "https://files.pythonhosted.org/packages/db/30/addf808816833de5e497a1bf9d10221b1fc8048703b3de2c21ac6450326f/blake3-1.0.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ef46ca98068d0153a8fd12215df8a24844800762fd5a3145658d89a3335990c", size = 387936, upload-time = "2025-09-29T16:40:08.214Z" }, + { url = "https://files.pythonhosted.org/packages/b3/70/48588e41c62d4ae1164acab51ba17bb15eca7499e18731645831a2e3459f/blake3-1.0.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f4c327bdb172c08b935a98d1f527c07437d825ab317469f35ea43243b58898bc", size = 550142, upload-time = "2025-09-29T16:40:09.898Z" }, + { url = "https://files.pythonhosted.org/packages/73/48/1aa83fd4671886eee2ab1497dcfa7089797975b927c79aefc418a2b928f3/blake3-1.0.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bbfad4c447e8f910b4f2442e79768e5a3f6d9b1a31453fc780bab25e3e03d9de", size = 553949, upload-time = "2025-09-29T16:40:11.183Z" }, + { url = "https://files.pythonhosted.org/packages/ad/84/b81b27ab8624125f0fdfea528c225d4ea42f59373d278dc36cdbff1d89c0/blake3-1.0.7-cp312-cp312-win32.whl", hash = "sha256:3df6e36afea67f5d13f54b4547b06c2bde38f31d32bbc80c471344e0a5f4c2c6", size = 227548, upload-time = "2025-09-29T16:40:12.589Z" }, + { url = "https://files.pythonhosted.org/packages/88/7d/be50ad1435d5cca65fc8d6b89a9504c8bbcc04d6cc2f09974e2c33caf044/blake3-1.0.7-cp312-cp312-win_amd64.whl", hash = "sha256:0aa03b72567ee73fe683d1efca8a02a3ff8a1ba47ba8bca5f2b897c00e759cd6", size = 214502, upload-time = "2025-09-29T16:40:14.166Z" }, +] + +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + +[[package]] +name = "boto3" +version = "1.40.41" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/f7/d652732fd8fc28f427f3e698e488b7422ede535fe4d813c7988642c7734b/boto3-1.40.41.tar.gz", hash = "sha256:2ea2463fc42812f3cab66b53114579b1f4b9a378ee48921d4385511a94307b24", size = 111621, upload-time = "2025-09-29T19:21:14.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/a8/d5c924e3dfce8804ff7119c1e84f423414851a82d30d744e81f4b61ddbff/boto3-1.40.41-py3-none-any.whl", hash = "sha256:02eac942aaa9f3a1c8a11f77e6f971b41c125973888f80f3eb177c2f21ad7a01", size = 139342, upload-time = "2025-09-29T19:21:12.728Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.41" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/55/fa23104e4352f28a457c452a3621b00fe2a7a1e096d04e2f4a9890250448/botocore-1.40.41.tar.gz", hash = "sha256:320873c6a34bfd64fb9bbc55e8ac38e7904a574cfc634d1f0f66b1490c62b89d", size = 14365601, upload-time = "2025-09-29T19:21:03.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/d2/1b5bc59c746413d3eca16bd659b8c00cf2d4eeb6274053d1ff3110d13670/botocore-1.40.41-py3-none-any.whl", hash = "sha256:8246bf73a2e20647cf1d4dae1e9a7c40f97f38a34a6a1fbfd49aa6b3dce5ffaa", size = 14034516, upload-time = "2025-09-29T19:20:59.792Z" }, +] + +[[package]] +name = "browsergym" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-assistantbench" }, + { name = "browsergym-core" }, + { name = "browsergym-experiments" }, + { name = "browsergym-miniwob" }, + { name = "browsergym-visualwebarena" }, + { name = "browsergym-webarena" }, + { name = "browsergym-workarena" }, + { name = "weblinx-browsergym" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/03/05c99a8148c5574c41027d078134989762a4109cf4506966b36f1d56b473/browsergym-0.13.3.tar.gz", hash = "sha256:c3ee2ac41cf7a13abe71e0f9c63c28b37fee348dcc64fa1a6d2b5e513f9929e0", size = 1516, upload-time = "2024-11-27T19:55:51.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/93/3380168262005a948f87f2f0c1d2d3f3166462be072e77ae9af83a3781a1/browsergym-0.13.3-py3-none-any.whl", hash = "sha256:4f1f8284ca3eb82e5bafb8fa24557ccdd98aaee55971cfa136ad7857011abb20", size = 1438, upload-time = "2024-11-27T19:55:40.726Z" }, +] + +[[package]] +name = "browsergym-assistantbench" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-core" }, + { name = "datasets" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/33/dab9ce9c271bb849e02af145150d16a87b72afbf5de5ba40dee5c792592f/browsergym_assistantbench-0.13.3.tar.gz", hash = "sha256:46d784c7dcfc7b07836e4378d20275998b185b6c2ca6d0973500ab0333fde981", size = 8931, upload-time = "2024-11-27T19:55:52.019Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/28/da851395fc513ff1f00ceeaf8f2bb149446c39febfd3be92024ade3ef7dc/browsergym_assistantbench-0.13.3-py3-none-any.whl", hash = "sha256:33f40b590f2baa521e05c1b32b063d867e9cd901c40dda5cb30cb203035236b7", size = 11225, upload-time = "2024-11-27T19:55:42.472Z" }, +] + +[[package]] +name = "browsergym-core" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "gymnasium" }, + { name = "lxml" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pillow" }, + { name = "playwright" }, + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/f8/935287c081a9c94bb0bcafdd7d0f630c3a7b26bde88ed826885d1b750cff/browsergym_core-0.13.3.tar.gz", hash = "sha256:ac5036b574c8c14ac4a0c09da578a0a00b584d6f5b5ed9bf7a247e24f4d9d2f8", size = 179244, upload-time = "2024-11-27T19:55:52.878Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/73/a0328b4b0a4dfaf34854f351cad153b35989476b36a004ac7f51ce1447b1/browsergym_core-0.13.3-py3-none-any.whl", hash = "sha256:db806c64deb819a51501f0466ecb51533fbc7b6edb5f7dbdcb865e7564a86719", size = 186538, upload-time = "2024-11-27T19:55:44.214Z" }, +] + +[[package]] +name = "browsergym-experiments" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-core" }, + { name = "dataclasses-json" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/19/c0330d222cf43d958b44284d4954487f5ee1669096d0e5eb42562d4de50e/browsergym_experiments-0.13.3.tar.gz", hash = "sha256:96842e7700e27380746ac57ffc647a1dd56d449f925441ed9bc87675cddfff08", size = 206625, upload-time = "2024-11-27T19:55:54.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/04/d2495e033b87673e5b52b898985cf51f375b13f0520bf4ee733742db3f47/browsergym_experiments-0.13.3-py3-none-any.whl", hash = "sha256:61963e747eb2c3d04f4f0b5bb5a2f61208025fe2f94faf23f1b86b98dfce3218", size = 237011, upload-time = "2024-11-27T19:55:46.038Z" }, +] + +[[package]] +name = "browsergym-miniwob" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/67/461f5c75a08c61a6d4aca46098edb8d93767e3552c7daabb5456be1d2423/browsergym_miniwob-0.13.3.tar.gz", hash = "sha256:0e22797a83d4664636364b2400c5ea0eca16ddd3f50d3003891b0892da1ff40e", size = 8918, upload-time = "2024-11-27T19:55:56.128Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/da/f337c0e7b710d8211467f897aab6ccdb25c88eee3717fc36d3c6b335efcb/browsergym_miniwob-0.13.3-py3-none-any.whl", hash = "sha256:353b9f8849b7f637e17a928021a93ce962ca9b828434cfe68cebdbe2f11f4a2f", size = 9114, upload-time = "2024-11-27T19:55:47.903Z" }, +] + +[[package]] +name = "browsergym-visualwebarena" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-core" }, + { name = "browsergym-webarena" }, + { name = "libvisualwebarena" }, + { name = "requests" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/1e/392963bc51d1ad397bd652f0f69bfebcfa8711f386d63afbe4be43e6e7f0/browsergym_visualwebarena-0.13.3.tar.gz", hash = "sha256:635b4a71c8ff6bff3e84c0fecc7a10b9e932fe2929d4bf8e2e9a5bf2e29438e4", size = 8380, upload-time = "2024-11-27T19:55:56.951Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5a/5022fb0e073b51caa8190b839d2055743f2f645de52af3d6afd67d22a71b/browsergym_visualwebarena-0.13.3-py3-none-any.whl", hash = "sha256:a42c200023497a4970290fce39b419a93aadfc9e92c02ae602704d2957e5e531", size = 9222, upload-time = "2024-11-27T19:55:48.946Z" }, +] + +[[package]] +name = "browsergym-webarena" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-core" }, + { name = "libwebarena" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/70/80144be60bd4c453c5fc55aa0f95f76ab9697a88525137271b1a40f57d36/browsergym_webarena-0.13.3.tar.gz", hash = "sha256:60347edfd8d16e9b6b34a03b3ccb0e058ff11b83f3308ac5ead60321a9cc6462", size = 7734, upload-time = "2024-11-27T19:55:58.355Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/1a/77c8b1d1a6a9c4dee52cd322885f5bbb172f55bf27dcdd70d08e1c8e2288/browsergym_webarena-0.13.3-py3-none-any.whl", hash = "sha256:28098690f7c4a513c06e9da0d95f13e5c7bc70ec4bcfcfb7f83311b4081af0c9", size = 8095, upload-time = "2024-11-27T19:55:50.034Z" }, +] + +[[package]] +name = "browsergym-workarena" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-core" }, + { name = "english-words" }, + { name = "faker" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/4c/af644b7ad90f03d22e1cb5aefc14d29eaf918e5f68372c860fd653597efa/browsergym_workarena-0.4.4.tar.gz", hash = "sha256:c6f6f1d4ed48c3ef785fc35133aae306d09e4f25f3d9eb4a6fb8015294d9ea56", size = 6608813, upload-time = "2025-08-13T18:05:37.185Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/44/ffc42a7e4cebc248f80311f5808b1fee07ac92353e364436d24445cae22c/browsergym_workarena-0.4.4-py3-none-any.whl", hash = "sha256:8160361a04c0b9c1d7e7ec63089adc86474a1846d90b93ba20fa5c7a729e59d5", size = 6781913, upload-time = "2025-08-13T18:05:35.094Z" }, +] + +[[package]] +name = "build" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "(os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux') or (os_name == 'nt' and sys_platform == 'darwin' and extra == 'extra-8-skyagent-skyrl-train') or (os_name == 'nt' and sys_platform == 'darwin' and extra != 'extra-8-skyagent-verl') or (os_name != 'nt' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/1c/23e33405a7c9eac261dff640926b8b5adaed6a6eb3e1767d441ed611d0c0/build-1.3.0.tar.gz", hash = "sha256:698edd0ea270bde950f53aed21f3a0135672206f3911e0176261a31e0e07b397", size = 48544, upload-time = "2025-08-01T21:27:09.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/8c/2b30c12155ad8de0cf641d76a8b396a16d2c36bc6d50b621a62b7c4567c1/build-1.3.0-py3-none-any.whl", hash = "sha256:7145f0b5061ba90a1500d60bd1b13ca0a8a4cebdd0cc16ed8adf1c0e739f43b4", size = 23382, upload-time = "2025-08-01T21:27:07.844Z" }, +] + +[[package]] +name = "cachecontrol" +version = "0.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/3a/0cbeb04ea57d2493f3ec5a069a117ab467f85e4a10017c6d854ddcbff104/cachecontrol-0.14.3.tar.gz", hash = "sha256:73e7efec4b06b20d9267b441c1f733664f989fb8688391b670ca812d70795d11", size = 28985, upload-time = "2025-04-30T16:45:06.135Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/4c/800b0607b00b3fd20f1087f80ab53d6b4d005515b0f773e4831e37cfa83f/cachecontrol-0.14.3-py3-none-any.whl", hash = "sha256:b35e44a3113f17d2a31c1e6b27b9de6d4405f84ae51baa8c1d3cc5b633010cae", size = 21802, upload-time = "2025-04-30T16:45:03.863Z" }, +] + +[package.optional-dependencies] +filecache = [ + { name = "filelock" }, +] + +[[package]] +name = "cachetools" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, +] + +[[package]] +name = "cbor2" +version = "5.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/89/01df16cdc9c60c07956756c90fe92c684021003079e358a78e213bce45a2/cbor2-5.7.0.tar.gz", hash = "sha256:3f6d843f4db4d0ec501c46453c22a4fbebb1abfb5b740e1bcab34c615cd7406b", size = 102374, upload-time = "2025-08-14T08:59:47.294Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/b1/d54c41b1bc71b8dea0bad3409d2a497df35f7b5ae5db70c1cc9ebc8d556d/cbor2-5.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7ad36f0537b75c1aa2c7a462cbdbeec5e8ba02802ea985e0b9fe5deee3b946f4", size = 69020, upload-time = "2025-08-14T08:59:02.276Z" }, + { url = "https://files.pythonhosted.org/packages/f4/e0/45368d5d78b520caaa9ca5a09f55365bc9933d43bce978a528922654ca9f/cbor2-5.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5fc9b335cf28e63d9eed4ae03d1e8f90f1a6b287cabc8d29bfddf73fa70643e9", size = 68950, upload-time = "2025-08-14T08:59:03.882Z" }, + { url = "https://files.pythonhosted.org/packages/1e/6a/9aed5b716407c1d48425ba55c6022a01a9abdbf58a691f50416461fa371d/cbor2-5.7.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16bea83598a1eeedbd50c2e9fdf3685bae78ca9d9ec8cd8010777db14a315578", size = 285685, upload-time = "2025-08-14T08:59:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6e/3499eaa2b858c7695a447b6311303f06ffc90fc2c45851337121661f1f5c/cbor2-5.7.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e837825a16e60ace6e284095aa9fbe504bf87a8f4494bf7d95931e37fb01a70", size = 284948, upload-time = "2025-08-14T08:59:06.64Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3e/ae67866ef65717665e0acf2873d466c5d4a1d965b0d0348f2269b73f28fb/cbor2-5.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:27396c5e275ff7c7cd87fe8aaadf781e6194903921f250934af7c86d5efec82e", size = 276375, upload-time = "2025-08-14T08:59:08.845Z" }, + { url = "https://files.pythonhosted.org/packages/b6/3d/2f8e9671111661dd571de206344ecd7706f6d458aab191e06834c89aa58e/cbor2-5.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c84bfef78c4e9c81eb0a10cec340222ba4e39498a63fc2e3d5f982a3f4efa4a7", size = 277680, upload-time = "2025-08-14T08:59:10.292Z" }, + { url = "https://files.pythonhosted.org/packages/85/03/27a9fefa4e084c1129d7180727791a166629fdae39e0609508401d322626/cbor2-5.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:f64270a24aaadb15dd31cbd64a98d99fca8e0398a65b1570ba07f3c259eb5516", size = 68354, upload-time = "2025-08-14T08:59:11.561Z" }, + { url = "https://files.pythonhosted.org/packages/25/d9/b856d078696542a0d7486d1ece5c936e937bebe5b114674db18d76feb131/cbor2-5.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:73ef321d7b580f08c9fadc41c3d2a218aa3f01e163be9793c6969aadee07f57a", size = 63896, upload-time = "2025-08-14T08:59:12.977Z" }, + { url = "https://files.pythonhosted.org/packages/41/cc/0ce73676d2a0c9e5a9330c301940c50eb325dacf5f6d9690fd43a8817fe9/cbor2-5.7.0-py3-none-any.whl", hash = "sha256:a871e7a6f7cba1ddb02503ea974f15f6524c95078fbfe0b860fd4193d7c8f27a", size = 23828, upload-time = "2025-08-14T08:59:46.129Z" }, +] + +[[package]] +name = "certifi" +version = "2025.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, +] + +[[package]] +name = "cfgv" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114, upload-time = "2023-08-12T20:38:17.776Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" }, +] + +[[package]] +name = "chardet" +version = "5.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, + { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, + { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, + { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +] + +[[package]] +name = "cleo" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "crashtest" }, + { name = "rapidfuzz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/30/f7960ed7041b158301c46774f87620352d50a9028d111b4211187af13783/cleo-2.1.0.tar.gz", hash = "sha256:0b2c880b5d13660a7ea651001fb4acb527696c01f15c9ee650f377aa543fd523", size = 79957, upload-time = "2023-10-30T18:54:12.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/f5/6bbead8b880620e5a99e0e4bb9e22e67cca16ff48d54105302a3e7821096/cleo-2.1.0-py3-none-any.whl", hash = "sha256:4a31bd4dd45695a64ee3c4758f583f134267c2bc518d8ae9a29cf237d009b07e", size = 78711, upload-time = "2023-10-30T18:54:08.557Z" }, +] + +[[package]] +name = "click" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, +] + +[[package]] +name = "clr-loader" +version = "0.2.7.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/b3/8ae917e458394e2cebdbf17bed0a8204f8d4ffc79a093a7b1141c7731d3c/clr_loader-0.2.7.post0.tar.gz", hash = "sha256:b7a8b3f8fbb1bcbbb6382d887e21d1742d4f10b5ea209e4ad95568fe97e1c7c6", size = 56701, upload-time = "2024-12-12T20:15:15.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/c0/06e64a54bced4e8b885c1e7ec03ee1869e52acf69e87da40f92391a214ad/clr_loader-0.2.7.post0-py3-none-any.whl", hash = "sha256:e0b9fcc107d48347a4311a28ffe3ae78c4968edb216ffb6564cb03f7ace0bb47", size = 50649, upload-time = "2024-12-12T20:15:13.714Z" }, +] + +[[package]] +name = "codetiming" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/4e/c40bf151af20ba2748bd6ea24e484d7b6196b1056ba3a1a4ee33b6939c37/codetiming-1.4.0.tar.gz", hash = "sha256:4937bf913a2814258b87eaaa43d9a1bb24711ffd3557a9ab6934fa1fe3ba0dbc", size = 14899, upload-time = "2022-11-08T07:12:29.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/91/e4a2b7c64e738beefddfa24b409d6eecb16c378bde01578918b6ea722a09/codetiming-1.4.0-py3-none-any.whl", hash = "sha256:3b80f409bef00941a9755c5524071ce2f72eaa4520f4bc35b33869cde024ccbd", size = 7165, upload-time = "2022-11-08T07:12:23.553Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "comm" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, +] + +[[package]] +name = "compressed-tensors" +version = "0.9.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "pydantic", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "transformers", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/3e/f74c5dcca6552e15a00df4a78c6e4a8776a7c901acc5a8c1dd371698ef54/compressed_tensors-0.9.3.tar.gz", hash = "sha256:5bdc7774a6c217496cba7d6a4fca6ffac943e68adae0481ead6d036660c1b340", size = 66354, upload-time = "2025-04-02T17:05:52.263Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/87/9c7eb4b57f89a51a65bee166cc079cd1bc1b398823da4f3b3c12f1021af8/compressed_tensors-0.9.3-py3-none-any.whl", hash = "sha256:5fcc3e4e7aa828036c2aeb130a610f9745a2e4890692cad6f6b5a2f960b21cc1", size = 98449, upload-time = "2025-04-02T17:05:49.642Z" }, +] + +[[package]] +name = "compressed-tensors" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "pydantic", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "transformers", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/86/d43d369abc81ec63ec7b8f6f27fc8b113ea0fd18a4116ae12063387b8b34/compressed_tensors-0.10.2.tar.gz", hash = "sha256:6de13ac535d7ffdd8890fad3d229444c33076170acaa8fab6bab8ecfa96c1d8f", size = 173459, upload-time = "2025-06-23T13:19:06.135Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ac/56bb4b6b3150783119479e2f05e32ebfc39ca6ff8e6fcd45eb178743b39e/compressed_tensors-0.10.2-py3-none-any.whl", hash = "sha256:e1b4d9bc2006e3fd3a938e59085f318fdb280c5af64688a4792bf1bc263e579d", size = 169030, upload-time = "2025-06-23T13:19:03.487Z" }, +] + +[[package]] +name = "compressed-tensors" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "frozendict", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pydantic", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "transformers", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/99/3fdabfc95609d6efdf02fa7f1ed0245524cb1209d3d4a17109d3205d2eed/compressed_tensors-0.11.0.tar.gz", hash = "sha256:95ddf19699f775df6494dd864e5f52e8a24f8015496520190c1a22c6cfc44b1f", size = 187566, upload-time = "2025-08-19T18:59:31.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/81/e3073017a8f5c75169e79108eda209e6089e3f96c9f197d307cbda7df71c/compressed_tensors-0.11.0-py3-none-any.whl", hash = "sha256:e1cbc46e1ae032b7ceea915fe18c8d2de5a54d3a50a607969b6bdfe703b6cb83", size = 179951, upload-time = "2025-08-19T18:59:29.308Z" }, +] + +[[package]] +name = "contourpy" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb", size = 293419, upload-time = "2025-07-26T12:01:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6", size = 273979, upload-time = "2025-07-26T12:01:22.448Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69", size = 185018, upload-time = "2025-07-26T12:01:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b", size = 226567, upload-time = "2025-07-26T12:01:36.804Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc", size = 193655, upload-time = "2025-07-26T12:01:37.999Z" }, +] + +[[package]] +name = "crashtest" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/5d/d79f51058e75948d6c9e7a3d679080a47be61c84d3cc8f71ee31255eb22b/crashtest-0.4.1.tar.gz", hash = "sha256:80d7b1f316ebfbd429f648076d6275c877ba30ba48979de4191714a75266f0ce", size = 4708, upload-time = "2022-11-02T21:15:13.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/5c/3ba7d12e7a79566f97b8f954400926d7b6eb33bcdccc1315a857f200f1f1/crashtest-0.4.1-py3-none-any.whl", hash = "sha256:8d23eac5fa660409f57472e3851dab7ac18aba459a8d19cbbba86d3d5aecd2a5", size = 7558, upload-time = "2022-11-02T21:15:12.437Z" }, +] + +[[package]] +name = "cryptography" +version = "46.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/62/e3664e6ffd7743e1694b244dde70b43a394f6f7fbcacf7014a8ff5197c73/cryptography-46.0.1.tar.gz", hash = "sha256:ed570874e88f213437f5cf758f9ef26cbfc3f336d889b1e592ee11283bb8d1c7", size = 749198, upload-time = "2025-09-17T00:10:35.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/8c/44ee01267ec01e26e43ebfdae3f120ec2312aa72fa4c0507ebe41a26739f/cryptography-46.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:1cd6d50c1a8b79af1a6f703709d8973845f677c8e97b1268f5ff323d38ce8475", size = 7285044, upload-time = "2025-09-17T00:08:36.807Z" }, + { url = "https://files.pythonhosted.org/packages/22/59/9ae689a25047e0601adfcb159ec4f83c0b4149fdb5c3030cc94cd218141d/cryptography-46.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ff483716be32690c14636e54a1f6e2e1b7bf8e22ca50b989f88fa1b2d287080", size = 4308182, upload-time = "2025-09-17T00:08:39.388Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ee/ca6cc9df7118f2fcd142c76b1da0f14340d77518c05b1ebfbbabca6b9e7d/cryptography-46.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9873bf7c1f2a6330bdfe8621e7ce64b725784f9f0c3a6a55c3047af5849f920e", size = 4572393, upload-time = "2025-09-17T00:08:41.663Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a3/0f5296f63815d8e985922b05c31f77ce44787b3127a67c0b7f70f115c45f/cryptography-46.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0dfb7c88d4462a0cfdd0d87a3c245a7bc3feb59de101f6ff88194f740f72eda6", size = 4308400, upload-time = "2025-09-17T00:08:43.559Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/74fcda3e4e01be1d32775d5b4dd841acaac3c1b8fa4d0774c7ac8d52463d/cryptography-46.0.1-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e22801b61613ebdebf7deb18b507919e107547a1d39a3b57f5f855032dd7cfb8", size = 4015786, upload-time = "2025-09-17T00:08:45.758Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b8/85d23287baeef273b0834481a3dd55bbed3a53587e3b8d9f0898235b8f91/cryptography-46.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:757af4f6341ce7a1e47c326ca2a81f41d236070217e5fbbad61bbfe299d55d28", size = 4982606, upload-time = "2025-09-17T00:08:47.602Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d3/de61ad5b52433b389afca0bc70f02a7a1f074651221f599ce368da0fe437/cryptography-46.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f7a24ea78de345cfa7f6a8d3bde8b242c7fac27f2bd78fa23474ca38dfaeeab9", size = 4604234, upload-time = "2025-09-17T00:08:49.879Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1f/dbd4d6570d84748439237a7478d124ee0134bf166ad129267b7ed8ea6d22/cryptography-46.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e8776dac9e660c22241b6587fae51a67b4b0147daa4d176b172c3ff768ad736", size = 4307669, upload-time = "2025-09-17T00:08:52.321Z" }, + { url = "https://files.pythonhosted.org/packages/ec/fd/ca0a14ce7f0bfe92fa727aacaf2217eb25eb7e4ed513b14d8e03b26e63ed/cryptography-46.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9f40642a140c0c8649987027867242b801486865277cbabc8c6059ddef16dc8b", size = 4947579, upload-time = "2025-09-17T00:08:54.697Z" }, + { url = "https://files.pythonhosted.org/packages/89/6b/09c30543bb93401f6f88fce556b3bdbb21e55ae14912c04b7bf355f5f96c/cryptography-46.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:449ef2b321bec7d97ef2c944173275ebdab78f3abdd005400cc409e27cd159ab", size = 4603669, upload-time = "2025-09-17T00:08:57.16Z" }, + { url = "https://files.pythonhosted.org/packages/23/9a/38cb01cb09ce0adceda9fc627c9cf98eb890fc8d50cacbe79b011df20f8a/cryptography-46.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2dd339ba3345b908fa3141ddba4025568fa6fd398eabce3ef72a29ac2d73ad75", size = 4435828, upload-time = "2025-09-17T00:08:59.606Z" }, + { url = "https://files.pythonhosted.org/packages/0f/53/435b5c36a78d06ae0bef96d666209b0ecd8f8181bfe4dda46536705df59e/cryptography-46.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7411c910fb2a412053cf33cfad0153ee20d27e256c6c3f14d7d7d1d9fec59fd5", size = 4709553, upload-time = "2025-09-17T00:09:01.832Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c4/0da6e55595d9b9cd3b6eb5dc22f3a07ded7f116a3ea72629cab595abb804/cryptography-46.0.1-cp311-abi3-win32.whl", hash = "sha256:cbb8e769d4cac884bb28e3ff620ef1001b75588a5c83c9c9f1fdc9afbe7f29b0", size = 3058327, upload-time = "2025-09-17T00:09:03.726Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/cd29a35e0d6e78a0ee61793564c8cff0929c38391cb0de27627bdc7525aa/cryptography-46.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:92e8cfe8bd7dd86eac0a677499894862cd5cc2fd74de917daa881d00871ac8e7", size = 3523893, upload-time = "2025-09-17T00:09:06.272Z" }, + { url = "https://files.pythonhosted.org/packages/f2/dd/eea390f3e78432bc3d2f53952375f8b37cb4d37783e626faa6a51e751719/cryptography-46.0.1-cp311-abi3-win_arm64.whl", hash = "sha256:db5597a4c7353b2e5fb05a8e6cb74b56a4658a2b7bf3cb6b1821ae7e7fd6eaa0", size = 2932145, upload-time = "2025-09-17T00:09:08.568Z" }, + { url = "https://files.pythonhosted.org/packages/98/e5/fbd632385542a3311915976f88e0dfcf09e62a3fc0aff86fb6762162a24d/cryptography-46.0.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:d84c40bdb8674c29fa192373498b6cb1e84f882889d21a471b45d1f868d8d44b", size = 7255677, upload-time = "2025-09-17T00:09:42.407Z" }, + { url = "https://files.pythonhosted.org/packages/56/3e/13ce6eab9ad6eba1b15a7bd476f005a4c1b3f299f4c2f32b22408b0edccf/cryptography-46.0.1-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9ed64e5083fa806709e74fc5ea067dfef9090e5b7a2320a49be3c9df3583a2d8", size = 4301110, upload-time = "2025-09-17T00:09:45.614Z" }, + { url = "https://files.pythonhosted.org/packages/a2/67/65dc233c1ddd688073cf7b136b06ff4b84bf517ba5529607c9d79720fc67/cryptography-46.0.1-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:341fb7a26bc9d6093c1b124b9f13acc283d2d51da440b98b55ab3f79f2522ead", size = 4562369, upload-time = "2025-09-17T00:09:47.601Z" }, + { url = "https://files.pythonhosted.org/packages/17/db/d64ae4c6f4e98c3dac5bf35dd4d103f4c7c345703e43560113e5e8e31b2b/cryptography-46.0.1-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6ef1488967e729948d424d09c94753d0167ce59afba8d0f6c07a22b629c557b2", size = 4302126, upload-time = "2025-09-17T00:09:49.335Z" }, + { url = "https://files.pythonhosted.org/packages/3d/19/5f1eea17d4805ebdc2e685b7b02800c4f63f3dd46cfa8d4c18373fea46c8/cryptography-46.0.1-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7823bc7cdf0b747ecfb096d004cc41573c2f5c7e3a29861603a2871b43d3ef32", size = 4009431, upload-time = "2025-09-17T00:09:51.239Z" }, + { url = "https://files.pythonhosted.org/packages/81/b5/229ba6088fe7abccbfe4c5edb96c7a5ad547fac5fdd0d40aa6ea540b2985/cryptography-46.0.1-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:f736ab8036796f5a119ff8211deda416f8c15ce03776db704a7a4e17381cb2ef", size = 4980739, upload-time = "2025-09-17T00:09:54.181Z" }, + { url = "https://files.pythonhosted.org/packages/3a/9c/50aa38907b201e74bc43c572f9603fa82b58e831bd13c245613a23cff736/cryptography-46.0.1-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e46710a240a41d594953012213ea8ca398cd2448fbc5d0f1be8160b5511104a0", size = 4592289, upload-time = "2025-09-17T00:09:56.731Z" }, + { url = "https://files.pythonhosted.org/packages/5a/33/229858f8a5bb22f82468bb285e9f4c44a31978d5f5830bb4ea1cf8a4e454/cryptography-46.0.1-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:84ef1f145de5aee82ea2447224dc23f065ff4cc5791bb3b506615957a6ba8128", size = 4301815, upload-time = "2025-09-17T00:09:58.548Z" }, + { url = "https://files.pythonhosted.org/packages/52/cb/b76b2c87fbd6ed4a231884bea3ce073406ba8e2dae9defad910d33cbf408/cryptography-46.0.1-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9394c7d5a7565ac5f7d9ba38b2617448eba384d7b107b262d63890079fad77ca", size = 4943251, upload-time = "2025-09-17T00:10:00.475Z" }, + { url = "https://files.pythonhosted.org/packages/94/0f/f66125ecf88e4cb5b8017ff43f3a87ede2d064cb54a1c5893f9da9d65093/cryptography-46.0.1-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:ed957044e368ed295257ae3d212b95456bd9756df490e1ac4538857f67531fcc", size = 4591247, upload-time = "2025-09-17T00:10:02.874Z" }, + { url = "https://files.pythonhosted.org/packages/f6/22/9f3134ae436b63b463cfdf0ff506a0570da6873adb4bf8c19b8a5b4bac64/cryptography-46.0.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f7de12fa0eee6234de9a9ce0ffcfa6ce97361db7a50b09b65c63ac58e5f22fc7", size = 4428534, upload-time = "2025-09-17T00:10:04.994Z" }, + { url = "https://files.pythonhosted.org/packages/89/39/e6042bcb2638650b0005c752c38ea830cbfbcbb1830e4d64d530000aa8dc/cryptography-46.0.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7fab1187b6c6b2f11a326f33b036f7168f5b996aedd0c059f9738915e4e8f53a", size = 4699541, upload-time = "2025-09-17T00:10:06.925Z" }, + { url = "https://files.pythonhosted.org/packages/68/46/753d457492d15458c7b5a653fc9a84a1c9c7a83af6ebdc94c3fc373ca6e8/cryptography-46.0.1-cp38-abi3-win32.whl", hash = "sha256:45f790934ac1018adeba46a0f7289b2b8fe76ba774a88c7f1922213a56c98bc1", size = 3043779, upload-time = "2025-09-17T00:10:08.951Z" }, + { url = "https://files.pythonhosted.org/packages/2f/50/b6f3b540c2f6ee712feeb5fa780bb11fad76634e71334718568e7695cb55/cryptography-46.0.1-cp38-abi3-win_amd64.whl", hash = "sha256:7176a5ab56fac98d706921f6416a05e5aff7df0e4b91516f450f8627cda22af3", size = 3517226, upload-time = "2025-09-17T00:10:10.769Z" }, + { url = "https://files.pythonhosted.org/packages/ff/e8/77d17d00981cdd27cc493e81e1749a0b8bbfb843780dbd841e30d7f50743/cryptography-46.0.1-cp38-abi3-win_arm64.whl", hash = "sha256:efc9e51c3e595267ff84adf56e9b357db89ab2279d7e375ffcaf8f678606f3d9", size = 2923149, upload-time = "2025-09-17T00:10:13.236Z" }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, +] + +[[package]] +name = "dataclasses-json" +version = "0.6.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marshmallow" }, + { name = "typing-inspect" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227, upload-time = "2024-06-09T16:20:19.103Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, +] + +[[package]] +name = "datasets" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"] }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/9d/348ed92110ba5f9b70b51ca1078d4809767a835aa2b7ce7e74ad2b98323d/datasets-4.0.0.tar.gz", hash = "sha256:9657e7140a9050db13443ba21cb5de185af8af944479b00e7ff1e00a61c8dbf1", size = 569566, upload-time = "2025-07-09T14:35:52.431Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/62/eb8157afb21bd229c864521c1ab4fa8e9b4f1b06bafdd8c4668a7a31b5dd/datasets-4.0.0-py3-none-any.whl", hash = "sha256:7ef95e62025fd122882dbce6cb904c8cd3fbc829de6669a5eb939c77d50e203d", size = 494825, upload-time = "2025-07-09T14:35:50.658Z" }, +] + +[[package]] +name = "daytona-api-client" +version = "0.20.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/95/7eeb159500dfb11a281957049b2dda5279d11eaee1e5ccd0c5af032973be/daytona_api_client-0.20.1.tar.gz", hash = "sha256:ff2061f7e7dc9c935a9087216600be277cb9cf6b8c1eecdfe333ef30d6b208fd", size = 74452, upload-time = "2025-06-04T10:05:40.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/4c/bbe0bbe26bdb4edbdce09807202ab2281109efd15534912486e441bf7dff/daytona_api_client-0.20.1-py3-none-any.whl", hash = "sha256:4d5023108013365eba76bd0bd4704f30dee54c13e2ac5b62e8c88bcd4af5db92", size = 202997, upload-time = "2025-06-04T10:05:39.723Z" }, +] + +[[package]] +name = "daytona-sdk" +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "daytona-api-client" }, + { name = "deprecated" }, + { name = "environs" }, + { name = "httpx" }, + { name = "marshmallow" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/68/b1b693024cb968b85328c94c991454f702d99a95d3cf84aba0676927c185/daytona_sdk-0.16.1.tar.gz", hash = "sha256:97fa381e8363a74ce99bbdd62508709c9083f657d3816e7f24c0349b739784b2", size = 42781, upload-time = "2025-05-06T17:08:07.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/39/71fefdaf28c1c10a3804bb0cd3ab774b0b0786efa029b1403f3208c1d879/daytona_sdk-0.16.1-py3-none-any.whl", hash = "sha256:f3ab5c92688d1b767bb12c3ef38c56bdea9e77b2be7e7dd57df785a14489a425", size = 47749, upload-time = "2025-05-06T17:08:05.769Z" }, +] + +[[package]] +name = "debugpy" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/61/fe/0486b90b9ac0d9afced236fdfe6e54c2f45b7ef09225210090f23dc6e48a/debugpy-1.8.0.zip", hash = "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0", size = 4824504, upload-time = "2023-09-12T20:32:17.726Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/59/b9c5dbe60c3a3e4e4bb12b1968aa75e98b99ff8b65273a5e7de75cceef75/debugpy-1.8.0-py2.py3-none-any.whl", hash = "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4", size = 5036334, upload-time = "2023-09-12T20:33:17.263Z" }, +] + +[[package]] +name = "debugpy" +version = "1.8.17" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform == 'darwin' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +sdist = { url = "https://files.pythonhosted.org/packages/15/ad/71e708ff4ca377c4230530d6a7aa7992592648c122a2cd2b321cf8b35a76/debugpy-1.8.17.tar.gz", hash = "sha256:fd723b47a8c08892b1a16b2c6239a8b96637c62a59b94bb5dab4bac592a58a8e", size = 1644129, upload-time = "2025-09-17T16:33:20.633Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/2b/9d8e65beb2751876c82e1aceb32f328c43ec872711fa80257c7674f45650/debugpy-1.8.17-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:f14467edef672195c6f6b8e27ce5005313cb5d03c9239059bc7182b60c176e2d", size = 2549522, upload-time = "2025-09-17T16:33:38.466Z" }, + { url = "https://files.pythonhosted.org/packages/b4/78/eb0d77f02971c05fca0eb7465b18058ba84bd957062f5eec82f941ac792a/debugpy-1.8.17-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:24693179ef9dfa20dca8605905a42b392be56d410c333af82f1c5dff807a64cc", size = 4309417, upload-time = "2025-09-17T16:33:41.299Z" }, + { url = "https://files.pythonhosted.org/packages/37/42/c40f1d8cc1fed1e75ea54298a382395b8b937d923fcf41ab0797a554f555/debugpy-1.8.17-cp312-cp312-win32.whl", hash = "sha256:6a4e9dacf2cbb60d2514ff7b04b4534b0139facbf2abdffe0639ddb6088e59cf", size = 5277130, upload-time = "2025-09-17T16:33:43.554Z" }, + { url = "https://files.pythonhosted.org/packages/72/22/84263b205baad32b81b36eac076de0cdbe09fe2d0637f5b32243dc7c925b/debugpy-1.8.17-cp312-cp312-win_amd64.whl", hash = "sha256:e8f8f61c518952fb15f74a302e068b48d9c4691768ade433e4adeea961993464", size = 5319053, upload-time = "2025-09-17T16:33:53.033Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d0/89247ec250369fc76db477720a26b2fce7ba079ff1380e4ab4529d2fe233/debugpy-1.8.17-py2.py3-none-any.whl", hash = "sha256:60c7dca6571efe660ccb7a9508d73ca14b8796c4ed484c2002abba714226cfef", size = 5283210, upload-time = "2025-09-17T16:34:25.835Z" }, +] + +[[package]] +name = "decorator" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, +] + +[[package]] +name = "deprecated" +version = "1.2.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, +] + +[[package]] +name = "depyf" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "astor", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "dill", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/ee/43a4cbba615abfc1eb2e5ff5eed3f80f38d58645b4d13d0ea06b9ca1909d/depyf-0.18.0.tar.gz", hash = "sha256:b99f0c383be949ae45d5d606fe444c71f375b55a57b8d6b20e7856670d52130d", size = 43050, upload-time = "2024-12-07T00:42:40.198Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d8/efc291d5c69a9905515055d23977643dd0d482ebfeb0dbabef1947ee75d8/depyf-0.18.0-py3-none-any.whl", hash = "sha256:007294d5bac19a38a0767d747be0f49b9ffdcea0394a822644142df22b33a3e1", size = 38839, upload-time = "2024-12-07T00:42:38.83Z" }, +] + +[[package]] +name = "depyf" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +dependencies = [ + { name = "astor" }, + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/38/69157d711be575f1b9cf3177b64ef4ade44373fc02839f183fdd98ec2dd6/depyf-0.19.0.tar.gz", hash = "sha256:afed0916b32d141cc90fa6220df01885eda442ca43b297d5050eeb90b4a5cb44", size = 6171405, upload-time = "2025-04-20T08:07:41.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/4d/1192acbcdc5e843f5e5d51f6e8788f2b60a9fe0b578ac385ded67a0b0b26/depyf-0.19.0-py3-none-any.whl", hash = "sha256:040b35fc0997d49df024b7d094f2a7836f91e9ed02f49982dd37e70aa3285ad5", size = 39034, upload-time = "2025-04-20T08:07:37.036Z" }, +] + +[[package]] +name = "dill" +version = "0.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/17/4d/ac7ffa80c69ea1df30a8aa11b3578692a5118e7cd1aa157e3ef73b092d15/dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", size = 184847, upload-time = "2024-01-27T23:42:16.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, +] + +[[package]] +name = "dirhash" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "scantree" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/1f/c8bf92552b7f0a13b9f12b85e3de8df6d9814240e0f8ce8f37433df028b3/dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09", size = 13119, upload-time = "2024-08-03T22:14:11.688Z" }, +] + +[[package]] +name = "diskcache" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + +[[package]] +name = "docker" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, +] + +[[package]] +name = "docstring-parser" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/9d/c3b43da9515bd270df0f80548d9944e389870713cc1fe2b8fb35fe2bcefd/docstring_parser-0.17.0.tar.gz", hash = "sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912", size = 27442, upload-time = "2025-07-21T07:35:01.868Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/e2/2537ebcff11c1ee1ff17d8d0b6f4db75873e3b0fb32c2d4a2ee31ecb310a/docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708", size = 36896, upload-time = "2025-07-21T07:35:00.684Z" }, +] + +[[package]] +name = "duckdb" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/93/adc0d183642fc9a602ca9b97cb16754c84b8c1d92e5b99aec412e0c419a8/duckdb-1.4.0.tar.gz", hash = "sha256:bd5edee8bd5a73b5822f2b390668597b5fcdc2d3292c244d8d933bb87ad6ac4c", size = 18453175, upload-time = "2025-09-16T10:22:41.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/6d/0c774d6af1aed82dbe855d266cb000a1c09ea31ed7d6c3a79e2167a38e7a/duckdb-1.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:18b3a048fca6cc7bafe08b10e1b0ab1509d7a0381ffb2c70359e7dc56d8a705d", size = 31307425, upload-time = "2025-09-16T10:21:57.83Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c0/1fd7b7b2c0c53d8d748d2f28ea9096df5ee9dc39fa736cca68acabe69656/duckdb-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c1271cb85aeacccfd0b1284e816280a7450df1dd4dd85ccb2848563cfdf90e9", size = 17295727, upload-time = "2025-09-16T10:22:02.242Z" }, + { url = "https://files.pythonhosted.org/packages/98/d3/4d4c4bd667b7ada5f6c207c2f127591ebb8468333f207f8f10ff0532578e/duckdb-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55064dd2e25711eeaa6a72c25405bdd7994c81a3221657e94309a2faf65d25a6", size = 14826879, upload-time = "2025-09-16T10:22:05.162Z" }, + { url = "https://files.pythonhosted.org/packages/b0/48/e0c1b97d76fb7567c53db5739931323238fad54a642707008104f501db37/duckdb-1.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0536d7c81bc506532daccf373ddbc8c6add46aeb70ef3cd5ee70ad5c2b3165ea", size = 18417856, upload-time = "2025-09-16T10:22:07.919Z" }, + { url = "https://files.pythonhosted.org/packages/12/78/297b838f3b9511589badc8f472f70b31cf3bbf9eb99fa0a4d6e911d3114a/duckdb-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:784554e3ddfcfc5c5c7b1aa1f9925fedb7938f6628729adba48f7ea37554598f", size = 20427154, upload-time = "2025-09-16T10:22:10.216Z" }, + { url = "https://files.pythonhosted.org/packages/ea/57/500d251b886494f6c52d56eeab8a1860572ee62aed05d7d50c71ba2320f3/duckdb-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:c5d2aa4d6981f525ada95e6db41bb929403632bb5ff24bd6d6dd551662b1b613", size = 12290108, upload-time = "2025-09-16T10:22:12.668Z" }, +] + +[[package]] +name = "dulwich" +version = "0.24.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/d0/b1275f9609e0d32800daf72786e5bec5602e71dde8db4995d0420d6b5cec/dulwich-0.24.2.tar.gz", hash = "sha256:d474844cf81bf95a6537a80aeec59d714d5d77d8e83d6d37991e2bde54746ca7", size = 883284, upload-time = "2025-09-26T09:11:38.978Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/66/77aab91732fcec662e69560850588b1323e3d992dc89ad788dff36ff40d2/dulwich-0.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dd1d9afdf511c167d30dd0f8fef2a0d5e548211644c8cef006e25a531fc0173d", size = 1131262, upload-time = "2025-09-26T09:11:05.13Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c5/47067a051b6dc3b98b71954ca0aeb07018ae547daf2febfbf5f9b4d036b2/dulwich-0.24.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e918cdd05d03933334542eb7649066ff36f818ca90a9dc2027e1ea28192b74b4", size = 1215635, upload-time = "2025-09-26T09:11:06.868Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5a/64a46133ead333a1cecfec749c05d0c64d2d4d3905b4a91f6b49a8d5c722/dulwich-0.24.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:be1cb36f6bf4b335f17b70cabedad3aa3d097364febeaa8531947781612c2a6b", size = 1241567, upload-time = "2025-09-26T09:11:08.698Z" }, + { url = "https://files.pythonhosted.org/packages/14/d1/38d9b62307f7c97793b3128ce9e141eba437f920c14499eb60158cec6e37/dulwich-0.24.2-cp312-cp312-win32.whl", hash = "sha256:fdb3fa61f3b98e6900aa57f73d2c4dd37822c66bb3e61ae3424f620c6353ee76", size = 816769, upload-time = "2025-09-26T09:11:10.723Z" }, + { url = "https://files.pythonhosted.org/packages/6b/4c/62102098777250ad5b6212351a38e1d4b550b68fb486608a16940690e685/dulwich-0.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:f7992e193d7cb5df1a043e41f0edfc9929c845f68cf7fdf09673d680ffa594ab", size = 833899, upload-time = "2025-09-26T09:11:12.699Z" }, + { url = "https://files.pythonhosted.org/packages/ec/42/58ce26920e32f17c39541fe0b8dd7b758bbf59063ff9bacb2496201c006b/dulwich-0.24.2-py3-none-any.whl", hash = "sha256:07426d209e870bd869d6eefb03573cf4879e62b30f690c53cc525b602ea413e5", size = 495365, upload-time = "2025-09-26T09:11:37.402Z" }, +] + +[[package]] +name = "e2b" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "httpcore" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/cc/811bf0d20aefdecb72905068107e33728c353e3deedbb6daeae355a40504/e2b-1.3.3.tar.gz", hash = "sha256:b0afb2e3b8edaade44a50d40dbf8934cbd9e2bc3c1c214ce11dbfd9e1d6815b0", size = 51402, upload-time = "2025-04-03T23:12:08.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/5b/5826c80e4de1a99bdde3a10995bf56c7c7d779fb58476c93fa542a626f40/e2b-1.3.3-py3-none-any.whl", hash = "sha256:1f851345fd22abc1f16fb82197e88786909323edfd140a7a17b8a6ed4bd4ef03", size = 96021, upload-time = "2025-04-03T23:12:06.491Z" }, +] + +[[package]] +name = "einops" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/81/df4fbe24dff8ba3934af99044188e20a98ed441ad17a274539b74e82e126/einops-0.8.1.tar.gz", hash = "sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84", size = 54805, upload-time = "2025-02-09T03:17:00.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/62/9773de14fe6c45c23649e98b83231fffd7b9892b6cf863251dc2afa73643/einops-0.8.1-py3-none-any.whl", hash = "sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737", size = 64359, upload-time = "2025-02-09T03:17:01.998Z" }, +] + +[[package]] +name = "email-validator" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, +] + +[[package]] +name = "english-words" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/95/12d90f2a3af18305b13d7c09401b59bd235db4cfb63d76ddf594748c2de8/english-words-2.0.2.tar.gz", hash = "sha256:e51d05607fe92ebf65047a2359e977c78a46aa16d1d92a3cba0bac3e34957d8e", size = 8152008, upload-time = "2025-08-14T21:56:27.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/6f/42700113073ca6775fb12a260112f0cb36449a2c2e42dcd53d3fcc6970b4/english_words-2.0.2-py3-none-any.whl", hash = "sha256:a65b900b17201526d8ab2d414f45e5a37afc714a1857377b30182c754ae6e0d6", size = 8196368, upload-time = "2025-08-14T21:56:24.865Z" }, +] + +[[package]] +name = "environs" +version = "9.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marshmallow" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/e3/c3c6c76f3dbe3e019e9a451b35bf9f44690026a5bb1232f7b77097b72ff5/environs-9.5.0.tar.gz", hash = "sha256:a76307b36fbe856bdca7ee9161e6c466fd7fcffc297109a118c59b54e27e30c9", size = 20795, upload-time = "2022-01-30T18:07:56.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/5e/f0f217dc393372681bfe05c50f06a212e78d0a3fee907a74ab451ec1dcdb/environs-9.5.0-py2.py3-none-any.whl", hash = "sha256:1e549569a3de49c05f856f40bce86979e7d5ffbbc4398e7f338574c220189124", size = 12548, upload-time = "2022-01-30T18:07:54.874Z" }, +] + +[[package]] +name = "eval-type-backport" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/ea/8b0ac4469d4c347c6a385ff09dc3c048c2d021696664e26c7ee6791631b5/eval_type_backport-0.2.2.tar.gz", hash = "sha256:f0576b4cf01ebb5bd358d02314d31846af5e07678387486e2c798af0e7d849c1", size = 9079, upload-time = "2024-12-21T20:09:46.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/31/55cd413eaccd39125368be33c46de24a1f639f2e12349b0361b4678f3915/eval_type_backport-0.2.2-py3-none-any.whl", hash = "sha256:cb6ad7c393517f476f96d456d0412ea80f0a8cf96f6892834cd9340149111b0a", size = 5830, upload-time = "2024-12-21T20:09:44.175Z" }, +] + +[[package]] +name = "evaluate" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "datasets" }, + { name = "dill" }, + { name = "fsspec", extra = ["http"] }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/d0/0c17a8e6e8dc7245f22dea860557c32bae50fc4d287ae030cb0e8ab8720f/evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21", size = 65716, upload-time = "2025-09-18T13:06:30.581Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/af/3e990d8d4002bbc9342adb4facd59506e653da93b2417de0fa6027cb86b1/evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29", size = 84069, upload-time = "2025-09-18T13:06:29.265Z" }, +] + +[[package]] +name = "executing" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, +] + +[[package]] +name = "faker" +version = "37.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/da/1336008d39e5d4076dddb4e0f3a52ada41429274bf558a3cc28030d324a3/faker-37.8.0.tar.gz", hash = "sha256:090bb5abbec2b30949a95ce1ba6b20d1d0ed222883d63483a0d4be4a970d6fb8", size = 1912113, upload-time = "2025-09-15T20:24:13.592Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/11/02ebebb09ff2104b690457cb7bc6ed700c9e0ce88cf581486bb0a5d3c88b/faker-37.8.0-py3-none-any.whl", hash = "sha256:b08233118824423b5fc239f7dd51f145e7018082b4164f8da6a9994e1f1ae793", size = 1953940, upload-time = "2025-09-15T20:24:11.482Z" }, +] + +[[package]] +name = "farama-notifications" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/2c/8384832b7a6b1fd6ba95bbdcae26e7137bb3eedc955c42fd5cdcc086cfbf/Farama-Notifications-0.0.4.tar.gz", hash = "sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18", size = 2131, upload-time = "2023-02-27T18:28:41.047Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/2c/ffc08c54c05cdce6fbed2aeebc46348dbe180c6d2c541c7af7ba0aa5f5f8/Farama_Notifications-0.0.4-py3-none-any.whl", hash = "sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae", size = 2511, upload-time = "2023-02-27T18:28:39.447Z" }, +] + +[[package]] +name = "fastapi" +version = "0.118.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/3c/2b9345a6504e4055eaa490e0b41c10e338ad61d9aeaae41d97807873cdf2/fastapi-0.118.0.tar.gz", hash = "sha256:5e81654d98c4d2f53790a7d32d25a7353b30c81441be7d0958a26b5d761fa1c8", size = 310536, upload-time = "2025-09-29T03:37:23.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/54e2bdaad22ca91a59455251998d43094d5c3d3567c52c7c04774b3f43f2/fastapi-0.118.0-py3-none-any.whl", hash = "sha256:705137a61e2ef71019d2445b123aa8845bd97273c395b744d5a7dfe559056855", size = 97694, upload-time = "2025-09-29T03:37:21.338Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "email-validator" }, + { name = "fastapi-cli", extra = ["standard"] }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "python-multipart" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[[package]] +name = "fastapi-cli" +version = "0.0.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rich-toolkit" }, + { name = "typer" }, + { name = "uvicorn", extra = ["standard"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/4e/3f61850012473b097fc5297d681bd85788e186fadb8555b67baf4c7707f4/fastapi_cli-0.0.13.tar.gz", hash = "sha256:312addf3f57ba7139457cf0d345c03e2170cc5a034057488259c33cd7e494529", size = 17780, upload-time = "2025-09-20T16:37:31.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/36/7432750f3638324b055496d2c952000bea824259fca70df5577a6a3c172f/fastapi_cli-0.0.13-py3-none-any.whl", hash = "sha256:219b73ccfde7622559cef1d43197da928516acb4f21f2ec69128c4b90057baba", size = 11142, upload-time = "2025-09-20T16:37:29.695Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "fastapi-cloud-cli" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[[package]] +name = "fastapi-cloud-cli" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "pydantic", extra = ["email"] }, + { name = "rich-toolkit" }, + { name = "rignore" }, + { name = "sentry-sdk" }, + { name = "typer" }, + { name = "uvicorn", extra = ["standard"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/38/1971f9dc8141e359d2435e6fae8bb228632adc55cff00cd00efed2a98456/fastapi_cloud_cli-0.2.1.tar.gz", hash = "sha256:aa22a4b867bf53165b6551d2f4eb21b079bad4aa74047cb889acf941e34699d9", size = 23676, upload-time = "2025-09-25T13:53:32.901Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/1f/5fa06afce6e4bb7fc7e54651236bad3b849340480967c54cbd7c13563c3f/fastapi_cloud_cli-0.2.1-py3-none-any.whl", hash = "sha256:245447bfb17b01ae5f7bc15dec0833bce85381ecf34532e8fa4bcf279ad1c361", size = 19894, upload-time = "2025-09-25T13:53:31.635Z" }, +] + +[[package]] +name = "fastcore" +version = "1.8.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/af/03bfd50e750139cb84c969ce628ad77ab06505d0834f791b090432f2486e/fastcore-1.8.12.tar.gz", hash = "sha256:c6febf2c689ea365d2edb708dcfce687dbc7e05437a5962c9187a9f170c82ff7", size = 81170, upload-time = "2025-09-29T00:56:34.855Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/56/202cddea9ea549be21867dbc1a98ead8660e61e943374a28339d02aa5b4a/fastcore-1.8.12-py3-none-any.whl", hash = "sha256:7b9bfb8c564f307313971bbb27a4542e945e48e8149361d8a5e4e14f7d436937", size = 84190, upload-time = "2025-09-29T00:56:33.086Z" }, +] + +[[package]] +name = "fastjsonschema" +version = "2.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, +] + +[[package]] +name = "fastuuid" +version = "0.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/80/3c16a1edad2e6cd82fbd15ac998cc1b881f478bf1f80ca717d941c441874/fastuuid-0.13.5.tar.gz", hash = "sha256:d4976821ab424d41542e1ea39bc828a9d454c3f8a04067c06fca123c5b95a1a1", size = 18255, upload-time = "2025-09-26T09:05:38.281Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/36/434f137c5970cac19e57834e1f7680e85301619d49891618c00666700c61/fastuuid-0.13.5-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:35fe8045e866bc6846f8de6fa05acb1de0c32478048484a995e96d31e21dff2a", size = 494638, upload-time = "2025-09-26T09:14:58.695Z" }, + { url = "https://files.pythonhosted.org/packages/ca/3c/083de2ac007b2b305523b9c006dba5051e5afd87a626ef1a39f76e2c6b82/fastuuid-0.13.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:02a460333f52d731a006d18a52ef6fcb2d295a1f5b1a5938d30744191b2f77b7", size = 253138, upload-time = "2025-09-26T09:13:33.283Z" }, + { url = "https://files.pythonhosted.org/packages/73/5e/630cffa1c8775db526e39e9e4c5c7db0c27be0786bb21ba82c912ae19f63/fastuuid-0.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:74b0e4f8c307b9f477a5d7284db4431ce53a3c1e3f4173db7a97db18564a6202", size = 244521, upload-time = "2025-09-26T09:14:40.682Z" }, + { url = "https://files.pythonhosted.org/packages/4d/51/55d78705f4fbdadf88fb40f382f508d6c7a4941ceddd7825fafebb4cc778/fastuuid-0.13.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6955a99ef455c2986f3851f4e0ccc35dec56ac1a7720f2b92e88a75d6684512e", size = 271557, upload-time = "2025-09-26T09:15:09.75Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2b/1b89e90a8635e5587ccdbbeb169c590672ce7637880f2c047482a0359950/fastuuid-0.13.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f10c77b826738c1a27dcdaa92ea4dc1ec9d869748a99e1fde54f1379553d4854", size = 272334, upload-time = "2025-09-26T09:07:48.865Z" }, + { url = "https://files.pythonhosted.org/packages/0c/06/4c8207894eeb30414999e5c3f66ac039bc4003437eb4060d8a1bceb4cc6f/fastuuid-0.13.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bb25dccbeb249d16d5e664f65f17ebec05136821d5ef462c4110e3f76b86fb86", size = 290594, upload-time = "2025-09-26T09:12:54.124Z" }, + { url = "https://files.pythonhosted.org/packages/50/69/96d221931a31d77a47cc2487bdfacfb3091edfc2e7a04b1795df1aec05df/fastuuid-0.13.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a5becc646a3eeafb76ce0a6783ba190cd182e3790a8b2c78ca9db2b5e87af952", size = 452835, upload-time = "2025-09-26T09:14:00.994Z" }, + { url = "https://files.pythonhosted.org/packages/25/ef/bf045f0a47dcec96247497ef3f7a31d86ebc074330e2dccc34b8dbc0468a/fastuuid-0.13.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:69b34363752d06e9bb0dbdf02ae391ec56ac948c6f2eb00be90dad68e80774b9", size = 468225, upload-time = "2025-09-26T09:13:38.585Z" }, + { url = "https://files.pythonhosted.org/packages/30/46/4817ab5a3778927155a4bde92540d4c4fa996161ec8b8e080c8928b0984e/fastuuid-0.13.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57d0768afcad0eab8770c9b8cf904716bd3c547e8b9a4e755ee8a673b060a3a3", size = 444907, upload-time = "2025-09-26T09:14:30.163Z" }, + { url = "https://files.pythonhosted.org/packages/80/27/ab284117ce4dc9b356a7196bdbf220510285f201d27f1f078592cdc8187b/fastuuid-0.13.5-cp312-cp312-win32.whl", hash = "sha256:8ac6c6f5129d52eaa6ef9ea4b6e2f7c69468a053f3ab8e439661186b9c06bb85", size = 145415, upload-time = "2025-09-26T09:08:59.494Z" }, + { url = "https://files.pythonhosted.org/packages/f4/0c/f970a4222773b248931819f8940800b760283216ca3dda173ed027e94bdd/fastuuid-0.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:ad630e97715beefef07ec37c9c162336e500400774e2c1cbe1a0df6f80d15b9a", size = 150840, upload-time = "2025-09-26T09:13:46.115Z" }, +] + +[[package]] +name = "filelock" +version = "3.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, +] + +[[package]] +name = "findpython" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/17/5a72566eecc9cbc1609459befe9f7dc65e101b66519a79999cc48044993c/findpython-0.7.0.tar.gz", hash = "sha256:8b31647c76352779a3c1a0806699b68e6a7bdc0b5c2ddd9af2a07a0d40c673dc", size = 18858, upload-time = "2025-07-02T09:22:24.977Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/e5/00c2aec929c3068eb6b18d4da4a13a12fd6de040a50c78bbc3c222aebdca/findpython-0.7.0-py3-none-any.whl", hash = "sha256:f53cfcc29536f5b83c962cf922bba8ff6d6a3c2a05fda6a45aa58a47d005d8fc", size = 21955, upload-time = "2025-07-02T09:22:23.753Z" }, +] + +[[package]] +name = "flake8" +version = "7.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mccabe" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/af/fbfe3c4b5a657d79e5c47a2827a362f9e1b763336a52f926126aa6dc7123/flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872", size = 48326, upload-time = "2025-06-20T19:31:35.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/56/13ab06b4f93ca7cac71078fbe37fcea175d3216f31f85c3168a6bbd0bb9a/flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e", size = 57922, upload-time = "2025-06-20T19:31:34.425Z" }, +] + +[[package]] +name = "flash-attn" +version = "2.7.4.post1" +source = { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "einops" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" } }, +] +wheels = [ + { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl", hash = "sha256:7e0b07913d56782d0f3f2ee76bd39587557742628983f6ab9ec2527c99437476" }, +] + +[package.metadata] +requires-dist = [ + { name = "einops" }, + { name = "torch" }, +] + +[[package]] +name = "flash-attn" +version = "2.8.0.post2" +source = { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.0.post2/flash_attn-2.8.0.post2+cu12torch2.7cxx11abiFALSE-cp312-cp312-linux_x86_64.whl" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "einops" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +wheels = [ + { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.0.post2/flash_attn-2.8.0.post2+cu12torch2.7cxx11abiFALSE-cp312-cp312-linux_x86_64.whl", hash = "sha256:5ebe77f75267d300d9f3d27cddccf6623fcef10f702c85b7414829d0eb7b9bf9" }, +] + +[package.metadata] +requires-dist = [ + { name = "einops" }, + { name = "torch" }, +] + +[[package]] +name = "flashinfer-python" +version = "0.2.6.post1+cu128torch2.7" +source = { url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" } +dependencies = [ + { name = "ninja" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "requests" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl", hash = "sha256:47b6db5e7bdcaff1f44d97166aa780627c9c244567c667b959359d862d3d5335" }, +] + +[package.metadata] +requires-dist = [ + { name = "ninja" }, + { name = "numpy" }, + { name = "requests" }, + { name = "torch" }, + { name = "torch", specifier = "==2.7.*" }, +] + +[[package]] +name = "flask" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "markupsafe" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/6d/cfe3c0fcc5e477df242b98bfe186a4c34357b4847e87ecaef04507332dab/flask-3.1.2.tar.gz", hash = "sha256:bf656c15c80190ed628ad08cdfd3aaa35beb087855e2f494910aa3774cc4fd87", size = 720160, upload-time = "2025-08-19T21:03:21.205Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c", size = 103308, upload-time = "2025-08-19T21:03:19.499Z" }, +] + +[[package]] +name = "fonttools" +version = "4.60.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/42/97a13e47a1e51a5a7142475bbcf5107fe3a68fc34aef331c897d5fb98ad0/fonttools-4.60.1.tar.gz", hash = "sha256:ef00af0439ebfee806b25f24c8f92109157ff3fac5731dc7867957812e87b8d9", size = 3559823, upload-time = "2025-09-29T21:13:27.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/f7/a10b101b7a6f8836a5adb47f2791f2075d044a6ca123f35985c42edc82d8/fonttools-4.60.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7b0c6d57ab00dae9529f3faf187f2254ea0aa1e04215cf2f1a8ec277c96661bc", size = 2832953, upload-time = "2025-09-29T21:11:39.616Z" }, + { url = "https://files.pythonhosted.org/packages/ed/fe/7bd094b59c926acf2304d2151354ddbeb74b94812f3dc943c231db09cb41/fonttools-4.60.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:839565cbf14645952d933853e8ade66a463684ed6ed6c9345d0faf1f0e868877", size = 2352706, upload-time = "2025-09-29T21:11:41.826Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ca/4bb48a26ed95a1e7eba175535fe5805887682140ee0a0d10a88e1de84208/fonttools-4.60.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8177ec9676ea6e1793c8a084a90b65a9f778771998eb919d05db6d4b1c0b114c", size = 4923716, upload-time = "2025-09-29T21:11:43.893Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9f/2cb82999f686c1d1ddf06f6ae1a9117a880adbec113611cc9d22b2fdd465/fonttools-4.60.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:996a4d1834524adbb423385d5a629b868ef9d774670856c63c9a0408a3063401", size = 4968175, upload-time = "2025-09-29T21:11:46.439Z" }, + { url = "https://files.pythonhosted.org/packages/18/79/be569699e37d166b78e6218f2cde8c550204f2505038cdd83b42edc469b9/fonttools-4.60.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a46b2f450bc79e06ef3b6394f0c68660529ed51692606ad7f953fc2e448bc903", size = 4911031, upload-time = "2025-09-29T21:11:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9f/89411cc116effaec5260ad519162f64f9c150e5522a27cbb05eb62d0c05b/fonttools-4.60.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6ec722ee589e89a89f5b7574f5c45604030aa6ae24cb2c751e2707193b466fed", size = 5062966, upload-time = "2025-09-29T21:11:54.344Z" }, + { url = "https://files.pythonhosted.org/packages/62/a1/f888221934b5731d46cb9991c7a71f30cb1f97c0ef5fcf37f8da8fce6c8e/fonttools-4.60.1-cp312-cp312-win32.whl", hash = "sha256:b2cf105cee600d2de04ca3cfa1f74f1127f8455b71dbad02b9da6ec266e116d6", size = 2218750, upload-time = "2025-09-29T21:11:56.601Z" }, + { url = "https://files.pythonhosted.org/packages/88/8f/a55b5550cd33cd1028601df41acd057d4be20efa5c958f417b0c0613924d/fonttools-4.60.1-cp312-cp312-win_amd64.whl", hash = "sha256:992775c9fbe2cf794786fa0ffca7f09f564ba3499b8fe9f2f80bd7197db60383", size = 2267026, upload-time = "2025-09-29T21:11:58.852Z" }, + { url = "https://files.pythonhosted.org/packages/c7/93/0dd45cd283c32dea1545151d8c3637b4b8c53cdb3a625aeb2885b184d74d/fonttools-4.60.1-py3-none-any.whl", hash = "sha256:906306ac7afe2156fcf0042173d6ebbb05416af70f6b370967b47f8f00103bbb", size = 1143175, upload-time = "2025-09-29T21:13:24.134Z" }, +] + +[[package]] +name = "frozendict" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/59/19eb300ba28e7547538bdf603f1c6c34793240a90e1a7b61b65d8517e35e/frozendict-2.4.6.tar.gz", hash = "sha256:df7cd16470fbd26fc4969a208efadc46319334eb97def1ddf48919b351192b8e", size = 316416, upload-time = "2024-10-13T12:15:32.449Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/13/d9839089b900fa7b479cce495d62110cddc4bd5630a04d8469916c0e79c5/frozendict-2.4.6-py311-none-any.whl", hash = "sha256:d065db6a44db2e2375c23eac816f1a022feb2fa98cbb50df44a9e83700accbea", size = 16148, upload-time = "2024-10-13T12:15:26.839Z" }, + { url = "https://files.pythonhosted.org/packages/ba/d0/d482c39cee2ab2978a892558cf130681d4574ea208e162da8958b31e9250/frozendict-2.4.6-py312-none-any.whl", hash = "sha256:49344abe90fb75f0f9fdefe6d4ef6d4894e640fadab71f11009d52ad97f370b9", size = 16146, upload-time = "2024-10-13T12:15:28.16Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" }, + { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" }, + { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" }, + { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" }, + { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" }, + { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" }, + { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" }, + { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" }, + { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" }, + { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" }, + { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" }, + { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" }, + { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" }, + { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" }, + { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, +] + +[[package]] +name = "fsspec" +version = "2025.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/f4/5721faf47b8c499e776bc34c6a8fc17efdf7fdef0b00f398128bc5dcb4ac/fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972", size = 298491, upload-time = "2025-03-07T21:47:56.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/53/eb690efa8513166adef3e0669afd31e95ffde69fb3c52ec2ac7223ed6018/fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3", size = 193615, upload-time = "2025-03-07T21:47:54.809Z" }, +] + +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, +] + +[[package]] +name = "func-timeout" +version = "4.3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/0d/bf0567477f7281d9a3926c582bfef21bff7498fc0ffd3e9de21811896a0b/func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd", size = 44264, upload-time = "2019-08-19T21:32:07.43Z" } + +[[package]] +name = "gdown" +version = "5.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "filelock" }, + { name = "requests", extra = ["socks"] }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/6a/37e6b70c5bda3161e40265861e63b64a86bfc6ca6a8f1c35328a675c84fd/gdown-5.2.0.tar.gz", hash = "sha256:2145165062d85520a3cd98b356c9ed522c5e7984d408535409fd46f94defc787", size = 284647, upload-time = "2024-05-12T06:45:12.725Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/70/e07c381e6488a77094f04c85c9caf1c8008cdc30778f7019bc52e5285ef0/gdown-5.2.0-py3-none-any.whl", hash = "sha256:33083832d82b1101bdd0e9df3edd0fbc0e1c5f14c9d8c38d2a35bf1683b526d6", size = 18235, upload-time = "2024-05-12T06:45:10.017Z" }, +] + +[[package]] +name = "gguf" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pyyaml" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/08/7de1ca4b71e7bf33b547f82bb22505e221b5fa42f67d635e200e0ad22ad6/gguf-0.17.1.tar.gz", hash = "sha256:36ad71aad900a3e75fc94ebe96ea6029f03a4e44be7627ef7ad3d03e8c7bcb53", size = 89338, upload-time = "2025-06-19T14:00:33.705Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/31/6a93a887617ee7deeaa602ca3d02d1c12a6cb8a742a695de5d128f5fa46a/gguf-0.17.1-py3-none-any.whl", hash = "sha256:7bc5aa7eeb1931f7d39b48fdc5b38fda6b294b9dca75cf607ac69557840a3943", size = 96224, upload-time = "2025-06-19T14:00:32.88Z" }, +] + +[[package]] +name = "ghapi" +version = "1.0.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastcore" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/45/714e944ca610758c5fd7cece1aea366fb88e318983d6a8e52857598b5b09/ghapi-1.0.8.tar.gz", hash = "sha256:9ab02dcd06b3c622ea2d9b21a2efee316076a744ce7847251a2fe9f542f381df", size = 72049, upload-time = "2025-09-16T23:57:48.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/cd/63129b50c44da8461f663f09f8e4801acaf0ff5bc8b201d75cee82d5b35c/ghapi-1.0.8-py3-none-any.whl", hash = "sha256:3e4023f475ec966995dd3feeacd3f42f9e296dd23148e6f28d15e80487300e66", size = 68569, upload-time = "2025-09-16T23:57:47.017Z" }, +] + +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, +] + +[[package]] +name = "google-ai-generativelanguage" +version = "0.6.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/d1/48fe5d7a43d278e9f6b5ada810b0a3530bbeac7ed7fcbcd366f932f05316/google_ai_generativelanguage-0.6.15.tar.gz", hash = "sha256:8f6d9dc4c12b065fe2d0289026171acea5183ebf2d0b11cefe12f3821e159ec3", size = 1375443, upload-time = "2025-01-13T21:50:47.459Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/a3/67b8a6ff5001a1d8864922f2d6488dc2a14367ceb651bc3f09a947f2f306/google_ai_generativelanguage-0.6.15-py3-none-any.whl", hash = "sha256:5a03ef86377aa184ffef3662ca28f19eeee158733e45d7947982eb953c6ebb6c", size = 1327356, upload-time = "2025-01-13T21:50:44.174Z" }, +] + +[[package]] +name = "google-api-core" +version = "2.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/21/e9d043e88222317afdbdb567165fdbc3b0aad90064c7e0c9eb0ad9955ad8/google_api_core-2.25.1.tar.gz", hash = "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8", size = 165443, upload-time = "2025-06-12T20:52:20.439Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/4b/ead00905132820b623732b175d66354e9d3e69fcf2a5dcdab780664e7896/google_api_core-2.25.1-py3-none-any.whl", hash = "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7", size = 160807, upload-time = "2025-06-12T20:52:19.334Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + +[[package]] +name = "google-api-python-client" +version = "2.183.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "httplib2" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/1f/49a2c83fc6dcd8b127cc9efbecf7d5fc36109c2028ba22ed6cb4d072fca4/google_api_python_client-2.183.0.tar.gz", hash = "sha256:abae37e04fecf719388e5c02f707ed9cdf952f10b217c79a3e76c636762e3ea9", size = 13645623, upload-time = "2025-09-23T22:27:00.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/06/1974f937172854bc7622eff5c2390f33542ceb843f305922922c8f5f7f17/google_api_python_client-2.183.0-py3-none-any.whl", hash = "sha256:2005b6e86c27be1db1a43f43e047a0f8e004159f3cceddecb08cf1624bddba31", size = 14214837, upload-time = "2025-09-23T22:26:57.758Z" }, +] + +[[package]] +name = "google-auth" +version = "2.41.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/c5/87742f5b5f055514c67f970f7174a876fccff2289a69d460b0614cc7ccfb/google_auth-2.41.0.tar.gz", hash = "sha256:c9d7b534ea4a5d9813c552846797fafb080312263cd4994d6622dd50992ae101", size = 292282, upload-time = "2025-09-29T21:36:35.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/ff/a1c426fc9bea7268230bf92340da7d112fae18cf946cafe13ab17d14e6ee/google_auth-2.41.0-py2.py3-none-any.whl", hash = "sha256:d8bed9b53ab63b7b0374656b8e1bef051f95bb14ecc0cf21ba49de7911d62e09", size = 221168, upload-time = "2025-09-29T21:36:33.925Z" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + +[[package]] +name = "google-auth-httplib2" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "httplib2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/be/217a598a818567b28e859ff087f347475c807a5649296fb5a817c58dacef/google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05", size = 10842, upload-time = "2023-12-12T17:40:30.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/8a/fe34d2f3f9470a27b01c9e76226965863f153d5fbe276f83608562e49c04/google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d", size = 9253, upload-time = "2023-12-12T17:40:13.055Z" }, +] + +[[package]] +name = "google-auth-oauthlib" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/87/e10bf24f7bcffc1421b84d6f9c3377c30ec305d082cd737ddaa6d8f77f7c/google_auth_oauthlib-1.2.2.tar.gz", hash = "sha256:11046fb8d3348b296302dd939ace8af0a724042e8029c1b872d87fabc9f41684", size = 20955, upload-time = "2025-04-22T16:40:29.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/84/40ee070be95771acd2f4418981edb834979424565c3eec3cd88b6aa09d24/google_auth_oauthlib-1.2.2-py3-none-any.whl", hash = "sha256:fd619506f4b3908b5df17b65f39ca8d66ea56986e5472eb5978fd8f3786f00a2", size = 19072, upload-time = "2025-04-22T16:40:28.174Z" }, +] + +[[package]] +name = "google-cloud-aiplatform" +version = "1.117.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docstring-parser" }, + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "google-cloud-bigquery" }, + { name = "google-cloud-resource-manager" }, + { name = "google-cloud-storage" }, + { name = "google-genai" }, + { name = "packaging" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "shapely" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/73/7ac970c14b6118eb1b29189bfb07f8ee20d0d75ffa2a44d3acf82dcae286/google_cloud_aiplatform-1.117.0.tar.gz", hash = "sha256:93d397b85559d136d2cdd4b569defce593e11dabd307779272480f015efe6378", size = 9659839, upload-time = "2025-09-25T21:11:26.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/a5/61f3615aecd5bdb502f4b095b102695da2f70ed7061b643c56c6f6e3482f/google_cloud_aiplatform-1.117.0-py2.py3-none-any.whl", hash = "sha256:3d7a5346bcceaa9b8fac35a7cc3e180ce89900a15f3c8e91404e80ccb913ac7f", size = 8038334, upload-time = "2025-09-25T21:11:22.973Z" }, +] + +[[package]] +name = "google-cloud-bigquery" +version = "3.38.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-resumable-media" }, + { name = "packaging" }, + { name = "python-dateutil" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/b2/a17e40afcf9487e3d17db5e36728ffe75c8d5671c46f419d7b6528a5728a/google_cloud_bigquery-3.38.0.tar.gz", hash = "sha256:8afcb7116f5eac849097a344eb8bfda78b7cfaae128e60e019193dd483873520", size = 503666, upload-time = "2025-09-17T20:33:33.47Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/3c/c8cada9ec282b29232ed9aed5a0b5cca6cf5367cb2ffa8ad0d2583d743f1/google_cloud_bigquery-3.38.0-py3-none-any.whl", hash = "sha256:e06e93ff7b245b239945ef59cb59616057598d369edac457ebf292bd61984da6", size = 259257, upload-time = "2025-09-17T20:33:31.404Z" }, +] + +[[package]] +name = "google-cloud-core" +version = "2.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/b8/2b53838d2acd6ec6168fd284a990c76695e84c65deee79c9f3a4276f6b4f/google_cloud_core-2.4.3.tar.gz", hash = "sha256:1fab62d7102844b278fe6dead3af32408b1df3eb06f5c7e8634cbd40edc4da53", size = 35861, upload-time = "2025-03-10T21:05:38.948Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/86/bda7241a8da2d28a754aad2ba0f6776e35b67e37c36ae0c45d49370f1014/google_cloud_core-2.4.3-py2.py3-none-any.whl", hash = "sha256:5130f9f4c14b4fafdff75c79448f9495cfade0d8775facf1b09c3bf67e027f6e", size = 29348, upload-time = "2025-03-10T21:05:37.785Z" }, +] + +[[package]] +name = "google-cloud-resource-manager" +version = "1.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "grpc-google-iam-v1" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/ca/a4648f5038cb94af4b3942815942a03aa9398f9fb0bef55b3f1585b9940d/google_cloud_resource_manager-1.14.2.tar.gz", hash = "sha256:962e2d904c550d7bac48372607904ff7bb3277e3bb4a36d80cc9a37e28e6eb74", size = 446370, upload-time = "2025-03-17T11:35:56.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/ea/a92631c358da377af34d3a9682c97af83185c2d66363d5939ab4a1169a7f/google_cloud_resource_manager-1.14.2-py3-none-any.whl", hash = "sha256:d0fa954dedd1d2b8e13feae9099c01b8aac515b648e612834f9942d2795a9900", size = 394344, upload-time = "2025-03-17T11:35:54.722Z" }, +] + +[[package]] +name = "google-cloud-storage" +version = "2.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-crc32c" }, + { name = "google-resumable-media" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/76/4d965702e96bb67976e755bed9828fa50306dca003dbee08b67f41dd265e/google_cloud_storage-2.19.0.tar.gz", hash = "sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2", size = 5535488, upload-time = "2024-12-05T01:35:06.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/94/6db383d8ee1adf45dc6c73477152b82731fa4c4a46d9c1932cc8757e0fd4/google_cloud_storage-2.19.0-py2.py3-none-any.whl", hash = "sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba", size = 131787, upload-time = "2024-12-05T01:35:04.736Z" }, +] + +[[package]] +name = "google-crc32c" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/ae/87802e6d9f9d69adfaedfcfd599266bf386a54d0be058b532d04c794f76d/google_crc32c-1.7.1.tar.gz", hash = "sha256:2bff2305f98846f3e825dbeec9ee406f89da7962accdb29356e4eadc251bd472", size = 14495, upload-time = "2025-03-26T14:29:13.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/b7/787e2453cf8639c94b3d06c9d61f512234a82e1d12d13d18584bd3049904/google_crc32c-1.7.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2d73a68a653c57281401871dd4aeebbb6af3191dcac751a76ce430df4d403194", size = 30470, upload-time = "2025-03-26T14:34:31.655Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b4/6042c2b0cbac3ec3a69bb4c49b28d2f517b7a0f4a0232603c42c58e22b44/google_crc32c-1.7.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:22beacf83baaf59f9d3ab2bbb4db0fb018da8e5aebdce07ef9f09fce8220285e", size = 30315, upload-time = "2025-03-26T15:01:54.634Z" }, + { url = "https://files.pythonhosted.org/packages/29/ad/01e7a61a5d059bc57b702d9ff6a18b2585ad97f720bd0a0dbe215df1ab0e/google_crc32c-1.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19eafa0e4af11b0a4eb3974483d55d2d77ad1911e6cf6f832e1574f6781fd337", size = 33180, upload-time = "2025-03-26T14:41:32.168Z" }, + { url = "https://files.pythonhosted.org/packages/3b/a5/7279055cf004561894ed3a7bfdf5bf90a53f28fadd01af7cd166e88ddf16/google_crc32c-1.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d86616faaea68101195c6bdc40c494e4d76f41e07a37ffdef270879c15fb65", size = 32794, upload-time = "2025-03-26T14:41:33.264Z" }, + { url = "https://files.pythonhosted.org/packages/0f/d6/77060dbd140c624e42ae3ece3df53b9d811000729a5c821b9fd671ceaac6/google_crc32c-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:b7491bdc0c7564fcf48c0179d2048ab2f7c7ba36b84ccd3a3e1c3f7a72d3bba6", size = 33477, upload-time = "2025-03-26T14:29:10.94Z" }, +] + +[[package]] +name = "google-genai" +version = "1.39.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "google-auth" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/3e/25b88bda07ca237043f1be45d13c49ffbc73f9edf45d3232345802f67197/google_genai-1.39.1.tar.gz", hash = "sha256:4721704b43d170fc3f1b1cb5494bee1a7f7aae20de3a5383cdf6a129139df80b", size = 244631, upload-time = "2025-09-26T20:56:19.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/c3/12c1f386184d2fcd694b73adeabc3714a5ed65c01cc183b4e3727a26b9d1/google_genai-1.39.1-py3-none-any.whl", hash = "sha256:6ca36c7e40db6fcba7049dfdd102c86da326804f34403bd7d90fa613a45e5a78", size = 244681, upload-time = "2025-09-26T20:56:17.527Z" }, +] + +[[package]] +name = "google-generativeai" +version = "0.8.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-ai-generativelanguage" }, + { name = "google-api-core" }, + { name = "google-api-python-client" }, + { name = "google-auth" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/40/c42ff9ded9f09ec9392879a8e6538a00b2dc185e834a3392917626255419/google_generativeai-0.8.5-py3-none-any.whl", hash = "sha256:22b420817fb263f8ed520b33285f45976d5b21e904da32b80d4fd20c055123a2", size = 155427, upload-time = "2025-04-17T00:40:00.67Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/5a/0efdc02665dca14e0837b62c8a1a93132c264bd02054a15abb2218afe0ae/google_resumable_media-2.7.2.tar.gz", hash = "sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0", size = 2163099, upload-time = "2024-08-07T22:20:38.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/35/b8d3baf8c46695858cb9d8835a53baa1eeb9906ddaf2f728a5f5b640fd1e/google_resumable_media-2.7.2-py2.py3-none-any.whl", hash = "sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa", size = 81251, upload-time = "2024-08-07T22:20:36.409Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, +] + +[[package]] +name = "greenlet" +version = "3.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/b8/704d753a5a45507a7aab61f18db9509302ed3d0a27ac7e0359ec2905b1a6/greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d", size = 188260, upload-time = "2025-08-07T13:24:33.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, + { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, + { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, + { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, + { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, +] + +[[package]] +name = "grep-ast" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pathspec" }, + { name = "tree-sitter-languages" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/31/728334e2f053b4a6c4cd2912b3e40218f9e12e158c2039a458814dd22e1f/grep_ast-0.3.3.tar.gz", hash = "sha256:42b8887d57301dc55634368f8d549e9c49c913dafb4d19c9b54c3ddb604fccf4", size = 11140, upload-time = "2024-07-30T15:36:25.493Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/9d/3db2dff3abe7706ba4c71fcb4f60ac5f8049a476c56392a8f6942de6c3fc/grep_ast-0.3.3-py3-none-any.whl", hash = "sha256:515cb889bffefefa26c4ab1377b9a75b3fc678aa5fa02bf9aa4f8f20999a83ad", size = 11833, upload-time = "2024-07-30T15:36:23.864Z" }, +] + +[[package]] +name = "grpc-google-iam-v1" +version = "0.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos", extra = ["grpc"] }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/4e/8d0ca3b035e41fe0b3f31ebbb638356af720335e5a11154c330169b40777/grpc_google_iam_v1-0.14.2.tar.gz", hash = "sha256:b3e1fc387a1a329e41672197d0ace9de22c78dd7d215048c4c78712073f7bd20", size = 16259, upload-time = "2025-03-17T11:40:23.586Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/6f/dd9b178aee7835b96c2e63715aba6516a9d50f6bebbd1cc1d32c82a2a6c3/grpc_google_iam_v1-0.14.2-py3-none-any.whl", hash = "sha256:a3171468459770907926d56a440b2bb643eec1d7ba215f48f3ecece42b4d8351", size = 19242, upload-time = "2025-03-17T11:40:22.648Z" }, +] + +[[package]] +name = "grpcio" +version = "1.75.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/f7/8963848164c7604efb3a3e6ee457fdb3a469653e19002bd24742473254f8/grpcio-1.75.1.tar.gz", hash = "sha256:3e81d89ece99b9ace23a6916880baca613c03a799925afb2857887efa8b1b3d2", size = 12731327, upload-time = "2025-09-26T09:03:36.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/81/42be79e73a50aaa20af66731c2defeb0e8c9008d9935a64dd8ea8e8c44eb/grpcio-1.75.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:7b888b33cd14085d86176b1628ad2fcbff94cfbbe7809465097aa0132e58b018", size = 5668314, upload-time = "2025-09-26T09:01:55.424Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/3686ed15822fedc58c22f82b3a7403d9faf38d7c33de46d4de6f06e49426/grpcio-1.75.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:8775036efe4ad2085975531d221535329f5dac99b6c2a854a995456098f99546", size = 11476125, upload-time = "2025-09-26T09:01:57.927Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/21c71d674f03345ab183c634ecd889d3330177e27baea8d5d247a89b6442/grpcio-1.75.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb658f703468d7fbb5dcc4037c65391b7dc34f808ac46ed9136c24fc5eeb041d", size = 6246335, upload-time = "2025-09-26T09:02:00.76Z" }, + { url = "https://files.pythonhosted.org/packages/fd/db/3beb661bc56a385ae4fa6b0e70f6b91ac99d47afb726fe76aaff87ebb116/grpcio-1.75.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4b7177a1cdb3c51b02b0c0a256b0a72fdab719600a693e0e9037949efffb200b", size = 6916309, upload-time = "2025-09-26T09:02:02.894Z" }, + { url = "https://files.pythonhosted.org/packages/1e/9c/eda9fe57f2b84343d44c1b66cf3831c973ba29b078b16a27d4587a1fdd47/grpcio-1.75.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4fa6ccc3ec2e68a04f7b883d354d7fea22a34c44ce535a2f0c0049cf626ddf", size = 6435419, upload-time = "2025-09-26T09:02:05.055Z" }, + { url = "https://files.pythonhosted.org/packages/c3/b8/090c98983e0a9d602e3f919a6e2d4e470a8b489452905f9a0fa472cac059/grpcio-1.75.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d86880ecaeb5b2f0a8afa63824de93adb8ebe4e49d0e51442532f4e08add7d6", size = 7064893, upload-time = "2025-09-26T09:02:07.275Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c0/6d53d4dbbd00f8bd81571f5478d8a95528b716e0eddb4217cc7cb45aae5f/grpcio-1.75.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a8041d2f9e8a742aeae96f4b047ee44e73619f4f9d24565e84d5446c623673b6", size = 8011922, upload-time = "2025-09-26T09:02:09.527Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7c/48455b2d0c5949678d6982c3e31ea4d89df4e16131b03f7d5c590811cbe9/grpcio-1.75.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3652516048bf4c314ce12be37423c79829f46efffb390ad64149a10c6071e8de", size = 7466181, upload-time = "2025-09-26T09:02:12.279Z" }, + { url = "https://files.pythonhosted.org/packages/fd/12/04a0e79081e3170b6124f8cba9b6275871276be06c156ef981033f691880/grpcio-1.75.1-cp312-cp312-win32.whl", hash = "sha256:44b62345d8403975513af88da2f3d5cc76f73ca538ba46596f92a127c2aea945", size = 3938543, upload-time = "2025-09-26T09:02:14.77Z" }, + { url = "https://files.pythonhosted.org/packages/5f/d7/11350d9d7fb5adc73d2b0ebf6ac1cc70135577701e607407fe6739a90021/grpcio-1.75.1-cp312-cp312-win_amd64.whl", hash = "sha256:b1e191c5c465fa777d4cafbaacf0c01e0d5278022082c0abbd2ee1d6454ed94d", size = 4641938, upload-time = "2025-09-26T09:02:16.927Z" }, +] + +[[package]] +name = "grpcio-status" +version = "1.62.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/d7/013ef01c5a1c2fd0932c27c904934162f69f41ca0f28396d3ffe4d386123/grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485", size = 13063, upload-time = "2024-08-06T00:37:08.003Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/40/972271de05f9315c0d69f9f7ebbcadd83bc85322f538637d11bb8c67803d/grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8", size = 14448, upload-time = "2024-08-06T00:30:15.702Z" }, +] + +[[package]] +name = "grpclib" +version = "0.4.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h2" }, + { name = "multidict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/b9/55936e462a5925190d7427e880b3033601d1effd13809b483d13a926061a/grpclib-0.4.7.tar.gz", hash = "sha256:2988ef57c02b22b7a2e8e961792c41ccf97efc2ace91ae7a5b0de03c363823c3", size = 61254, upload-time = "2023-12-24T17:47:41.572Z" } + +[[package]] +name = "gymnasium" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpickle" }, + { name = "farama-notifications" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/de/b923d09654df8f8ee29a3cc7ec7829ac057efd0d969cc3da0c8a7b219d59/gymnasium-1.2.1.tar.gz", hash = "sha256:4e6480273528523a90b3db99befb6111b13f15fa0866de88c4b675770495b66c", size = 827852, upload-time = "2025-09-23T08:22:39.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/73/85bc0412f15388e3068dc93331c858df6c8fc635b1e3cb30c7f7070ca481/gymnasium-1.2.1-py3-none-any.whl", hash = "sha256:85cd1c16351db0b89f73be54e952ddfece97b56d1e5400d2dcd59f58b7707963", size = 951141, upload-time = "2025-09-23T08:22:38.117Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "h2" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, +] + +[[package]] +name = "hf-transfer" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/eb/8fc64f40388c29ce8ce3b2b180a089d4d6b25b1d0d232d016704cb852104/hf_transfer-0.1.9.tar.gz", hash = "sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf", size = 25201, upload-time = "2025-01-07T10:05:12.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/f5/461d2e5f307e5048289b1168d5c642ae3bb2504e88dff1a38b92ed990a21/hf_transfer-0.1.9-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e66acf91df4a8b72f60223059df3003062a5ae111757187ed1a06750a30e911b", size = 1393046, upload-time = "2025-01-07T10:04:51.003Z" }, + { url = "https://files.pythonhosted.org/packages/41/ba/8d9fd9f1083525edfcb389c93738c802f3559cb749324090d7109c8bf4c2/hf_transfer-0.1.9-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:8669dbcc7a3e2e8d61d42cd24da9c50d57770bd74b445c65123291ca842a7e7a", size = 1348126, upload-time = "2025-01-07T10:04:45.712Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a2/cd7885bc9959421065a6fae0fe67b6c55becdeda4e69b873e52976f9a9f0/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fd0167c4407a3bc4cdd0307e65ada2294ec04f1813d8a69a5243e379b22e9d8", size = 3728604, upload-time = "2025-01-07T10:04:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/f6/2e/a072cf196edfeda3310c9a5ade0a0fdd785e6154b3ce24fc738c818da2a7/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f", size = 3064995, upload-time = "2025-01-07T10:04:18.663Z" }, + { url = "https://files.pythonhosted.org/packages/c2/84/aec9ef4c0fab93c1ea2b1badff38c78b4b2f86f0555b26d2051dbc920cde/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5828057e313de59300dd1abb489444bc452efe3f479d3c55b31a8f680936ba42", size = 3580908, upload-time = "2025-01-07T10:04:32.834Z" }, + { url = "https://files.pythonhosted.org/packages/29/63/b560d39651a56603d64f1a0212d0472a44cbd965db2fa62b99d99cb981bf/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d", size = 3400839, upload-time = "2025-01-07T10:04:26.122Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d8/f87ea6f42456254b48915970ed98e993110521e9263472840174d32c880d/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdca9bfb89e6f8f281890cc61a8aff2d3cecaff7e1a4d275574d96ca70098557", size = 3552664, upload-time = "2025-01-07T10:04:40.123Z" }, + { url = "https://files.pythonhosted.org/packages/d6/56/1267c39b65fc8f4e2113b36297320f102718bf5799b544a6cbe22013aa1d/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:89a23f58b7b7effbc047b8ca286f131b17728c99a9f972723323003ffd1bb916", size = 4073732, upload-time = "2025-01-07T10:04:55.624Z" }, + { url = "https://files.pythonhosted.org/packages/82/1a/9c748befbe3decf7cb415e34f8a0c3789a0a9c55910dea73d581e48c0ce5/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:dc7fff1345980d6c0ebb92c811d24afa4b98b3e07ed070c8e38cc91fd80478c5", size = 3390096, upload-time = "2025-01-07T10:04:59.98Z" }, + { url = "https://files.pythonhosted.org/packages/72/85/4c03da147b6b4b7cb12e074d3d44eee28604a387ed0eaf7eaaead5069c57/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:1a6bd16c667ebe89a069ca163060127a794fa3a3525292c900b8c8cc47985b0d", size = 3664743, upload-time = "2025-01-07T10:05:05.416Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6e/e597b04f753f1b09e6893075d53a82a30c13855cbaa791402695b01e369f/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d2fde99d502093ade3ab1b53f80da18480e9902aa960dab7f74fb1b9e5bc5746", size = 3695243, upload-time = "2025-01-07T10:05:11.411Z" }, + { url = "https://files.pythonhosted.org/packages/09/89/d4e234727a26b2546c8fb70a276cd924260d60135f2165bf8b9ed67bb9a4/hf_transfer-0.1.9-cp38-abi3-win32.whl", hash = "sha256:435cc3cdc8524ce57b074032b8fd76eed70a4224d2091232fa6a8cef8fd6803e", size = 1086605, upload-time = "2025-01-07T10:05:18.873Z" }, + { url = "https://files.pythonhosted.org/packages/a1/14/f1e15b851d1c2af5b0b1a82bf8eb10bda2da62d98180220ba6fd8879bb5b/hf_transfer-0.1.9-cp38-abi3-win_amd64.whl", hash = "sha256:16f208fc678911c37e11aa7b586bc66a37d02e636208f18b6bc53d29b5df40ad", size = 1160240, upload-time = "2025-01-07T10:05:14.324Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.1.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/31/feeddfce1748c4a233ec1aa5b7396161c07ae1aa9b7bdbc9a72c3c7dd768/hf_xet-1.1.10.tar.gz", hash = "sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97", size = 487910, upload-time = "2025-09-12T20:10:27.12Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/a2/343e6d05de96908366bdc0081f2d8607d61200be2ac802769c4284cc65bd/hf_xet-1.1.10-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d", size = 2761466, upload-time = "2025-09-12T20:10:22.836Z" }, + { url = "https://files.pythonhosted.org/packages/31/f9/6215f948ac8f17566ee27af6430ea72045e0418ce757260248b483f4183b/hf_xet-1.1.10-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b", size = 2623807, upload-time = "2025-09-12T20:10:21.118Z" }, + { url = "https://files.pythonhosted.org/packages/15/07/86397573efefff941e100367bbda0b21496ffcdb34db7ab51912994c32a2/hf_xet-1.1.10-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435", size = 3186960, upload-time = "2025-09-12T20:10:19.336Z" }, + { url = "https://files.pythonhosted.org/packages/01/a7/0b2e242b918cc30e1f91980f3c4b026ff2eedaf1e2ad96933bca164b2869/hf_xet-1.1.10-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c", size = 3087167, upload-time = "2025-09-12T20:10:17.255Z" }, + { url = "https://files.pythonhosted.org/packages/4a/25/3e32ab61cc7145b11eee9d745988e2f0f4fafda81b25980eebf97d8cff15/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06", size = 3248612, upload-time = "2025-09-12T20:10:24.093Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3d/ab7109e607ed321afaa690f557a9ada6d6d164ec852fd6bf9979665dc3d6/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f", size = 3353360, upload-time = "2025-09-12T20:10:25.563Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691, upload-time = "2025-09-12T20:10:28.433Z" }, +] + +[[package]] +name = "hpack" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, +] + +[[package]] +name = "html2text" +version = "2025.4.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/27/e158d86ba1e82967cc2f790b0cb02030d4a8bef58e0c79a8590e9678107f/html2text-2025.4.15.tar.gz", hash = "sha256:948a645f8f0bc3abe7fd587019a2197a12436cd73d0d4908af95bfc8da337588", size = 64316, upload-time = "2025-04-15T04:02:30.045Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/84/1a0f9555fd5f2b1c924ff932d99b40a0f8a6b12f6dd625e2a47f415b00ea/html2text-2025.4.15-py3-none-any.whl", hash = "sha256:00569167ffdab3d7767a4cdf589b7f57e777a5ed28d12907d8c58769ec734acc", size = 34656, upload-time = "2025-04-15T04:02:28.44Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httplib2" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/77/6653db69c1f7ecfe5e3f9726fdadc981794656fcd7d98c4209fecfea9993/httplib2-0.31.0.tar.gz", hash = "sha256:ac7ab497c50975147d4f7b1ade44becc7df2f8954d42b38b3d69c515f531135c", size = 250759, upload-time = "2025-09-11T12:16:03.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/a2/0d269db0f6163be503775dc8b6a6fa15820cc9fdc866f6ba608d86b721f2/httplib2-0.31.0-py3-none-any.whl", hash = "sha256:b9cd78abea9b4e43a7714c6e0f8b6b8561a6fc1e95d5dbd367f5bf0ef35f5d24", size = 91148, upload-time = "2025-09-11T12:16:01.803Z" }, +] + +[[package]] +name = "httptools" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/9a/ce5e1f7e131522e6d3426e8e7a490b3a01f39a6696602e1c4f33f9e94277/httptools-0.6.4.tar.gz", hash = "sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c", size = 240639, upload-time = "2024-10-16T19:45:08.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/0e/d0b71465c66b9185f90a091ab36389a7352985fe857e352801c39d6127c8/httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2", size = 200683, upload-time = "2024-10-16T19:44:30.175Z" }, + { url = "https://files.pythonhosted.org/packages/e2/b8/412a9bb28d0a8988de3296e01efa0bd62068b33856cdda47fe1b5e890954/httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44", size = 104337, upload-time = "2024-10-16T19:44:31.786Z" }, + { url = "https://files.pythonhosted.org/packages/9b/01/6fb20be3196ffdc8eeec4e653bc2a275eca7f36634c86302242c4fbb2760/httptools-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1", size = 508796, upload-time = "2024-10-16T19:44:32.825Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d8/b644c44acc1368938317d76ac991c9bba1166311880bcc0ac297cb9d6bd7/httptools-0.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2", size = 510837, upload-time = "2024-10-16T19:44:33.974Z" }, + { url = "https://files.pythonhosted.org/packages/52/d8/254d16a31d543073a0e57f1c329ca7378d8924e7e292eda72d0064987486/httptools-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81", size = 485289, upload-time = "2024-10-16T19:44:35.111Z" }, + { url = "https://files.pythonhosted.org/packages/5f/3c/4aee161b4b7a971660b8be71a92c24d6c64372c1ab3ae7f366b3680df20f/httptools-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f", size = 489779, upload-time = "2024-10-16T19:44:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/12/b7/5cae71a8868e555f3f67a50ee7f673ce36eac970f029c0c5e9d584352961/httptools-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970", size = 88634, upload-time = "2024-10-16T19:44:37.357Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "httpx-sse" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/fa/66bd985dd0b7c109a3bcb89272ee0bfb7e2b4d06309ad7b38ff866734b2a/httpx_sse-0.4.1.tar.gz", hash = "sha256:8f44d34414bc7b21bf3602713005c5df4917884f76072479b21f68befa4ea26e", size = 12998, upload-time = "2025-06-24T13:21:05.71Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/0a/6269e3473b09aed2dab8aa1a600c70f31f00ae1349bee30658f7e358a159/httpx_sse-0.4.1-py3-none-any.whl", hash = "sha256:cba42174344c3a5b06f255ce65b350880f962d99ead85e776f23c6618a377a37", size = 8054, upload-time = "2025-06-24T13:21:04.772Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "0.35.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/a0/651f93d154cb72323358bf2bbae3e642bdb5d2f1bfc874d096f7cb159fa0/huggingface_hub-0.35.3-py3-none-any.whl", hash = "sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba", size = 564262, upload-time = "2025-09-29T14:29:55.813Z" }, +] + +[package.optional-dependencies] +hf-xet = [ + { name = "hf-xet", marker = "extra == 'extra-8-skyagent-verl'" }, +] + +[[package]] +name = "hydra-core" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "omegaconf" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/8e/07e42bc434a847154083b315779b0a81d567154504624e181caf2c71cd98/hydra-core-1.3.2.tar.gz", hash = "sha256:8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824", size = 3263494, upload-time = "2023-02-23T18:33:43.03Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/50/e0edd38dcd63fb26a8547f13d28f7a008bc4a3fd4eb4ff030673f22ad41a/hydra_core-1.3.2-py3-none-any.whl", hash = "sha256:fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b", size = 154547, upload-time = "2023-02-23T18:33:40.801Z" }, +] + +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + +[[package]] +name = "identify" +version = "2.6.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/c4/62963f25a678f6a050fb0505a65e9e726996171e6dbe1547f79619eefb15/identify-2.6.14.tar.gz", hash = "sha256:663494103b4f717cb26921c52f8751363dc89db64364cd836a9bf1535f53cd6a", size = 99283, upload-time = "2025-09-06T19:30:52.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/ae/2ad30f4652712c82f1c23423d79136fbce338932ad166d70c1efb86a5998/identify-2.6.14-py2.py3-none-any.whl", hash = "sha256:11a073da82212c6646b1f39bb20d4483bfb9543bd5566fec60053c4bb309bf2e", size = 99172, upload-time = "2025-09-06T19:30:51.759Z" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "imageio" +version = "2.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/47/57e897fb7094afb2d26e8b2e4af9a45c7cf1a405acdeeca001fdf2c98501/imageio-2.37.0.tar.gz", hash = "sha256:71b57b3669666272c818497aebba2b4c5f20d5b37c81720e5e1a56d59c492996", size = 389963, upload-time = "2025-01-20T02:42:37.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/bd/b394387b598ed84d8d0fa90611a90bee0adc2021820ad5729f7ced74a8e2/imageio-2.37.0-py3-none-any.whl", hash = "sha256:11efa15b87bc7871b61590326b2d635439acc321cf7f8ce996f812543ce10eed", size = 315796, upload-time = "2025-01-20T02:42:34.931Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/ff/bd28f70283b9cca0cbf0c2a6082acbecd822d1962ae7b2a904861b9965f8/importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812", size = 52667, upload-time = "2024-06-25T18:38:04.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/ef/38766b2edb096260d9b1b6ad35adaa0bce3b0567abb452b21eb074af88c4/importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f", size = 24769, upload-time = "2024-06-25T18:38:02.324Z" }, +] + +[[package]] +name = "installer" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/18/ceeb4e3ab3aa54495775775b38ae42b10a92f42ce42dfa44da684289b8c8/installer-0.7.0.tar.gz", hash = "sha256:a26d3e3116289bb08216e0d0f7d925fcef0b0194eedfa0c944bcaaa106c4b631", size = 474349, upload-time = "2023-03-17T20:39:38.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", hash = "sha256:05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", size = 453838, upload-time = "2023-03-17T20:39:36.219Z" }, +] + +[[package]] +name = "interegular" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/9d/8b6dde58a028a3962ce17e84d5fe73758df61378e00ef8ac3d85da34b0ff/interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600", size = 24705, upload-time = "2024-01-06T23:01:22.372Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/01/72d6472f80651673716d1deda2a5bbb633e563ecf94f4479da5519d69d25/interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c", size = 23635, upload-time = "2024-01-06T23:01:20.829Z" }, +] + +[[package]] +name = "ipykernel" +version = "6.30.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "appnope", marker = "sys_platform == 'darwin' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "comm" }, + { name = "debugpy", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "debugpy", version = "1.8.17", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl' or extra != 'extra-8-skyagent-skyrl-train'" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/76/11082e338e0daadc89c8ff866185de11daf67d181901038f9e139d109761/ipykernel-6.30.1.tar.gz", hash = "sha256:6abb270161896402e76b91394fcdce5d1be5d45f456671e5080572f8505be39b", size = 166260, upload-time = "2025-08-04T15:47:35.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/c7/b445faca8deb954fe536abebff4ece5b097b923de482b26e78448c89d1dd/ipykernel-6.30.1-py3-none-any.whl", hash = "sha256:aa6b9fb93dca949069d8b85b6c79b2518e32ac583ae9c7d37c51d119e18b3fb4", size = 117484, upload-time = "2025-08-04T15:47:32.622Z" }, +] + +[[package]] +name = "ipython" +version = "9.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform == 'win32' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/34/29b18c62e39ee2f7a6a3bba7efd952729d8aadd45ca17efc34453b717665/ipython-9.6.0.tar.gz", hash = "sha256:5603d6d5d356378be5043e69441a072b50a5b33b4503428c77b04cb8ce7bc731", size = 4396932, upload-time = "2025-09-29T10:55:53.948Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/c5/d5e07995077e48220269c28a221e168c91123ad5ceee44d548f54a057fc0/ipython-9.6.0-py3-none-any.whl", hash = "sha256:5f77efafc886d2f023442479b8149e7d86547ad0a979e9da9f045d252f648196", size = 616170, upload-time = "2025-09-29T10:55:47.676Z" }, +] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, +] + +[[package]] +name = "ipywidgets" +version = "8.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm" }, + { name = "ipython" }, + { name = "jupyterlab-widgets" }, + { name = "traitlets" }, + { name = "widgetsnbextension" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/48/d3dbac45c2814cb73812f98dd6b38bbcc957a4e7bb31d6ea9c03bf94ed87/ipywidgets-8.1.7.tar.gz", hash = "sha256:15f1ac050b9ccbefd45dccfbb2ef6bed0029d8278682d569d71b8dd96bee0376", size = 116721, upload-time = "2025-05-05T12:42:03.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/6a/9166369a2f092bd286d24e6307de555d63616e8ddb373ebad2b5635ca4cd/ipywidgets-8.1.7-py3-none-any.whl", hash = "sha256:764f2602d25471c213919b8a1997df04bef869251db4ca8efba1b76b1bd9f7bb", size = 139806, upload-time = "2025-05-05T12:41:56.833Z" }, +] + +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, +] + +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/ad/f3777b81bf0b6e7bc7514a1656d3e637b2e8e15fab2ce3235730b3e7a4e6/jaraco_context-6.0.1.tar.gz", hash = "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3", size = 13912, upload-time = "2024-08-20T03:39:27.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/db/0c52c4cf5e4bd9f5d7135ec7669a3a767af21b3a308e1ed3674881e52b62/jaraco.context-6.0.1-py3-none-any.whl", hash = "sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4", size = 6825, upload-time = "2024-08-20T03:39:25.966Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/ed/1aa2d585304ec07262e1a83a9889880701079dde796ac7b1d1826f40c63d/jaraco_functools-4.3.0.tar.gz", hash = "sha256:cfd13ad0dd2c47a3600b439ef72d8615d482cedcff1632930d6f28924d92f294", size = 19755, upload-time = "2025-08-18T20:05:09.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/09/726f168acad366b11e420df31bf1c702a54d373a83f968d94141a8c3fde0/jaraco_functools-4.3.0-py3-none-any.whl", hash = "sha256:227ff8ed6f7b8f62c56deff101545fa7543cf2c8e7b82a7c2116e672f29c26e8", size = 10408, upload-time = "2025-08-18T20:05:08.69Z" }, +] + +[[package]] +name = "jaxtyping" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wadler-lindig" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/a8/416bd7ea110ec6b68e8868b0f99c985c735adcf7badc491d3c343937260a/jaxtyping-0.3.2.tar.gz", hash = "sha256:f30483fac4b42e915db8ad2414a85c3b63284aa7d3c100b96b59f755cf4a86ad", size = 44989, upload-time = "2025-04-23T09:48:16.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/b9/281e10e2d967ea5e481683eaec99f55ac5a61085ee60551c36942ef32bef/jaxtyping-0.3.2-py3-none-any.whl", hash = "sha256:6a020fd276226ddb5ac4f5725323843dd65e3c7e85c64fd62431e5f738c74e04", size = 55409, upload-time = "2025-04-23T09:48:15.924Z" }, +] + +[[package]] +name = "jedi" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jiter" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/c0/a3bb4cc13aced219dd18191ea66e874266bd8aa7b96744e495e1c733aa2d/jiter-0.11.0.tar.gz", hash = "sha256:1d9637eaf8c1d6a63d6562f2a6e5ab3af946c66037eb1b894e8fad75422266e4", size = 167094, upload-time = "2025-09-15T09:20:38.212Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/b5/3009b112b8f673e568ef79af9863d8309a15f0a8cdcc06ed6092051f377e/jiter-0.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:2fb7b377688cc3850bbe5c192a6bd493562a0bc50cbc8b047316428fbae00ada", size = 305510, upload-time = "2025-09-15T09:19:25.893Z" }, + { url = "https://files.pythonhosted.org/packages/fe/82/15514244e03b9e71e086bbe2a6de3e4616b48f07d5f834200c873956fb8c/jiter-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a1b7cbe3f25bd0d8abb468ba4302a5d45617ee61b2a7a638f63fee1dc086be99", size = 316521, upload-time = "2025-09-15T09:19:27.525Z" }, + { url = "https://files.pythonhosted.org/packages/92/94/7a2e905f40ad2d6d660e00b68d818f9e29fb87ffe82774f06191e93cbe4a/jiter-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0a7f0ec81d5b7588c5cade1eb1925b91436ae6726dc2df2348524aeabad5de6", size = 338214, upload-time = "2025-09-15T09:19:28.727Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9c/5791ed5bdc76f12110158d3316a7a3ec0b1413d018b41c5ed399549d3ad5/jiter-0.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07630bb46ea2a6b9c6ed986c6e17e35b26148cce2c535454b26ee3f0e8dcaba1", size = 361280, upload-time = "2025-09-15T09:19:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7f/b7d82d77ff0d2cb06424141000176b53a9e6b16a1125525bb51ea4990c2e/jiter-0.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7764f27d28cd4a9cbc61704dfcd80c903ce3aad106a37902d3270cd6673d17f4", size = 487895, upload-time = "2025-09-15T09:19:31.424Z" }, + { url = "https://files.pythonhosted.org/packages/42/44/10a1475d46f1fc1fd5cc2e82c58e7bca0ce5852208e0fa5df2f949353321/jiter-0.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d4a6c4a737d486f77f842aeb22807edecb4a9417e6700c7b981e16d34ba7c72", size = 378421, upload-time = "2025-09-15T09:19:32.746Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5f/0dc34563d8164d31d07bc09d141d3da08157a68dcd1f9b886fa4e917805b/jiter-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf408d2a0abd919b60de8c2e7bc5eeab72d4dafd18784152acc7c9adc3291591", size = 347932, upload-time = "2025-09-15T09:19:34.612Z" }, + { url = "https://files.pythonhosted.org/packages/f7/de/b68f32a4fcb7b4a682b37c73a0e5dae32180140cd1caf11aef6ad40ddbf2/jiter-0.11.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cdef53eda7d18e799625023e1e250dbc18fbc275153039b873ec74d7e8883e09", size = 386959, upload-time = "2025-09-15T09:19:35.994Z" }, + { url = "https://files.pythonhosted.org/packages/76/0a/c08c92e713b6e28972a846a81ce374883dac2f78ec6f39a0dad9f2339c3a/jiter-0.11.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:53933a38ef7b551dd9c7f1064f9d7bb235bb3168d0fa5f14f0798d1b7ea0d9c5", size = 517187, upload-time = "2025-09-15T09:19:37.426Z" }, + { url = "https://files.pythonhosted.org/packages/89/b5/4a283bec43b15aad54fcae18d951f06a2ec3f78db5708d3b59a48e9c3fbd/jiter-0.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11840d2324c9ab5162fc1abba23bc922124fedcff0d7b7f85fffa291e2f69206", size = 509461, upload-time = "2025-09-15T09:19:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/34/a5/f8bad793010534ea73c985caaeef8cc22dfb1fedb15220ecdf15c623c07a/jiter-0.11.0-cp312-cp312-win32.whl", hash = "sha256:4f01a744d24a5f2bb4a11657a1b27b61dc038ae2e674621a74020406e08f749b", size = 206664, upload-time = "2025-09-15T09:19:40.096Z" }, + { url = "https://files.pythonhosted.org/packages/ed/42/5823ec2b1469395a160b4bf5f14326b4a098f3b6898fbd327366789fa5d3/jiter-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:29fff31190ab3a26de026da2f187814f4b9c6695361e20a9ac2123e4d4378a4c", size = 203520, upload-time = "2025-09-15T09:19:41.798Z" }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/5d/447af5ea094b9e4c4054f82e223ada074c552335b9b4b2d14bd9b35a67c4/joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55", size = 331077, upload-time = "2025-08-27T12:15:46.575Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/e8/685f47e0d754320684db4425a0967f7d3fa70126bffd76110b7009a0090f/joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241", size = 308396, upload-time = "2025-08-27T12:15:45.188Z" }, +] + +[[package]] +name = "json-repair" +version = "0.51.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/3a/f30f3c92da3a285dcbe469c50b058f2d349dc9a20fc1b60c3219befda53f/json_repair-0.51.0.tar.gz", hash = "sha256:487e00042d5bc5cc4897ea9c3cccd4f6641e926b732cc09f98691a832485098a", size = 35289, upload-time = "2025-09-19T04:23:16.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/fc/eb15e39547b29dbf2b786bbbd1e79e7f1d87ec4e7c9ea61786f093181481/json_repair-0.51.0-py3-none-any.whl", hash = "sha256:871f7651ee82abf72efc50a80d3a9af0ade8abf5b4541b418eeeabe4e677e314", size = 26263, upload-time = "2025-09-19T04:23:15.064Z" }, +] + +[[package]] +name = "json5" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/12/ae/929aee9619e9eba9015207a9d2c1c54db18311da7eb4dcf6d41ad6f0eb67/json5-0.12.1.tar.gz", hash = "sha256:b2743e77b3242f8d03c143dd975a6ec7c52e2f2afe76ed934e53503dd4ad4990", size = 52191, upload-time = "2025-08-12T19:47:42.583Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/e2/05328bd2621be49a6fed9e3030b1e51a2d04537d3f816d211b9cc53c5262/json5-0.12.1-py3-none-any.whl", hash = "sha256:d9c9b3bc34a5f54d43c35e11ef7cb87d8bdd098c6ace87117a7b7e83e705c1d5", size = 36119, upload-time = "2025-08-12T19:47:41.131Z" }, +] + +[[package]] +name = "jsonlines" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/87/bcda8e46c88d0e34cad2f09ee2d0c7f5957bccdb9791b0b934ec84d84be4/jsonlines-4.0.0.tar.gz", hash = "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74", size = 11359, upload-time = "2023-09-01T12:34:44.187Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/62/d9ba6323b9202dd2fe166beab8a86d29465c41a0288cbe229fac60c1ab8d/jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55", size = 8701, upload-time = "2023-09-01T12:34:42.563Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/69/f7185de793a29082a9f3c7728268ffb31cb5095131a9c139a74078e27336/jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85", size = 357342, upload-time = "2025-08-18T17:03:50.038Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63", size = 90040, upload-time = "2025-08-18T17:03:48.373Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "jupyter-client" +version = "8.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/22/bf9f12fdaeae18019a468b68952a60fe6dbab5d67cd2a103cac7659b41ca/jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419", size = 342019, upload-time = "2024-09-17T10:44:17.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/85/b0394e0b6fcccd2c1eeefc230978a6f8cb0c5df1e4cd3e7625735a0d7d1e/jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f", size = 106105, upload-time = "2024-09-17T10:44:15.218Z" }, +] + +[[package]] +name = "jupyter-core" +version = "5.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "platformdirs" }, + { name = "pywin32", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (platform_python_implementation == 'PyPy' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'win32' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/1b/72906d554acfeb588332eaaa6f61577705e9ec752ddb486f302dafa292d9/jupyter_core-5.8.1.tar.gz", hash = "sha256:0a5f9706f70e64786b75acba995988915ebd4601c8a52e534a40b51c95f59941", size = 88923, upload-time = "2025-05-27T07:38:16.655Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/57/6bffd4b20b88da3800c5d691e0337761576ee688eb01299eae865689d2df/jupyter_core-5.8.1-py3-none-any.whl", hash = "sha256:c28d268fc90fb53f1338ded2eb410704c5449a358406e8a948b75706e24863d0", size = 28880, upload-time = "2025-05-27T07:38:15.137Z" }, +] + +[[package]] +name = "jupyterlab-widgets" +version = "3.0.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/7d/160595ca88ee87ac6ba95d82177d29ec60aaa63821d3077babb22ce031a5/jupyterlab_widgets-3.0.15.tar.gz", hash = "sha256:2920888a0c2922351a9202817957a68c07d99673504d6cd37345299e971bb08b", size = 213149, upload-time = "2025-05-05T12:32:31.004Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/6a/ca128561b22b60bd5a0c4ea26649e68c8556b82bc70a0c396eebc977fe86/jupyterlab_widgets-3.0.15-py3-none-any.whl", hash = "sha256:d59023d7d7ef71400d51e6fee9a88867f6e65e10a4201605d2d7f3e8f012a31c", size = 216571, upload-time = "2025-05-05T12:32:29.534Z" }, +] + +[[package]] +name = "keyring" +version = "25.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "secretstorage", marker = "sys_platform == 'linux' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/09/d904a6e96f76ff214be59e7aa6ef7190008f52a0ab6689760a98de0bf37d/keyring-25.6.0.tar.gz", hash = "sha256:0b39998aa941431eb3d9b0d4b2460bc773b9df6fed7621c2dfb291a7e0187a66", size = 62750, upload-time = "2024-12-25T15:26:45.782Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/32/da7f44bcb1105d3e88a0b74ebdca50c59121d2ddf71c9e34ba47df7f3a56/keyring-25.6.0-py3-none-any.whl", hash = "sha256:552a3f7af126ece7ed5c89753650eec89c7eaae8617d0aa4d9ad2b75111266bd", size = 39085, upload-time = "2024-12-25T15:26:44.377Z" }, +] + +[[package]] +name = "kiwisolver" +version = "1.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/3c/85844f1b0feb11ee581ac23fe5fce65cd049a200c1446708cc1b7f922875/kiwisolver-1.4.9.tar.gz", hash = "sha256:c3b22c26c6fd6811b0ae8363b95ca8ce4ea3c202d3d0975b2914310ceb1bcc4d", size = 97564, upload-time = "2025-08-10T21:27:49.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/c9/13573a747838aeb1c76e3267620daa054f4152444d1f3d1a2324b78255b5/kiwisolver-1.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ac5a486ac389dddcc5bef4f365b6ae3ffff2c433324fb38dd35e3fab7c957999", size = 123686, upload-time = "2025-08-10T21:26:10.034Z" }, + { url = "https://files.pythonhosted.org/packages/51/ea/2ecf727927f103ffd1739271ca19c424d0e65ea473fbaeea1c014aea93f6/kiwisolver-1.4.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2ba92255faa7309d06fe44c3a4a97efe1c8d640c2a79a5ef728b685762a6fd2", size = 66460, upload-time = "2025-08-10T21:26:11.083Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5a/51f5464373ce2aeb5194508298a508b6f21d3867f499556263c64c621914/kiwisolver-1.4.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a2899935e724dd1074cb568ce7ac0dce28b2cd6ab539c8e001a8578eb106d14", size = 64952, upload-time = "2025-08-10T21:26:12.058Z" }, + { url = "https://files.pythonhosted.org/packages/70/90/6d240beb0f24b74371762873e9b7f499f1e02166a2d9c5801f4dbf8fa12e/kiwisolver-1.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f6008a4919fdbc0b0097089f67a1eb55d950ed7e90ce2cc3e640abadd2757a04", size = 1474756, upload-time = "2025-08-10T21:26:13.096Z" }, + { url = "https://files.pythonhosted.org/packages/12/42/f36816eaf465220f683fb711efdd1bbf7a7005a2473d0e4ed421389bd26c/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67bb8b474b4181770f926f7b7d2f8c0248cbcb78b660fdd41a47054b28d2a752", size = 1276404, upload-time = "2025-08-10T21:26:14.457Z" }, + { url = "https://files.pythonhosted.org/packages/2e/64/bc2de94800adc830c476dce44e9b40fd0809cddeef1fde9fcf0f73da301f/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2327a4a30d3ee07d2fbe2e7933e8a37c591663b96ce42a00bc67461a87d7df77", size = 1294410, upload-time = "2025-08-10T21:26:15.73Z" }, + { url = "https://files.pythonhosted.org/packages/5f/42/2dc82330a70aa8e55b6d395b11018045e58d0bb00834502bf11509f79091/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a08b491ec91b1d5053ac177afe5290adacf1f0f6307d771ccac5de30592d198", size = 1343631, upload-time = "2025-08-10T21:26:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/22/fd/f4c67a6ed1aab149ec5a8a401c323cee7a1cbe364381bb6c9c0d564e0e20/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8fc5c867c22b828001b6a38d2eaeb88160bf5783c6cb4a5e440efc981ce286d", size = 2224963, upload-time = "2025-08-10T21:26:18.737Z" }, + { url = "https://files.pythonhosted.org/packages/45/aa/76720bd4cb3713314677d9ec94dcc21ced3f1baf4830adde5bb9b2430a5f/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3b3115b2581ea35bb6d1f24a4c90af37e5d9b49dcff267eeed14c3893c5b86ab", size = 2321295, upload-time = "2025-08-10T21:26:20.11Z" }, + { url = "https://files.pythonhosted.org/packages/80/19/d3ec0d9ab711242f56ae0dc2fc5d70e298bb4a1f9dfab44c027668c673a1/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858e4c22fb075920b96a291928cb7dea5644e94c0ee4fcd5af7e865655e4ccf2", size = 2487987, upload-time = "2025-08-10T21:26:21.49Z" }, + { url = "https://files.pythonhosted.org/packages/39/e9/61e4813b2c97e86b6fdbd4dd824bf72d28bcd8d4849b8084a357bc0dd64d/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ed0fecd28cc62c54b262e3736f8bb2512d8dcfdc2bcf08be5f47f96bf405b145", size = 2291817, upload-time = "2025-08-10T21:26:22.812Z" }, + { url = "https://files.pythonhosted.org/packages/a0/41/85d82b0291db7504da3c2defe35c9a8a5c9803a730f297bd823d11d5fb77/kiwisolver-1.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:f68208a520c3d86ea51acf688a3e3002615a7f0238002cccc17affecc86a8a54", size = 73895, upload-time = "2025-08-10T21:26:24.37Z" }, + { url = "https://files.pythonhosted.org/packages/e2/92/5f3068cf15ee5cb624a0c7596e67e2a0bb2adee33f71c379054a491d07da/kiwisolver-1.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:2c1a4f57df73965f3f14df20b80ee29e6a7930a57d2d9e8491a25f676e197c60", size = 64992, upload-time = "2025-08-10T21:26:25.732Z" }, +] + +[[package]] +name = "lark" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, +] + +[[package]] +name = "latex2sympy2-extended" +version = "1.10.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "sympy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "sympy", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/de/472f9115c14c6f6d8a5889cabe3418283d708bde62ce00402c29441deed4/latex2sympy2_extended-1.10.2.tar.gz", hash = "sha256:41a517ffcc5a140e910a7d1646ce6ff440817e5f9d48fc8279d88bd0925bc389", size = 206188, upload-time = "2025-07-02T15:26:06.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/60/dfbbf40e3a371388c0e03ff65b01319b7d4023e883df6d7261125772ffdc/latex2sympy2_extended-1.10.2-py3-none-any.whl", hash = "sha256:f910442c5b02a466c1046f47d05cc5285181068b882399281f30102715337fb7", size = 207855, upload-time = "2025-07-02T15:26:04.88Z" }, +] + +[[package]] +name = "lazy-loader" +version = "0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/6b/c875b30a1ba490860c93da4cabf479e03f584eba06fe5963f6f6644653d8/lazy_loader-0.4.tar.gz", hash = "sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1", size = 15431, upload-time = "2024-04-05T13:03:12.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/60/d497a310bde3f01cb805196ac61b7ad6dc5dcf8dce66634dc34364b20b4f/lazy_loader-0.4-py3-none-any.whl", hash = "sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc", size = 12097, upload-time = "2024-04-05T13:03:10.514Z" }, +] + +[[package]] +name = "libtmux" +version = "0.39.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/c4/e35564de320c876c6c370b8b9303a31ca2fcd632051aa8ac14a2d4700b68/libtmux-0.39.0.tar.gz", hash = "sha256:59346aeef3c0d6017f3bc5e23248d43cdf50f32b775b9cb5d9ff5e2e5f3059f4", size = 314810, upload-time = "2024-11-26T12:53:25.72Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/f8/afdcf3715c7388962c1eef39c7ba415a170f6772808ad83f60f8fdfdc6d0/libtmux-0.39.0-py3-none-any.whl", hash = "sha256:6b6e338be2727f67aa6b7eb67fa134368fa3c3eac5df27565396467692891c1e", size = 56219, upload-time = "2024-11-26T12:53:22.561Z" }, +] + +[[package]] +name = "libvisualwebarena" +version = "0.0.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiolimiter" }, + { name = "beartype" }, + { name = "evaluate" }, + { name = "flask" }, + { name = "gymnasium" }, + { name = "nltk" }, + { name = "openai" }, + { name = "pillow" }, + { name = "playwright" }, + { name = "scikit-image" }, + { name = "text-generation" }, + { name = "tiktoken" }, + { name = "transformers" }, + { name = "types-tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/ce/1ae6d0a6ba418863399c3e15fc235a9e235eb27c3fbd5b09b65d5bfe361c/libvisualwebarena-0.0.15.tar.gz", hash = "sha256:8be874353ad773e2878260d84fb5f9f11a3d142695c328009f6684bc2603c38b", size = 7545246, upload-time = "2024-11-14T17:39:50.808Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/de/321b533a7ec8538396addbc3206eef0ada4153208f783679e6091d94b2f7/libvisualwebarena-0.0.15-py3-none-any.whl", hash = "sha256:02c2f66b6a7ccb98e11c6decd8b00e8fc262223b002b80a13464e20bcdfe2999", size = 7551760, upload-time = "2024-11-14T17:39:48.459Z" }, +] + +[[package]] +name = "libwebarena" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiolimiter" }, + { name = "beartype" }, + { name = "evaluate" }, + { name = "flask" }, + { name = "gymnasium" }, + { name = "nltk" }, + { name = "openai" }, + { name = "pillow" }, + { name = "playwright" }, + { name = "text-generation" }, + { name = "tiktoken" }, + { name = "transformers" }, + { name = "types-tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/48/b3734dbe7058a48bda2065f5af99ac3d4d2cb0938ea5a4501b08e98e9317/libwebarena-0.0.4.tar.gz", hash = "sha256:78f76a415402cebf13b269b74b0a22c1888ae5513a2bde4b4dab781cb9445afd", size = 107046, upload-time = "2024-11-14T20:21:13.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/86/495d0d95cd5a1a16496c70f71171ee9a62fb09f2e31ae02443db0bd6aa5f/libwebarena-0.0.4-py3-none-any.whl", hash = "sha256:9ebee3b4371502c4f0f7e727a72e5846235d6750d420db9a3b8a168107654feb", size = 116253, upload-time = "2024-11-14T20:21:11.242Z" }, +] + +[[package]] +name = "liger-kernel" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" } }, + { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "triton", version = "3.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'linux' and extra == 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "triton", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/23/be0b4dcac42d77f99406c906567cde22a7a3d71b3f3ffdfda2ac6153ec36/liger_kernel-0.6.2.tar.gz", hash = "sha256:5c5bcffffa769bc26ae838f5a4954170dd5cacde036abb1b383039f39fa5fd69", size = 3679495, upload-time = "2025-08-22T00:15:28.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/2c/68d992835e8630c1b94cdcb246ea7eecad790a955037ca3f19b6c01e8215/liger_kernel-0.6.2-py3-none-any.whl", hash = "sha256:303b9bbf5c10f9289c3139afb41e4d989e8c809516624a106b89b064163d971d", size = 192815, upload-time = "2025-08-22T00:15:27.04Z" }, +] + +[[package]] +name = "litellm" +version = "1.77.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "click" }, + { name = "fastuuid" }, + { name = "httpx" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "openai" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "tiktoken" }, + { name = "tokenizers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/a3/85fc92d998ec9645c9fac108618681ef411ca4b338cc7544d6b3aad57699/litellm-1.77.5.tar.gz", hash = "sha256:8e8a83b49c4a6ae044b1a1c01adfbdef72b0031b86f1463dd743e267fa1d7b99", size = 10351819, upload-time = "2025-09-28T07:17:39.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/4c/89553f7e375ef39497d86f2266a0cdb37371a07e9e0aa8949f33c15a4198/litellm-1.77.5-py3-none-any.whl", hash = "sha256:07f53964c08d555621d4376cc42330458301ae889bfb6303155dcabc51095fbf", size = 9165458, upload-time = "2025-09-28T07:17:35.474Z" }, +] + +[[package]] +name = "llguidance" +version = "0.7.30" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/38/d1ef3ae08d8d857e5e0690c5b1e07bf7eb4a1cae5881d87215826dc6cadb/llguidance-0.7.30.tar.gz", hash = "sha256:e93bf75f2b6e48afb86a5cee23038746975e1654672bf5ba0ae75f7d4d4a2248", size = 1055528, upload-time = "2025-06-23T00:23:49.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/e1/694c89986fcae7777184fc8b22baa0976eba15a6847221763f6ad211fc1f/llguidance-0.7.30-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c80af02c118d2b0526bcecaab389af2ed094537a069b0fc724cd2a2f2ba3990f", size = 3327974, upload-time = "2025-06-23T00:23:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/ab7a548ae189dc23900fdd37803c115c2339b1223af9e8eb1f4329b5935a/llguidance-0.7.30-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:00a256d532911d2cf5ba4ef63e182944e767dd2402f38d63002016bc37755958", size = 3210709, upload-time = "2025-06-23T00:23:45.872Z" }, + { url = "https://files.pythonhosted.org/packages/9c/5b/6a166564b14f9f805f0ea01ec233a84f55789cb7eeffe1d6224ccd0e6cdd/llguidance-0.7.30-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af8741c867e4bc7e42f7cdc68350c076b4edd0ca10ecefbde75f15a9f6bc25d0", size = 14867038, upload-time = "2025-06-23T00:23:39.571Z" }, + { url = "https://files.pythonhosted.org/packages/17/ec/69507bdb36767f9b6ff2e290660a9b5afdda0fb8a7903faa37f37c6c2a72/llguidance-0.7.30-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4a327a30dd37d86dd6347861ac8de3521fc1dbef9475296c06744e5b40ffc54", size = 15142936, upload-time = "2025-06-23T00:23:41.944Z" }, + { url = "https://files.pythonhosted.org/packages/af/80/5a40b9689f17612434b820854cba9b8cabd5142072c491b5280fe5f7a35e/llguidance-0.7.30-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9edc409b9decd6cffba5f5bf3b4fbd7541f95daa8cbc9510cbf96c6ab1ffc153", size = 15004926, upload-time = "2025-06-23T00:23:43.965Z" }, + { url = "https://files.pythonhosted.org/packages/bb/bc/2d2f9b446bb3e51e4dd4db290590afee03ae29163f417168569f0361204c/llguidance-0.7.30-cp39-abi3-win32.whl", hash = "sha256:a0d52b8d1b2d3b0e661e3f953ecccfa16644f302026b3067a4815c1baa2ae643", size = 2585627, upload-time = "2025-06-23T00:23:52.39Z" }, + { url = "https://files.pythonhosted.org/packages/99/47/58e49a118b514855b245f8a962c6aaf9a5cc95a0f61eac7e230e691c7b7e/llguidance-0.7.30-cp39-abi3-win_amd64.whl", hash = "sha256:05234ecceea7c9c6ff13b9739112043173a3bcb88cae860249b20335a07b3075", size = 2796878, upload-time = "2025-06-23T00:23:51Z" }, +] + +[[package]] +name = "llvmlite" +version = "0.44.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/6a/95a3d3610d5c75293d5dbbb2a76480d5d4eeba641557b69fe90af6c5b84e/llvmlite-0.44.0.tar.gz", hash = "sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4", size = 171880, upload-time = "2025-01-20T11:14:41.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/86/e3c3195b92e6e492458f16d233e58a1a812aa2bfbef9bdd0fbafcec85c60/llvmlite-0.44.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:1d671a56acf725bf1b531d5ef76b86660a5ab8ef19bb6a46064a705c6ca80aad", size = 28132297, upload-time = "2025-01-20T11:13:32.57Z" }, + { url = "https://files.pythonhosted.org/packages/d6/53/373b6b8be67b9221d12b24125fd0ec56b1078b660eeae266ec388a6ac9a0/llvmlite-0.44.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f79a728e0435493611c9f405168682bb75ffd1fbe6fc360733b850c80a026db", size = 26201105, upload-time = "2025-01-20T11:13:38.744Z" }, + { url = "https://files.pythonhosted.org/packages/cb/da/8341fd3056419441286c8e26bf436923021005ece0bff5f41906476ae514/llvmlite-0.44.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0143a5ef336da14deaa8ec26c5449ad5b6a2b564df82fcef4be040b9cacfea9", size = 42361901, upload-time = "2025-01-20T11:13:46.711Z" }, + { url = "https://files.pythonhosted.org/packages/53/ad/d79349dc07b8a395a99153d7ce8b01d6fcdc9f8231355a5df55ded649b61/llvmlite-0.44.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d752f89e31b66db6f8da06df8b39f9b91e78c5feea1bf9e8c1fba1d1c24c065d", size = 41184247, upload-time = "2025-01-20T11:13:56.159Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3b/a9a17366af80127bd09decbe2a54d8974b6d8b274b39bf47fbaedeec6307/llvmlite-0.44.0-cp312-cp312-win_amd64.whl", hash = "sha256:eae7e2d4ca8f88f89d315b48c6b741dcb925d6a1042da694aa16ab3dd4cbd3a1", size = 30332380, upload-time = "2025-01-20T11:14:02.442Z" }, +] + +[[package]] +name = "lm-format-enforcer" +version = "0.10.12" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform == 'darwin' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +dependencies = [ + { name = "interegular", marker = "extra == 'extra-8-skyagent-skyrl-train' or extra == 'extra-8-skyagent-verl'" }, + { name = "packaging", marker = "extra == 'extra-8-skyagent-skyrl-train' or extra == 'extra-8-skyagent-verl'" }, + { name = "pydantic", marker = "extra == 'extra-8-skyagent-skyrl-train' or extra == 'extra-8-skyagent-verl'" }, + { name = "pyyaml", marker = "extra == 'extra-8-skyagent-skyrl-train' or extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/e0/bdbfad8f5d319de5d05cc2b70d579b49eb8ce3a09989cd0999b8c138c068/lm_format_enforcer-0.10.12.tar.gz", hash = "sha256:130bd7ce8a6b224f25b6314ba9ae78ee4b48594db1767c74391c9182e2902a6c", size = 39481, upload-time = "2025-08-04T21:13:45.727Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/1c/7bb80fe2dff9a9c38b180571ca867f518eb9110f79d4b670ea124e153680/lm_format_enforcer-0.10.12-py3-none-any.whl", hash = "sha256:267c2b421c77f7cd51ac2e0e3af8db278a373704d834b49ff55f18a2c05e9800", size = 44327, upload-time = "2025-08-04T21:13:44.492Z" }, +] + +[[package]] +name = "lm-format-enforcer" +version = "0.11.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "interegular", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "packaging", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pydantic", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pyyaml", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/ef/11292bb0b85cf4c93447cab5a29f64576ed14d3ab4280e35ddd23486594a/lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f", size = 45418, upload-time = "2025-08-24T19:37:46.325Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + +[[package]] +name = "lxml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887, upload-time = "2025-09-22T04:01:17.265Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818, upload-time = "2025-09-22T04:01:19.688Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807, upload-time = "2025-09-22T04:01:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179, upload-time = "2025-09-22T04:01:23.32Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044, upload-time = "2025-09-22T04:01:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685, upload-time = "2025-09-22T04:01:27.398Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127, upload-time = "2025-09-22T04:01:29.629Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958, upload-time = "2025-09-22T04:01:31.535Z" }, + { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541, upload-time = "2025-09-22T04:01:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426, upload-time = "2025-09-22T04:01:35.639Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917, upload-time = "2025-09-22T04:01:37.448Z" }, + { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795, upload-time = "2025-09-22T04:01:39.165Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759, upload-time = "2025-09-22T04:01:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666, upload-time = "2025-09-22T04:01:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989, upload-time = "2025-09-22T04:01:45.215Z" }, + { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456, upload-time = "2025-09-22T04:01:48.243Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793, upload-time = "2025-09-22T04:01:50.042Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" }, +] + +[[package]] +name = "markdown" +version = "3.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, +] + +[[package]] +name = "marshmallow" +version = "3.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/5e/5e53d26b42ab75491cda89b871dab9e97c840bf12c63ec58a1919710cd06/marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6", size = 221825, upload-time = "2025-02-03T15:32:25.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878, upload-time = "2025-02-03T15:32:22.295Z" }, +] + +[[package]] +name = "math-verify" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "latex2sympy2-extended" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/b5/b1db6fa6b6c28ebbe1889ee11a4703a72a2ca7750ec415f4559c758cf01a/math_verify-0.8.0.tar.gz", hash = "sha256:3295e0adb94bfe553ff6e3189c44f1916a85aa24ab5d1900f2086a706e28f7c4", size = 60191, upload-time = "2025-07-02T15:52:07.209Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/9f/59979f699b5c97334298f1295bc9fcdc9904d98d2276479bffff863d23b1/math_verify-0.8.0-py3-none-any.whl", hash = "sha256:31ca651296d817a9bb3fd58ca1fd0d192dcea709b1e5ecf2d0a4514c16f89087", size = 29994, upload-time = "2025-07-02T15:52:05.023Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.10.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a0/59/c3e6453a9676ffba145309a73c462bb407f4400de7de3f2b41af70720a3c/matplotlib-3.10.6.tar.gz", hash = "sha256:ec01b645840dd1996df21ee37f208cd8ba57644779fa20464010638013d3203c", size = 34804264, upload-time = "2025-08-30T00:14:25.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/1a/7042f7430055d567cc3257ac409fcf608599ab27459457f13772c2d9778b/matplotlib-3.10.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:31ca662df6a80bd426f871105fdd69db7543e28e73a9f2afe80de7e531eb2347", size = 8272404, upload-time = "2025-08-30T00:12:59.112Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5d/1d5f33f5b43f4f9e69e6a5fe1fb9090936ae7bc8e2ff6158e7a76542633b/matplotlib-3.10.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1678bb61d897bb4ac4757b5ecfb02bfb3fddf7f808000fb81e09c510712fda75", size = 8128262, upload-time = "2025-08-30T00:13:01.141Z" }, + { url = "https://files.pythonhosted.org/packages/67/c3/135fdbbbf84e0979712df58e5e22b4f257b3f5e52a3c4aacf1b8abec0d09/matplotlib-3.10.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:56cd2d20842f58c03d2d6e6c1f1cf5548ad6f66b91e1e48f814e4fb5abd1cb95", size = 8697008, upload-time = "2025-08-30T00:13:03.24Z" }, + { url = "https://files.pythonhosted.org/packages/9c/be/c443ea428fb2488a3ea7608714b1bd85a82738c45da21b447dc49e2f8e5d/matplotlib-3.10.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:662df55604a2f9a45435566d6e2660e41efe83cd94f4288dfbf1e6d1eae4b0bb", size = 9530166, upload-time = "2025-08-30T00:13:05.951Z" }, + { url = "https://files.pythonhosted.org/packages/a9/35/48441422b044d74034aea2a3e0d1a49023f12150ebc58f16600132b9bbaf/matplotlib-3.10.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:08f141d55148cd1fc870c3387d70ca4df16dee10e909b3b038782bd4bda6ea07", size = 9593105, upload-time = "2025-08-30T00:13:08.356Z" }, + { url = "https://files.pythonhosted.org/packages/45/c3/994ef20eb4154ab84cc08d033834555319e4af970165e6c8894050af0b3c/matplotlib-3.10.6-cp312-cp312-win_amd64.whl", hash = "sha256:590f5925c2d650b5c9d813c5b3b5fc53f2929c3f8ef463e4ecfa7e052044fb2b", size = 8122784, upload-time = "2025-08-30T00:13:10.367Z" }, + { url = "https://files.pythonhosted.org/packages/57/b8/5c85d9ae0e40f04e71bedb053aada5d6bab1f9b5399a0937afb5d6b02d98/matplotlib-3.10.6-cp312-cp312-win_arm64.whl", hash = "sha256:f44c8d264a71609c79a78d50349e724f5d5fc3684ead7c2a473665ee63d868aa", size = 7992823, upload-time = "2025-08-30T00:13:12.24Z" }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/5b/a36a337438a14116b16480db471ad061c36c3694df7c2084a0da7ba538b7/matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90", size = 8159, upload-time = "2024-04-15T13:44:44.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899, upload-time = "2024-04-15T13:44:43.265Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "mcp" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-multipart" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/ae/588691c45b38f4fbac07fa3d6d50cea44cc6b35d16ddfdf26e17a0467ab2/mcp-1.7.1.tar.gz", hash = "sha256:eb4f1f53bd717f75dda8a1416e00804b831a8f3c331e23447a03b78f04b43a6e", size = 230903, upload-time = "2025-05-02T17:01:56.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/79/fe0e20c3358997a80911af51bad927b5ea2f343ef95ab092b19c9cc48b59/mcp-1.7.1-py3-none-any.whl", hash = "sha256:f7e6108977db6d03418495426c7ace085ba2341b75197f8727f96f9cfd30057a", size = 100365, upload-time = "2025-05-02T17:01:54.674Z" }, +] + +[[package]] +name = "mcpm" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "duckdb" }, + { name = "mcp" }, + { name = "prompt-toolkit" }, + { name = "psutil" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "rich" }, + { name = "ruamel-yaml" }, + { name = "watchfiles" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/f7/b7fff1838c836868e3d6e0c14b4eaa992341f428a8683fc7e5f9dee592d6/mcpm-1.9.0.tar.gz", hash = "sha256:97c112cb6d40e9bbcb4091c1db79da4eeda256bfa48083fa1f3abb260b814686", size = 1816041, upload-time = "2025-05-05T17:26:24.785Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/3e/0457ba71fa93844c65193beed66981ca505e1419e9d224ecf367cff40a1f/mcpm-1.9.0-py3-none-any.whl", hash = "sha256:fc9efe355329bef6a30d201668f9752d6fbc46f9d3a2affda8d45b9c5240475e", size = 102226, upload-time = "2025-05-05T17:26:22.866Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "memory-profiler" +version = "0.61.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "psutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/88/e1907e1ca3488f2d9507ca8b0ae1add7b1cd5d3ca2bc8e5b329382ea2c7b/memory_profiler-0.61.0.tar.gz", hash = "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0", size = 35935, upload-time = "2022-11-15T17:57:28.994Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/26/aaca612a0634ceede20682e692a6c55e35a94c21ba36b807cc40fe910ae1/memory_profiler-0.61.0-py3-none-any.whl", hash = "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84", size = 31803, upload-time = "2022-11-15T17:57:27.031Z" }, +] + +[[package]] +name = "minio" +version = "7.2.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/99/1ad8733fa3f2fa82726e470f8c321e17f9321083b234ab45ad6b59d80d9f/minio-7.2.18.tar.gz", hash = "sha256:173402a5716099159c5659f9de75be204ebe248557b9f1cc9cf45aa70e9d3024", size = 135544, upload-time = "2025-09-29T17:00:28.238Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl", hash = "sha256:f23a6edbff8d0bc4b5c1a61b2628a01c5a3342aefc613ff9c276012e6321108f", size = 93120, upload-time = "2025-09-29T17:00:26.86Z" }, +] + +[[package]] +name = "mistral-common" +version = "1.8.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "pydantic-extra-types", extra = ["pycountry"] }, + { name = "requests" }, + { name = "tiktoken" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/ff/1992a00ccc936f2c6e69ecb1f2cac678e0fd46c53c71bdab99eda4f89dfd/mistral_common-1.8.5.tar.gz", hash = "sha256:9f6204ede9c807f09040a208a9381ae78ef93e2e5a9cd5202dc12e712a025de8", size = 6331923, upload-time = "2025-09-12T06:43:01.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/4a/54e19c5e75939fd9418c7b806c21d12cf252ea2ba38f122b597272b459dd/mistral_common-1.8.5-py3-none-any.whl", hash = "sha256:f3cf87b61958a00485e603f3fe0530eb509d7e9b2f7178329dcd260e307eced1", size = 6515140, upload-time = "2025-09-12T06:42:59.622Z" }, +] + +[package.optional-dependencies] +audio = [ + { name = "soundfile" }, + { name = "soxr" }, +] +image = [ + { name = "opencv-python-headless", version = "4.12.0.88", source = { registry = "https://pypi.org/simple" } }, +] +opencv = [ + { name = "opencv-python-headless", version = "4.11.0.86", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, +] + +[[package]] +name = "modal" +version = "0.77.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "certifi" }, + { name = "click" }, + { name = "grpclib" }, + { name = "protobuf" }, + { name = "rich" }, + { name = "synchronicity" }, + { name = "toml" }, + { name = "typer" }, + { name = "types-certifi" }, + { name = "types-toml" }, + { name = "typing-extensions" }, + { name = "watchfiles" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/06/acd53f8fb8d4f0f7a1d5a5bfdb962fcbbe917ed6f703d2a6dfdbfb490c34/modal-0.77.0.tar.gz", hash = "sha256:0a3c2b438a5d26c4b2a1100a7cecdd47e09f715e2a6e0de2a50aad235ddc92c3", size = 508716, upload-time = "2025-05-13T22:02:01.514Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/a2/06b274bbec3501de530e8be5da02e62258f568201200bcb115976ae3b68e/modal-0.77.0-py3-none-any.whl", hash = "sha256:f646d47e9950b929a03918124c4b12663fdc146287af3c0bd96de3ce23776399", size = 576782, upload-time = "2025-05-13T22:01:58.734Z" }, +] + +[[package]] +name = "more-itertools" +version = "10.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "msgpack" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/26/389b9c593eda2b8551b2e7126ad3a06af6f9b44274eb3a4f054d48ff7e47/msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238", size = 82359, upload-time = "2025-06-13T06:52:03.909Z" }, + { url = "https://files.pythonhosted.org/packages/ab/65/7d1de38c8a22cf8b1551469159d4b6cf49be2126adc2482de50976084d78/msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157", size = 79172, upload-time = "2025-06-13T06:52:05.246Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bd/cacf208b64d9577a62c74b677e1ada005caa9b69a05a599889d6fc2ab20a/msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce", size = 425013, upload-time = "2025-06-13T06:52:06.341Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ec/fd869e2567cc9c01278a736cfd1697941ba0d4b81a43e0aa2e8d71dab208/msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a", size = 426905, upload-time = "2025-06-13T06:52:07.501Z" }, + { url = "https://files.pythonhosted.org/packages/55/2a/35860f33229075bce803a5593d046d8b489d7ba2fc85701e714fc1aaf898/msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c", size = 407336, upload-time = "2025-06-13T06:52:09.047Z" }, + { url = "https://files.pythonhosted.org/packages/8c/16/69ed8f3ada150bf92745fb4921bd621fd2cdf5a42e25eb50bcc57a5328f0/msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b", size = 409485, upload-time = "2025-06-13T06:52:10.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b6/0c398039e4c6d0b2e37c61d7e0e9d13439f91f780686deb8ee64ecf1ae71/msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef", size = 412182, upload-time = "2025-06-13T06:52:11.644Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/0cf4a6ecb9bc960d624c93effaeaae75cbf00b3bc4a54f35c8507273cda1/msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a", size = 419883, upload-time = "2025-06-13T06:52:12.806Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/9697c211720fa71a2dfb632cad6196a8af3abea56eece220fde4674dc44b/msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c", size = 65406, upload-time = "2025-06-13T06:52:14.271Z" }, + { url = "https://files.pythonhosted.org/packages/c0/23/0abb886e80eab08f5e8c485d6f13924028602829f63b8f5fa25a06636628/msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4", size = 72558, upload-time = "2025-06-13T06:52:15.252Z" }, +] + +[[package]] +name = "msgspec" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cf/9b/95d8ce458462b8b71b8a70fa94563b2498b89933689f3a7b8911edfae3d7/msgspec-0.19.0.tar.gz", hash = "sha256:604037e7cd475345848116e89c553aa9a233259733ab51986ac924ab1b976f8e", size = 216934, upload-time = "2024-12-27T17:40:28.597Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/5f/a70c24f075e3e7af2fae5414c7048b0e11389685b7f717bb55ba282a34a7/msgspec-0.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f98bd8962ad549c27d63845b50af3f53ec468b6318400c9f1adfe8b092d7b62f", size = 190485, upload-time = "2024-12-27T17:39:44.974Z" }, + { url = "https://files.pythonhosted.org/packages/89/b0/1b9763938cfae12acf14b682fcf05c92855974d921a5a985ecc197d1c672/msgspec-0.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:43bbb237feab761b815ed9df43b266114203f53596f9b6e6f00ebd79d178cdf2", size = 183910, upload-time = "2024-12-27T17:39:46.401Z" }, + { url = "https://files.pythonhosted.org/packages/87/81/0c8c93f0b92c97e326b279795f9c5b956c5a97af28ca0fbb9fd86c83737a/msgspec-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cfc033c02c3e0aec52b71710d7f84cb3ca5eb407ab2ad23d75631153fdb1f12", size = 210633, upload-time = "2024-12-27T17:39:49.099Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/c5422ce8af73928d194a6606f8ae36e93a52fd5e8df5abd366903a5ca8da/msgspec-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d911c442571605e17658ca2b416fd8579c5050ac9adc5e00c2cb3126c97f73bc", size = 213594, upload-time = "2024-12-27T17:39:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/19/2b/4137bc2ed45660444842d042be2cf5b18aa06efd2cda107cff18253b9653/msgspec-0.19.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:757b501fa57e24896cf40a831442b19a864f56d253679f34f260dcb002524a6c", size = 214053, upload-time = "2024-12-27T17:39:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e6/8ad51bdc806aac1dc501e8fe43f759f9ed7284043d722b53323ea421c360/msgspec-0.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5f0f65f29b45e2816d8bded36e6b837a4bf5fb60ec4bc3c625fa2c6da4124537", size = 219081, upload-time = "2024-12-27T17:39:55.142Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ef/27dd35a7049c9a4f4211c6cd6a8c9db0a50647546f003a5867827ec45391/msgspec-0.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:067f0de1c33cfa0b6a8206562efdf6be5985b988b53dd244a8e06f993f27c8c0", size = 187467, upload-time = "2024-12-27T17:39:56.531Z" }, +] + +[[package]] +name = "multidict" +version = "6.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" }, + { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" }, + { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" }, + { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" }, + { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" }, + { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" }, + { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" }, + { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" }, + { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" }, + { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" }, + { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" }, + { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" }, +] + +[[package]] +name = "multiprocess" +version = "0.70.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" }, + { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "nanobind" +version = "2.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/48/edccb2ec318dee2f979765ca8e565e867569522bb55ad384d57b01c3da03/nanobind-2.9.2.tar.gz", hash = "sha256:e7608472de99d375759814cab3e2c94aba3f9ec80e62cfef8ced495ca5c27d6e", size = 984270, upload-time = "2025-09-04T12:52:45.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/20/f8944f162df48a20f8f533f4e4faed63891aa368ceca8cc8b32d4c785ed4/nanobind-2.9.2-py3-none-any.whl", hash = "sha256:c37957ffd5eac7eda349cff3622ecd32e5ee1244ecc912c99b5bc8188bafd16e", size = 243830, upload-time = "2025-09-04T12:52:43.652Z" }, +] + +[[package]] +name = "narwhals" +version = "2.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/dd/40ff412dabf90ef6b99266b0b74f217bb88733541733849e0153a108c750/narwhals-2.6.0.tar.gz", hash = "sha256:5c9e2ba923e6a0051017e146184e49fb793548936f978ce130c9f55a9a81240e", size = 561649, upload-time = "2025-09-29T09:08:56.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/3b/0e2c535c3e6970cfc5763b67f6cc31accaab35a7aa3e322fb6a12830450f/narwhals-2.6.0-py3-none-any.whl", hash = "sha256:3215ea42afb452c6c8527e79cefbe542b674aa08d7e2e99d46b2c9708870e0d4", size = 408435, upload-time = "2025-09-29T09:08:54.503Z" }, +] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, +] + +[[package]] +name = "ninja" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/73/79a0b22fc731989c708068427579e840a6cf4e937fe7ae5c5d0b7356ac22/ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978", size = 242558, upload-time = "2025-08-11T15:10:19.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/74/d02409ed2aa865e051b7edda22ad416a39d81a84980f544f8de717cab133/ninja-1.13.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:fa2a8bfc62e31b08f83127d1613d10821775a0eb334197154c4d6067b7068ff1", size = 310125, upload-time = "2025-08-11T15:09:50.971Z" }, + { url = "https://files.pythonhosted.org/packages/8e/de/6e1cd6b84b412ac1ef327b76f0641aeb5dcc01e9d3f9eee0286d0c34fd93/ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630", size = 177467, upload-time = "2025-08-11T15:09:52.767Z" }, + { url = "https://files.pythonhosted.org/packages/c8/83/49320fb6e58ae3c079381e333575fdbcf1cca3506ee160a2dcce775046fa/ninja-1.13.0-py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:be7f478ff9f96a128b599a964fc60a6a87b9fa332ee1bd44fa243ac88d50291c", size = 187834, upload-time = "2025-08-11T15:09:54.115Z" }, + { url = "https://files.pythonhosted.org/packages/56/c7/ba22748fb59f7f896b609cd3e568d28a0a367a6d953c24c461fe04fc4433/ninja-1.13.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:60056592cf495e9a6a4bea3cd178903056ecb0943e4de45a2ea825edb6dc8d3e", size = 202736, upload-time = "2025-08-11T15:09:55.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/22/d1de07632b78ac8e6b785f41fa9aad7a978ec8c0a1bf15772def36d77aac/ninja-1.13.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1c97223cdda0417f414bf864cfb73b72d8777e57ebb279c5f6de368de0062988", size = 179034, upload-time = "2025-08-11T15:09:57.394Z" }, + { url = "https://files.pythonhosted.org/packages/ed/de/0e6edf44d6a04dabd0318a519125ed0415ce437ad5a1ec9b9be03d9048cf/ninja-1.13.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa", size = 180716, upload-time = "2025-08-11T15:09:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/54/28/938b562f9057aaa4d6bfbeaa05e81899a47aebb3ba6751e36c027a7f5ff7/ninja-1.13.0-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4be9c1b082d244b1ad7ef41eb8ab088aae8c109a9f3f0b3e56a252d3e00f42c1", size = 146843, upload-time = "2025-08-11T15:10:00.046Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fb/d06a3838de4f8ab866e44ee52a797b5491df823901c54943b2adb0389fbb/ninja-1.13.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6739d3352073341ad284246f81339a384eec091d9851a886dfa5b00a6d48b3e2", size = 154402, upload-time = "2025-08-11T15:10:01.657Z" }, + { url = "https://files.pythonhosted.org/packages/31/bf/0d7808af695ceddc763cf251b84a9892cd7f51622dc8b4c89d5012779f06/ninja-1.13.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11be2d22027bde06f14c343f01d31446747dbb51e72d00decca2eb99be911e2f", size = 552388, upload-time = "2025-08-11T15:10:03.349Z" }, + { url = "https://files.pythonhosted.org/packages/9d/70/c99d0c2c809f992752453cce312848abb3b1607e56d4cd1b6cded317351a/ninja-1.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aa45b4037b313c2f698bc13306239b8b93b4680eb47e287773156ac9e9304714", size = 472501, upload-time = "2025-08-11T15:10:04.735Z" }, + { url = "https://files.pythonhosted.org/packages/9f/43/c217b1153f0e499652f5e0766da8523ce3480f0a951039c7af115e224d55/ninja-1.13.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f8e1e8a1a30835eeb51db05cf5a67151ad37542f5a4af2a438e9490915e5b72", size = 638280, upload-time = "2025-08-11T15:10:06.512Z" }, + { url = "https://files.pythonhosted.org/packages/8c/45/9151bba2c8d0ae2b6260f71696330590de5850e5574b7b5694dce6023e20/ninja-1.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:3d7d7779d12cb20c6d054c61b702139fd23a7a964ec8f2c823f1ab1b084150db", size = 642420, upload-time = "2025-08-11T15:10:08.35Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/95752eb635bb8ad27d101d71bef15bc63049de23f299e312878fc21cb2da/ninja-1.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:d741a5e6754e0bda767e3274a0f0deeef4807f1fec6c0d7921a0244018926ae5", size = 585106, upload-time = "2025-08-11T15:10:09.818Z" }, + { url = "https://files.pythonhosted.org/packages/c1/31/aa56a1a286703800c0cbe39fb4e82811c277772dc8cd084f442dd8e2938a/ninja-1.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:e8bad11f8a00b64137e9b315b137d8bb6cbf3086fbdc43bf1f90fd33324d2e96", size = 707138, upload-time = "2025-08-11T15:10:11.366Z" }, + { url = "https://files.pythonhosted.org/packages/34/6f/5f5a54a1041af945130abdb2b8529cbef0cdcbbf9bcf3f4195378319d29a/ninja-1.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4f2a072db3c0f944c32793e91532d8948d20d9ab83da9c0c7c15b5768072200", size = 581758, upload-time = "2025-08-11T15:10:13.295Z" }, + { url = "https://files.pythonhosted.org/packages/95/97/51359c77527d45943fe7a94d00a3843b81162e6c4244b3579fe8fc54cb9c/ninja-1.13.0-py3-none-win32.whl", hash = "sha256:8cfbb80b4a53456ae8a39f90ae3d7a2129f45ea164f43fadfa15dc38c4aef1c9", size = 267201, upload-time = "2025-08-11T15:10:15.158Z" }, + { url = "https://files.pythonhosted.org/packages/29/45/c0adfbfb0b5895aa18cec400c535b4f7ff3e52536e0403602fc1a23f7de9/ninja-1.13.0-py3-none-win_amd64.whl", hash = "sha256:fb8ee8719f8af47fed145cced4a85f0755dd55d45b2bddaf7431fa89803c5f3e", size = 309975, upload-time = "2025-08-11T15:10:16.697Z" }, + { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, +] + +[[package]] +name = "nltk" +version = "3.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/87/db8be88ad32c2d042420b6fd9ffd4a149f9a0d7f0e86b3f543be2eeeedd2/nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868", size = 2904691, upload-time = "2024-08-18T19:48:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/66/7d9e26593edda06e8cb531874633f7c2372279c3b0f46235539fe546df8b/nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1", size = 1505442, upload-time = "2024-08-18T19:48:21.909Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, +] + +[[package]] +name = "numba" +version = "0.61.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/a0/e21f57604304aa03ebb8e098429222722ad99176a4f979d34af1d1ee80da/numba-0.61.2.tar.gz", hash = "sha256:8750ee147940a6637b80ecf7f95062185ad8726c8c28a2295b8ec1160a196f7d", size = 2820615, upload-time = "2025-04-09T02:58:07.659Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/a0/c6b7b9c615cfa3b98c4c63f4316e3f6b3bbe2387740277006551784218cd/numba-0.61.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2", size = 2776626, upload-time = "2025-04-09T02:57:51.857Z" }, + { url = "https://files.pythonhosted.org/packages/92/4a/fe4e3c2ecad72d88f5f8cd04e7f7cff49e718398a2fac02d2947480a00ca/numba-0.61.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ddce10009bc097b080fc96876d14c051cc0c7679e99de3e0af59014dab7dfe8", size = 2779287, upload-time = "2025-04-09T02:57:53.658Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2d/e518df036feab381c23a624dac47f8445ac55686ec7f11083655eb707da3/numba-0.61.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b1bb509d01f23d70325d3a5a0e237cbc9544dd50e50588bc581ba860c213546", size = 3885928, upload-time = "2025-04-09T02:57:55.206Z" }, + { url = "https://files.pythonhosted.org/packages/10/0f/23cced68ead67b75d77cfcca3df4991d1855c897ee0ff3fe25a56ed82108/numba-0.61.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:48a53a3de8f8793526cbe330f2a39fe9a6638efcbf11bd63f3d2f9757ae345cd", size = 3577115, upload-time = "2025-04-09T02:57:56.818Z" }, + { url = "https://files.pythonhosted.org/packages/68/1d/ddb3e704c5a8fb90142bf9dc195c27db02a08a99f037395503bfbc1d14b3/numba-0.61.2-cp312-cp312-win_amd64.whl", hash = "sha256:97cf4f12c728cf77c9c1d7c23707e4d8fb4632b46275f8f3397de33e5877af18", size = 2831929, upload-time = "2025-04-09T02:57:58.45Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, + { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, + { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.4.5.8" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/7f/7fbae15a3982dc9595e49ce0f19332423b260045d0a6afe93cdbe2f1f624/nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0f8aa1706812e00b9f19dfe0cdb3999b092ccb8ca168c0db5b8ea712456fd9b3", size = 363333771, upload-time = "2024-06-18T19:28:09.881Z" }, + { url = "https://files.pythonhosted.org/packages/ae/71/1c91302526c45ab494c23f61c7a84aa568b8c1f9d196efa5993957faf906/nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b", size = 363438805, upload-time = "2024-04-03T20:57:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/e2/2a/4f27ca96232e8b5269074a72e03b4e0d43aa68c9b965058b1684d07c6ff8/nvidia_cublas_cu12-12.4.5.8-py3-none-win_amd64.whl", hash = "sha256:5a796786da89203a0657eda402bcdcec6180254a8ac22d72213abc42069522dc", size = 396895858, upload-time = "2024-04-03T21:03:31.996Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.3.14" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/63/684a6f72f52671ea222c12ecde9bdf748a0ba025e2ad3ec374e466c26eb6/nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:93a4e0e386cc7f6e56c822531396de8170ed17068a1e18f987574895044cd8c3", size = 604900717, upload-time = "2025-01-23T17:52:55.486Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/4b01f10069e23c641f116c62fc31e31e8dc361a153175d81561d15c8143b/nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:3f0e05e7293598cf61933258b73e66a160c27d59c4422670bf0b79348c04be44", size = 609620630, upload-time = "2025-01-23T17:55:00.753Z" }, + { url = "https://files.pythonhosted.org/packages/6c/54/fbfa3315b936d3358517f7da5f9f2557c279bf210e5261f0cf66cc0f9832/nvidia_cublas_cu12-12.8.3.14-py3-none-win_amd64.whl", hash = "sha256:9ae5eae500aead01fc4bdfc458209df638b1a3551557ce11a78eea9ece602ae9", size = 578387959, upload-time = "2025-01-23T18:08:00.662Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/99/db44d685f0e257ff0e213ade1964fc459b4a690a73293220e98feb3307cf/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0", size = 590537124, upload-time = "2025-03-07T01:43:53.556Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, + { url = "https://files.pythonhosted.org/packages/70/61/7d7b3c70186fb651d0fbd35b01dbfc8e755f69fd58f817f3d0f642df20c3/nvidia_cublas_cu12-12.8.4.1-py3-none-win_amd64.whl", hash = "sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af", size = 567544208, upload-time = "2025-03-07T01:53:30.535Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.4.127" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/b5/9fb3d00386d3361b03874246190dfec7b206fd74e6e287b26a8fcb359d95/nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:79279b35cf6f91da114182a5ce1864997fd52294a87a16179ce275773799458a", size = 12354556, upload-time = "2024-06-18T19:30:40.546Z" }, + { url = "https://files.pythonhosted.org/packages/67/42/f4f60238e8194a3106d06a058d494b18e006c10bb2b915655bd9f6ea4cb1/nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb", size = 13813957, upload-time = "2024-04-03T20:55:01.564Z" }, + { url = "https://files.pythonhosted.org/packages/f3/79/8cf313ec17c58ccebc965568e5bcb265cdab0a1df99c4e674bb7a3b99bfe/nvidia_cuda_cupti_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:5688d203301ab051449a2b1cb6690fbe90d2b372f411521c86018b950f3d7922", size = 9938035, upload-time = "2024-04-03T21:01:01.109Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.57" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/53/458956a65283c55c22ba40a65745bbe9ff20c10b68ea241bc575e20c0465/nvidia_cuda_cupti_cu12-12.8.57-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff154211724fd824e758ce176b66007b558eea19c9a5135fc991827ee147e317", size = 9526469, upload-time = "2025-01-23T17:47:33.104Z" }, + { url = "https://files.pythonhosted.org/packages/39/6f/3683ecf4e38931971946777d231c2df00dd5c1c4c2c914c42ad8f9f4dca6/nvidia_cuda_cupti_cu12-12.8.57-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e0b2eb847de260739bee4a3f66fac31378f4ff49538ff527a38a01a9a39f950", size = 10237547, upload-time = "2025-01-23T17:47:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2a/cabe033045427beb042b70b394ac3fd7cfefe157c965268824011b16af67/nvidia_cuda_cupti_cu12-12.8.57-py3-none-win_amd64.whl", hash = "sha256:bbed719c52a476958a74cfc42f2b95a3fd6b3fd94eb40134acc4601feb4acac3", size = 7002337, upload-time = "2025-01-23T18:04:35.34Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/1f/b3bd73445e5cb342727fd24fe1f7b748f690b460acadc27ea22f904502c8/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed", size = 9533318, upload-time = "2025-03-07T01:40:10.421Z" }, + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, + { url = "https://files.pythonhosted.org/packages/41/bc/83f5426095d93694ae39fe1311431b5d5a9bb82e48bf0dd8e19be2765942/nvidia_cuda_cupti_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e", size = 7015759, upload-time = "2025-03-07T01:51:11.355Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.4.127" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/aa/083b01c427e963ad0b314040565ea396f914349914c298556484f799e61b/nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0eedf14185e04b76aa05b1fea04133e59f465b6f960c0cbf4e37c3cb6b0ea198", size = 24133372, upload-time = "2024-06-18T19:32:00.576Z" }, + { url = "https://files.pythonhosted.org/packages/2c/14/91ae57cd4db3f9ef7aa99f4019cfa8d54cb4caa7e00975df6467e9725a9f/nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338", size = 24640306, upload-time = "2024-04-03T20:56:01.463Z" }, + { url = "https://files.pythonhosted.org/packages/7c/30/8c844bfb770f045bcd8b2c83455c5afb45983e1a8abf0c4e5297b481b6a5/nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:a961b2f1d5f17b14867c619ceb99ef6fcec12e46612711bcec78eb05068a60ec", size = 19751955, upload-time = "2024-04-03T21:01:51.133Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.61" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/22/32029d4583f7b19cfe75c84399cbcfd23f2aaf41c66fc8db4da460104fff/nvidia_cuda_nvrtc_cu12-12.8.61-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a0fa9c2a21583105550ebd871bd76e2037205d56f33f128e69f6d2a55e0af9ed", size = 88024585, upload-time = "2025-01-23T17:50:10.722Z" }, + { url = "https://files.pythonhosted.org/packages/f1/98/29f98d57fc40d6646337e942d37509c6d5f8abe29012671f7a6eb9978ebe/nvidia_cuda_nvrtc_cu12-12.8.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1f376bf58111ca73dde4fd4df89a462b164602e074a76a2c29c121ca478dcd4", size = 43097015, upload-time = "2025-01-23T17:49:44.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5b/052d05aa068e4752415ad03bac58e852ea8bc17c9321e08546b3f261e47e/nvidia_cuda_nvrtc_cu12-12.8.61-py3-none-win_amd64.whl", hash = "sha256:9c8887bf5e5dffc441018ba8c5dc59952372a6f4806819e8c1f03d62637dbeea", size = 73567440, upload-time = "2025-01-23T18:05:51.036Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d1/e50d0acaab360482034b84b6e27ee83c6738f7d32182b987f9c7a4e32962/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8", size = 43106076, upload-time = "2025-03-07T01:41:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/45/51/52a3d84baa2136cc8df15500ad731d74d3a1114d4c123e043cb608d4a32b/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909", size = 73586838, upload-time = "2025-03-07T01:52:13.483Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.4.127" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/aa/b656d755f474e2084971e9a297def515938d56b466ab39624012070cb773/nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:961fe0e2e716a2a1d967aab7caee97512f71767f852f67432d572e36cb3a11f3", size = 894177, upload-time = "2024-06-18T19:32:52.877Z" }, + { url = "https://files.pythonhosted.org/packages/ea/27/1795d86fe88ef397885f2e580ac37628ed058a92ed2c39dc8eac3adf0619/nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5", size = 883737, upload-time = "2024-04-03T20:54:51.355Z" }, + { url = "https://files.pythonhosted.org/packages/a8/8b/450e93fab75d85a69b50ea2d5fdd4ff44541e0138db16f9cd90123ef4de4/nvidia_cuda_runtime_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:09c2e35f48359752dfa822c09918211844a3d93c100a715d79b59591130c5e1e", size = 878808, upload-time = "2024-04-03T21:00:49.77Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.57" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/9d/e77ec4227e70c6006195bdf410370f2d0e5abfa2dc0d1d315cacd57c5c88/nvidia_cuda_runtime_cu12-12.8.57-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:534ccebd967b6a44292678fa5da4f00666029cb2ed07a79515ea41ef31fe3ec7", size = 965264, upload-time = "2025-01-23T17:47:11.759Z" }, + { url = "https://files.pythonhosted.org/packages/16/f6/0e1ef31f4753a44084310ba1a7f0abaf977ccd810a604035abb43421c057/nvidia_cuda_runtime_cu12-12.8.57-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75342e28567340b7428ce79a5d6bb6ca5ff9d07b69e7ce00d2c7b4dc23eff0be", size = 954762, upload-time = "2025-01-23T17:47:22.21Z" }, + { url = "https://files.pythonhosted.org/packages/16/ee/52508c74bee2a3de8d59c6fd9af4ca2f216052fa2bc916da3a6a7bb998af/nvidia_cuda_runtime_cu12-12.8.57-py3-none-win_amd64.whl", hash = "sha256:89be637e3ee967323865b85e0f147d75f9a5bd98360befa37481b02dd57af8f5", size = 944309, upload-time = "2025-01-23T18:04:23.143Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/75/f865a3b236e4647605ea34cc450900854ba123834a5f1598e160b9530c3a/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d", size = 965265, upload-time = "2025-03-07T01:39:43.533Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, + { url = "https://files.pythonhosted.org/packages/30/a5/a515b7600ad361ea14bfa13fb4d6687abf500adc270f19e89849c0590492/nvidia_cuda_runtime_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8", size = 944318, upload-time = "2025-03-07T01:51:01.794Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.1.0.70" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.4.5.8", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741, upload-time = "2024-04-22T15:24:15.253Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d0/f90ee6956a628f9f04bf467932c0a25e5a7e706a684b896593c06c82f460/nvidia_cudnn_cu12-9.1.0.70-py3-none-win_amd64.whl", hash = "sha256:6278562929433d68365a07a4a1546c237ba2849852c0d4b2262a486e805b977a", size = 679925892, upload-time = "2024-04-22T15:24:53.333Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.7.1.26" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.3.14", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/2e/ec5dda717eeb1de3afbbbb611ca556f9d6d057470759c6abd36d72f0063b/nvidia_cudnn_cu12-9.7.1.26-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:848a61d40ef3b32bd4e1fadb599f0cf04a4b942fbe5fb3be572ad75f9b8c53ef", size = 725862213, upload-time = "2025-02-06T22:14:57.169Z" }, + { url = "https://files.pythonhosted.org/packages/25/dc/dc825c4b1c83b538e207e34f48f86063c88deaa35d46c651c7c181364ba2/nvidia_cudnn_cu12-9.7.1.26-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:6d011159a158f3cfc47bf851aea79e31bcff60d530b70ef70474c84cac484d07", size = 726851421, upload-time = "2025-02-06T22:18:29.812Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ea/636cda41b3865caa0d43c34f558167304acde3d2c5f6c54c00a550e69ecd/nvidia_cudnn_cu12-9.7.1.26-py3-none-win_amd64.whl", hash = "sha256:7b805b9a4cf9f3da7c5f4ea4a9dff7baf62d1a612d6154a7e0d2ea51ed296241", size = 715962100, upload-time = "2025-02-06T22:21:32.431Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/41/e79269ce215c857c935fd86bcfe91a451a584dfc27f1e068f568b9ad1ab7/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8", size = 705026878, upload-time = "2025-06-06T21:52:51.348Z" }, + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, + { url = "https://files.pythonhosted.org/packages/3d/90/0bd6e586701b3a890fd38aa71c387dab4883d619d6e5ad912ccbd05bfd67/nvidia_cudnn_cu12-9.10.2.21-py3-none-win_amd64.whl", hash = "sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e", size = 692992268, upload-time = "2025-06-06T21:55:18.114Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.2.1.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/8a/0e728f749baca3fbeffad762738276e5df60851958be7783af121a7221e7/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399", size = 211422548, upload-time = "2024-06-18T19:33:39.396Z" }, + { url = "https://files.pythonhosted.org/packages/27/94/3266821f65b92b3138631e9c8e7fe1fb513804ac934485a8d05776e1dd43/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9", size = 211459117, upload-time = "2024-04-03T20:57:40.402Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ee/3f3f8e9874f0be5bbba8fb4b62b3de050156d159f8b6edc42d6f1074113b/nvidia_cufft_cu12-11.2.1.3-py3-none-win_amd64.whl", hash = "sha256:d802f4954291101186078ccbe22fc285a902136f974d369540fd4a5333d1440b", size = 210576476, upload-time = "2024-04-03T21:04:06.422Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.41" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/95/6157cb45a49f5090a470de42353a22a0ed5b13077886dca891b4b0e350fe/nvidia_cufft_cu12-11.3.3.41-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68509dcd7e3306e69d0e2d8a6d21c8b25ed62e6df8aac192ce752f17677398b5", size = 193108626, upload-time = "2025-01-23T17:55:49.192Z" }, + { url = "https://files.pythonhosted.org/packages/ac/26/b53c493c38dccb1f1a42e1a21dc12cba2a77fbe36c652f7726d9ec4aba28/nvidia_cufft_cu12-11.3.3.41-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:da650080ab79fcdf7a4b06aa1b460e99860646b176a43f6208099bdc17836b6a", size = 193118795, upload-time = "2025-01-23T17:56:30.536Z" }, + { url = "https://files.pythonhosted.org/packages/32/f3/f6248aa119c2726b1bdd02d472332cae274133bd32ca5fa8822efb0c308c/nvidia_cufft_cu12-11.3.3.41-py3-none-win_amd64.whl", hash = "sha256:f9760612886786601d27a0993bb29ce1f757e6b8b173499d0ecfa850d31b50f8", size = 192216738, upload-time = "2025-01-23T18:08:51.102Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/bc/7771846d3a0272026c416fbb7e5f4c1f146d6d80704534d0b187dd6f4800/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a", size = 193109211, upload-time = "2025-03-07T01:44:56.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ec/ce1629f1e478bb5ccd208986b5f9e0316a78538dd6ab1d0484f012f8e2a1/nvidia_cufft_cu12-11.3.3.83-py3-none-win_amd64.whl", hash = "sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7", size = 192216559, upload-time = "2025-03-07T01:53:57.106Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.0.11" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/9c/1f3264d0a84c8a031487fb7f59780fc78fa6f1c97776233956780e3dc3ac/nvidia_cufile_cu12-1.13.0.11-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:483f434c541806936b98366f6d33caef5440572de8ddf38d453213729da3e7d4", size = 1197801, upload-time = "2025-01-23T17:57:07.247Z" }, + { url = "https://files.pythonhosted.org/packages/35/80/f6a0fc90ab6fa4ac916f3643e5b620fd19724626c59ae83b74f5efef0349/nvidia_cufile_cu12-1.13.0.11-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:2acbee65dc2eaf58331f0798c5e6bcdd790c4acb26347530297e63528c9eba5d", size = 1120660, upload-time = "2025-01-23T17:56:56.608Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f5/5607710447a6fe9fd9b3283956fceeee8a06cda1d2f56ce31371f595db2a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a", size = 1120705, upload-time = "2025-03-07T01:45:41.434Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.5.147" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/9c/a79180e4d70995fdf030c6946991d0171555c6edf95c265c6b2bf7011112/nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1f173f09e3e3c76ab084aba0de819c49e56614feae5c12f69883f4ae9bb5fad9", size = 56314811, upload-time = "2024-06-18T19:34:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6d/44ad094874c6f1b9c654f8ed939590bdc408349f137f9b98a3a23ccec411/nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b", size = 56305206, upload-time = "2024-04-03T20:58:08.722Z" }, + { url = "https://files.pythonhosted.org/packages/1c/22/2573503d0d4e45673c263a313f79410e110eb562636b0617856fdb2ff5f6/nvidia_curand_cu12-10.3.5.147-py3-none-win_amd64.whl", hash = "sha256:f307cc191f96efe9e8f05a87096abc20d08845a841889ef78cb06924437f6771", size = 55799918, upload-time = "2024-04-03T21:04:34.45Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.55" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/13/bbcf48e2f8a6a9adef58f130bc968810528a4e66bbbe62fad335241e699f/nvidia_curand_cu12-10.3.9.55-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b6bb90c044fa9b07cedae2ef29077c4cf851fb6fdd6d862102321f359dca81e9", size = 63623836, upload-time = "2025-01-23T17:57:22.319Z" }, + { url = "https://files.pythonhosted.org/packages/bd/fc/7be5d0082507269bb04ac07cc614c84b78749efb96e8cf4100a8a1178e98/nvidia_curand_cu12-10.3.9.55-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8387d974240c91f6a60b761b83d4b2f9b938b7e0b9617bae0f0dafe4f5c36b86", size = 63618038, upload-time = "2025-01-23T17:57:41.838Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f0/91252f3cffe3f3c233a8e17262c21b41534652edfe783c1e58ea1c92c115/nvidia_curand_cu12-10.3.9.55-py3-none-win_amd64.whl", hash = "sha256:570d82475fe7f3d8ed01ffbe3b71796301e0e24c98762ca018ff8ce4f5418e1f", size = 62761446, upload-time = "2025-01-23T18:09:21.663Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/5e/92aa15eca622a388b80fbf8375d4760738df6285b1e92c43d37390a33a9a/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd", size = 63625754, upload-time = "2025-03-07T01:46:10.735Z" }, + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, + { url = "https://files.pythonhosted.org/packages/b9/75/70c05b2f3ed5be3bb30b7102b6eb78e100da4bbf6944fd6725c012831cab/nvidia_curand_cu12-10.3.9.90-py3-none-win_amd64.whl", hash = "sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec", size = 62765309, upload-time = "2025-03-07T01:54:20.478Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.6.1.9" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.4.5.8", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparse-cu12", version = "12.3.1.170", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/6b/a5c33cf16af09166845345275c34ad2190944bcc6026797a39f8e0a282e0/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e", size = 127634111, upload-time = "2024-06-18T19:35:01.793Z" }, + { url = "https://files.pythonhosted.org/packages/3a/e1/5b9089a4b2a4790dfdea8b3a006052cfecff58139d5a4e34cb1a51df8d6f/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260", size = 127936057, upload-time = "2024-04-03T20:58:28.735Z" }, + { url = "https://files.pythonhosted.org/packages/f2/be/d435b7b020e854d5d5a682eb5de4328fd62f6182507406f2818280e206e2/nvidia_cusolver_cu12-11.6.1.9-py3-none-win_amd64.whl", hash = "sha256:e77314c9d7b694fcebc84f58989f3aa4fb4cb442f12ca1a9bde50f5e8f6d1b9c", size = 125224015, upload-time = "2024-04-03T21:04:53.339Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.2.55" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.3.14", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.7.53", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/ce/4214a892e804b20bf66d04f04a473006fc2d3dac158160ef85f1bc906639/nvidia_cusolver_cu12-11.7.2.55-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:0fd9e98246f43c15bee5561147ad235dfdf2d037f5d07c9d41af3f7f72feb7cc", size = 260094827, upload-time = "2025-01-23T17:58:17.586Z" }, + { url = "https://files.pythonhosted.org/packages/c2/08/953675873a136d96bb12f93b49ba045d1107bc94d2551c52b12fa6c7dec3/nvidia_cusolver_cu12-11.7.2.55-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4d1354102f1e922cee9db51920dba9e2559877cf6ff5ad03a00d853adafb191b", size = 260373342, upload-time = "2025-01-23T17:58:56.406Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/e0e6f8b7aecd13e0f9e937d116fb3211329a0a92b9bea9624b1368de307a/nvidia_cusolver_cu12-11.7.2.55-py3-none-win_amd64.whl", hash = "sha256:a5a516c55da5c5aba98420d9bc9bcab18245f21ec87338cc1f930eb18dd411ac", size = 249600787, upload-time = "2025-01-23T18:10:07.641Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/32/f7cd6ce8a7690544d084ea21c26e910a97e077c9b7f07bf5de623ee19981/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0", size = 267229841, upload-time = "2025-03-07T01:46:54.356Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, + { url = "https://files.pythonhosted.org/packages/13/c0/76ca8551b8a84146ffa189fec81c26d04adba4bc0dbe09cd6e6fd9b7de04/nvidia_cusolver_cu12-11.7.3.90-py3-none-win_amd64.whl", hash = "sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34", size = 256720438, upload-time = "2025-03-07T01:54:39.898Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.3.1.170" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/a9/c0d2f83a53d40a4a41be14cea6a0bf9e668ffcf8b004bd65633f433050c0/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3", size = 207381987, upload-time = "2024-06-18T19:35:32.989Z" }, + { url = "https://files.pythonhosted.org/packages/db/f7/97a9ea26ed4bbbfc2d470994b8b4f338ef663be97b8f677519ac195e113d/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1", size = 207454763, upload-time = "2024-04-03T20:58:59.995Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e0/3155ca539760a8118ec94cc279b34293309bcd14011fc724f87f31988843/nvidia_cusparse_cu12-12.3.1.170-py3-none-win_amd64.whl", hash = "sha256:9bc90fb087bc7b4c15641521f31c0371e9a612fc2ba12c338d3ae032e6b6797f", size = 204684315, upload-time = "2024-04-03T21:05:26.031Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.7.53" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/a2/313db0453087f5324a5900380ca2e57e050c8de76f407b5e11383dc762ae/nvidia_cusparse_cu12-12.5.7.53-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d869c6146ca80f4305b62e02d924b4aaced936f8173e3cef536a67eed2a91af1", size = 291963692, upload-time = "2025-01-23T17:59:40.325Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ab/31e8149c66213b846c082a3b41b1365b831f41191f9f40c6ddbc8a7d550e/nvidia_cusparse_cu12-12.5.7.53-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3c1b61eb8c85257ea07e9354606b26397612627fdcd327bfd91ccf6155e7c86d", size = 292064180, upload-time = "2025-01-23T18:00:23.233Z" }, + { url = "https://files.pythonhosted.org/packages/7c/48/64b01653919a3d1d9b5117c156806ab0db8312c7496ff646477a5c1545bf/nvidia_cusparse_cu12-12.5.7.53-py3-none-win_amd64.whl", hash = "sha256:82c201d6781bacf6bb7c654f0446728d0fe596dfdd82ef4a04c204ce3e107441", size = 288767123, upload-time = "2025-01-23T18:11:01.543Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/cd777c4109681367721b00a106f491e0d0d15cfa1fd59672ce580ce42a97/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc", size = 288117129, upload-time = "2025-03-07T01:47:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, + { url = "https://files.pythonhosted.org/packages/62/07/f3b2ad63f8e3d257a599f422ae34eb565e70c41031aecefa3d18b62cabd1/nvidia_cusparse_cu12-12.5.8.93-py3-none-win_amd64.whl", hash = "sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd", size = 284937404, upload-time = "2025-03-07T01:55:07.742Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/8e/675498726c605c9441cf46653bd29cb1b8666da1fb1469ffa25f67f20c58/nvidia_cusparselt_cu12-0.6.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:067a7f6d03ea0d4841c85f0c6f1991c5dda98211f6302cb83a4ab234ee95bef8", size = 149422781, upload-time = "2024-07-23T17:35:27.203Z" }, + { url = "https://files.pythonhosted.org/packages/78/a8/bcbb63b53a4b1234feeafb65544ee55495e1bb37ec31b999b963cbccfd1d/nvidia_cusparselt_cu12-0.6.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:df2c24502fd76ebafe7457dbc4716b2fec071aabaed4fb7691a201cde03704d9", size = 150057751, upload-time = "2024-07-23T02:35:53.074Z" }, + { url = "https://files.pythonhosted.org/packages/56/8f/2c33082238b6c5e783a877dc8786ab62619e3e6171c083bd3bba6e3fe75e/nvidia_cusparselt_cu12-0.6.2-py3-none-win_amd64.whl", hash = "sha256:0057c91d230703924c0422feabe4ce768841f9b4b44d28586b6f6d2eb86fbe70", size = 148755794, upload-time = "2024-07-23T02:35:00.261Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/da/4de092c61c6dea1fc9c936e69308a02531d122e12f1f649825934ad651b5/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1", size = 156402859, upload-time = "2024-10-16T02:23:17.184Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9a/72ef35b399b0e183bc2e8f6f558036922d453c4d8237dab26c666a04244b/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46", size = 156785796, upload-time = "2024-10-15T21:29:17.709Z" }, + { url = "https://files.pythonhosted.org/packages/46/3e/9e1e394a02a06f694be2c97bbe47288bb7c90ea84c7e9cf88f7b28afe165/nvidia_cusparselt_cu12-0.6.3-py3-none-win_amd64.whl", hash = "sha256:3b325bcbd9b754ba43df5a311488fca11a6b5dc3d11df4d190c000cf1a0765c7", size = 155595972, upload-time = "2024-10-15T22:58:35.426Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/b9/598f6ff36faaece4b3c50d26f50e38661499ff34346f00e057760b35cc9d/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5", size = 283835557, upload-time = "2025-02-26T00:16:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d8/a6b0d0d0c2435e9310f3e2bb0d9c9dd4c33daef86aa5f30b3681defd37ea/nvidia_cusparselt_cu12-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075", size = 271020911, upload-time = "2025-02-26T00:14:47.204Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.21.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/99/12cd266d6233f47d00daf3a72739872bdc10267d0383508b0b9c84a18bb6/nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0", size = 188654414, upload-time = "2024-04-03T15:32:57.427Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.26.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/5b/ca2f213f637305633814ae8c36b153220e40a07ea001966dcd87391f3acb/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522", size = 291671495, upload-time = "2025-03-13T00:30:07.805Z" }, + { url = "https://files.pythonhosted.org/packages/67/ca/f42388aed0fddd64ade7493dbba36e1f534d4e6fdbdd355c6a90030ae028/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6", size = 201319755, upload-time = "2025-03-13T00:29:55.296Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/7b/8354b784cf73b0ba51e566b4baba3ddd44fe8288a3d39ef1e06cd5417226/nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9ddf1a245abc36c550870f26d537a9b6087fb2e2e3d6e0ef03374c6fd19d984f", size = 322397768, upload-time = "2025-06-03T21:57:30.234Z" }, + { url = "https://files.pythonhosted.org/packages/5c/5b/4e4fff7bad39adf89f735f2bc87248c81db71205b62bcc0d5ca5b606b3c3/nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039", size = 322364134, upload-time = "2025-06-03T21:58:04.013Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.4.127" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/45/239d52c05074898a80a900f49b1615d81c07fceadd5ad6c4f86a987c0bc4/nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83", size = 20552510, upload-time = "2024-06-18T20:20:13.871Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ff/847841bacfbefc97a00036e0fce5a0f086b640756dc38caea5e1bb002655/nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57", size = 21066810, upload-time = "2024-04-03T20:59:46.957Z" }, + { url = "https://files.pythonhosted.org/packages/81/19/0babc919031bee42620257b9a911c528f05fb2688520dcd9ca59159ffea8/nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1", size = 95336325, upload-time = "2024-04-03T21:06:25.073Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.61" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/f8/9d85593582bd99b8d7c65634d2304780aefade049b2b94d96e44084be90b/nvidia_nvjitlink_cu12-12.8.61-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:45fd79f2ae20bd67e8bc411055939049873bfd8fac70ff13bd4865e0b9bdab17", size = 39243473, upload-time = "2025-01-23T18:03:03.509Z" }, + { url = "https://files.pythonhosted.org/packages/af/53/698f3758f48c5fcb1112721e40cc6714da3980d3c7e93bae5b29dafa9857/nvidia_nvjitlink_cu12-12.8.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b80ecab31085dda3ce3b41d043be0ec739216c3fc633b8abe212d5a30026df0", size = 38374634, upload-time = "2025-01-23T18:02:35.812Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c6/0d1b2bfeb2ef42c06db0570c4d081e5cde4450b54c09e43165126cfe6ff6/nvidia_nvjitlink_cu12-12.8.61-py3-none-win_amd64.whl", hash = "sha256:1166a964d25fdc0eae497574d38824305195a5283324a21ccb0ce0c802cbf41c", size = 268514099, upload-time = "2025-01-23T18:12:33.874Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a2/8cee5da30d13430e87bf99bb33455d2724d0a4a9cb5d7926d80ccb96d008/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7", size = 38386204, upload-time = "2025-03-07T01:49:43.612Z" }, + { url = "https://files.pythonhosted.org/packages/ed/d7/34f02dad2e30c31b10a51f6b04e025e5dd60e5f936af9045a9b858a05383/nvidia_nvjitlink_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f", size = 268553710, upload-time = "2025-03-07T01:56:24.13Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.4.127" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/39/471f581edbb7804b39e8063d92fc8305bdc7a80ae5c07dbe6ea5c50d14a5/nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3", size = 100417, upload-time = "2024-06-18T20:16:22.484Z" }, + { url = "https://files.pythonhosted.org/packages/87/20/199b8713428322a2f22b722c62b8cc278cc53dffa9705d744484b5035ee9/nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a", size = 99144, upload-time = "2024-04-03T20:56:12.406Z" }, + { url = "https://files.pythonhosted.org/packages/54/1b/f77674fbb73af98843be25803bbd3b9a4f0a96c75b8d33a2854a5c7d2d77/nvidia_nvtx_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:641dccaaa1139f3ffb0d3164b4b84f9d253397e38246a4f2f36728b48566d485", size = 66307, upload-time = "2024-04-03T21:02:01.959Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.55" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/e8/ae6ecbdade8bb9174d75db2b302c57c1c27d9277d6531c62aafde5fb32a3/nvidia_nvtx_cu12-12.8.55-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c38405335fbc0f0bf363eaeaeb476e8dfa8bae82fada41d25ace458b9ba9f3db", size = 91103, upload-time = "2025-01-23T17:50:24.664Z" }, + { url = "https://files.pythonhosted.org/packages/8d/cd/0e8c51b2ae3a58f054f2e7fe91b82d201abfb30167f2431e9bd92d532f42/nvidia_nvtx_cu12-12.8.55-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dd0780f1a55c21d8e06a743de5bd95653de630decfff40621dbde78cc307102", size = 89896, upload-time = "2025-01-23T17:50:44.487Z" }, + { url = "https://files.pythonhosted.org/packages/e5/14/84d46e62bfde46dd20cfb041e0bb5c2ec454fd6a384696e7fa3463c5bb59/nvidia_nvtx_cu12-12.8.55-py3-none-win_amd64.whl", hash = "sha256:9022681677aef1313458f88353ad9c0d2fbbe6402d6b07c9f00ba0e3ca8774d3", size = 56435, upload-time = "2025-01-23T18:06:06.268Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/c0/1b303feea90d296f6176f32a2a70b5ef230f9bdeb3a72bddb0dc922dc137/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615", size = 91161, upload-time = "2025-03-07T01:42:23.922Z" }, + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/9f/99/4c9c0c329bf9fc125008c3b54c7c94c0023518d06fc025ae36431375e1fe/nvidia_nvtx_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e", size = 56492, upload-time = "2025-03-07T01:52:24.69Z" }, +] + +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + +[[package]] +name = "omegaconf" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, +] + +[[package]] +name = "openai" +version = "1.109.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/a1/a303104dc55fc546a3f6914c842d3da471c64eec92043aef8f652eb6c524/openai-1.109.1.tar.gz", hash = "sha256:d173ed8dbca665892a6db099b4a2dfac624f94d20a93f46eb0b56aae940ed869", size = 564133, upload-time = "2025-09-24T13:00:53.075Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/2a/7dd3d207ec669cacc1f186fd856a0f61dbc255d24f6fdc1a6715d6051b0f/openai-1.109.1-py3-none-any.whl", hash = "sha256:6bcaf57086cf59159b8e27447e4e7dd019db5d29a438072fbd49c290c7e65315", size = 948627, upload-time = "2025-09-24T13:00:50.754Z" }, +] + +[[package]] +name = "openai-harmony" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/94/01509d510bebf6606614e51113e5a415ced15b8f34aa98a8bf2539314650/openai_harmony-0.0.4.tar.gz", hash = "sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c", size = 279848, upload-time = "2025-08-09T01:43:24.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/3e/6bb75a4d15a6aad0ba1b23193ca0d2c202cc1f3364ba840833374b7c9c1a/openai_harmony-0.0.4-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3586d90c899cd41f8624e7b82a48c289f6e4be56c66304ecaf3a0ba88963a73f", size = 2772770, upload-time = "2025-08-09T01:43:14.839Z" }, + { url = "https://files.pythonhosted.org/packages/34/41/2f256fba6762d028ed6f935f0015f71d81927a52b9a1c873679a409b72bf/openai_harmony-0.0.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ef21a1e2384a65c62d5ec5e1cded9fe026f1d032d5c5d725110d1a8d330d8f54", size = 2633682, upload-time = "2025-08-09T01:43:12.681Z" }, + { url = "https://files.pythonhosted.org/packages/05/88/ade63bd8f36603610040e7cc086bc134d57a99a742e05f7fcddfdf822ee1/openai_harmony-0.0.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf2344366f10981bbc0f6d9949a0b2bb87151d209ed295943ed6ad8eda37932", size = 2963206, upload-time = "2025-08-09T01:43:02.433Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ef/a65a0ff177fdf67bc0afd18bb9e7ad690d1b553a8eb5ebf27f601b22dbd0/openai_harmony-0.0.4-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2d8d16d84702059833fb03b841b28c25600c54e83cadccef79af44e1c81166b1", size = 2724854, upload-time = "2025-08-09T01:43:04.606Z" }, + { url = "https://files.pythonhosted.org/packages/8a/a1/ebaf0f55601a98609641283884d52dbfe9a1cf34b04f1cf80acb1560ab74/openai_harmony-0.0.4-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97f1fe3909733212cc6b36f0f199b1421a9c57b79ec665f0322bd604cec47340", size = 2984312, upload-time = "2025-08-09T01:43:08.908Z" }, + { url = "https://files.pythonhosted.org/packages/45/24/246f6f470bfbc89a117714b68f27cdaee12b31166237a227cc657780cc1d/openai_harmony-0.0.4-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:567cc568b6bf7b4d041b0c9aa7d6b2c9394f8af6065bc87fa6d23f207b5af9a7", size = 3447870, upload-time = "2025-08-09T01:43:06.734Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ec/dcdcace0ffcf3a532cca910e0c351b62d3a7decf0b091ea8cf856d2a67a6/openai_harmony-0.0.4-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31e9bcac0902a309e2fc688e52f247eec7fffcd00d17e958b9a83a8fea6519c2", size = 3049306, upload-time = "2025-08-09T01:43:11.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/39/172f1048d935db1523a82b45fee5231ad6c622645e566706e6bcf3731da8/openai_harmony-0.0.4-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:96a63199c0d81095b5d5d1ae8ca82b64c1c13d18d4e30323ae9e8ab31bc80a3d", size = 3121347, upload-time = "2025-08-09T01:43:16.705Z" }, + { url = "https://files.pythonhosted.org/packages/6b/36/8ee4ca5d0b25587121fd3621e6a6106fba80218cb6d159e1670aeb2b22ef/openai_harmony-0.0.4-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d38f2639f6bf7c3c34a5dfd79e29075811ae2fa9b895a63e76767f74a47a971e", size = 2952326, upload-time = "2025-08-09T01:43:18.841Z" }, + { url = "https://files.pythonhosted.org/packages/ae/a0/ec8906393968679e269e23e957e11ff419978d1d077fb9af9561b161c988/openai_harmony-0.0.4-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:038f1d6772d1be5213b36ae76e5d042022395ec35c428a73ccb8b839b2cecf6a", size = 3015832, upload-time = "2025-08-09T01:43:21.076Z" }, + { url = "https://files.pythonhosted.org/packages/a8/bd/aa9e6e5cf140716dbcae17402fac2a81a9ebb3f934059ac0eec61cb447fc/openai_harmony-0.0.4-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:15e6d53a66502491a3675a536df30e271f976e6c5efe68250a65191efcb85c4f", size = 3221129, upload-time = "2025-08-09T01:43:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/5a/22/2c7e1728689c7fa98a259ca2d14e718ea7af964516a617a9784f0d35d88a/openai_harmony-0.0.4-cp38-abi3-win32.whl", hash = "sha256:b9ee9e9ab6a237cebbe16563c787a6e83f3fcc034075c3d321dab94448426282", size = 2077125, upload-time = "2025-08-09T01:43:28.91Z" }, + { url = "https://files.pythonhosted.org/packages/e7/93/3a08a06ff3bde7f4c264f86d437e6a5c49792a6e362383b3a669f39c9690/openai_harmony-0.0.4-cp38-abi3-win_amd64.whl", hash = "sha256:746f751de5033b3dbcfcd4a726a4c56ce452c593ad3d54472d8597ce8d8b6d44", size = 2444821, upload-time = "2025-08-09T01:43:26.846Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "4.11.0.86" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929, upload-time = "2025-01-16T13:53:40.22Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/53/2c50afa0b1e05ecdb4603818e85f7d174e683d874ef63a6abe3ac92220c8/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:48128188ade4a7e517237c8e1e11a9cdf5c282761473383e77beb875bb1e61ca", size = 37326460, upload-time = "2025-01-16T13:52:57.015Z" }, + { url = "https://files.pythonhosted.org/packages/3b/43/68555327df94bb9b59a1fd645f63fafb0762515344d2046698762fc19d58/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81", size = 56723330, upload-time = "2025-01-16T13:55:45.731Z" }, + { url = "https://files.pythonhosted.org/packages/45/be/1438ce43ebe65317344a87e4b150865c5585f4c0db880a34cdae5ac46881/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6efabcaa9df731f29e5ea9051776715b1bdd1845d7c9530065c7951d2a2899eb", size = 29487060, upload-time = "2025-01-16T13:51:59.625Z" }, + { url = "https://files.pythonhosted.org/packages/dd/5c/c139a7876099916879609372bfa513b7f1257f7f1a908b0bdc1c2328241b/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b", size = 49969856, upload-time = "2025-01-16T13:53:29.654Z" }, + { url = "https://files.pythonhosted.org/packages/95/dd/ed1191c9dc91abcc9f752b499b7928aacabf10567bb2c2535944d848af18/opencv_python_headless-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b", size = 29324425, upload-time = "2025-01-16T13:52:49.048Z" }, + { url = "https://files.pythonhosted.org/packages/86/8a/69176a64335aed183529207ba8bc3d329c2999d852b4f3818027203f50e6/opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca", size = 39402386, upload-time = "2025-01-16T13:52:56.418Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "4.12.0.88" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/63/6861102ec149c3cd298f4d1ea7ce9d6adbc7529221606ff1dab991a19adb/opencv-python-headless-4.12.0.88.tar.gz", hash = "sha256:cfdc017ddf2e59b6c2f53bc12d74b6b0be7ded4ec59083ea70763921af2b6c09", size = 95379675, upload-time = "2025-07-07T09:21:06.815Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/7d/414e243c5c8216a5277afd104a319cc1291c5e23f5eeef512db5629ee7f4/opencv_python_headless-4.12.0.88-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:1e58d664809b3350c1123484dd441e1667cd7bed3086db1b9ea1b6f6cb20b50e", size = 37877864, upload-time = "2025-07-07T09:14:41.693Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/7e162714beed1cd5e7b5eb66fcbcba2f065c51b1d9da2463024c84d2f7c0/opencv_python_headless-4.12.0.88-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:365bb2e486b50feffc2d07a405b953a8f3e8eaa63865bc650034e5c71e7a5154", size = 57326608, upload-time = "2025-07-07T09:14:51.885Z" }, + { url = "https://files.pythonhosted.org/packages/69/4e/116720df7f1f7f3b59abc608ca30fbec9d2b3ae810afe4e4d26483d9dfa0/opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:aeb4b13ecb8b4a0beb2668ea07928160ea7c2cd2d9b5ef571bbee6bafe9cc8d0", size = 33145800, upload-time = "2025-07-07T09:15:00.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/53/e19c21e0c4eb1275c3e2c97b081103b6dfb3938172264d283a519bf728b9/opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:236c8df54a90f4d02076e6f9c1cc763d794542e886c576a6fee46ec8ff75a7a9", size = 54023419, upload-time = "2025-07-07T09:15:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9c/a76fd5414de6ec9f21f763a600058a0c3e290053cea87e0275692b1375c0/opencv_python_headless-4.12.0.88-cp37-abi3-win32.whl", hash = "sha256:fde2cf5c51e4def5f2132d78e0c08f9c14783cd67356922182c6845b9af87dbd", size = 30225230, upload-time = "2025-07-07T09:15:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/f2/35/0858e9e71b36948eafbc5e835874b63e515179dc3b742cbe3d76bc683439/opencv_python_headless-4.12.0.88-cp37-abi3-win_amd64.whl", hash = "sha256:86b413bdd6c6bf497832e346cd5371995de148e579b9774f8eba686dee3f5528", size = 38923559, upload-time = "2025-07-07T09:15:25.229Z" }, +] + +[[package]] +name = "openhands-aci" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "binaryornot" }, + { name = "cachetools" }, + { name = "charset-normalizer" }, + { name = "flake8" }, + { name = "gitpython" }, + { name = "grep-ast" }, + { name = "pydantic" }, + { name = "tree-sitter" }, + { name = "tree-sitter-javascript" }, + { name = "tree-sitter-python" }, + { name = "tree-sitter-ruby" }, + { name = "tree-sitter-typescript" }, + { name = "whatthepatch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/1a/64ef2cab9012dba0b95cc2b7930fab5811c863f0b4d0400ef8cba2d185c1/openhands_aci-0.2.13.tar.gz", hash = "sha256:a34a73a5d2a5230a096578b71b98af2d2724e6ea47db6dec61b3f981e8cc3b38", size = 18854, upload-time = "2025-05-12T14:12:15.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/06/50b98f0013949dcbca4453e81f66963999ef69d5dcbdf64ae0c4cd690419/openhands_aci-0.2.13-py3-none-any.whl", hash = "sha256:f95447b23b5b66dd8f6be6c112f25c4fbe8d916d4b178864fa2ffbc8522e4367", size = 24509, upload-time = "2025-05-12T14:12:14.746Z" }, +] + +[[package]] +name = "openhands-ai" +version = "0.38.0" +source = { editable = "SkyRL-OpenHands" } +dependencies = [ + { name = "aiodocker" }, + { name = "aiohttp" }, + { name = "anthropic", extra = ["vertex"] }, + { name = "anyio" }, + { name = "bashlex" }, + { name = "boto3" }, + { name = "browsergym-core" }, + { name = "daytona-sdk" }, + { name = "dirhash" }, + { name = "docker" }, + { name = "e2b" }, + { name = "fastapi" }, + { name = "google-api-python-client" }, + { name = "google-auth-httplib2" }, + { name = "google-auth-oauthlib" }, + { name = "google-cloud-aiplatform" }, + { name = "google-generativeai" }, + { name = "html2text" }, + { name = "ipywidgets" }, + { name = "jinja2" }, + { name = "joblib" }, + { name = "json-repair" }, + { name = "libtmux" }, + { name = "litellm" }, + { name = "mcp" }, + { name = "mcpm" }, + { name = "memory-profiler" }, + { name = "minio" }, + { name = "modal" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "openhands-aci" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "pathspec" }, + { name = "pexpect" }, + { name = "poetry" }, + { name = "prompt-toolkit" }, + { name = "protobuf" }, + { name = "psutil" }, + { name = "pydantic-settings" }, + { name = "pygithub" }, + { name = "pyjwt" }, + { name = "pylatexenc" }, + { name = "pylcs" }, + { name = "pypdf2" }, + { name = "python-docx" }, + { name = "python-dotenv" }, + { name = "python-frontmatter" }, + { name = "python-json-logger" }, + { name = "python-multipart" }, + { name = "python-pptx" }, + { name = "python-socketio" }, + { name = "pythonnet" }, + { name = "qtconsole" }, + { name = "redis" }, + { name = "runloop-api-client" }, + { name = "sse-starlette" }, + { name = "stripe" }, + { name = "tenacity" }, + { name = "termcolor" }, + { name = "toml" }, + { name = "tornado" }, + { name = "tree-sitter" }, + { name = "types-toml" }, + { name = "uvicorn" }, + { name = "whatthepatch" }, + { name = "zope-interface" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiodocker" }, + { name = "aiohttp", specifier = ">=3.9.0,!=3.11.13" }, + { name = "anthropic", extras = ["vertex"] }, + { name = "anyio", specifier = "==4.9.0" }, + { name = "bashlex", specifier = ">=0.18,<0.19" }, + { name = "boto3" }, + { name = "browsergym-core", specifier = "==0.13.3" }, + { name = "daytona-sdk", specifier = "==0.16.1" }, + { name = "dirhash" }, + { name = "docker" }, + { name = "e2b", specifier = ">=1.0.5,<1.4.0" }, + { name = "fastapi" }, + { name = "google-api-python-client", specifier = ">=2.164.0,<3.0.0" }, + { name = "google-auth-httplib2" }, + { name = "google-auth-oauthlib" }, + { name = "google-cloud-aiplatform" }, + { name = "google-generativeai" }, + { name = "html2text" }, + { name = "ipywidgets", specifier = ">=8.1.5,<9.0.0" }, + { name = "jinja2", specifier = ">=3.1.3,<4.0.0" }, + { name = "joblib" }, + { name = "json-repair" }, + { name = "libtmux", specifier = ">=0.37,<0.40" }, + { name = "litellm", specifier = ">=1.60.0,!=1.64.4,!=1.67.*,<2.0.0" }, + { name = "mcp", specifier = "==1.7.1" }, + { name = "mcpm", specifier = "==1.9.0" }, + { name = "memory-profiler", specifier = ">=0.61.0,<0.62.0" }, + { name = "minio", specifier = ">=7.2.8,<8.0.0" }, + { name = "modal", specifier = ">=0.66.26,<0.78.0" }, + { name = "numpy" }, + { name = "openhands-aci", specifier = "==0.2.13" }, + { name = "opentelemetry-api", specifier = "==1.25.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = "==1.25.0" }, + { name = "pathspec", specifier = ">=0.12.1,<0.13.0" }, + { name = "pexpect" }, + { name = "poetry", specifier = ">=2.1.2,<3.0.0" }, + { name = "prompt-toolkit", specifier = ">=3.0.50,<4.0.0" }, + { name = "protobuf", specifier = ">=4.21.6,<5.0.0" }, + { name = "psutil" }, + { name = "pydantic-settings" }, + { name = "pygithub", specifier = ">=2.5.0,<3.0.0" }, + { name = "pyjwt", specifier = ">=2.9.0,<3.0.0" }, + { name = "pylatexenc" }, + { name = "pylcs", specifier = ">=0.1.1,<0.2.0" }, + { name = "pypdf2" }, + { name = "python-docx" }, + { name = "python-dotenv" }, + { name = "python-frontmatter", specifier = ">=1.1.0,<2.0.0" }, + { name = "python-json-logger", specifier = ">=3.2.1,<4.0.0" }, + { name = "python-multipart" }, + { name = "python-pptx" }, + { name = "python-socketio", specifier = ">=5.11.4,<6.0.0" }, + { name = "pythonnet" }, + { name = "qtconsole", specifier = ">=5.6.1,<6.0.0" }, + { name = "redis", specifier = ">=5.2,<7.0" }, + { name = "runloop-api-client", specifier = "==0.32.0" }, + { name = "sse-starlette", specifier = ">=2.1.3,<3.0.0" }, + { name = "stripe", specifier = ">=11.5,<13.0" }, + { name = "tenacity", specifier = ">=8.5,<10.0" }, + { name = "termcolor" }, + { name = "toml" }, + { name = "tornado" }, + { name = "tree-sitter", specifier = ">=0.24.0,<0.25.0" }, + { name = "types-toml" }, + { name = "uvicorn" }, + { name = "whatthepatch", specifier = ">=1.0.6,<2.0.0" }, + { name = "zope-interface", specifier = "==7.2" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "importlib-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/d4/e9a0ddef6eed086c96e8265d864a46da099611b7be153b0cfb63fd47e1b4/opentelemetry_api-1.26.0.tar.gz", hash = "sha256:2bd639e4bed5b18486fef0b5a520aaffde5a18fc225e808a1ac4df363f43a1ce", size = 60904, upload-time = "2024-07-25T04:02:03.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/a7/6322d1d7a1fb926e8b99208c27730f21217da2f1e0e11dab48a78a0427a4/opentelemetry_api-1.26.0-py3-none-any.whl", hash = "sha256:7d7ea33adf2ceda2dd680b18b1677e4152000b37ca76e679da71ff103b943064", size = 61533, upload-time = "2024-07-25T04:01:38.504Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/99/80edf6286f9040fadf065f9a11869fda34449a61e62a5372cb84d5a6f53b/opentelemetry_exporter_otlp-1.26.0.tar.gz", hash = "sha256:cf0e093f080011951d9f97431a83869761e4d4ebe83a4195ee92d7806223299c", size = 6168, upload-time = "2024-07-25T04:02:05.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/71/b9221af6af61213c522401b5f46a5eaa41d8dd7daeb0740dc5604f5c3980/opentelemetry_exporter_otlp-1.26.0-py3-none-any.whl", hash = "sha256:f839989f54bda85ee33c5dae033c44dcec9ccbb0dafc6a43d585df44da1d2036", size = 7001, upload-time = "2024-07-25T04:01:41.651Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/cd/ed9eaa1d80facb6609d02af6c393b02ce3797a15742361be4859db6fdc17/opentelemetry_exporter_otlp_proto_common-1.26.0.tar.gz", hash = "sha256:bdbe50e2e22a1c71acaa0c8ba6efaadd58882e5a5978737a44a4c4b10d304c92", size = 17815, upload-time = "2024-07-25T04:02:06.537Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/2f/0f7e0a73fd901c9abc6ea680d7f19a803dac830c450f21e1123d3a3ec488/opentelemetry_exporter_otlp_proto_common-1.26.0-py3-none-any.whl", hash = "sha256:ee4d8f8891a1b9c372abf8d109409e5b81947cf66423fd998e56880057afbc71", size = 17837, upload-time = "2024-07-25T04:01:42.942Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a0/23/cac89aca97ecb8f7498a875dc2ac89224b4f3345bcb8ffff643b59886196/opentelemetry_exporter_otlp_proto_grpc-1.26.0.tar.gz", hash = "sha256:a65b67a9a6b06ba1ec406114568e21afe88c1cdb29c464f2507d529eb906d8ae", size = 25239, upload-time = "2024-07-25T04:02:07.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/0c/e4473692fec8076008c7926dfcef7223fc6d2785f04ad9d8402347a4eba9/opentelemetry_exporter_otlp_proto_grpc-1.26.0-py3-none-any.whl", hash = "sha256:e2be5eff72ebcb010675b818e8d7c2e7d61ec451755b8de67a140bc49b9b0280", size = 18228, upload-time = "2024-07-25T04:01:44.308Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "googleapis-common-protos", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-api", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-exporter-otlp-proto-common", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-proto", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-sdk", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "requests", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/d2/4e6e2066b87626966f99f8fc7fcb9414e7548779d751def7db54c9d25b1c/opentelemetry_exporter_otlp_proto_http-1.26.0.tar.gz", hash = "sha256:5801ebbcf7b527377883e6cbbdda35ee712dc55114fff1e93dfee210be56c908", size = 14451, upload-time = "2024-07-25T04:02:08.192Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/d3/0b7217b61903249035d219fbe93a8558287f86aead340c7b2dc1226b8ad4/opentelemetry_exporter_otlp_proto_http-1.26.0-py3-none-any.whl", hash = "sha256:ee72a87c48ec977421b02f16c52ea8d884122470e0be573905237b540f4ee562", size = 16795, upload-time = "2024-07-25T04:01:45.645Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/06/9505ef04e527fa711ebffb47f3f56cac6015405953ff688fc349d170fb9c/opentelemetry_proto-1.26.0.tar.gz", hash = "sha256:c5c18796c0cab3751fc3b98dee53855835e90c0422924b484432ac852d93dc1e", size = 34749, upload-time = "2024-07-25T04:02:16.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/f4/66a3892eea913cded9bac0fdd3fb1a412fa2da8eb50014ec87a52648444a/opentelemetry_proto-1.26.0-py3-none-any.whl", hash = "sha256:6c4d7b4d4d9c88543bcf8c28ae3f8f0448a753dc291c18c5390444c90b76a725", size = 52466, upload-time = "2024-07-25T04:01:58.287Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/85/8ca0d5ebfe708287b091dffcd15553b74bbfe4532f8dd42662b78b2e0cab/opentelemetry_sdk-1.26.0.tar.gz", hash = "sha256:c90d2868f8805619535c05562d699e2f4fb1f00dbd55a86dcefca4da6fa02f85", size = 143139, upload-time = "2024-07-25T04:02:17.52Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/f1/a9b550d0f9c049653dd2eab45cecf8fe4baa9795ed143d87834056ffabaf/opentelemetry_sdk-1.26.0-py3-none-any.whl", hash = "sha256:feb5056a84a88670c041ea0ded9921fca559efec03905dddeb3885525e0af897", size = 109475, upload-time = "2024-07-25T04:01:59.997Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.47b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "opentelemetry-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/85/edef14d10ad00ddd9fffb20e4d3d938f4c5c1247e11a175066fe2b4a72f8/opentelemetry_semantic_conventions-0.47b0.tar.gz", hash = "sha256:a8d57999bbe3495ffd4d510de26a97dadc1dace53e0275001b2c1b2f67992a7e", size = 83994, upload-time = "2024-07-25T04:02:19.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/c2/ca5cef8e4cd8eec5a95deed95ec3f6005e499fd9d17ca08731ced03a6921/opentelemetry_semantic_conventions-0.47b0-py3-none-any.whl", hash = "sha256:4ff9d595b85a59c1c1413f02bba320ce7ea6bf9e2ead2b0913c4395c7bbc1063", size = 138027, upload-time = "2024-07-25T04:02:01.7Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions-ai" +version = "0.4.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/e6/40b59eda51ac47009fb47afcdf37c6938594a0bd7f3b9fadcbc6058248e3/opentelemetry_semantic_conventions_ai-0.4.13.tar.gz", hash = "sha256:94efa9fb4ffac18c45f54a3a338ffeb7eedb7e1bb4d147786e77202e159f0036", size = 5368, upload-time = "2025-08-22T10:14:17.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/b5/cf25da2218910f0d6cdf7f876a06bed118c4969eacaf60a887cbaef44f44/opentelemetry_semantic_conventions_ai-0.4.13-py3-none-any.whl", hash = "sha256:883a30a6bb5deaec0d646912b5f9f6dcbb9f6f72557b73d0f2560bf25d13e2d5", size = 6080, upload-time = "2025-08-22T10:14:16.477Z" }, +] + +[[package]] +name = "orjson" +version = "3.11.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/4d/8df5f83256a809c22c4d6792ce8d43bb503be0fb7a8e4da9025754b09658/orjson-3.11.3.tar.gz", hash = "sha256:1c0603b1d2ffcd43a411d64797a19556ef76958aef1c182f22dc30860152a98a", size = 5482394, upload-time = "2025-08-26T17:46:43.171Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/b0/a7edab2a00cdcb2688e1c943401cb3236323e7bfd2839815c6131a3742f4/orjson-3.11.3-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8c752089db84333e36d754c4baf19c0e1437012242048439c7e80eb0e6426e3b", size = 238259, upload-time = "2025-08-26T17:45:15.093Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c6/ff4865a9cc398a07a83342713b5932e4dc3cb4bf4bc04e8f83dedfc0d736/orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:9b8761b6cf04a856eb544acdd82fc594b978f12ac3602d6374a7edb9d86fd2c2", size = 127633, upload-time = "2025-08-26T17:45:16.417Z" }, + { url = "https://files.pythonhosted.org/packages/6e/e6/e00bea2d9472f44fe8794f523e548ce0ad51eb9693cf538a753a27b8bda4/orjson-3.11.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b13974dc8ac6ba22feaa867fc19135a3e01a134b4f7c9c28162fed4d615008a", size = 123061, upload-time = "2025-08-26T17:45:17.673Z" }, + { url = "https://files.pythonhosted.org/packages/54/31/9fbb78b8e1eb3ac605467cb846e1c08d0588506028b37f4ee21f978a51d4/orjson-3.11.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f83abab5bacb76d9c821fd5c07728ff224ed0e52d7a71b7b3de822f3df04e15c", size = 127956, upload-time = "2025-08-26T17:45:19.172Z" }, + { url = "https://files.pythonhosted.org/packages/36/88/b0604c22af1eed9f98d709a96302006915cfd724a7ebd27d6dd11c22d80b/orjson-3.11.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6fbaf48a744b94091a56c62897b27c31ee2da93d826aa5b207131a1e13d4064", size = 130790, upload-time = "2025-08-26T17:45:20.586Z" }, + { url = "https://files.pythonhosted.org/packages/0e/9d/1c1238ae9fffbfed51ba1e507731b3faaf6b846126a47e9649222b0fd06f/orjson-3.11.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc779b4f4bba2847d0d2940081a7b6f7b5877e05408ffbb74fa1faf4a136c424", size = 132385, upload-time = "2025-08-26T17:45:22.036Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b5/c06f1b090a1c875f337e21dd71943bc9d84087f7cdf8c6e9086902c34e42/orjson-3.11.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd4b909ce4c50faa2192da6bb684d9848d4510b736b0611b6ab4020ea6fd2d23", size = 135305, upload-time = "2025-08-26T17:45:23.4Z" }, + { url = "https://files.pythonhosted.org/packages/a0/26/5f028c7d81ad2ebbf84414ba6d6c9cac03f22f5cd0d01eb40fb2d6a06b07/orjson-3.11.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:524b765ad888dc5518bbce12c77c2e83dee1ed6b0992c1790cc5fb49bb4b6667", size = 132875, upload-time = "2025-08-26T17:45:25.182Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d4/b8df70d9cfb56e385bf39b4e915298f9ae6c61454c8154a0f5fd7efcd42e/orjson-3.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:84fd82870b97ae3cdcea9d8746e592b6d40e1e4d4527835fc520c588d2ded04f", size = 130940, upload-time = "2025-08-26T17:45:27.209Z" }, + { url = "https://files.pythonhosted.org/packages/da/5e/afe6a052ebc1a4741c792dd96e9f65bf3939d2094e8b356503b68d48f9f5/orjson-3.11.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fbecb9709111be913ae6879b07bafd4b0785b44c1eb5cac8ac76da048b3885a1", size = 403852, upload-time = "2025-08-26T17:45:28.478Z" }, + { url = "https://files.pythonhosted.org/packages/f8/90/7bbabafeb2ce65915e9247f14a56b29c9334003536009ef5b122783fe67e/orjson-3.11.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9dba358d55aee552bd868de348f4736ca5a4086d9a62e2bfbbeeb5629fe8b0cc", size = 146293, upload-time = "2025-08-26T17:45:29.86Z" }, + { url = "https://files.pythonhosted.org/packages/27/b3/2d703946447da8b093350570644a663df69448c9d9330e5f1d9cce997f20/orjson-3.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eabcf2e84f1d7105f84580e03012270c7e97ecb1fb1618bda395061b2a84a049", size = 135470, upload-time = "2025-08-26T17:45:31.243Z" }, + { url = "https://files.pythonhosted.org/packages/38/70/b14dcfae7aff0e379b0119c8a812f8396678919c431efccc8e8a0263e4d9/orjson-3.11.3-cp312-cp312-win32.whl", hash = "sha256:3782d2c60b8116772aea8d9b7905221437fdf53e7277282e8d8b07c220f96cca", size = 136248, upload-time = "2025-08-26T17:45:32.567Z" }, + { url = "https://files.pythonhosted.org/packages/35/b8/9e3127d65de7fff243f7f3e53f59a531bf6bb295ebe5db024c2503cc0726/orjson-3.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:79b44319268af2eaa3e315b92298de9a0067ade6e6003ddaef72f8e0bedb94f1", size = 131437, upload-time = "2025-08-26T17:45:34.949Z" }, + { url = "https://files.pythonhosted.org/packages/51/92/a946e737d4d8a7fd84a606aba96220043dcc7d6988b9e7551f7f6d5ba5ad/orjson-3.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:0e92a4e83341ef79d835ca21b8bd13e27c859e4e9e4d7b63defc6e58462a3710", size = 125978, upload-time = "2025-08-26T17:45:36.422Z" }, +] + +[[package]] +name = "outlines" +version = "0.1.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "airportsdata", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "cloudpickle", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "diskcache", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "interegular", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "jinja2", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "jsonschema", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "lark", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "nest-asyncio", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "outlines-core", version = "0.1.26", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "pycountry", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "pydantic", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "referencing", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "requests", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "tqdm", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "typing-extensions", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/d0/d59ae830bf7026425942899e3d48e77b58a713cff946a695e5405808da1b/outlines-0.1.11.tar.gz", hash = "sha256:0997bd9da1cc050e430bd08995dc7d4bd855918bafa4531e49d3f37110a23aba", size = 2488858, upload-time = "2024-12-13T07:24:08.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/b4/99ea4a122bef60e3fd6402d19665aff1f928e0daf8fac3044d0b73f72003/outlines-0.1.11-py3-none-any.whl", hash = "sha256:f5a5f2242ed9802d3aab7a92789bf4008d734c576be9258cc0a297f690124727", size = 87623, upload-time = "2024-12-13T07:24:05.817Z" }, +] + +[[package]] +name = "outlines-core" +version = "0.1.26" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "interegular", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "jsonschema", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/f3/274d07f4702728b43581235a77e545ec602b25f9b0098b288a0f3052521d/outlines_core-0.1.26.tar.gz", hash = "sha256:481c4301341e77cc8f1832d616784adb4d461b4fec65878e7c0d2cba7163a189", size = 75139, upload-time = "2024-12-12T23:38:50.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/86/0fb40746e579db38d89f127122a3900d9e0350f76aae8cb61adeaff44cc2/outlines_core-0.1.26-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f54633bca50055d42ea4d94ae06dcbe52d3d76a9b621b75723b1177d0d952953", size = 321874, upload-time = "2024-12-12T23:38:26.834Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0c/b91f7bc03843796c1d643ee030b6cd8fd5a8ba2cd4856c855f140c878976/outlines_core-0.1.26-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9525321b48700dcaaabf60bcdc951e45f9357ba3fb3e1bfc81b662d7d4170e7c", size = 301995, upload-time = "2024-12-12T23:38:29.625Z" }, + { url = "https://files.pythonhosted.org/packages/ad/db/fa91a2d54288b900de82d86eda3adb2417b3b5b2db6256854a5e8bc85c32/outlines_core-0.1.26-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f409f72c11f6ffadb57066950dd384d5388015028c1a1a615c9a64988dae3e", size = 321050, upload-time = "2024-12-12T23:38:32.274Z" }, + { url = "https://files.pythonhosted.org/packages/e2/1d/a36292b6198986bd9c3ff8c24355deb82ed5475403379ee40b5b5473e2e3/outlines_core-0.1.26-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86a1bb46adc5cbf6dfd7a7fe4105e0e2a4c6e041732a053126b41c521a1f223", size = 343201, upload-time = "2024-12-12T23:38:34.631Z" }, + { url = "https://files.pythonhosted.org/packages/08/63/5dd2b5a364412f674b6edcb59b0c21513bdb07cdcc7613b064c1a0660d01/outlines_core-0.1.26-cp312-cp312-win32.whl", hash = "sha256:19f462f6b00935708677ad27cb4df55e0e17f6ffe713ab750f5f2683b090f95d", size = 233970, upload-time = "2024-12-12T23:38:37.318Z" }, + { url = "https://files.pythonhosted.org/packages/a5/56/8adf0b7446d1e975c2314454813c59eb7b195889908a2932ed34148c113c/outlines_core-0.1.26-cp312-cp312-win_amd64.whl", hash = "sha256:9b36bff12779e58883747116893a17b3551bbd10865878b951b03a44d112229a", size = 243578, upload-time = "2024-12-12T23:38:39.964Z" }, +] + +[[package]] +name = "outlines-core" +version = "0.2.10" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/13/e2/de74a5fd00299215270a750f356ba7cb427ba5d3e495cab07cfc3110ca86/outlines_core-0.2.10.tar.gz", hash = "sha256:c9ee7be195ac18dda5acce41d8805c2fb550a4affd525414511662cfa7097dfe", size = 197140, upload-time = "2025-05-12T18:20:27.301Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/d6/95a02a498ee09cdc7fbb69b780ccdecaf3afd7033e8510b0f5e715d0b269/outlines_core-0.2.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:32615f6fe4286d80699e9e6537eecbde387bf73d87751858f7a0693947381cdc", size = 1946785, upload-time = "2025-05-12T18:19:41.847Z" }, + { url = "https://files.pythonhosted.org/packages/89/02/eb2bbcb0d9b860e0aedd7ea4c65361a15c2b8180f495ecda4361a8f8d218/outlines_core-0.2.10-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:db3e07c999ee17035114f20263c7200bf5bea0f643d2d026671eb5cfc2a9cf71", size = 2121685, upload-time = "2025-05-12T18:19:44.7Z" }, + { url = "https://files.pythonhosted.org/packages/ea/61/e3acafae52fba50e9134e685c98e89dc84c4a80776b8991937308807cb6c/outlines_core-0.2.10-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e39847ab495ec9923dc1a59ccab04ef7888b5e066bc5856b5cb7fe98e9663f3d", size = 1945900, upload-time = "2025-05-12T18:19:46.359Z" }, + { url = "https://files.pythonhosted.org/packages/07/25/be941feb461df2af76a4301b0c4dfc210a939a316f3219c4adc97371d135/outlines_core-0.2.10-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:f543f23b263c0b010860ab5ea760b2be566b604315e6a89499632758ca177a5d", size = 2117245, upload-time = "2025-05-12T18:19:48.069Z" }, + { url = "https://files.pythonhosted.org/packages/c2/83/db792ce386d1c13d875a03d6ff5ba31612cfb558ecf5b945910db9505574/outlines_core-0.2.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8666735ec367a06e26331e164a80a4c2936b349713ac05ab53650e2997a30891", size = 2273188, upload-time = "2025-05-12T18:19:49.759Z" }, + { url = "https://files.pythonhosted.org/packages/86/82/dd116270e984f5a3936ec747404676cc4c306a92f4f0fcc077ac6be90024/outlines_core-0.2.10-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:faf5b43181b1d033871364e74e9d348362c6a77b1d054d7af35e09fdfcff5b16", size = 2101406, upload-time = "2025-05-12T18:19:51.54Z" }, + { url = "https://files.pythonhosted.org/packages/41/3a/63e9729dadd962e83dabbd52d600d9b8dd7154d92b6f351c10b514bd121f/outlines_core-0.2.10-cp312-cp312-win32.whl", hash = "sha256:b37e192de974fdbfe20332720a4a9cdda92719536dface60b48dc8eeeda24390", size = 1761638, upload-time = "2025-05-12T18:19:53.252Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e4/e6cb03a092043997babcee5d52d37b59254b265751bcf0f0e75aba11fc1d/outlines_core-0.2.10-cp312-cp312-win_amd64.whl", hash = "sha256:f895834da0a577120dcb8d979c12c0690fe912095413bf0070a73e9ff363b7bf", size = 2053750, upload-time = "2025-05-12T18:19:55.587Z" }, +] + +[[package]] +name = "outlines-core" +version = "0.2.11" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/d3/e04e9145f8f806723dec9b9e5227ad695a3efcd3ced7794cf7c22b15df5e/outlines_core-0.2.11.tar.gz", hash = "sha256:dfce56f717ff5083e54cbcfdb66cad243365437fccbb5509adaa7e31e030f1d8", size = 197263, upload-time = "2025-05-19T10:12:51.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/2c/c7636823244c70e2960060bf9bd978248dffb55c5e7c91c46d18354b2a24/outlines_core-0.2.11-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4a9db4872bae083631d720994f4cee603bce0536b33d5a988814576863b657cf", size = 1957668, upload-time = "2025-05-19T10:12:18.29Z" }, + { url = "https://files.pythonhosted.org/packages/c7/09/5c62047da139d722317a444a4d01cd5f11943a8c2eaecce784341dd0844a/outlines_core-0.2.11-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:8359a45c59f6a8f2eb717245806501a59044c75f6ea8bd08faaa131cc8cdec45", size = 2130493, upload-time = "2025-05-19T10:12:19.537Z" }, + { url = "https://files.pythonhosted.org/packages/89/7a/d6a2810f90e37d550168e0c0a9a915086ea721444727e3ca2c630898d1ef/outlines_core-0.2.11-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:5d26a46591377340e0b870b8a96ea8341058341a62ee0bded9098e0c88dd24f4", size = 1956804, upload-time = "2025-05-19T10:12:20.755Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ea/339e6c273b5581128c3b7ca27d428d8993c3085912af1a467aa32ef0e9d1/outlines_core-0.2.11-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:ae460a34675fb11d92a5c605a480fbae4cd6c1b2d11b3698da64a7fcaba64dcf", size = 2127085, upload-time = "2025-05-19T10:12:22.02Z" }, + { url = "https://files.pythonhosted.org/packages/92/c7/a65d1fddf49830ebc41422294eacde35286d9f68994a8aa905cb14f5aade/outlines_core-0.2.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86df9740368866295077346440d911df4972da2b3f1f54b8125e6f329e8a8891", size = 2287677, upload-time = "2025-05-19T10:12:24.24Z" }, + { url = "https://files.pythonhosted.org/packages/23/79/8795aed8be9b77dd69d78e7cfbfcf28c179e6b08da6e56bbbf48a09fe55f/outlines_core-0.2.11-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:96ce4dd78f106799be4a0a5795cefd1352806162973756a4b6fce4bb6eddd7e4", size = 2113000, upload-time = "2025-05-19T10:12:25.446Z" }, + { url = "https://files.pythonhosted.org/packages/59/e3/cbe9294b06d92ee1892dbb6f2125d833d68e8629d45d080d6daba54eec2d/outlines_core-0.2.11-cp312-cp312-win32.whl", hash = "sha256:358db161cce3650ba822e118dcf0a1efa571c7deb4864ab9d64ca2c9cca7425d", size = 1765703, upload-time = "2025-05-19T10:12:26.693Z" }, + { url = "https://files.pythonhosted.org/packages/1d/c9/ed3cf362515fac16e313368b9b2f2497051f4ded88679205830b6f889f54/outlines_core-0.2.11-cp312-cp312-win_amd64.whl", hash = "sha256:231f9d20d2630c70665345821780d7808b29539620a75c99f65113b518c51032", size = 2060945, upload-time = "2025-05-19T10:12:28.294Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, + { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, + { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, + { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, + { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, +] + +[[package]] +name = "parso" +version = "0.8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/de/53e0bcf53d13e005bd8c92e7855142494f41171b34c2536b86187474184d/parso-0.8.5.tar.gz", hash = "sha256:034d7354a9a018bdce352f48b2a8a450f05e9d6ee85db84764e9b6bd96dafe5a", size = 401205, upload-time = "2025-08-23T15:15:28.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl", hash = "sha256:646204b5ee239c396d040b90f9e272e9a8017c630092bf59980beb62fd033887", size = 106668, upload-time = "2025-08-23T15:15:25.663Z" }, +] + +[[package]] +name = "partial-json-parser" +version = "0.2.1.1.post6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/13/459e86c9c67a006651803a3df3d0b08f7708bc5483fdc482582d75562949/partial_json_parser-0.2.1.1.post6.tar.gz", hash = "sha256:43896b68929678224cbbe4884a6a5fe9251ded4b30b8b7d7eb569e5feea93afc", size = 10299, upload-time = "2025-06-23T17:51:45.372Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/40/1f922794af3dc7503f19319a8804b398a161a2cd54183cff8b12225b8d85/partial_json_parser-0.2.1.1.post6-py3-none-any.whl", hash = "sha256:abc332f09b13ef5233384dbfe7128a0e9ea3fa4b8f8be9b37ac1b433c810e99e", size = 10876, upload-time = "2025-06-23T17:51:44.332Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "pbs-installer" +version = "2025.9.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/db/077daf042cbbb627275c4345d56fe3f8c2d394673d2b5c510958f84e4b0c/pbs_installer-2025.9.18.tar.gz", hash = "sha256:c0a51a7c1e015723bd8396f02e15b5876e439f74b0f45bbac436b189f903219f", size = 59190, upload-time = "2025-09-19T22:03:12.987Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/fd/a8bab37d8429e2fe2bbda7c1382087d8a8fbe9cf387e4ee5245402aee181/pbs_installer-2025.9.18-py3-none-any.whl", hash = "sha256:8ef55d7675698747505c237015d14c81759bd66a0d4c8b20cec9a2dc96e8434c", size = 60891, upload-time = "2025-09-19T22:03:11.365Z" }, +] + +[package.optional-dependencies] +download = [ + { name = "httpx" }, +] +install = [ + { name = "zstandard" }, +] + +[[package]] +name = "peft" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "accelerate" }, + { name = "huggingface-hub" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "tqdm" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/b8/2e79377efaa1e5f0d70a497db7914ffd355846e760ffa2f7883ab0f600fb/peft-0.17.1.tar.gz", hash = "sha256:e6002b42517976c290b3b8bbb9829a33dd5d470676b2dec7cb4df8501b77eb9f", size = 568192, upload-time = "2025-08-21T09:25:22.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/fe/a2da1627aa9cb6310b6034598363bd26ac301c4a99d21f415b1b2855891e/peft-0.17.1-py3-none-any.whl", hash = "sha256:3d129d64def3d74779c32a080d2567e5f7b674e77d546e3585138216d903f99e", size = 504896, upload-time = "2025-08-21T09:25:18.974Z" }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + +[[package]] +name = "pillow" +version = "11.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/d0d6dea55cd152ce3d6767bb38a8fc10e33796ba4ba210cbab9354b6d238/pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523", size = 47113069, upload-time = "2025-07-01T09:16:30.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/fe/1bc9b3ee13f68487a99ac9529968035cca2f0a51ec36892060edcc51d06a/pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4", size = 5278800, upload-time = "2025-07-01T09:14:17.648Z" }, + { url = "https://files.pythonhosted.org/packages/2c/32/7e2ac19b5713657384cec55f89065fb306b06af008cfd87e572035b27119/pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69", size = 4686296, upload-time = "2025-07-01T09:14:19.828Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1e/b9e12bbe6e4c2220effebc09ea0923a07a6da1e1f1bfbc8d7d29a01ce32b/pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d", size = 5871726, upload-time = "2025-07-03T13:10:04.448Z" }, + { url = "https://files.pythonhosted.org/packages/8d/33/e9200d2bd7ba00dc3ddb78df1198a6e80d7669cce6c2bdbeb2530a74ec58/pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6", size = 7644652, upload-time = "2025-07-03T13:10:10.391Z" }, + { url = "https://files.pythonhosted.org/packages/41/f1/6f2427a26fc683e00d985bc391bdd76d8dd4e92fac33d841127eb8fb2313/pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7", size = 5977787, upload-time = "2025-07-01T09:14:21.63Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c9/06dd4a38974e24f932ff5f98ea3c546ce3f8c995d3f0985f8e5ba48bba19/pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024", size = 6645236, upload-time = "2025-07-01T09:14:23.321Z" }, + { url = "https://files.pythonhosted.org/packages/40/e7/848f69fb79843b3d91241bad658e9c14f39a32f71a301bcd1d139416d1be/pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809", size = 6086950, upload-time = "2025-07-01T09:14:25.237Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1a/7cff92e695a2a29ac1958c2a0fe4c0b2393b60aac13b04a4fe2735cad52d/pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d", size = 6723358, upload-time = "2025-07-01T09:14:27.053Z" }, + { url = "https://files.pythonhosted.org/packages/26/7d/73699ad77895f69edff76b0f332acc3d497f22f5d75e5360f78cbcaff248/pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149", size = 6275079, upload-time = "2025-07-01T09:14:30.104Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ce/e7dfc873bdd9828f3b6e5c2bbb74e47a98ec23cc5c74fc4e54462f0d9204/pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d", size = 6986324, upload-time = "2025-07-01T09:14:31.899Z" }, + { url = "https://files.pythonhosted.org/packages/16/8f/b13447d1bf0b1f7467ce7d86f6e6edf66c0ad7cf44cf5c87a37f9bed9936/pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542", size = 2423067, upload-time = "2025-07-01T09:14:33.709Z" }, +] + +[[package]] +name = "pkginfo" +version = "1.12.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/03/e26bf3d6453b7fda5bd2b84029a426553bb373d6277ef6b5ac8863421f87/pkginfo-1.12.1.2.tar.gz", hash = "sha256:5cd957824ac36f140260964eba3c6be6442a8359b8c48f4adf90210f33a04b7b", size = 451828, upload-time = "2025-02-19T15:27:37.188Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/3d/f4f2ba829efb54b6cd2d91349c7463316a9cc55a43fc980447416c88540f/pkginfo-1.12.1.2-py3-none-any.whl", hash = "sha256:c783ac885519cab2c34927ccfa6bf64b5a704d7c69afaea583dd9b7afe969343", size = 32717, upload-time = "2025-02-19T15:27:33.071Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, +] + +[[package]] +name = "playwright" +version = "1.55.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/3a/c81ff76df266c62e24f19718df9c168f49af93cabdbc4608ae29656a9986/playwright-1.55.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:d7da108a95001e412effca4f7610de79da1637ccdf670b1ae3fdc08b9694c034", size = 40428109, upload-time = "2025-08-28T15:46:20.357Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f5/bdb61553b20e907196a38d864602a9b4a461660c3a111c67a35179b636fa/playwright-1.55.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8290cf27a5d542e2682ac274da423941f879d07b001f6575a5a3a257b1d4ba1c", size = 38687254, upload-time = "2025-08-28T15:46:23.925Z" }, + { url = "https://files.pythonhosted.org/packages/4a/64/48b2837ef396487807e5ab53c76465747e34c7143fac4a084ef349c293a8/playwright-1.55.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:25b0d6b3fd991c315cca33c802cf617d52980108ab8431e3e1d37b5de755c10e", size = 40428108, upload-time = "2025-08-28T15:46:27.119Z" }, + { url = "https://files.pythonhosted.org/packages/08/33/858312628aa16a6de97839adc2ca28031ebc5391f96b6fb8fdf1fcb15d6c/playwright-1.55.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c6d4d8f6f8c66c483b0835569c7f0caa03230820af8e500c181c93509c92d831", size = 45905643, upload-time = "2025-08-28T15:46:30.312Z" }, + { url = "https://files.pythonhosted.org/packages/83/83/b8d06a5b5721931aa6d5916b83168e28bd891f38ff56fe92af7bdee9860f/playwright-1.55.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29a0777c4ce1273acf90c87e4ae2fe0130182100d99bcd2ae5bf486093044838", size = 45296647, upload-time = "2025-08-28T15:46:33.221Z" }, + { url = "https://files.pythonhosted.org/packages/06/2e/9db64518aebcb3d6ef6cd6d4d01da741aff912c3f0314dadb61226c6a96a/playwright-1.55.0-py3-none-win32.whl", hash = "sha256:29e6d1558ad9d5b5c19cbec0a72f6a2e35e6353cd9f262e22148685b86759f90", size = 35476046, upload-time = "2025-08-28T15:46:36.184Z" }, + { url = "https://files.pythonhosted.org/packages/46/4f/9ba607fa94bb9cee3d4beb1c7b32c16efbfc9d69d5037fa85d10cafc618b/playwright-1.55.0-py3-none-win_amd64.whl", hash = "sha256:7eb5956473ca1951abb51537e6a0da55257bb2e25fc37c2b75af094a5c93736c", size = 35476048, upload-time = "2025-08-28T15:46:38.867Z" }, + { url = "https://files.pythonhosted.org/packages/21/98/5ca173c8ec906abde26c28e1ecb34887343fd71cc4136261b90036841323/playwright-1.55.0-py3-none-win_arm64.whl", hash = "sha256:012dc89ccdcbd774cdde8aeee14c08e0dd52ddb9135bf10e9db040527386bd76", size = 31225543, upload-time = "2025-08-28T15:46:41.613Z" }, +] + +[[package]] +name = "poetry" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "build" }, + { name = "cachecontrol", extra = ["filecache"] }, + { name = "cleo" }, + { name = "dulwich" }, + { name = "fastjsonschema" }, + { name = "findpython" }, + { name = "installer" }, + { name = "keyring" }, + { name = "packaging" }, + { name = "pbs-installer", extra = ["download", "install"] }, + { name = "pkginfo" }, + { name = "platformdirs" }, + { name = "poetry-core" }, + { name = "pyproject-hooks" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "shellingham" }, + { name = "tomlkit" }, + { name = "trove-classifiers" }, + { name = "virtualenv" }, + { name = "xattr", marker = "sys_platform == 'darwin' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/28/f790e21769afaaa1f326d9634f9a5c700c4cdb4c468a1707c6db0b350505/poetry-2.2.1.tar.gz", hash = "sha256:bef9aa4bb00ce4c10b28b25e7bac724094802d6958190762c45df6c12749b37c", size = 3441978, upload-time = "2025-09-21T14:51:49.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/52/abfc9449312877622aa87ece9e66b2f2d9290661d0023a963aedefda4099/poetry-2.2.1-py3-none-any.whl", hash = "sha256:f5958b908b96c5824e2acbb8b19cdef8a3351c62142d7ecff2d705396c8ca34c", size = 281560, upload-time = "2025-09-21T14:51:46.483Z" }, +] + +[[package]] +name = "poetry-core" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ef/a16c11de95b638341961765e072dfdd4c9a0be51d6b22d594c5f3255e4bb/poetry_core-2.2.1.tar.gz", hash = "sha256:97e50d8593c8729d3f49364b428583e044087ee3def1e010c6496db76bd65ac5", size = 378867, upload-time = "2025-09-21T14:27:58.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/d8/bb2f602f5e012e177e1c8560125f9770945d36622595990cf1cb794477c2/poetry_core-2.2.1-py3-none-any.whl", hash = "sha256:bdfce710edc10bfcf9ab35041605c480829be4ab23f5bc01202cfe5db8f125ab", size = 338598, upload-time = "2025-09-21T14:27:56.86Z" }, +] + +[[package]] +name = "polars" +version = "1.33.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/da/8246f1d69d7e49f96f0c5529057a19af1536621748ef214bbd4112c83b8e/polars-1.33.1.tar.gz", hash = "sha256:fa3fdc34eab52a71498264d6ff9b0aa6955eb4b0ae8add5d3cb43e4b84644007", size = 4822485, upload-time = "2025-09-09T08:37:49.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/79/c51e7e1d707d8359bcb76e543a8315b7ae14069ecf5e75262a0ecb32e044/polars-1.33.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3881c444b0f14778ba94232f077a709d435977879c1b7d7bd566b55bd1830bb5", size = 39132875, upload-time = "2025-09-09T08:36:38.609Z" }, + { url = "https://files.pythonhosted.org/packages/f8/15/1094099a1b9cb4fbff58cd8ed3af8964f4d22a5b682ea0b7bb72bf4bc3d9/polars-1.33.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:29200b89c9a461e6f06fc1660bc9c848407640ee30fe0e5ef4947cfd49d55337", size = 35638783, upload-time = "2025-09-09T08:36:43.748Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b9/9ac769e4d8e8f22b0f2e974914a63dd14dec1340cd23093de40f0d67d73b/polars-1.33.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:444940646e76342abaa47f126c70e3e40b56e8e02a9e89e5c5d1c24b086db58a", size = 39742297, upload-time = "2025-09-09T08:36:47.132Z" }, + { url = "https://files.pythonhosted.org/packages/7a/26/4c5da9f42fa067b2302fe62bcbf91faac5506c6513d910fae9548fc78d65/polars-1.33.1-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:094a37d06789286649f654f229ec4efb9376630645ba8963b70cb9c0b008b3e1", size = 36684940, upload-time = "2025-09-09T08:36:50.561Z" }, + { url = "https://files.pythonhosted.org/packages/06/a6/dc535da476c93b2efac619e04ab81081e004e4b4553352cd10e0d33a015d/polars-1.33.1-cp39-abi3-win_amd64.whl", hash = "sha256:c9781c704432a2276a185ee25898aa427f39a904fbe8fde4ae779596cdbd7a9e", size = 39456676, upload-time = "2025-09-09T08:36:54.612Z" }, + { url = "https://files.pythonhosted.org/packages/cb/4e/a4300d52dd81b58130ccadf3873f11b3c6de54836ad4a8f32bac2bd2ba17/polars-1.33.1-cp39-abi3-win_arm64.whl", hash = "sha256:c3cfddb3b78eae01a218222bdba8048529fef7e14889a71e33a5198644427642", size = 35445171, upload-time = "2025-09-09T08:36:58.043Z" }, +] + +[[package]] +name = "portalocker" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" }, +] + +[[package]] +name = "prometheus-client" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, +] + +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prometheus-client" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/6d/24d53033cf93826aa7857699a4450c1c67e5b9c710e925b1ed2b320c04df/prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e", size = 20220, upload-time = "2025-03-19T19:35:05.351Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/72/0824c18f3bc75810f55dacc2dd933f6ec829771180245ae3cc976195dec0/prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9", size = 19296, upload-time = "2025-03-19T19:35:04.323Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + +[[package]] +name = "propcache" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" }, + { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" }, + { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" }, + { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" }, + { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" }, + { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" }, + { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" }, + { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" }, + { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" }, + { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" }, + { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" }, +] + +[[package]] +name = "proto-plus" +version = "1.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/ac/87285f15f7cce6d4a008f33f1757fb5a13611ea8914eb58c3d0d26243468/proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012", size = 56142, upload-time = "2025-03-10T15:54:38.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/6d/280c4c2ce28b1593a19ad5239c8b826871fc6ec275c21afc8e1820108039/proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", size = 50163, upload-time = "2025-03-10T15:54:37.335Z" }, +] + +[[package]] +name = "protobuf" +version = "4.25.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/01/34c8d2b6354906d728703cb9d546a0e534de479e25f1b581e4094c4a85cc/protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd", size = 380920, upload-time = "2025-05-28T14:22:25.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/ff/05f34305fe6b85bbfbecbc559d423a5985605cad5eda4f47eae9e9c9c5c5/protobuf-4.25.8-cp310-abi3-win32.whl", hash = "sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0", size = 392745, upload-time = "2025-05-28T14:22:10.524Z" }, + { url = "https://files.pythonhosted.org/packages/08/35/8b8a8405c564caf4ba835b1fdf554da869954712b26d8f2a98c0e434469b/protobuf-4.25.8-cp310-abi3-win_amd64.whl", hash = "sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9", size = 413736, upload-time = "2025-05-28T14:22:13.156Z" }, + { url = "https://files.pythonhosted.org/packages/28/d7/ab27049a035b258dab43445eb6ec84a26277b16105b277cbe0a7698bdc6c/protobuf-4.25.8-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f", size = 394537, upload-time = "2025-05-28T14:22:14.768Z" }, + { url = "https://files.pythonhosted.org/packages/bd/6d/a4a198b61808dd3d1ee187082ccc21499bc949d639feb948961b48be9a7e/protobuf-4.25.8-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7", size = 294005, upload-time = "2025-05-28T14:22:16.052Z" }, + { url = "https://files.pythonhosted.org/packages/d6/c6/c9deaa6e789b6fc41b88ccbdfe7a42d2b82663248b715f55aa77fbc00724/protobuf-4.25.8-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0", size = 294924, upload-time = "2025-05-28T14:22:17.105Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c1/6aece0ab5209981a70cd186f164c133fdba2f51e124ff92b73de7fd24d78/protobuf-4.25.8-py3-none-any.whl", hash = "sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59", size = 156757, upload-time = "2025-05-28T14:22:24.135Z" }, +] + +[[package]] +name = "psutil" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/31/4723d756b59344b643542936e37a31d1d3204bcdc42a7daa8ee9eb06fb50/psutil-7.1.0.tar.gz", hash = "sha256:655708b3c069387c8b77b072fc429a57d0e214221d01c0a772df7dfedcb3bcd2", size = 497660, upload-time = "2025-09-17T20:14:52.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/62/ce4051019ee20ce0ed74432dd73a5bb087a6704284a470bb8adff69a0932/psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76168cef4397494250e9f4e73eb3752b146de1dd950040b29186d0cce1d5ca13", size = 245242, upload-time = "2025-09-17T20:14:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/38/61/f76959fba841bf5b61123fbf4b650886dc4094c6858008b5bf73d9057216/psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:5d007560c8c372efdff9e4579c2846d71de737e4605f611437255e81efcca2c5", size = 246682, upload-time = "2025-09-17T20:14:58.25Z" }, + { url = "https://files.pythonhosted.org/packages/88/7a/37c99d2e77ec30d63398ffa6a660450b8a62517cabe44b3e9bae97696e8d/psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22e4454970b32472ce7deaa45d045b34d3648ce478e26a04c7e858a0a6e75ff3", size = 287994, upload-time = "2025-09-17T20:14:59.901Z" }, + { url = "https://files.pythonhosted.org/packages/9d/de/04c8c61232f7244aa0a4b9a9fbd63a89d5aeaf94b2fc9d1d16e2faa5cbb0/psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c70e113920d51e89f212dd7be06219a9b88014e63a4cec69b684c327bc474e3", size = 291163, upload-time = "2025-09-17T20:15:01.481Z" }, + { url = "https://files.pythonhosted.org/packages/f4/58/c4f976234bf6d4737bc8c02a81192f045c307b72cf39c9e5c5a2d78927f6/psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d4a113425c037300de3ac8b331637293da9be9713855c4fc9d2d97436d7259d", size = 293625, upload-time = "2025-09-17T20:15:04.492Z" }, + { url = "https://files.pythonhosted.org/packages/79/87/157c8e7959ec39ced1b11cc93c730c4fb7f9d408569a6c59dbd92ceb35db/psutil-7.1.0-cp37-abi3-win32.whl", hash = "sha256:09ad740870c8d219ed8daae0ad3b726d3bf9a028a198e7f3080f6a1888b99bca", size = 244812, upload-time = "2025-09-17T20:15:07.462Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e9/b44c4f697276a7a95b8e94d0e320a7bf7f3318521b23de69035540b39838/psutil-7.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:57f5e987c36d3146c0dd2528cd42151cf96cd359b9d67cfff836995cc5df9a3d", size = 247965, upload-time = "2025-09-17T20:15:09.673Z" }, + { url = "https://files.pythonhosted.org/packages/26/65/1070a6e3c036f39142c2820c4b52e9243246fcfc3f96239ac84472ba361e/psutil-7.1.0-cp37-abi3-win_arm64.whl", hash = "sha256:6937cb68133e7c97b6cc9649a570c9a18ba0efebed46d8c5dae4c07fa1b67a07", size = 244971, upload-time = "2025-09-17T20:15:12.262Z" }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, +] + +[[package]] +name = "py" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/ff/fec109ceb715d2a6b4c4a85a61af3b40c723a961e8828319fbcb15b868dc/py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", size = 207796, upload-time = "2021-11-04T17:17:01.377Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/f0/10642828a8dfb741e5f3fbaac830550a518a775c7fff6f04a007259b0548/py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378", size = 98708, upload-time = "2021-11-04T17:17:00.152Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + +[[package]] +name = "pyarrow" +version = "21.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487, upload-time = "2025-07-18T00:57:31.761Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305, upload-time = "2025-07-18T00:55:35.373Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264, upload-time = "2025-07-18T00:55:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099, upload-time = "2025-07-18T00:55:42.889Z" }, + { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529, upload-time = "2025-07-18T00:55:47.069Z" }, + { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883, upload-time = "2025-07-18T00:55:53.069Z" }, + { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802, upload-time = "2025-07-18T00:55:57.714Z" }, + { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175, upload-time = "2025-07-18T00:56:01.364Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/e9/01f1a64245b89f039897cb0130016d79f77d52669aae6ee7b159a6c4c018/pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034", size = 145322, upload-time = "2024-09-10T22:41:42.55Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + +[[package]] +name = "pybase64" +version = "1.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/14/43297a7b7f0c1bf0c00b596f754ee3ac946128c64d21047ccf9c9bbc5165/pybase64-1.4.2.tar.gz", hash = "sha256:46cdefd283ed9643315d952fe44de80dc9b9a811ce6e3ec97fd1827af97692d0", size = 137246, upload-time = "2025-07-27T13:08:57.808Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/6d/11ede991e800797b9f5ebd528013b34eee5652df93de61ffb24503393fa5/pybase64-1.4.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2c75d1388855b5a1015b65096d7dbcc708e7de3245dcbedeb872ec05a09326", size = 38326, upload-time = "2025-07-27T13:03:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/fe/84/87f1f565f42e2397e2aaa2477c86419f5173c3699881c42325c090982f0a/pybase64-1.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b621a972a01841368fdb9dedc55fd3c6e0c7217d0505ba3b1ebe95e7ef1b493", size = 31661, upload-time = "2025-07-27T13:03:10.295Z" }, + { url = "https://files.pythonhosted.org/packages/cb/2a/a24c810e7a61d2cc6f73fe9ee4872a03030887fa8654150901b15f376f65/pybase64-1.4.2-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f48c32ac6a16cbf57a5a96a073fef6ff7e3526f623cd49faa112b7f9980bafba", size = 68192, upload-time = "2025-07-27T13:03:11.467Z" }, + { url = "https://files.pythonhosted.org/packages/ee/87/d9baf98cbfc37b8657290ad4421f3a3c36aa0eafe4872c5859cfb52f3448/pybase64-1.4.2-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ace8b23093a6bb862477080d9059b784096ab2f97541e8bfc40d42f062875149", size = 71587, upload-time = "2025-07-27T13:03:12.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/89/3df043cc56ef3b91b7aa0c26ae822a2d7ec8da0b0fd7c309c879b0eb5988/pybase64-1.4.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1772c7532a7fb6301baea3dd3e010148dbf70cd1136a83c2f5f91bdc94822145", size = 59910, upload-time = "2025-07-27T13:03:14.266Z" }, + { url = "https://files.pythonhosted.org/packages/75/4f/6641e9edf37aeb4d4524dc7ba2168eff8d96c90e77f6283c2be3400ab380/pybase64-1.4.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:f86f7faddcba5cbfea475f8ab96567834c28bf09ca6c7c3d66ee445adac80d8f", size = 56701, upload-time = "2025-07-27T13:03:15.6Z" }, + { url = "https://files.pythonhosted.org/packages/2d/7f/20d8ac1046f12420a0954a45a13033e75f98aade36eecd00c64e3549b071/pybase64-1.4.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0b8c8e275b5294089f314814b4a50174ab90af79d6a4850f6ae11261ff6a7372", size = 59288, upload-time = "2025-07-27T13:03:16.823Z" }, + { url = "https://files.pythonhosted.org/packages/17/ea/9c0ca570e3e50b3c6c3442e280c83b321a0464c86a9db1f982a4ff531550/pybase64-1.4.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:864d85a0470c615807ae8b97d724d068b940a2d10ac13a5f1b9e75a3ce441758", size = 60267, upload-time = "2025-07-27T13:03:18.132Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ac/46894929d71ccedebbfb0284173b0fea96bc029cd262654ba8451a7035d6/pybase64-1.4.2-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:47254d97ed2d8351e30ecfdb9e2414547f66ba73f8a09f932c9378ff75cd10c5", size = 54801, upload-time = "2025-07-27T13:03:19.669Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/02c95218ea964f0b2469717c2c69b48e63f4ca9f18af01a5b2a29e4c1216/pybase64-1.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:264b65ecc4f0ee73f3298ab83bbd8008f7f9578361b8df5b448f985d8c63e02a", size = 58599, upload-time = "2025-07-27T13:03:20.951Z" }, + { url = "https://files.pythonhosted.org/packages/15/45/ccc21004930789b8fb439d43e3212a6c260ccddb2bf450c39a20db093f33/pybase64-1.4.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fbcc2b30cd740c16c9699f596f22c7a9e643591311ae72b1e776f2d539e9dd9d", size = 52388, upload-time = "2025-07-27T13:03:23.064Z" }, + { url = "https://files.pythonhosted.org/packages/c4/45/22e46e549710c4c237d77785b6fb1bc4c44c288a5c44237ba9daf5c34b82/pybase64-1.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cda9f79c22d51ee4508f5a43b673565f1d26af4330c99f114e37e3186fdd3607", size = 68802, upload-time = "2025-07-27T13:03:24.673Z" }, + { url = "https://files.pythonhosted.org/packages/55/0c/232c6261b81296e5593549b36e6e7884a5da008776d12665923446322c36/pybase64-1.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0c91c6d2a7232e2a1cd10b3b75a8bb657defacd4295a1e5e80455df2dfc84d4f", size = 57841, upload-time = "2025-07-27T13:03:25.948Z" }, + { url = "https://files.pythonhosted.org/packages/20/8a/b35a615ae6f04550d696bb179c414538b3b477999435fdd4ad75b76139e4/pybase64-1.4.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a370dea7b1cee2a36a4d5445d4e09cc243816c5bc8def61f602db5a6f5438e52", size = 54320, upload-time = "2025-07-27T13:03:27.495Z" }, + { url = "https://files.pythonhosted.org/packages/d3/a9/8bd4f9bcc53689f1b457ecefed1eaa080e4949d65a62c31a38b7253d5226/pybase64-1.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9aa4de83f02e462a6f4e066811c71d6af31b52d7484de635582d0e3ec3d6cc3e", size = 56482, upload-time = "2025-07-27T13:03:28.942Z" }, + { url = "https://files.pythonhosted.org/packages/75/e5/4a7735b54a1191f61c3f5c2952212c85c2d6b06eb5fb3671c7603395f70c/pybase64-1.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83a1c2f9ed00fee8f064d548c8654a480741131f280e5750bb32475b7ec8ee38", size = 70959, upload-time = "2025-07-27T13:03:30.171Z" }, + { url = "https://files.pythonhosted.org/packages/d3/67/e2b6cb32c782e12304d467418e70da0212567f42bd4d3b5eb1fdf64920ad/pybase64-1.4.2-cp312-cp312-win32.whl", hash = "sha256:a6e5688b18d558e8c6b8701cc8560836c4bbeba61d33c836b4dba56b19423716", size = 33683, upload-time = "2025-07-27T13:03:31.775Z" }, + { url = "https://files.pythonhosted.org/packages/4f/bc/d5c277496063a09707486180f17abbdbdebbf2f5c4441b20b11d3cb7dc7c/pybase64-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:c995d21b8bd08aa179cd7dd4db0695c185486ecc72da1e8f6c37ec86cadb8182", size = 35817, upload-time = "2025-07-27T13:03:32.99Z" }, + { url = "https://files.pythonhosted.org/packages/e6/69/e4be18ae685acff0ae77f75d4586590f29d2cd187bf603290cf1d635cad4/pybase64-1.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:e254b9258c40509c2ea063a7784f6994988f3f26099d6e08704e3c15dfed9a55", size = 30900, upload-time = "2025-07-27T13:03:34.499Z" }, +] + +[[package]] +name = "pybind11" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/7b/a6d8dcb83c457e24a9df1e4d8fd5fb8034d4bbc62f3c324681e8a9ba57c2/pybind11-3.0.1.tar.gz", hash = "sha256:9c0f40056a016da59bab516efb523089139fcc6f2ba7e4930854c61efb932051", size = 546914, upload-time = "2025-08-22T20:09:27.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/8a/37362fc2b949d5f733a8b0f2ff51ba423914cabefe69f1d1b6aab710f5fe/pybind11-3.0.1-py3-none-any.whl", hash = "sha256:aa8f0aa6e0a94d3b64adfc38f560f33f15e589be2175e103c0a33c6bce55ee89", size = 293611, upload-time = "2025-08-22T20:09:25.235Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/e0/abfd2a0d2efe47670df87f3e3a0e2edda42f055053c85361f19c0e2c1ca8/pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", size = 39472, upload-time = "2025-06-20T18:49:48.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, +] + +[[package]] +name = "pycountry" +version = "24.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/57/c389fa68c50590881a75b7883eeb3dc15e9e73a0fdc001cdd45c13290c92/pycountry-24.6.1.tar.gz", hash = "sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221", size = 6043910, upload-time = "2024-06-01T04:12:15.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/ec/1fb891d8a2660716aadb2143235481d15ed1cbfe3ad669194690b0604492/pycountry-24.6.1-py3-none-any.whl", hash = "sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f", size = 6335189, upload-time = "2024-06-01T04:11:49.711Z" }, +] + +[[package]] +name = "pycparser" +version = "2.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, +] + +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, +] + +[[package]] +name = "pydantic" +version = "2.11.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/5d/09a551ba512d7ca404d785072700d3f6727a02f6f3c24ecfd081c7cf0aa8/pydantic-2.11.9.tar.gz", hash = "sha256:6b8ffda597a14812a7975c90b82a8a2e777d9257aba3453f973acd3c032a18e2", size = 788495, upload-time = "2025-09-13T11:26:39.325Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/d3/108f2006987c58e76691d5ae5d200dd3e0f532cb4e5fa3560751c3a1feba/pydantic-2.11.9-py3-none-any.whl", hash = "sha256:c42dd626f5cfc1c6950ce6205ea58c93efa406da65f479dcb4029d5934857da2", size = 444855, upload-time = "2025-09-13T11:26:36.909Z" }, +] + +[package.optional-dependencies] +email = [ + { name = "email-validator" }, +] + +[[package]] +name = "pydantic-core" +version = "2.33.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/8a/2b41c97f554ec8c71f2a8a5f85cb56a8b0956addfe8b0efb5b3d77e8bdc3/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc", size = 2009000, upload-time = "2025-04-23T18:31:25.863Z" }, + { url = "https://files.pythonhosted.org/packages/a1/02/6224312aacb3c8ecbaa959897af57181fb6cf3a3d7917fd44d0f2917e6f2/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7", size = 1847996, upload-time = "2025-04-23T18:31:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/d6/46/6dcdf084a523dbe0a0be59d054734b86a981726f221f4562aed313dbcb49/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025", size = 1880957, upload-time = "2025-04-23T18:31:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6b/1ec2c03837ac00886ba8160ce041ce4e325b41d06a034adbef11339ae422/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011", size = 1964199, upload-time = "2025-04-23T18:31:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1d/6bf34d6adb9debd9136bd197ca72642203ce9aaaa85cfcbfcf20f9696e83/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f", size = 2120296, upload-time = "2025-04-23T18:31:32.514Z" }, + { url = "https://files.pythonhosted.org/packages/e0/94/2bd0aaf5a591e974b32a9f7123f16637776c304471a0ab33cf263cf5591a/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88", size = 2676109, upload-time = "2025-04-23T18:31:33.958Z" }, + { url = "https://files.pythonhosted.org/packages/f9/41/4b043778cf9c4285d59742281a769eac371b9e47e35f98ad321349cc5d61/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1", size = 2002028, upload-time = "2025-04-23T18:31:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d5/7bb781bf2748ce3d03af04d5c969fa1308880e1dca35a9bd94e1a96a922e/pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b", size = 2100044, upload-time = "2025-04-23T18:31:41.034Z" }, + { url = "https://files.pythonhosted.org/packages/fe/36/def5e53e1eb0ad896785702a5bbfd25eed546cdcf4087ad285021a90ed53/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1", size = 2058881, upload-time = "2025-04-23T18:31:42.757Z" }, + { url = "https://files.pythonhosted.org/packages/01/6c/57f8d70b2ee57fc3dc8b9610315949837fa8c11d86927b9bb044f8705419/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6", size = 2227034, upload-time = "2025-04-23T18:31:44.304Z" }, + { url = "https://files.pythonhosted.org/packages/27/b9/9c17f0396a82b3d5cbea4c24d742083422639e7bb1d5bf600e12cb176a13/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea", size = 2234187, upload-time = "2025-04-23T18:31:45.891Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6a/adf5734ffd52bf86d865093ad70b2ce543415e0e356f6cacabbc0d9ad910/pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290", size = 1892628, upload-time = "2025-04-23T18:31:47.819Z" }, + { url = "https://files.pythonhosted.org/packages/43/e4/5479fecb3606c1368d496a825d8411e126133c41224c1e7238be58b87d7e/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2", size = 1955866, upload-time = "2025-04-23T18:31:49.635Z" }, + { url = "https://files.pythonhosted.org/packages/0d/24/8b11e8b3e2be9dd82df4b11408a67c61bb4dc4f8e11b5b0fc888b38118b5/pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab", size = 1888894, upload-time = "2025-04-23T18:31:51.609Z" }, +] + +[[package]] +name = "pydantic-extra-types" +version = "2.10.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/ba/4178111ec4116c54e1dc7ecd2a1ff8f54256cdbd250e576882911e8f710a/pydantic_extra_types-2.10.5.tar.gz", hash = "sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8", size = 138429, upload-time = "2025-06-02T09:31:52.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/1a/5f4fd9e7285f10c44095a4f9fe17d0f358d1702a7c74a9278c794e8a7537/pydantic_extra_types-2.10.5-py3-none-any.whl", hash = "sha256:b60c4e23d573a69a4f1a16dd92888ecc0ef34fb0e655b4f305530377fa70e7a8", size = 38315, upload-time = "2025-06-02T09:31:51.229Z" }, +] + +[package.optional-dependencies] +pycountry = [ + { name = "pycountry" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/c5/dbbc27b814c71676593d1c3f718e6cd7d4f00652cefa24b75f7aa3efb25e/pydantic_settings-2.11.0.tar.gz", hash = "sha256:d0e87a1c7d33593beb7194adb8470fc426e95ba02af83a0f23474a04c9a08180", size = 188394, upload-time = "2025-09-24T14:19:11.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/d6/887a1ff844e64aa823fb4905978d882a633cfe295c32eacad582b78a7d8b/pydantic_settings-2.11.0-py3-none-any.whl", hash = "sha256:fe2cea3413b9530d10f3a5875adffb17ada5c1e1bab0b2885546d7310415207c", size = 48608, upload-time = "2025-09-24T14:19:10.015Z" }, +] + +[[package]] +name = "pydeck" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/ca/40e14e196864a0f61a92abb14d09b3d3da98f94ccb03b49cf51688140dab/pydeck-0.9.1.tar.gz", hash = "sha256:f74475ae637951d63f2ee58326757f8d4f9cd9f2a457cf42950715003e2cb605", size = 3832240, upload-time = "2024-05-10T15:36:21.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/4c/b888e6cf58bd9db9c93f40d1c6be8283ff49d88919231afe93a6bcf61626/pydeck-0.9.1-py2.py3-none-any.whl", hash = "sha256:b3f75ba0d273fc917094fa61224f3f6076ca8752b93d46faf3bcfd9f9d59b038", size = 6900403, upload-time = "2024-05-10T15:36:17.36Z" }, +] + +[[package]] +name = "pyee" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/03/1fd98d5841cd7964a27d729ccf2199602fe05eb7a405c1462eb7277945ed/pyee-13.0.0.tar.gz", hash = "sha256:b391e3c5a434d1f5118a25615001dbc8f669cf410ab67d04c4d4e07c55481c37", size = 31250, upload-time = "2025-03-17T18:53:15.955Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/4d/b9add7c84060d4c1906abe9a7e5359f2a60f7a9a4f67268b2766673427d8/pyee-13.0.0-py3-none-any.whl", hash = "sha256:48195a3cddb3b1515ce0695ed76036b5ccc2ef3a9f963ff9f77aec0139845498", size = 15730, upload-time = "2025-03-17T18:53:14.532Z" }, +] + +[[package]] +name = "pyflakes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/dc/fd034dc20b4b264b3d015808458391acbf9df40b1e54750ef175d39180b1/pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58", size = 64669, upload-time = "2025-06-20T18:45:27.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/2f/81d580a0fb83baeb066698975cb14a618bdbed7720678566f1b046a95fe8/pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f", size = 63551, upload-time = "2025-06-20T18:45:26.937Z" }, +] + +[[package]] +name = "pygithub" +version = "2.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyjwt", extra = ["crypto"] }, + { name = "pynacl" }, + { name = "requests" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/74/e560bdeffea72ecb26cff27f0fad548bbff5ecc51d6a155311ea7f9e4c4c/pygithub-2.8.1.tar.gz", hash = "sha256:341b7c78521cb07324ff670afd1baa2bf5c286f8d9fd302c1798ba594a5400c9", size = 2246994, upload-time = "2025-09-02T17:41:54.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/ba/7049ce39f653f6140aac4beb53a5aaf08b4407b6a3019aae394c1c5244ff/pygithub-2.8.1-py3-none-any.whl", hash = "sha256:23a0a5bca93baef082e03411bf0ce27204c32be8bfa7abc92fe4a3e132936df0", size = 432709, upload-time = "2025-09-02T17:41:52.947Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785, upload-time = "2024-11-28T03:43:29.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[[package]] +name = "pylatexenc" +version = "2.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/ab/34ec41718af73c00119d0351b7a2531d2ebddb51833a36448fc7b862be60/pylatexenc-2.10.tar.gz", hash = "sha256:3dd8fd84eb46dc30bee1e23eaab8d8fb5a7f507347b23e5f38ad9675c84f40d3", size = 162597, upload-time = "2021-04-06T07:56:07.854Z" } + +[[package]] +name = "pylcs" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pybind11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/7f/9ca900387de8f3d3658dbf7d0aba96aa2a69f4f5329c83af9be98dc7307d/pylcs-0.1.1.tar.gz", hash = "sha256:632c69235d77cda0ba524d82796878801d2f46131fc59e730c98767fc4ce1307", size = 11274, upload-time = "2023-10-24T23:56:24.576Z" } + +[[package]] +name = "pynacl" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c6/a3124dee667a423f2c637cfd262a54d67d8ccf3e160f3c50f622a85b7723/pynacl-1.6.0.tar.gz", hash = "sha256:cb36deafe6e2bce3b286e5d1f3e1c246e0ccdb8808ddb4550bb2792f2df298f2", size = 3505641, upload-time = "2025-09-10T23:39:22.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/37/87c72df19857c5b3b47ace6f211a26eb862ada495cc96daa372d96048fca/pynacl-1.6.0-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:f4b3824920e206b4f52abd7de621ea7a44fd3cb5c8daceb7c3612345dfc54f2e", size = 382610, upload-time = "2025-09-10T23:38:49.459Z" }, + { url = "https://files.pythonhosted.org/packages/0c/64/3ce958a5817fd3cc6df4ec14441c43fd9854405668d73babccf77f9597a3/pynacl-1.6.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16dd347cdc8ae0b0f6187a2608c0af1c8b7ecbbe6b4a06bff8253c192f696990", size = 798744, upload-time = "2025-09-10T23:38:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/e4/8a/3f0dd297a0a33fa3739c255feebd0206bb1df0b44c52fbe2caf8e8bc4425/pynacl-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16c60daceee88d04f8d41d0a4004a7ed8d9a5126b997efd2933e08e93a3bd850", size = 1397879, upload-time = "2025-09-10T23:39:00.44Z" }, + { url = "https://files.pythonhosted.org/packages/41/94/028ff0434a69448f61348d50d2c147dda51aabdd4fbc93ec61343332174d/pynacl-1.6.0-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25720bad35dfac34a2bcdd61d9e08d6bfc6041bebc7751d9c9f2446cf1e77d64", size = 833907, upload-time = "2025-09-10T23:38:50.936Z" }, + { url = "https://files.pythonhosted.org/packages/52/bc/a5cff7f8c30d5f4c26a07dfb0bcda1176ab8b2de86dda3106c00a02ad787/pynacl-1.6.0-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bfaa0a28a1ab718bad6239979a5a57a8d1506d0caf2fba17e524dbb409441cf", size = 1436649, upload-time = "2025-09-10T23:38:52.783Z" }, + { url = "https://files.pythonhosted.org/packages/7a/20/c397be374fd5d84295046e398de4ba5f0722dc14450f65db76a43c121471/pynacl-1.6.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ef214b90556bb46a485b7da8258e59204c244b1b5b576fb71848819b468c44a7", size = 817142, upload-time = "2025-09-10T23:38:54.4Z" }, + { url = "https://files.pythonhosted.org/packages/12/30/5efcef3406940cda75296c6d884090b8a9aad2dcc0c304daebb5ae99fb4a/pynacl-1.6.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:49c336dd80ea54780bcff6a03ee1a476be1612423010472e60af83452aa0f442", size = 1401794, upload-time = "2025-09-10T23:38:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/be/e1/a8fe1248cc17ccb03b676d80fa90763760a6d1247da434844ea388d0816c/pynacl-1.6.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f3482abf0f9815e7246d461fab597aa179b7524628a4bc36f86a7dc418d2608d", size = 772161, upload-time = "2025-09-10T23:39:01.93Z" }, + { url = "https://files.pythonhosted.org/packages/a3/76/8a62702fb657d6d9104ce13449db221a345665d05e6a3fdefb5a7cafd2ad/pynacl-1.6.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:140373378e34a1f6977e573033d1dd1de88d2a5d90ec6958c9485b2fd9f3eb90", size = 1370720, upload-time = "2025-09-10T23:39:03.531Z" }, + { url = "https://files.pythonhosted.org/packages/6d/38/9e9e9b777a1c4c8204053733e1a0269672c0bd40852908c9ad6b6eaba82c/pynacl-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6b393bc5e5a0eb86bb85b533deb2d2c815666665f840a09e0aa3362bb6088736", size = 791252, upload-time = "2025-09-10T23:39:05.058Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d972ce3d92ae05c9091363cf185e8646933f91c376e97b8be79ea6e96c22/pynacl-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a25cfede801f01e54179b8ff9514bd7b5944da560b7040939732d1804d25419", size = 1362910, upload-time = "2025-09-10T23:39:06.924Z" }, + { url = "https://files.pythonhosted.org/packages/35/2c/ee0b373a1861f66a7ca8bdb999331525615061320dd628527a50ba8e8a60/pynacl-1.6.0-cp38-abi3-win32.whl", hash = "sha256:dcdeb41c22ff3c66eef5e63049abf7639e0db4edee57ba70531fc1b6b133185d", size = 226461, upload-time = "2025-09-10T23:39:11.894Z" }, + { url = "https://files.pythonhosted.org/packages/75/f7/41b6c0b9dd9970173b6acc026bab7b4c187e4e5beef2756d419ad65482da/pynacl-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:cf831615cc16ba324240de79d925eacae8265b7691412ac6b24221db157f6bd1", size = 238802, upload-time = "2025-09-10T23:39:08.966Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0f/462326910c6172fa2c6ed07922b22ffc8e77432b3affffd9e18f444dbfbb/pynacl-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:84709cea8f888e618c21ed9a0efdb1a59cc63141c403db8bf56c469b71ad56f2", size = 183846, upload-time = "2025-09-10T23:39:10.552Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.2.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/a5/181488fc2b9d093e3972d2a472855aae8a03f000592dbfce716a512b3359/pyparsing-3.2.5.tar.gz", hash = "sha256:2df8d5b7b2802ef88e8d016a2eb9c7aeaa923529cd251ed0fe4608275d4105b6", size = 1099274, upload-time = "2025-09-21T04:11:06.277Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/5e/1aa9a93198c6b64513c9d7752de7422c06402de6600a8767da1524f9570b/pyparsing-3.2.5-py3-none-any.whl", hash = "sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e", size = 113890, upload-time = "2025-09-21T04:11:04.117Z" }, +] + +[[package]] +name = "pypdf2" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/bb/18dc3062d37db6c491392007dfd1a7f524bb95886eb956569ac38a23a784/PyPDF2-3.0.1.tar.gz", hash = "sha256:a74408f69ba6271f71b9352ef4ed03dc53a31aa404d29b5d31f53bfecfee1440", size = 227419, upload-time = "2022-12-31T10:36:13.13Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/5e/c86a5643653825d3c913719e788e41386bee415c2b87b4f955432f2de6b2/pypdf2-3.0.1-py3-none-any.whl", hash = "sha256:d16e4205cfee272fbdc0568b68d82be796540b1537508cef59388f839c191928", size = 232572, upload-time = "2022-12-31T10:36:10.327Z" }, +] + +[[package]] +name = "pyproject-hooks" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/82/28175b2414effca1cdac8dc99f76d660e7a4fb0ceefa4b4ab8f5f6742925/pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", size = 19228, upload-time = "2024-09-29T09:24:13.293Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, +] + +[[package]] +name = "pysocks" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-docx" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/b0/4bc07ccd3572a2f9df7e6782f52b0c6c90dcbb803ac4a167702d7d0dfe1e/python_dotenv-1.1.1.tar.gz", hash = "sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab", size = 41978, upload-time = "2025-06-24T04:21:07.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/ed/539768cf28c661b5b068d66d96a2f155c4971a5d55684a514c1a0e0dec2f/python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc", size = 20556, upload-time = "2025-06-24T04:21:06.073Z" }, +] + +[[package]] +name = "python-engineio" +version = "4.12.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "simple-websocket" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/d8/63e5535ab21dc4998ba1cfe13690ccf122883a38f025dca24d6e56c05eba/python_engineio-4.12.3.tar.gz", hash = "sha256:35633e55ec30915e7fc8f7e34ca8d73ee0c080cec8a8cd04faf2d7396f0a7a7a", size = 91910, upload-time = "2025-09-28T06:31:36.765Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/f0/c5aa0a69fd9326f013110653543f36ece4913c17921f3e1dbd78e1b423ee/python_engineio-4.12.3-py3-none-any.whl", hash = "sha256:7c099abb2a27ea7ab429c04da86ab2d82698cdd6c52406cb73766fe454feb7e1", size = 59637, upload-time = "2025-09-28T06:31:35.354Z" }, +] + +[[package]] +name = "python-frontmatter" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/de/910fa208120314a12f9a88ea63e03707261692af782c99283f1a2c8a5e6f/python-frontmatter-1.1.0.tar.gz", hash = "sha256:7118d2bd56af9149625745c58c9b51fb67e8d1294a0c76796dafdc72c36e5f6d", size = 16256, upload-time = "2024-01-16T18:50:04.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/87/3c8da047b3ec5f99511d1b4d7a5bc72d4b98751c7e78492d14dc736319c5/python_frontmatter-1.1.0-py3-none-any.whl", hash = "sha256:335465556358d9d0e6c98bbeb69b1c969f2a4a21360587b9873bfc3b213407c1", size = 9834, upload-time = "2024-01-16T18:50:00.911Z" }, +] + +[[package]] +name = "python-json-logger" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/de/d3144a0bceede957f961e975f3752760fbe390d57fbe194baf709d8f1f7b/python_json_logger-3.3.0.tar.gz", hash = "sha256:12b7e74b17775e7d565129296105bbe3910842d9d0eb083fc83a6a617aa8df84", size = 16642, upload-time = "2025-03-07T07:08:27.301Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/20/0f2523b9e50a8052bc6a8b732dfc8568abbdc42010aef03a2d750bdab3b2/python_json_logger-3.3.0-py3-none-any.whl", hash = "sha256:dd980fae8cffb24c13caf6e158d3d61c0d6d22342f932cb6e9deedab3d35eec7", size = 15163, upload-time = "2025-03-07T07:08:25.627Z" }, +] + +[[package]] +name = "python-multipart" +version = "0.0.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158, upload-time = "2024-12-16T19:45:46.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, +] + +[[package]] +name = "python-pptx" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "pillow" }, + { name = "typing-extensions" }, + { name = "xlsxwriter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, +] + +[[package]] +name = "python-socketio" +version = "5.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bidict" }, + { name = "python-engineio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/1a/396d50ccf06ee539fa758ce5623b59a9cb27637fc4b2dc07ed08bf495e77/python_socketio-5.13.0.tar.gz", hash = "sha256:ac4e19a0302ae812e23b712ec8b6427ca0521f7c582d6abb096e36e24a263029", size = 121125, upload-time = "2025-04-12T15:46:59.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/32/b4fb8585d1be0f68bde7e110dffbcf354915f77ad8c778563f0ad9655c02/python_socketio-5.13.0-py3-none-any.whl", hash = "sha256:51f68d6499f2df8524668c24bcec13ba1414117cfb3a90115c559b601ab10caf", size = 77800, upload-time = "2025-04-12T15:46:58.412Z" }, +] + +[[package]] +name = "pythonnet" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "clr-loader" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/d6/1afd75edd932306ae9bd2c2d961d603dc2b52fcec51b04afea464f1f6646/pythonnet-3.0.5.tar.gz", hash = "sha256:48e43ca463941b3608b32b4e236db92d8d40db4c58a75ace902985f76dac21cf", size = 239212, upload-time = "2024-12-13T08:30:44.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/f1/bfb6811df4745f92f14c47a29e50e89a36b1533130fcc56452d4660bd2d6/pythonnet-3.0.5-py3-none-any.whl", hash = "sha256:f6702d694d5d5b163c9f3f5cc34e0bed8d6857150237fae411fefb883a656d20", size = 297506, upload-time = "2024-12-13T08:30:40.661Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "pyvers" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/39/c5432f541e6ea1d616dfd6ef42ce02792f7eb42dd44f5ed4439dbe17a58b/pyvers-0.1.0-py3-none-any.whl", hash = "sha256:065249805ae537ddf9a2d1a8dffc6d0a12474a347d2eaa2f35ebdae92c0c8199", size = 10092, upload-time = "2025-06-08T23:46:46.219Z" }, +] + +[[package]] +name = "pywin32" +version = "311" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543, upload-time = "2025-07-14T20:13:20.765Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040, upload-time = "2025-07-14T20:13:22.543Z" }, + { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102, upload-time = "2025-07-14T20:13:24.682Z" }, +] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "pyzmq" +version = "27.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, + { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" }, + { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" }, +] + +[[package]] +name = "qtconsole" +version = "5.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipykernel" }, + { name = "ipython-pygments-lexers" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "qtpy" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/47/dc/c8cb80a72af781cbdba46ba9416de33c501ecb4644e36191fbb27d592c16/qtconsole-5.7.0.tar.gz", hash = "sha256:0d33371ce9ef554c7022ee300564ba9ffbd615e304ee615f6769d4068e063171", size = 436635, upload-time = "2025-09-10T18:58:19.363Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/d2/3cb0980a6a4a2581facc4d47b2e7d99912a1f7065407f007f27419e19979/qtconsole-5.7.0-py3-none-any.whl", hash = "sha256:42ff0734269a77129097131ce96059f97dc864e6e061d7ed919609ccfe18dd22", size = 125653, upload-time = "2025-09-10T18:58:17.306Z" }, +] + +[[package]] +name = "qtpy" +version = "2.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/01/392eba83c8e47b946b929d7c46e0f04b35e9671f8bb6fc36b6f7945b4de8/qtpy-2.4.3.tar.gz", hash = "sha256:db744f7832e6d3da90568ba6ccbca3ee2b3b4a890c3d6fbbc63142f6e4cdf5bb", size = 66982, upload-time = "2025-02-11T15:09:25.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/76/37c0ccd5ab968a6a438f9c623aeecc84c202ab2fabc6a8fd927580c15b5a/QtPy-2.4.3-py3-none-any.whl", hash = "sha256:72095afe13673e017946cc258b8d5da43314197b741ed2890e563cf384b51aa1", size = 95045, upload-time = "2025-02-11T15:09:24.162Z" }, +] + +[[package]] +name = "rapidfuzz" +version = "3.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/fc/a98b616db9a42dcdda7c78c76bdfdf6fe290ac4c5ffbb186f73ec981ad5b/rapidfuzz-3.14.1.tar.gz", hash = "sha256:b02850e7f7152bd1edff27e9d584505b84968cacedee7a734ec4050c655a803c", size = 57869570, upload-time = "2025-09-08T21:08:15.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/77/2f4887c9b786f203e50b816c1cde71f96642f194e6fa752acfa042cf53fd/rapidfuzz-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:809515194f628004aac1b1b280c3734c5ea0ccbd45938c9c9656a23ae8b8f553", size = 1932216, upload-time = "2025-09-08T21:06:09.342Z" }, + { url = "https://files.pythonhosted.org/packages/de/bd/b5e445d156cb1c2a87d36d8da53daf4d2a1d1729b4851660017898b49aa0/rapidfuzz-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0afcf2d6cb633d0d4260d8df6a40de2d9c93e9546e2c6b317ab03f89aa120ad7", size = 1393414, upload-time = "2025-09-08T21:06:10.959Z" }, + { url = "https://files.pythonhosted.org/packages/de/bd/98d065dd0a4479a635df855616980eaae1a1a07a876db9400d421b5b6371/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1c3d07d53dcafee10599da8988d2b1f39df236aee501ecbd617bd883454fcd", size = 1377194, upload-time = "2025-09-08T21:06:12.471Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8a/1265547b771128b686f3c431377ff1db2fa073397ed082a25998a7b06d4e/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e9ee3e1eb0a027717ee72fe34dc9ac5b3e58119f1bd8dd15bc19ed54ae3e62b", size = 1669573, upload-time = "2025-09-08T21:06:14.016Z" }, + { url = "https://files.pythonhosted.org/packages/a8/57/e73755c52fb451f2054196404ccc468577f8da023b3a48c80bce29ee5d4a/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:70c845b64a033a20c44ed26bc890eeb851215148cc3e696499f5f65529afb6cb", size = 2217833, upload-time = "2025-09-08T21:06:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/20/14/7399c18c460e72d1b754e80dafc9f65cb42a46cc8f29cd57d11c0c4acc94/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26db0e815213d04234298dea0d884d92b9cb8d4ba954cab7cf67a35853128a33", size = 3159012, upload-time = "2025-09-08T21:06:17.631Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5e/24f0226ddb5440cabd88605d2491f99ae3748a6b27b0bc9703772892ced7/rapidfuzz-3.14.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:6ad3395a416f8b126ff11c788531f157c7debeb626f9d897c153ff8980da10fb", size = 1227032, upload-time = "2025-09-08T21:06:21.06Z" }, + { url = "https://files.pythonhosted.org/packages/40/43/1d54a4ad1a5fac2394d5f28a3108e2bf73c26f4f23663535e3139cfede9b/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:61c5b9ab6f730e6478aa2def566223712d121c6f69a94c7cc002044799442afd", size = 2395054, upload-time = "2025-09-08T21:06:23.482Z" }, + { url = "https://files.pythonhosted.org/packages/0c/71/e9864cd5b0f086c4a03791f5dfe0155a1b132f789fe19b0c76fbabd20513/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:13e0ea3d0c533969158727d1bb7a08c2cc9a816ab83f8f0dcfde7e38938ce3e6", size = 2524741, upload-time = "2025-09-08T21:06:26.825Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0c/53f88286b912faf4a3b2619a60df4f4a67bd0edcf5970d7b0c1143501f0c/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6325ca435b99f4001aac919ab8922ac464999b100173317defb83eae34e82139", size = 2785311, upload-time = "2025-09-08T21:06:29.471Z" }, + { url = "https://files.pythonhosted.org/packages/53/9a/229c26dc4f91bad323f07304ee5ccbc28f0d21c76047a1e4f813187d0bad/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:07a9fad3247e68798424bdc116c1094e88ecfabc17b29edf42a777520347648e", size = 3303630, upload-time = "2025-09-08T21:06:31.094Z" }, + { url = "https://files.pythonhosted.org/packages/05/de/20e330d6d58cbf83da914accd9e303048b7abae2f198886f65a344b69695/rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8ff5dbe78db0a10c1f916368e21d328935896240f71f721e073cf6c4c8cdedd", size = 4262364, upload-time = "2025-09-08T21:06:32.877Z" }, + { url = "https://files.pythonhosted.org/packages/1f/10/2327f83fad3534a8d69fe9cd718f645ec1fe828b60c0e0e97efc03bf12f8/rapidfuzz-3.14.1-cp312-cp312-win32.whl", hash = "sha256:9c83270e44a6ae7a39fc1d7e72a27486bccc1fa5f34e01572b1b90b019e6b566", size = 1711927, upload-time = "2025-09-08T21:06:34.669Z" }, + { url = "https://files.pythonhosted.org/packages/78/8d/199df0370133fe9f35bc72f3c037b53c93c5c1fc1e8d915cf7c1f6bb8557/rapidfuzz-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:e06664c7fdb51c708e082df08a6888fce4c5c416d7e3cc2fa66dd80eb76a149d", size = 1542045, upload-time = "2025-09-08T21:06:36.364Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c6/cc5d4bd1b16ea2657c80b745d8b1c788041a31fad52e7681496197b41562/rapidfuzz-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:6c7c26025f7934a169a23dafea6807cfc3fb556f1dd49229faf2171e5d8101cc", size = 813170, upload-time = "2025-09-08T21:06:38.001Z" }, +] + +[[package]] +name = "ray" +version = "2.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "jsonschema" }, + { name = "msgpack" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "pyyaml" }, + { name = "requests" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/53/0d105e1baa6c8c9582f90154ba3f0ca08d58129384ea2707b2e59449b03b/ray-2.48.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:8de799f3b0896f48d306d5e4a04fc6037a08c495d45f9c79935344e5693e3cf8", size = 67302857, upload-time = "2025-07-18T22:33:06.414Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/7de1e9d92a45b1805fe828dcbd18b4c5a1f35ab3cad9134efeb20a3ab3e5/ray-2.48.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5a6f57126eac9dd3286289e07e91e87b054792f9698b6f7ccab88b624816b542", size = 69823198, upload-time = "2025-07-18T22:33:12.494Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a6/e7c969bd371c65b7c233d86f23610489e15164ee7eadb3eb78f9d55eda4d/ray-2.48.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f1cf33d260316f92f77558185f1c36fc35506d76ee7fdfed9f5b70f9c4bdba7f", size = 69151702, upload-time = "2025-07-18T22:33:18.655Z" }, + { url = "https://files.pythonhosted.org/packages/61/02/1894be2ab930b599de0f1f77f785b86c78bda4873c6c2dd65d1de5b40837/ray-2.48.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:a42ed3b640f4b599a3fc8067c83ee60497c0f03d070d7a7df02a388fa17a546b", size = 70124265, upload-time = "2025-07-18T22:33:25.155Z" }, + { url = "https://files.pythonhosted.org/packages/79/8c/d3653d17337fc787af108411d9c9a38333c9fbdf247283ee56dd096d3360/ray-2.48.0-cp312-cp312-win_amd64.whl", hash = "sha256:e15fdffa6b60d5729f6025691396b8a01dc3461ba19dc92bba354ec1813ed6b1", size = 26745570, upload-time = "2025-07-18T22:33:31.328Z" }, +] + +[[package]] +name = "redis" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" }, +] + +[[package]] +name = "referencing" +version = "0.36.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" }, +] + +[[package]] +name = "regex" +version = "2025.9.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/d3/eaa0d28aba6ad1827ad1e716d9a93e1ba963ada61887498297d3da715133/regex-2025.9.18.tar.gz", hash = "sha256:c5ba23274c61c6fef447ba6a39333297d0c247f53059dba0bca415cac511edc4", size = 400917, upload-time = "2025-09-19T00:38:35.79Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/99/05859d87a66ae7098222d65748f11ef7f2dff51bfd7482a4e2256c90d72b/regex-2025.9.18-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:436e1b31d7efd4dcd52091d076482031c611dde58bf9c46ca6d0a26e33053a7e", size = 486335, upload-time = "2025-09-19T00:36:03.661Z" }, + { url = "https://files.pythonhosted.org/packages/97/7e/d43d4e8b978890932cf7b0957fce58c5b08c66f32698f695b0c2c24a48bf/regex-2025.9.18-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c190af81e5576b9c5fdc708f781a52ff20f8b96386c6e2e0557a78402b029f4a", size = 289720, upload-time = "2025-09-19T00:36:05.471Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3b/ff80886089eb5dcf7e0d2040d9aaed539e25a94300403814bb24cc775058/regex-2025.9.18-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4121f1ce2b2b5eec4b397cc1b277686e577e658d8f5870b7eb2d726bd2300ab", size = 287257, upload-time = "2025-09-19T00:36:07.072Z" }, + { url = "https://files.pythonhosted.org/packages/ee/66/243edf49dd8720cba8d5245dd4d6adcb03a1defab7238598c0c97cf549b8/regex-2025.9.18-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:300e25dbbf8299d87205e821a201057f2ef9aa3deb29caa01cd2cac669e508d5", size = 797463, upload-time = "2025-09-19T00:36:08.399Z" }, + { url = "https://files.pythonhosted.org/packages/df/71/c9d25a1142c70432e68bb03211d4a82299cd1c1fbc41db9409a394374ef5/regex-2025.9.18-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b47fcf9f5316c0bdaf449e879407e1b9937a23c3b369135ca94ebc8d74b1742", size = 862670, upload-time = "2025-09-19T00:36:10.101Z" }, + { url = "https://files.pythonhosted.org/packages/f8/8f/329b1efc3a64375a294e3a92d43372bf1a351aa418e83c21f2f01cf6ec41/regex-2025.9.18-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:57a161bd3acaa4b513220b49949b07e252165e6b6dc910ee7617a37ff4f5b425", size = 910881, upload-time = "2025-09-19T00:36:12.223Z" }, + { url = "https://files.pythonhosted.org/packages/35/9e/a91b50332a9750519320ed30ec378b74c996f6befe282cfa6bb6cea7e9fd/regex-2025.9.18-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f130c3a7845ba42de42f380fff3c8aebe89a810747d91bcf56d40a069f15352", size = 802011, upload-time = "2025-09-19T00:36:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1d/6be3b8d7856b6e0d7ee7f942f437d0a76e0d5622983abbb6d21e21ab9a17/regex-2025.9.18-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5f96fa342b6f54dcba928dd452e8d8cb9f0d63e711d1721cd765bb9f73bb048d", size = 786668, upload-time = "2025-09-19T00:36:15.391Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ce/4a60e53df58bd157c5156a1736d3636f9910bdcc271d067b32b7fcd0c3a8/regex-2025.9.18-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0f0d676522d68c207828dcd01fb6f214f63f238c283d9f01d85fc664c7c85b56", size = 856578, upload-time = "2025-09-19T00:36:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/86/e8/162c91bfe7217253afccde112868afb239f94703de6580fb235058d506a6/regex-2025.9.18-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:40532bff8a1a0621e7903ae57fce88feb2e8a9a9116d341701302c9302aef06e", size = 849017, upload-time = "2025-09-19T00:36:18.597Z" }, + { url = "https://files.pythonhosted.org/packages/35/34/42b165bc45289646ea0959a1bc7531733e90b47c56a72067adfe6b3251f6/regex-2025.9.18-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:039f11b618ce8d71a1c364fdee37da1012f5a3e79b1b2819a9f389cd82fd6282", size = 788150, upload-time = "2025-09-19T00:36:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/79/5d/cdd13b1f3c53afa7191593a7ad2ee24092a5a46417725ffff7f64be8342d/regex-2025.9.18-cp312-cp312-win32.whl", hash = "sha256:e1dd06f981eb226edf87c55d523131ade7285137fbde837c34dc9d1bf309f459", size = 264536, upload-time = "2025-09-19T00:36:21.922Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f5/4a7770c9a522e7d2dc1fa3ffc83ab2ab33b0b22b447e62cffef186805302/regex-2025.9.18-cp312-cp312-win_amd64.whl", hash = "sha256:3d86b5247bf25fa3715e385aa9ff272c307e0636ce0c9595f64568b41f0a9c77", size = 275501, upload-time = "2025-09-19T00:36:23.4Z" }, + { url = "https://files.pythonhosted.org/packages/df/05/9ce3e110e70d225ecbed455b966003a3afda5e58e8aec2964042363a18f4/regex-2025.9.18-cp312-cp312-win_arm64.whl", hash = "sha256:032720248cbeeae6444c269b78cb15664458b7bb9ed02401d3da59fe4d68c3a5", size = 268601, upload-time = "2025-09-19T00:36:25.092Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[package.optional-dependencies] +socks = [ + { name = "pysocks" }, +] + +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, +] + +[[package]] +name = "retry" +version = "0.9.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, + { name = "py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/72/75d0b85443fbc8d9f38d08d2b1b67cc184ce35280e4a3813cda2f445f3a4/retry-0.9.2.tar.gz", hash = "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4", size = 6448, upload-time = "2016-05-11T13:58:51.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/0d/53aea75710af4528a25ed6837d71d117602b01946b307a3912cb3cfcbcba/retry-0.9.2-py2.py3-none-any.whl", hash = "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606", size = 7986, upload-time = "2016-05-11T13:58:39.925Z" }, +] + +[[package]] +name = "rich" +version = "14.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" }, +] + +[[package]] +name = "rich-toolkit" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/33/1a18839aaa8feef7983590c05c22c9c09d245ada6017d118325bbfcc7651/rich_toolkit-0.15.1.tar.gz", hash = "sha256:6f9630eb29f3843d19d48c3bd5706a086d36d62016687f9d0efa027ddc2dd08a", size = 115322, upload-time = "2025-09-04T09:28:11.789Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/49/42821d55ead7b5a87c8d121edf323cb393d8579f63e933002ade900b784f/rich_toolkit-0.15.1-py3-none-any.whl", hash = "sha256:36a0b1d9a135d26776e4b78f1d5c2655da6e0ef432380b5c6b523c8d8ab97478", size = 29412, upload-time = "2025-09-04T09:28:10.587Z" }, +] + +[[package]] +name = "rignore" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/46/05a94dc55ac03cf931d18e43b86ecee5ee054cb88b7853fffd741e35009c/rignore-0.6.4.tar.gz", hash = "sha256:e893fdd2d7fdcfa9407d0b7600ef2c2e2df97f55e1c45d4a8f54364829ddb0ab", size = 11633, upload-time = "2025-07-19T19:24:46.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/6c/e5af4383cdd7829ef9aa63ac82a6507983e02dbc7c2e7b9aa64b7b8e2c7a/rignore-0.6.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:74720d074b79f32449d5d212ce732e0144a294a184246d1f1e7bcc1fc5c83b69", size = 885885, upload-time = "2025-07-19T19:23:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/89/3e/1b02a868830e464769aa417ee195ac352fe71ff818df8ce50c4b998edb9c/rignore-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a8184fcf567bd6b6d7b85a0c138d98dd40f63054141c96b175844414c5530d7", size = 819736, upload-time = "2025-07-19T19:23:46.565Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/b9be0c523d97c09f3c6508a67ce376aba4efe41c333c58903a0d7366439a/rignore-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcb0d7d7ecc3fbccf6477bb187c04a091579ea139f15f139abe0b3b48bdfef69", size = 892779, upload-time = "2025-07-19T19:22:35.167Z" }, + { url = "https://files.pythonhosted.org/packages/91/f4/3064b06233697f2993485d132f06fe95061fef71631485da75aed246c4fd/rignore-0.6.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:feac73377a156fb77b3df626c76f7e5893d9b4e9e886ac8c0f9d44f1206a2a91", size = 872116, upload-time = "2025-07-19T19:22:47.828Z" }, + { url = "https://files.pythonhosted.org/packages/99/94/cb8e7af9a3c0a665f10e2366144e0ebc66167cf846aca5f1ac31b3661598/rignore-0.6.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:465179bc30beb1f7a3439e428739a2b5777ed26660712b8c4e351b15a7c04483", size = 1163345, upload-time = "2025-07-19T19:23:00.557Z" }, + { url = "https://files.pythonhosted.org/packages/86/6b/49faa7ad85ceb6ccef265df40091d9992232d7f6055fa664fe0a8b13781c/rignore-0.6.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4a4877b4dca9cf31a4d09845b300c677c86267657540d0b4d3e6d0ce3110e6e9", size = 939967, upload-time = "2025-07-19T19:23:13.494Z" }, + { url = "https://files.pythonhosted.org/packages/80/c8/b91afda10bd5ca1e3a80463340b899c0dc26a7750a9f3c94f668585c7f40/rignore-0.6.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:456456802b1e77d1e2d149320ee32505b8183e309e228129950b807d204ddd17", size = 949717, upload-time = "2025-07-19T19:23:36.404Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f1/88bfdde58ae3fb1c1a92bb801f492eea8eafcdaf05ab9b75130023a4670b/rignore-0.6.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c1ff2fc223f1d9473d36923160af37bf765548578eb9d47a2f52e90da8ae408", size = 975534, upload-time = "2025-07-19T19:23:25.988Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8f/a80b4a2e48ceba56ba19e096d41263d844757e10aa36ede212571b5d8117/rignore-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e445fbc214ae18e0e644a78086ea5d0f579e210229a4fbe86367d11a4cd03c11", size = 1067837, upload-time = "2025-07-19T19:23:59.888Z" }, + { url = "https://files.pythonhosted.org/packages/7d/90/0905597af0e78748909ef58418442a480ddd93e9fc89b0ca9ab170c357c0/rignore-0.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e07d9c5270fc869bc431aadcfb6ed0447f89b8aafaa666914c077435dc76a123", size = 1134959, upload-time = "2025-07-19T19:24:12.396Z" }, + { url = "https://files.pythonhosted.org/packages/cc/7d/0fa29adf9183b61947ce6dc8a1a9779a8ea16573f557be28ec893f6ddbaa/rignore-0.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7a6ccc0ea83d2c0c6df6b166f2acacedcc220a516436490f41e99a5ae73b6019", size = 1109708, upload-time = "2025-07-19T19:24:24.176Z" }, + { url = "https://files.pythonhosted.org/packages/4e/a7/92892ed86b2e36da403dd3a0187829f2d880414cef75bd612bfdf4dedebc/rignore-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:536392c5ec91755db48389546c833c4ab1426fe03e5a8522992b54ef8a244e7e", size = 1120546, upload-time = "2025-07-19T19:24:36.377Z" }, + { url = "https://files.pythonhosted.org/packages/31/1b/d29ae1fe901d523741d6d1d3ffe0d630734dd0ed6b047628a69c1e15ea44/rignore-0.6.4-cp312-cp312-win32.whl", hash = "sha256:f5f9dca46fc41c0a1e236767f68be9d63bdd2726db13a0ae3a30f68414472969", size = 642005, upload-time = "2025-07-19T19:24:56.671Z" }, + { url = "https://files.pythonhosted.org/packages/1a/41/a224944824688995374e4525115ce85fecd82442fc85edd5bcd81f4f256d/rignore-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:e02eecb9e1b9f9bf7c9030ae73308a777bed3b2486204cc74dfcfbe699ab1497", size = 720358, upload-time = "2025-07-19T19:24:49.959Z" }, +] + +[[package]] +name = "rpds-py" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/dd/2c0cbe774744272b0ae725f44032c77bdcab6e8bcf544bffa3b6e70c8dba/rpds_py-0.27.1.tar.gz", hash = "sha256:26a1c73171d10b7acccbded82bf6a586ab8203601e565badc74bbbf8bc5a10f8", size = 27479, upload-time = "2025-08-27T12:16:36.024Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/fe/38de28dee5df58b8198c743fe2bea0c785c6d40941b9950bac4cdb71a014/rpds_py-0.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ae2775c1973e3c30316892737b91f9283f9908e3cc7625b9331271eaaed7dc90", size = 361887, upload-time = "2025-08-27T12:13:10.233Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/4b6c7eedc7dd90986bf0fab6ea2a091ec11c01b15f8ba0a14d3f80450468/rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2643400120f55c8a96f7c9d858f7be0c88d383cd4653ae2cf0d0c88f668073e5", size = 345795, upload-time = "2025-08-27T12:13:11.65Z" }, + { url = "https://files.pythonhosted.org/packages/6f/0e/e650e1b81922847a09cca820237b0edee69416a01268b7754d506ade11ad/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16323f674c089b0360674a4abd28d5042947d54ba620f72514d69be4ff64845e", size = 385121, upload-time = "2025-08-27T12:13:13.008Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ea/b306067a712988e2bff00dcc7c8f31d26c29b6d5931b461aa4b60a013e33/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a1f4814b65eacac94a00fc9a526e3fdafd78e439469644032032d0d63de4881", size = 398976, upload-time = "2025-08-27T12:13:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0a/26dc43c8840cb8fe239fe12dbc8d8de40f2365e838f3d395835dde72f0e5/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ba32c16b064267b22f1850a34051121d423b6f7338a12b9459550eb2096e7ec", size = 525953, upload-time = "2025-08-27T12:13:15.774Z" }, + { url = "https://files.pythonhosted.org/packages/22/14/c85e8127b573aaf3a0cbd7fbb8c9c99e735a4a02180c84da2a463b766e9e/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5c20f33fd10485b80f65e800bbe5f6785af510b9f4056c5a3c612ebc83ba6cb", size = 407915, upload-time = "2025-08-27T12:13:17.379Z" }, + { url = "https://files.pythonhosted.org/packages/ed/7b/8f4fee9ba1fb5ec856eb22d725a4efa3deb47f769597c809e03578b0f9d9/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466bfe65bd932da36ff279ddd92de56b042f2266d752719beb97b08526268ec5", size = 386883, upload-time = "2025-08-27T12:13:18.704Z" }, + { url = "https://files.pythonhosted.org/packages/86/47/28fa6d60f8b74fcdceba81b272f8d9836ac0340570f68f5df6b41838547b/rpds_py-0.27.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:41e532bbdcb57c92ba3be62c42e9f096431b4cf478da9bc3bc6ce5c38ab7ba7a", size = 405699, upload-time = "2025-08-27T12:13:20.089Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fd/c5987b5e054548df56953a21fe2ebed51fc1ec7c8f24fd41c067b68c4a0a/rpds_py-0.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f149826d742b406579466283769a8ea448eed82a789af0ed17b0cd5770433444", size = 423713, upload-time = "2025-08-27T12:13:21.436Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ba/3c4978b54a73ed19a7d74531be37a8bcc542d917c770e14d372b8daea186/rpds_py-0.27.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80c60cfb5310677bd67cb1e85a1e8eb52e12529545441b43e6f14d90b878775a", size = 562324, upload-time = "2025-08-27T12:13:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/b5/6c/6943a91768fec16db09a42b08644b960cff540c66aab89b74be6d4a144ba/rpds_py-0.27.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7ee6521b9baf06085f62ba9c7a3e5becffbc32480d2f1b351559c001c38ce4c1", size = 593646, upload-time = "2025-08-27T12:13:24.122Z" }, + { url = "https://files.pythonhosted.org/packages/11/73/9d7a8f4be5f4396f011a6bb7a19fe26303a0dac9064462f5651ced2f572f/rpds_py-0.27.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a512c8263249a9d68cac08b05dd59d2b3f2061d99b322813cbcc14c3c7421998", size = 558137, upload-time = "2025-08-27T12:13:25.557Z" }, + { url = "https://files.pythonhosted.org/packages/6e/96/6772cbfa0e2485bcceef8071de7821f81aeac8bb45fbfd5542a3e8108165/rpds_py-0.27.1-cp312-cp312-win32.whl", hash = "sha256:819064fa048ba01b6dadc5116f3ac48610435ac9a0058bbde98e569f9e785c39", size = 221343, upload-time = "2025-08-27T12:13:26.967Z" }, + { url = "https://files.pythonhosted.org/packages/67/b6/c82f0faa9af1c6a64669f73a17ee0eeef25aff30bb9a1c318509efe45d84/rpds_py-0.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:d9199717881f13c32c4046a15f024971a3b78ad4ea029e8da6b86e5aa9cf4594", size = 232497, upload-time = "2025-08-27T12:13:28.326Z" }, + { url = "https://files.pythonhosted.org/packages/e1/96/2817b44bd2ed11aebacc9251da03689d56109b9aba5e311297b6902136e2/rpds_py-0.27.1-cp312-cp312-win_arm64.whl", hash = "sha256:33aa65b97826a0e885ef6e278fbd934e98cdcfed80b63946025f01e2f5b29502", size = 222790, upload-time = "2025-08-27T12:13:29.71Z" }, +] + +[[package]] +name = "rsa" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, +] + +[[package]] +name = "ruamel-yaml" +version = "0.18.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ruamel-yaml-clib", marker = "platform_python_implementation == 'CPython' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/db/f3950f5e5031b618aae9f423a39bf81a55c148aecd15a34527898e752cf4/ruamel.yaml-0.18.15.tar.gz", hash = "sha256:dbfca74b018c4c3fba0b9cc9ee33e53c371194a9000e694995e620490fd40700", size = 146865, upload-time = "2025-08-19T11:15:10.694Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/e5/f2a0621f1781b76a38194acae72f01e37b1941470407345b6e8653ad7640/ruamel.yaml-0.18.15-py3-none-any.whl", hash = "sha256:148f6488d698b7a5eded5ea793a025308b25eca97208181b6a026037f391f701", size = 119702, upload-time = "2025-08-19T11:15:07.696Z" }, +] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/e9/39ec4d4b3f91188fad1842748f67d4e749c77c37e353c4e545052ee8e893/ruamel.yaml.clib-0.2.14.tar.gz", hash = "sha256:803f5044b13602d58ea378576dd75aa759f52116a0232608e8fdada4da33752e", size = 225394, upload-time = "2025-09-22T19:51:23.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/42/ccfb34a25289afbbc42017e4d3d4288e61d35b2e00cfc6b92974a6a1f94b/ruamel.yaml.clib-0.2.14-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6aeadc170090ff1889f0d2c3057557f9cd71f975f17535c26a5d37af98f19c27", size = 271775, upload-time = "2025-09-23T14:24:12.771Z" }, + { url = "https://files.pythonhosted.org/packages/82/73/e628a92e80197ff6a79ab81ec3fa00d4cc082d58ab78d3337b7ba7043301/ruamel.yaml.clib-0.2.14-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5e56ac47260c0eed992789fa0b8efe43404a9adb608608631a948cee4fc2b052", size = 138842, upload-time = "2025-09-22T19:50:49.156Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c5/346c7094344a60419764b4b1334d9e0285031c961176ff88ffb652405b0c/ruamel.yaml.clib-0.2.14-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:a911aa73588d9a8b08d662b9484bc0567949529824a55d3885b77e8dd62a127a", size = 647404, upload-time = "2025-09-22T19:50:52.921Z" }, + { url = "https://files.pythonhosted.org/packages/df/99/65080c863eb06d4498de3d6c86f3e90595e02e159fd8529f1565f56cfe2c/ruamel.yaml.clib-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a05ba88adf3d7189a974b2de7a9d56731548d35dc0a822ec3dc669caa7019b29", size = 753141, upload-time = "2025-09-22T19:50:50.294Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e3/0de85f3e3333f8e29e4b10244374a202a87665d1131798946ee22cf05c7c/ruamel.yaml.clib-0.2.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb04c5650de6668b853623eceadcdb1a9f2fee381f5d7b6bc842ee7c239eeec4", size = 703477, upload-time = "2025-09-22T19:50:51.508Z" }, + { url = "https://files.pythonhosted.org/packages/d9/25/0d2f09d8833c7fd77ab8efeff213093c16856479a9d293180a0d89f6bed9/ruamel.yaml.clib-0.2.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:df3ec9959241d07bc261f4983d25a1205ff37703faf42b474f15d54d88b4f8c9", size = 741157, upload-time = "2025-09-23T18:42:50.408Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8c/959f10c2e2153cbdab834c46e6954b6dd9e3b109c8f8c0a3cf1618310985/ruamel.yaml.clib-0.2.14-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fbc08c02e9b147a11dfcaa1ac8a83168b699863493e183f7c0c8b12850b7d259", size = 745859, upload-time = "2025-09-22T19:50:54.497Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6b/e580a7c18b485e1a5f30a32cda96b20364b0ba649d9d2baaf72f8bd21f83/ruamel.yaml.clib-0.2.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c099cafc1834d3c5dac305865d04235f7c21c167c8dd31ebc3d6bbc357e2f023", size = 770200, upload-time = "2025-09-22T19:50:55.718Z" }, + { url = "https://files.pythonhosted.org/packages/ef/44/3455eebc761dc8e8fdced90f2b0a3fa61e32ba38b50de4130e2d57db0f21/ruamel.yaml.clib-0.2.14-cp312-cp312-win32.whl", hash = "sha256:b5b0f7e294700b615a3bcf6d28b26e6da94e8eba63b079f4ec92e9ba6c0d6b54", size = 98829, upload-time = "2025-09-22T19:50:58.895Z" }, + { url = "https://files.pythonhosted.org/packages/76/ab/5121f7f3b651db93de546f8c982c241397aad0a4765d793aca1dac5eadee/ruamel.yaml.clib-0.2.14-cp312-cp312-win_amd64.whl", hash = "sha256:a37f40a859b503304dd740686359fcf541d6fb3ff7fc10f539af7f7150917c68", size = 115570, upload-time = "2025-09-22T19:50:57.981Z" }, +] + +[[package]] +name = "runloop-api-client" +version = "0.32.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/da/4d13de09e8090a5a84edb86b837eb3630766f9dc6a0efcc38989d42215a4/runloop_api_client-0.32.0.tar.gz", hash = "sha256:735a967d96b5c3e8a08b89072722adcbe2b10ed904268d3f45785b7cfd5420d1", size = 195611, upload-time = "2025-04-25T17:26:41.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/22/2aa82315662626b217bb51b037fde764b5e6a92eea28082b92d8bff8f080/runloop_api_client-0.32.0-py3-none-any.whl", hash = "sha256:37f156f711b1aa4cef86c0f779cc27afa43ce3d3f6b1976d7f68667466317a6d", size = 217312, upload-time = "2025-04-25T17:26:40.083Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, +] + +[[package]] +name = "sacrebleu" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "lxml" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "portalocker" }, + { name = "regex" }, + { name = "tabulate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/14/8526cf8a5b912b618e7d6ed319a5b1876788bebba1f9a660e1291832c1cc/sacrebleu-2.5.1.tar.gz", hash = "sha256:1a088cc1c74ffaff0759c3191a85db09eecfa7a52e09be244e319d8d64e2fb11", size = 1896900, upload-time = "2025-01-03T20:15:16.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/45/7b55a7bd7e5c5b573b40ad58ba43fa09962dc5c8d71b1f573d4aeaa54a7e/sacrebleu-2.5.1-py3-none-any.whl", hash = "sha256:7c9f7ee75bec3a5bf19dd87112dfd654952130e403ad30c48298fb7da3212d5d", size = 104107, upload-time = "2025-01-03T20:15:14.626Z" }, +] + +[[package]] +name = "safetensors" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/cc/738f3011628920e027a11754d9cae9abec1aed00f7ae860abbf843755233/safetensors-0.6.2.tar.gz", hash = "sha256:43ff2aa0e6fa2dc3ea5524ac7ad93a9839256b8703761e76e2d0b2a3fa4f15d9", size = 197968, upload-time = "2025-08-08T13:13:58.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/b1/3f5fd73c039fc87dba3ff8b5d528bfc5a32b597fea8e7a6a4800343a17c7/safetensors-0.6.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c85ede8ec58f120bad982ec47746981e210492a6db876882aa021446af8ffba", size = 454797, upload-time = "2025-08-08T13:13:52.066Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c9/bb114c158540ee17907ec470d01980957fdaf87b4aa07914c24eba87b9c6/safetensors-0.6.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6675cf4b39c98dbd7d940598028f3742e0375a6b4d4277e76beb0c35f4b843b", size = 432206, upload-time = "2025-08-08T13:13:50.931Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8e/f70c34e47df3110e8e0bb268d90db8d4be8958a54ab0336c9be4fe86dac8/safetensors-0.6.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d2d2b3ce1e2509c68932ca03ab8f20570920cd9754b05063d4368ee52833ecd", size = 473261, upload-time = "2025-08-08T13:13:41.259Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f5/be9c6a7c7ef773e1996dc214e73485286df1836dbd063e8085ee1976f9cb/safetensors-0.6.2-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:93de35a18f46b0f5a6a1f9e26d91b442094f2df02e9fd7acf224cfec4238821a", size = 485117, upload-time = "2025-08-08T13:13:43.506Z" }, + { url = "https://files.pythonhosted.org/packages/c9/55/23f2d0a2c96ed8665bf17a30ab4ce5270413f4d74b6d87dd663258b9af31/safetensors-0.6.2-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89a89b505f335640f9120fac65ddeb83e40f1fd081cb8ed88b505bdccec8d0a1", size = 616154, upload-time = "2025-08-08T13:13:45.096Z" }, + { url = "https://files.pythonhosted.org/packages/98/c6/affb0bd9ce02aa46e7acddbe087912a04d953d7a4d74b708c91b5806ef3f/safetensors-0.6.2-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc4d0d0b937e04bdf2ae6f70cd3ad51328635fe0e6214aa1fc811f3b576b3bda", size = 520713, upload-time = "2025-08-08T13:13:46.25Z" }, + { url = "https://files.pythonhosted.org/packages/fe/5d/5a514d7b88e310c8b146e2404e0dc161282e78634d9358975fd56dfd14be/safetensors-0.6.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8045db2c872db8f4cbe3faa0495932d89c38c899c603f21e9b6486951a5ecb8f", size = 485835, upload-time = "2025-08-08T13:13:49.373Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7b/4fc3b2ba62c352b2071bea9cfbad330fadda70579f617506ae1a2f129cab/safetensors-0.6.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:81e67e8bab9878bb568cffbc5f5e655adb38d2418351dc0859ccac158f753e19", size = 521503, upload-time = "2025-08-08T13:13:47.651Z" }, + { url = "https://files.pythonhosted.org/packages/5a/50/0057e11fe1f3cead9254315a6c106a16dd4b1a19cd247f7cc6414f6b7866/safetensors-0.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0e4d029ab0a0e0e4fdf142b194514695b1d7d3735503ba700cf36d0fc7136ce", size = 652256, upload-time = "2025-08-08T13:13:53.167Z" }, + { url = "https://files.pythonhosted.org/packages/e9/29/473f789e4ac242593ac1656fbece6e1ecd860bb289e635e963667807afe3/safetensors-0.6.2-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:fa48268185c52bfe8771e46325a1e21d317207bcabcb72e65c6e28e9ffeb29c7", size = 747281, upload-time = "2025-08-08T13:13:54.656Z" }, + { url = "https://files.pythonhosted.org/packages/68/52/f7324aad7f2df99e05525c84d352dc217e0fa637a4f603e9f2eedfbe2c67/safetensors-0.6.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:d83c20c12c2d2f465997c51b7ecb00e407e5f94d7dec3ea0cc11d86f60d3fde5", size = 692286, upload-time = "2025-08-08T13:13:55.884Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/cad1d9762868c7c5dc70c8620074df28ebb1a8e4c17d4c0cb031889c457e/safetensors-0.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d944cea65fad0ead848b6ec2c37cc0b197194bec228f8020054742190e9312ac", size = 655957, upload-time = "2025-08-08T13:13:57.029Z" }, + { url = "https://files.pythonhosted.org/packages/59/a7/e2158e17bbe57d104f0abbd95dff60dda916cf277c9f9663b4bf9bad8b6e/safetensors-0.6.2-cp38-abi3-win32.whl", hash = "sha256:cab75ca7c064d3911411461151cb69380c9225798a20e712b102edda2542ddb1", size = 308926, upload-time = "2025-08-08T13:14:01.095Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c3/c0be1135726618dc1e28d181b8c442403d8dbb9e273fd791de2d4384bcdd/safetensors-0.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:c7b214870df923cbc1593c3faee16bec59ea462758699bd3fee399d00aac072c", size = 320192, upload-time = "2025-08-08T13:13:59.467Z" }, +] + +[[package]] +name = "scantree" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "pathspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ce/828467ddfa0d2fe473673026442d2032d552a168e42cfbf25fd0e5264e0c/scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174", size = 20690, upload-time = "2024-08-03T20:08:58.137Z" }, +] + +[[package]] +name = "scikit-image" +version = "0.25.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "imageio" }, + { name = "lazy-loader" }, + { name = "networkx" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "scipy" }, + { name = "tifffile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/a8/3c0f256012b93dd2cb6fda9245e9f4bff7dc0486880b248005f15ea2255e/scikit_image-0.25.2.tar.gz", hash = "sha256:e5a37e6cd4d0c018a7a55b9d601357e3382826d3888c10d0213fc63bff977dde", size = 22693594, upload-time = "2025-02-18T18:05:24.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/8c/5df82881284459f6eec796a5ac2a0a304bb3384eec2e73f35cfdfcfbf20c/scikit_image-0.25.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8db8dd03663112783221bf01ccfc9512d1cc50ac9b5b0fe8f4023967564719fb", size = 13986000, upload-time = "2025-02-18T18:04:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e6/93bebe1abcdce9513ffec01d8af02528b4c41fb3c1e46336d70b9ed4ef0d/scikit_image-0.25.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:483bd8cc10c3d8a7a37fae36dfa5b21e239bd4ee121d91cad1f81bba10cfb0ed", size = 13235893, upload-time = "2025-02-18T18:04:51.049Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/eda616e33f67129e5979a9eb33c710013caa3aa8a921991e6cc0b22cea33/scikit_image-0.25.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d1e80107bcf2bf1291acfc0bf0425dceb8890abe9f38d8e94e23497cbf7ee0d", size = 14178389, upload-time = "2025-02-18T18:04:54.245Z" }, + { url = "https://files.pythonhosted.org/packages/6b/b5/b75527c0f9532dd8a93e8e7cd8e62e547b9f207d4c11e24f0006e8646b36/scikit_image-0.25.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a17e17eb8562660cc0d31bb55643a4da996a81944b82c54805c91b3fe66f4824", size = 15003435, upload-time = "2025-02-18T18:04:57.586Z" }, + { url = "https://files.pythonhosted.org/packages/34/e3/49beb08ebccda3c21e871b607c1cb2f258c3fa0d2f609fed0a5ba741b92d/scikit_image-0.25.2-cp312-cp312-win_amd64.whl", hash = "sha256:bdd2b8c1de0849964dbc54037f36b4e9420157e67e45a8709a80d727f52c7da2", size = 12899474, upload-time = "2025-02-18T18:05:01.166Z" }, +] + +[[package]] +name = "scipy" +version = "1.16.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/3b/546a6f0bfe791bbb7f8d591613454d15097e53f906308ec6f7c1ce588e8e/scipy-1.16.2.tar.gz", hash = "sha256:af029b153d243a80afb6eabe40b0a07f8e35c9adc269c019f364ad747f826a6b", size = 30580599, upload-time = "2025-09-11T17:48:08.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/8d/6396e00db1282279a4ddd507c5f5e11f606812b608ee58517ce8abbf883f/scipy-1.16.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:89d6c100fa5c48472047632e06f0876b3c4931aac1f4291afc81a3644316bb0d", size = 36646259, upload-time = "2025-09-11T17:40:39.329Z" }, + { url = "https://files.pythonhosted.org/packages/3b/93/ea9edd7e193fceb8eef149804491890bde73fb169c896b61aa3e2d1e4e77/scipy-1.16.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ca748936cd579d3f01928b30a17dc474550b01272d8046e3e1ee593f23620371", size = 28888976, upload-time = "2025-09-11T17:40:46.82Z" }, + { url = "https://files.pythonhosted.org/packages/91/4d/281fddc3d80fd738ba86fd3aed9202331180b01e2c78eaae0642f22f7e83/scipy-1.16.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:fac4f8ce2ddb40e2e3d0f7ec36d2a1e7f92559a2471e59aec37bd8d9de01fec0", size = 20879905, upload-time = "2025-09-11T17:40:52.545Z" }, + { url = "https://files.pythonhosted.org/packages/69/40/b33b74c84606fd301b2915f0062e45733c6ff5708d121dd0deaa8871e2d0/scipy-1.16.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:033570f1dcefd79547a88e18bccacff025c8c647a330381064f561d43b821232", size = 23553066, upload-time = "2025-09-11T17:40:59.014Z" }, + { url = "https://files.pythonhosted.org/packages/55/a7/22c739e2f21a42cc8f16bc76b47cff4ed54fbe0962832c589591c2abec34/scipy-1.16.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ea3421209bf00c8a5ef2227de496601087d8f638a2363ee09af059bd70976dc1", size = 33336407, upload-time = "2025-09-11T17:41:06.796Z" }, + { url = "https://files.pythonhosted.org/packages/53/11/a0160990b82999b45874dc60c0c183d3a3a969a563fffc476d5a9995c407/scipy-1.16.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f66bd07ba6f84cd4a380b41d1bf3c59ea488b590a2ff96744845163309ee8e2f", size = 35673281, upload-time = "2025-09-11T17:41:15.055Z" }, + { url = "https://files.pythonhosted.org/packages/96/53/7ef48a4cfcf243c3d0f1643f5887c81f29fdf76911c4e49331828e19fc0a/scipy-1.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e9feab931bd2aea4a23388c962df6468af3d808ddf2d40f94a81c5dc38f32ef", size = 36004222, upload-time = "2025-09-11T17:41:23.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7f/71a69e0afd460049d41c65c630c919c537815277dfea214031005f474d78/scipy-1.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03dfc75e52f72cf23ec2ced468645321407faad8f0fe7b1f5b49264adbc29cb1", size = 38664586, upload-time = "2025-09-11T17:41:31.021Z" }, + { url = "https://files.pythonhosted.org/packages/34/95/20e02ca66fb495a95fba0642fd48e0c390d0ece9b9b14c6e931a60a12dea/scipy-1.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:0ce54e07bbb394b417457409a64fd015be623f36e330ac49306433ffe04bc97e", size = 38550641, upload-time = "2025-09-11T17:41:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/92/ad/13646b9beb0a95528ca46d52b7babafbe115017814a611f2065ee4e61d20/scipy-1.16.2-cp312-cp312-win_arm64.whl", hash = "sha256:2a8ffaa4ac0df81a0b94577b18ee079f13fecdb924df3328fc44a7dc5ac46851", size = 25456070, upload-time = "2025-09-11T17:41:41.3Z" }, +] + +[[package]] +name = "seaborn" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, +] + +[[package]] +name = "secretstorage" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography", marker = "sys_platform == 'linux' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "jeepney", marker = "sys_platform == 'linux' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/9f/11ef35cf1027c1339552ea7bfe6aaa74a8516d8b5caf6e7d338daf54fd80/secretstorage-3.4.0.tar.gz", hash = "sha256:c46e216d6815aff8a8a18706a2fbfd8d53fcbb0dce99301881687a1b0289ef7c", size = 19748, upload-time = "2025-09-09T16:42:13.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/ff/2e2eed29e02c14a5cb6c57f09b2d5b40e65d6cc71f45b52e0be295ccbc2f/secretstorage-3.4.0-py3-none-any.whl", hash = "sha256:0e3b6265c2c63509fb7415717607e4b2c9ab767b7f344a57473b779ca13bd02e", size = 15272, upload-time = "2025-09-09T16:42:12.744Z" }, +] + +[[package]] +name = "sentencepiece" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/15/2e7a025fc62d764b151ae6d0f2a92f8081755ebe8d4a64099accc6f77ba6/sentencepiece-0.2.1.tar.gz", hash = "sha256:8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad", size = 3228515, upload-time = "2025-08-12T07:00:51.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/be/32ce495aa1d0e0c323dcb1ba87096037358edee539cac5baf8755a6bd396/sentencepiece-0.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:57cae326c8727de58c85977b175af132a7138d84c764635d7e71bbee7e774133", size = 1943152, upload-time = "2025-08-12T06:59:40.048Z" }, + { url = "https://files.pythonhosted.org/packages/88/7e/ff23008899a58678e98c6ff592bf4d368eee5a71af96d0df6b38a039dd4f/sentencepiece-0.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:56dd39a3c4d6493db3cdca7e8cc68c6b633f0d4195495cbadfcf5af8a22d05a6", size = 1325651, upload-time = "2025-08-12T06:59:41.536Z" }, + { url = "https://files.pythonhosted.org/packages/19/84/42eb3ce4796777a1b5d3699dfd4dca85113e68b637f194a6c8d786f16a04/sentencepiece-0.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9381351182ff9888cc80e41c632e7e274b106f450de33d67a9e8f6043da6f76", size = 1253645, upload-time = "2025-08-12T06:59:42.903Z" }, + { url = "https://files.pythonhosted.org/packages/89/fa/d3d5ebcba3cb9e6d3775a096251860c41a6bc53a1b9461151df83fe93255/sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99f955df238021bf11f0fc37cdb54fd5e5b5f7fd30ecc3d93fb48b6815437167", size = 1316273, upload-time = "2025-08-12T06:59:44.476Z" }, + { url = "https://files.pythonhosted.org/packages/04/88/14f2f4a2b922d8b39be45bf63d79e6cd3a9b2f248b2fcb98a69b12af12f5/sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0cdfecef430d985f1c2bcbfff3defd1d95dae876fbd0173376012d2d7d24044b", size = 1387881, upload-time = "2025-08-12T06:59:46.09Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b8/903e5ccb77b4ef140605d5d71b4f9e0ad95d456d6184688073ed11712809/sentencepiece-0.2.1-cp312-cp312-win32.whl", hash = "sha256:a483fd29a34c3e34c39ac5556b0a90942bec253d260235729e50976f5dba1068", size = 999540, upload-time = "2025-08-12T06:59:48.023Z" }, + { url = "https://files.pythonhosted.org/packages/2d/81/92df5673c067148c2545b1bfe49adfd775bcc3a169a047f5a0e6575ddaca/sentencepiece-0.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:4cdc7c36234fda305e85c32949c5211faaf8dd886096c7cea289ddc12a2d02de", size = 1054671, upload-time = "2025-08-12T06:59:49.895Z" }, + { url = "https://files.pythonhosted.org/packages/fe/02/c5e3bc518655d714622bec87d83db9cdba1cd0619a4a04e2109751c4f47f/sentencepiece-0.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:daeb5e9e9fcad012324807856113708614d534f596d5008638eb9b40112cd9e4", size = 1033923, upload-time = "2025-08-12T06:59:51.952Z" }, +] + +[[package]] +name = "sentry-sdk" +version = "2.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/72/43294fa4bdd75c51610b5104a3ff834459ba653abb415150aa7826a249dd/sentry_sdk-2.39.0.tar.gz", hash = "sha256:8c185854d111f47f329ab6bc35993f28f7a6b7114db64aa426b326998cfa14e9", size = 348556, upload-time = "2025-09-25T09:15:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/44/4356cc64246ba7b2b920f7c97a85c3c52748e213e250b512ee8152eb559d/sentry_sdk-2.39.0-py2.py3-none-any.whl", hash = "sha256:ba655ca5e57b41569b18e2a5552cb3375209760a5d332cdd87c6c3f28f729602", size = 370851, upload-time = "2025-09-25T09:15:36.35Z" }, +] + +[[package]] +name = "setproctitle" +version = "1.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/f0/2dc88e842077719d7384d86cc47403e5102810492b33680e7dadcee64cd8/setproctitle-1.3.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2dc99aec591ab6126e636b11035a70991bc1ab7a261da428491a40b84376654e", size = 18049, upload-time = "2025-09-05T12:49:36.241Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b4/50940504466689cda65680c9e9a1e518e5750c10490639fa687489ac7013/setproctitle-1.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdd8aa571b7aa39840fdbea620e308a19691ff595c3a10231e9ee830339dd798", size = 13079, upload-time = "2025-09-05T12:49:38.088Z" }, + { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/cee06af4ffcfb0e8aba047bd44f5262e644199ae7527ae2c1f672b86495c/setproctitle-1.3.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6915964a6dda07920a1159321dcd6d94fc7fc526f815ca08a8063aeca3c204f1", size = 33736, upload-time = "2025-09-05T12:49:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3a/50caca532a9343828e3bf5778c7a84d6c737a249b1796d50dd680290594d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7cb05bd446687ff816a3aaaf831047fc4c364feff7ada94a66024f1367b448c", size = 33143, upload-time = "2025-09-05T12:49:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, + { url = "https://files.pythonhosted.org/packages/ef/dc/ef76a81fac9bf27b84ed23df19c1f67391a753eed6e3c2254ebcb5133f56/setproctitle-1.3.7-cp312-cp312-win32.whl", hash = "sha256:b0304f905efc845829ac2bc791ddebb976db2885f6171f4a3de678d7ee3f7c9f", size = 12552, upload-time = "2025-09-05T12:49:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, +] + +[[package]] +name = "setuptools" +version = "79.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/71/b6365e6325b3290e14957b2c3a804a529968c77a049b2ed40c095f749707/setuptools-79.0.1.tar.gz", hash = "sha256:128ce7b8f33c3079fd1b067ecbb4051a66e8526e7b65f6cec075dfc650ddfa88", size = 1367909, upload-time = "2025-04-23T22:20:59.241Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/6d/b4752b044bf94cb802d88a888dc7d288baaf77d7910b7dedda74b5ceea0c/setuptools-79.0.1-py3-none-any.whl", hash = "sha256:e147c0549f27767ba362f9da434eab9c5dc0045d5304feb602a0af001089fc51", size = 1256281, upload-time = "2025-04-23T22:20:56.768Z" }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + +[[package]] +name = "shapely" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/c0/f3b6453cf2dfa99adc0ba6675f9aaff9e526d2224cbd7ff9c1a879238693/shapely-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe2533caae6a91a543dec62e8360fe86ffcdc42a7c55f9dfd0128a977a896b94", size = 1833550, upload-time = "2025-09-24T13:50:30.019Z" }, + { url = "https://files.pythonhosted.org/packages/86/07/59dee0bc4b913b7ab59ab1086225baca5b8f19865e6101db9ebb7243e132/shapely-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ba4d1333cc0bc94381d6d4308d2e4e008e0bd128bdcff5573199742ee3634359", size = 1643556, upload-time = "2025-09-24T13:50:32.291Z" }, + { url = "https://files.pythonhosted.org/packages/26/29/a5397e75b435b9895cd53e165083faed5d12fd9626eadec15a83a2411f0f/shapely-2.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0bd308103340030feef6c111d3eb98d50dc13feea33affc8a6f9fa549e9458a3", size = 2988308, upload-time = "2025-09-24T13:50:33.862Z" }, + { url = "https://files.pythonhosted.org/packages/b9/37/e781683abac55dde9771e086b790e554811a71ed0b2b8a1e789b7430dd44/shapely-2.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e7d4d7ad262a48bb44277ca12c7c78cb1b0f56b32c10734ec9a1d30c0b0c54b", size = 3099844, upload-time = "2025-09-24T13:50:35.459Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f3/9876b64d4a5a321b9dc482c92bb6f061f2fa42131cba643c699f39317cb9/shapely-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9eddfe513096a71896441a7c37db72da0687b34752c4e193577a145c71736fc", size = 3988842, upload-time = "2025-09-24T13:50:37.478Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a0/704c7292f7014c7e74ec84eddb7b109e1fbae74a16deae9c1504b1d15565/shapely-2.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:980c777c612514c0cf99bc8a9de6d286f5e186dcaf9091252fcd444e5638193d", size = 4152714, upload-time = "2025-09-24T13:50:39.9Z" }, + { url = "https://files.pythonhosted.org/packages/53/46/319c9dc788884ad0785242543cdffac0e6530e4d0deb6c4862bc4143dcf3/shapely-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9111274b88e4d7b54a95218e243282709b330ef52b7b86bc6aaf4f805306f454", size = 1542745, upload-time = "2025-09-24T13:50:41.414Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bf/cb6c1c505cb31e818e900b9312d514f381fbfa5c4363edfce0fcc4f8c1a4/shapely-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:743044b4cfb34f9a67205cee9279feaf60ba7d02e69febc2afc609047cb49179", size = 1722861, upload-time = "2025-09-24T13:50:43.35Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "sigtools" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/db/669ca14166814da187b3087b908ca924cf83f5b504fe23b3859a3ef67d4f/sigtools-4.0.1.tar.gz", hash = "sha256:4b8e135a9cd4d2ea00da670c093372d74e672ba3abb87f4c98d8e73dea54445c", size = 71910, upload-time = "2022-10-13T07:03:54.149Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/91/853dbf6ec096197dba9cd5fd0c836c5fc19142038b7db60ebe6332b1bab1/sigtools-4.0.1-py2.py3-none-any.whl", hash = "sha256:d216b4cf920bbab0fce636ddc429ed8463a5b533d9e1492acb45a2a1bc36ac6c", size = 76419, upload-time = "2022-10-13T07:03:52.658Z" }, +] + +[[package]] +name = "simple-websocket" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "skyagent" +version = "0.0.1" +source = { editable = "." } +dependencies = [ + { name = "browsergym" }, + { name = "browsergym-miniwob" }, + { name = "browsergym-visualwebarena" }, + { name = "browsergym-webarena" }, + { name = "codetiming" }, + { name = "datasets" }, + { name = "daytona-api-client" }, + { name = "dill" }, + { name = "evaluate" }, + { name = "func-timeout" }, + { name = "gdown" }, + { name = "hf-transfer" }, + { name = "hydra-core" }, + { name = "json5" }, + { name = "jsonlines" }, + { name = "loguru" }, + { name = "math-verify" }, + { name = "matplotlib" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "openhands-ai" }, + { name = "pandas" }, + { name = "polars" }, + { name = "pyarrow" }, + { name = "pybind11" }, + { name = "pylatexenc" }, + { name = "ray" }, + { name = "retry" }, + { name = "seaborn" }, + { name = "streamlit" }, + { name = "swebench" }, + { name = "swegym" }, + { name = "sympy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "sympy", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "tabulate" }, + { name = "together" }, + { name = "torchdata" }, + { name = "transformers" }, + { name = "vllm", version = "0.8.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "vllm", version = "0.10.1.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "vllm", version = "0.10.2", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "wandb" }, + { name = "whatthepatch" }, +] + +[package.optional-dependencies] +skyrl-train = [ + { name = "skyrl-train", extra = ["vllm"], marker = "extra == 'extra-8-skyagent-skyrl-train'" }, +] +verl = [ + { name = "verl", extra = ["gpu", "vllm"], marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "vllm", version = "0.8.5", source = { registry = "https://pypi.org/simple" } }, +] + +[package.metadata] +requires-dist = [ + { name = "browsergym", specifier = "==0.13.3" }, + { name = "browsergym-miniwob", specifier = "==0.13.3" }, + { name = "browsergym-visualwebarena", specifier = "==0.13.3" }, + { name = "browsergym-webarena", specifier = "==0.13.3" }, + { name = "codetiming" }, + { name = "datasets" }, + { name = "datasets", specifier = "==4.0.0" }, + { name = "daytona-api-client", specifier = "==0.20.1" }, + { name = "dill" }, + { name = "evaluate" }, + { name = "func-timeout" }, + { name = "gdown" }, + { name = "hf-transfer" }, + { name = "hydra-core" }, + { name = "json5" }, + { name = "jsonlines" }, + { name = "loguru" }, + { name = "math-verify" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "openhands-ai", editable = "SkyRL-OpenHands" }, + { name = "pandas" }, + { name = "polars" }, + { name = "pyarrow", specifier = ">=15.0.0" }, + { name = "pybind11" }, + { name = "pylatexenc" }, + { name = "ray", extras = ["default"], specifier = ">=2.10" }, + { name = "retry" }, + { name = "seaborn" }, + { name = "skyrl-train", extras = ["vllm"], marker = "extra == 'skyrl-train'", editable = "skyrl-train" }, + { name = "streamlit" }, + { name = "swebench", git = "https://github.com/SWE-Gym/SWE-Bench-Fork.git" }, + { name = "swegym", git = "https://github.com/SWE-Gym/SWE-Bench-Package.git" }, + { name = "sympy" }, + { name = "tabulate" }, + { name = "together" }, + { name = "torchdata" }, + { name = "transformers" }, + { name = "verl", extras = ["vllm", "gpu"], marker = "extra == 'verl'", editable = "verl" }, + { name = "vllm", specifier = ">=0.8.5" }, + { name = "vllm", marker = "extra == 'verl'", specifier = "==0.8.5" }, + { name = "wandb" }, + { name = "whatthepatch" }, +] +provides-extras = ["skyrl-train", "verl"] + +[[package]] +name = "skyrl-gym" +version = "0.1.1" +source = { editable = "skyrl-train/skyrl-gym" } +dependencies = [ + { name = "func-timeout" }, + { name = "omegaconf" }, + { name = "pandas" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "func-timeout" }, + { name = "omegaconf" }, + { name = "pandas" }, + { name = "pytest", marker = "extra == 'dev'" }, + { name = "requests" }, +] +provides-extras = ["dev"] + +[[package]] +name = "skyrl-train" +version = "0.1.0" +source = { editable = "skyrl-train" } +dependencies = [ + { name = "accelerate" }, + { name = "datasets" }, + { name = "debugpy", version = "1.8.0", source = { registry = "https://pypi.org/simple" } }, + { name = "flash-attn", version = "2.8.0.post2", source = { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.0.post2/flash_attn-2.8.0.post2+cu12torch2.7cxx11abiFALSE-cp312-cp312-linux_x86_64.whl" } }, + { name = "func-timeout" }, + { name = "hf-transfer" }, + { name = "hydra-core" }, + { name = "jaxtyping" }, + { name = "loguru" }, + { name = "omegaconf" }, + { name = "peft" }, + { name = "polars" }, + { name = "ray" }, + { name = "skyrl-gym" }, + { name = "tensorboard" }, + { name = "tensordict", version = "0.10.0", source = { registry = "https://pypi.org/simple" } }, + { name = "torchdata" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "wandb" }, +] + +[package.optional-dependencies] +vllm = [ + { name = "flashinfer-python" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.22.1", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "vllm", version = "0.10.1.1", source = { registry = "https://pypi.org/simple" } }, +] + +[package.metadata] +requires-dist = [ + { name = "accelerate" }, + { name = "black", marker = "extra == 'dev'", specifier = "==24.10.0" }, + { name = "datasets", specifier = "==4.0.0" }, + { name = "debugpy", specifier = "==1.8.0" }, + { name = "deepspeed", marker = "extra == 'deepspeed'", specifier = "==0.16.5" }, + { name = "fastapi", marker = "extra == 'dev'" }, + { name = "flash-attn", url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.0.post2/flash_attn-2.8.0.post2+cu12torch2.7cxx11abiFALSE-cp312-cp312-linux_x86_64.whl" }, + { name = "flashinfer-python", marker = "extra == 'flashrl' and extra != 'sglang' and extra != 'vllm'" }, + { name = "flashinfer-python", marker = "extra == 'flashrl' and extra == 'sglang' and extra != 'vllm'", url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" }, + { name = "flashinfer-python", marker = "extra == 'flashrl' and extra == 'vllm'", url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" }, + { name = "flashinfer-python", marker = "extra == 'sglang' and extra == 'vllm'", url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" }, + { name = "flashinfer-python", marker = "extra == 'sglang' and extra != 'vllm'", url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" }, + { name = "flashinfer-python", marker = "extra == 'vllm'", url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" }, + { name = "func-timeout" }, + { name = "hf-transfer" }, + { name = "hydra-core", specifier = "==1.3.2" }, + { name = "jaxtyping" }, + { name = "litellm", marker = "extra == 'dev'" }, + { name = "litellm", marker = "extra == 'miniswe'" }, + { name = "litellm", extras = ["proxy"], marker = "extra == 'sandboxes'", specifier = ">=1.67.5" }, + { name = "loguru" }, + { name = "mbridge", marker = "extra == 'mcore'", specifier = "==0.15.1" }, + { name = "megatron-core", marker = "extra == 'mcore'", git = "https://github.com/NVIDIA/Megatron-LM.git?rev=core_r0.13.0" }, + { name = "mini-swe-agent", marker = "extra == 'miniswe'", specifier = ">=1.12.0" }, + { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=2.0.0" }, + { name = "omegaconf" }, + { name = "peft" }, + { name = "polars" }, + { name = "pre-commit", marker = "extra == 'dev'" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=6.2.5" }, + { name = "pytest-asyncio", marker = "extra == 'dev'" }, + { name = "ray", specifier = "==2.48.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.11.9" }, + { name = "sglang", extras = ["srt", "openai", "torch-memory-saver"], marker = "extra == 'sglang'", specifier = "==0.4.8.post1" }, + { name = "skyrl-gym", editable = "skyrl-train/skyrl-gym" }, + { name = "sphinx", marker = "extra == 'docs'", specifier = ">=7.0.0" }, + { name = "sphinx-autobuild", marker = "extra == 'docs'", specifier = ">=2021.3.14" }, + { name = "sphinx-autodoc-typehints", marker = "extra == 'docs'", specifier = ">=1.25.0" }, + { name = "sphinx-copybutton", marker = "extra == 'docs'", specifier = ">=0.5.0" }, + { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = ">=2.0.0" }, + { name = "tensorboard" }, + { name = "tensordict" }, + { name = "torch", marker = "extra == 'flashrl'", specifier = "==2.7.0", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torch", marker = "extra == 'sglang'", specifier = "==2.7.1", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torch", marker = "extra == 'vllm'", specifier = "==2.7.1", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchdata" }, + { name = "torchvision", marker = "extra == 'flashrl'", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "extra == 'sglang'", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "extra == 'vllm'", index = "https://download.pytorch.org/whl/cu128" }, + { name = "tqdm" }, + { name = "transformers", specifier = ">=4.51.0" }, + { name = "uvicorn", marker = "extra == 'dev'" }, + { name = "vllm", marker = "extra == 'flashrl'", url = "https://github.com/NovaSky-AI/SkyRL/releases/download/skyrl_train-v0.1.0/vllm-0.1.dev7509+gcc487699a.d20250821-cp312-cp312-linux_x86_64.whl" }, + { name = "vllm", marker = "extra == 'vllm'", specifier = "==0.10.1.1" }, + { name = "wandb" }, +] +provides-extras = ["deepspeed", "dev", "docs", "sandboxes", "vllm", "sglang", "mcore", "flashrl", "miniswe"] + +[[package]] +name = "smmap" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "soundfile" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/41/9b873a8c055582859b239be17902a85339bec6a30ad162f98c9b0288a2cc/soundfile-0.13.1.tar.gz", hash = "sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b", size = 46156, upload-time = "2025-01-25T09:17:04.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/28/e2a36573ccbcf3d57c00626a21fe51989380636e821b341d36ccca0c1c3a/soundfile-0.13.1-py2.py3-none-any.whl", hash = "sha256:a23c717560da2cf4c7b5ae1142514e0fd82d6bbd9dfc93a50423447142f2c445", size = 25751, upload-time = "2025-01-25T09:16:44.235Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ab/73e97a5b3cc46bba7ff8650a1504348fa1863a6f9d57d7001c6b67c5f20e/soundfile-0.13.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:82dc664d19831933fe59adad199bf3945ad06d84bc111a5b4c0d3089a5b9ec33", size = 1142250, upload-time = "2025-01-25T09:16:47.583Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e5/58fd1a8d7b26fc113af244f966ee3aecf03cb9293cb935daaddc1e455e18/soundfile-0.13.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:743f12c12c4054921e15736c6be09ac26b3b3d603aef6fd69f9dde68748f2593", size = 1101406, upload-time = "2025-01-25T09:16:49.662Z" }, + { url = "https://files.pythonhosted.org/packages/58/ae/c0e4a53d77cf6e9a04179535766b3321b0b9ced5f70522e4caf9329f0046/soundfile-0.13.1-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9c9e855f5a4d06ce4213f31918653ab7de0c5a8d8107cd2427e44b42df547deb", size = 1235729, upload-time = "2025-01-25T09:16:53.018Z" }, + { url = "https://files.pythonhosted.org/packages/57/5e/70bdd9579b35003a489fc850b5047beeda26328053ebadc1fb60f320f7db/soundfile-0.13.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:03267c4e493315294834a0870f31dbb3b28a95561b80b134f0bd3cf2d5f0e618", size = 1313646, upload-time = "2025-01-25T09:16:54.872Z" }, + { url = "https://files.pythonhosted.org/packages/fe/df/8c11dc4dfceda14e3003bb81a0d0edcaaf0796dd7b4f826ea3e532146bba/soundfile-0.13.1-py2.py3-none-win32.whl", hash = "sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5", size = 899881, upload-time = "2025-01-25T09:16:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/6b761de83277f2f02ded7e7ea6f07828ec78e4b229b80e4ca55dd205b9dc/soundfile-0.13.1-py2.py3-none-win_amd64.whl", hash = "sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9", size = 1019162, upload-time = "2025-01-25T09:16:59.573Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/e6/21ccce3262dd4889aa3332e5a119a3491a95e8f60939870a3a035aabac0d/soupsieve-2.8.tar.gz", hash = "sha256:e2dd4a40a628cb5f28f6d4b0db8800b8f581b65bb380b97de22ba5ca8d72572f", size = 103472, upload-time = "2025-08-27T15:39:51.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/a0/bb38d3b76b8cae341dad93a2dd83ab7462e6dbcdd84d43f54ee60a8dc167/soupsieve-2.8-py3-none-any.whl", hash = "sha256:0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c", size = 36679, upload-time = "2025-08-27T15:39:50.179Z" }, +] + +[[package]] +name = "soxr" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/7e/f4b461944662ad75036df65277d6130f9411002bfb79e9df7dff40a31db9/soxr-1.0.0.tar.gz", hash = "sha256:e07ee6c1d659bc6957034f4800c60cb8b98de798823e34d2a2bba1caa85a4509", size = 171415, upload-time = "2025-09-07T13:22:21.317Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/c7/f92b81f1a151c13afb114f57799b86da9330bec844ea5a0d3fe6a8732678/soxr-1.0.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:abecf4e39017f3fadb5e051637c272ae5778d838e5c3926a35db36a53e3a607f", size = 205508, upload-time = "2025-09-07T13:22:01.252Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1d/c945fea9d83ea1f2be9d116b3674dbaef26ed090374a77c394b31e3b083b/soxr-1.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:e973d487ee46aa8023ca00a139db6e09af053a37a032fe22f9ff0cc2e19c94b4", size = 163568, upload-time = "2025-09-07T13:22:03.558Z" }, + { url = "https://files.pythonhosted.org/packages/b5/80/10640970998a1d2199bef6c4d92205f36968cddaf3e4d0e9fe35ddd405bd/soxr-1.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ce273cca101aff3d8c387db5a5a41001ba76ef1837883438d3c652507a9ccc", size = 204707, upload-time = "2025-09-07T13:22:05.125Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/2726603c13c2126cb8ded9e57381b7377f4f0df6ba4408e1af5ddbfdc3dd/soxr-1.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8f2a69686f2856d37823bbb7b78c3d44904f311fe70ba49b893af11d6b6047b", size = 238032, upload-time = "2025-09-07T13:22:06.428Z" }, + { url = "https://files.pythonhosted.org/packages/ce/04/530252227f4d0721a5524a936336485dfb429bb206a66baf8e470384f4a2/soxr-1.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:2a3b77b115ae7c478eecdbd060ed4f61beda542dfb70639177ac263aceda42a2", size = 172070, upload-time = "2025-09-07T13:22:07.62Z" }, +] + +[[package]] +name = "sse-starlette" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/3e/eae74d8d33e3262bae0a7e023bb43d8bdd27980aa3557333f4632611151f/sse_starlette-2.4.1.tar.gz", hash = "sha256:7c8a800a1ca343e9165fc06bbda45c78e4c6166320707ae30b416c42da070926", size = 18635, upload-time = "2025-07-06T09:41:33.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/f1/6c7eaa8187ba789a6dd6d74430307478d2a91c23a5452ab339b6fbe15a08/sse_starlette-2.4.1-py3-none-any.whl", hash = "sha256:08b77ea898ab1a13a428b2b6f73cfe6d0e607a7b4e15b9bb23e4a37b087fd39a", size = 10824, upload-time = "2025-07-06T09:41:32.321Z" }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, +] + +[[package]] +name = "starlette" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/a5/d6f429d43394057b67a6b5bbe6eae2f77a6bf7459d961fdb224bf206eee6/starlette-0.48.0.tar.gz", hash = "sha256:7e8cee469a8ab2352911528110ce9088fdc6a37d9876926e73da7ce4aa4c7a46", size = 2652949, upload-time = "2025-09-13T08:41:05.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/72/2db2f49247d0a18b4f1bb9a5a39a0162869acf235f3a96418363947b3d46/starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659", size = 73736, upload-time = "2025-09-13T08:41:03.869Z" }, +] + +[[package]] +name = "streamlit" +version = "1.50.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altair" }, + { name = "blinker" }, + { name = "cachetools" }, + { name = "click" }, + { name = "gitpython" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "pyarrow" }, + { name = "pydeck" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "tornado" }, + { name = "typing-extensions" }, + { name = "watchdog", marker = "sys_platform != 'darwin' or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/f6/f7d3a0146577c1918439d3163707040f7111a7d2e7e2c73fa7adeb169c06/streamlit-1.50.0.tar.gz", hash = "sha256:87221d568aac585274a05ef18a378b03df332b93e08103fffcf3cd84d852af46", size = 9664808, upload-time = "2025-09-23T19:24:00.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/38/991bbf9fa3ed3d9c8e69265fc449bdaade8131c7f0f750dbd388c3c477dc/streamlit-1.50.0-py3-none-any.whl", hash = "sha256:9403b8f94c0a89f80cf679c2fcc803d9a6951e0fba542e7611995de3f67b4bb3", size = 10068477, upload-time = "2025-09-23T19:23:57.245Z" }, +] + +[[package]] +name = "stripe" +version = "12.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/92/f1b72783e13662026efc7d3f0fc201230edab55edd1604326a97162d8ce4/stripe-12.5.1.tar.gz", hash = "sha256:5407d092c355c31393e767d2dcb2d5a8c3980caa9a073acb32d0ccb3c01b04b5", size = 1432826, upload-time = "2025-09-05T13:27:07.976Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/8d/f4c2f047eedc943aea573cb56af2c9481b2d97ee8de9819f8c5d87e9ba34/stripe-12.5.1-py2.py3-none-any.whl", hash = "sha256:39b47b331dfe933ce6cf6bd654e0ae9ba082caa9d1dd8d0ae9ae48293975f115", size = 1664067, upload-time = "2025-09-05T13:27:05.559Z" }, +] + +[[package]] +name = "swebench" +version = "2.0.13" +source = { git = "https://github.com/SWE-Gym/SWE-Bench-Fork.git#242429c188fcfd06aad13fce9a54d450470bf0ac" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "chardet" }, + { name = "datasets" }, + { name = "docker" }, + { name = "ghapi" }, + { name = "gitpython" }, + { name = "pre-commit" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "rich" }, + { name = "tqdm" }, + { name = "unidiff" }, +] + +[[package]] +name = "swegym" +version = "2.0.13" +source = { git = "https://github.com/SWE-Gym/SWE-Bench-Package.git#16dd480cce9b27bf111a362d280881c6def5d2a7" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "chardet" }, + { name = "datasets" }, + { name = "docker" }, + { name = "ghapi" }, + { name = "gitpython" }, + { name = "pre-commit" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "rich" }, + { name = "tqdm" }, + { name = "unidiff" }, +] + +[[package]] +name = "sympy" +version = "1.13.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "mpmath", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/99/5a5b6f19ff9f083671ddf7b9632028436167cd3d33e11015754e41b249a4/sympy-1.13.1.tar.gz", hash = "sha256:9cebf7e04ff162015ce31c9c6c9144daa34a93bd082f54fd8f12deca4f47515f", size = 7533040, upload-time = "2024-07-19T09:26:51.238Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/fe/81695a1aa331a842b582453b605175f419fe8540355886031328089d840a/sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8", size = 6189177, upload-time = "2024-07-19T09:26:48.863Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +dependencies = [ + { name = "mpmath", marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "synchronicity" +version = "0.9.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sigtools" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/75/0987c52929e352f0055705ece16614ce107db26856a4ee7850c6ae39413e/synchronicity-0.9.16.tar.gz", hash = "sha256:bc174b756d12330b1e6027f8293c3b944cf3684dec241af009cb7bb169a0072f", size = 53303, upload-time = "2025-06-16T14:07:14.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/db/772959498f87bc03f4c8ebe8ee0d07dee4cab2a862b66361d1ef3123e0d8/synchronicity-0.9.16-py3-none-any.whl", hash = "sha256:79dd3451c063a06489d466b0585808e1d5d257fc3c2304f5f733a142ef269e97", size = 37595, upload-time = "2025-06-16T14:07:12.261Z" }, +] + +[[package]] +name = "tabulate" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090, upload-time = "2022-10-06T17:21:48.54Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, +] + +[[package]] +name = "tensorboard" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "grpcio" }, + { name = "markdown" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "tensorboard-data-server" }, + { name = "werkzeug" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" }, +] + +[[package]] +name = "tensorboard-data-server" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/dabeaf902892922777492e1d253bb7e1264cadce3cea932f7ff599e53fea/tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60", size = 4823598, upload-time = "2023-10-23T21:23:33.714Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" }, +] + +[[package]] +name = "tensordict" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "cloudpickle" }, + { name = "importlib-metadata" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "orjson" }, + { name = "packaging" }, + { name = "pyvers" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/dd/2e1d044a5a7901fb2a94686299e7291482f6c5623149edb777be887a0658/tensordict-0.9.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:46f59e9db37af740a3cafb08ff98df6f252ffaa0598a9354d2546dcbb0e97d92", size = 739949, upload-time = "2025-07-14T12:52:15.177Z" }, + { url = "https://files.pythonhosted.org/packages/4c/24/dd6d1874cda92749cd8ddd9a77f82a758f2cd5ece0d376417f662543badf/tensordict-0.9.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5a5b436784d2d19f94a7e6023da8430b93efc202f4daadcb247985569b384461", size = 424861, upload-time = "2025-07-14T12:52:16.239Z" }, + { url = "https://files.pythonhosted.org/packages/89/68/0d801f339ff43aaaecad26d87f74bacb20503d9bcb1bdf7f76486c1f0b86/tensordict-0.9.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ba30eacb5630dc1ee47159a66baa9dc6c5cec3f130b39b7589ff712f9e45777a", size = 430242, upload-time = "2025-07-14T12:52:17.643Z" }, + { url = "https://files.pythonhosted.org/packages/8a/07/3950b9c61be10dc1bf44c8f686c88dd715b803cb78a02b8f0c378c9f80b0/tensordict-0.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:040c7d79f97dcc7e1f99ce7914a01b1fb74b4861d11291d998e4dc47cad4c106", size = 474256, upload-time = "2025-07-14T12:52:19.18Z" }, +] + +[[package]] +name = "tensordict" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "cloudpickle" }, + { name = "importlib-metadata" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "orjson" }, + { name = "packaging" }, + { name = "pyvers" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/89/2914b6d2796bdbe64ba8d42b568bf02c25673f187079e8795fc668c609fa/tensordict-0.10.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f52321ddec5da5acb3b90785c68b4fd4cce805aab6eb700ec59352e9f9e6214f", size = 801519, upload-time = "2025-09-08T10:07:18.954Z" }, + { url = "https://files.pythonhosted.org/packages/9e/88/2c1bf6c1abdc4d0bfcbdda2d1a5b19c7a9540f67ff6d20fe08d328d78305/tensordict-0.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6a6ba462cc4c04299eb3746e8994df4a625706e60e2dfb88cc5f9513b6cbad2f", size = 445427, upload-time = "2025-09-08T10:07:20.148Z" }, + { url = "https://files.pythonhosted.org/packages/5e/05/6e7d130c5e9af947fad25fb7d40a3aa2fd9ef9d37c9c7ddc94ba11853d23/tensordict-0.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6f0a52524c7c46778bf250444f1cd508f055735667b8d596a1a7e2fb38824e8c", size = 449961, upload-time = "2025-09-08T10:07:21.977Z" }, + { url = "https://files.pythonhosted.org/packages/f4/40/877fd0453c9c79a14063ecd21102a23460d033e3760e83eae7fd6c09b3ef/tensordict-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:76e7c1d6604addd08e026141c3943fa15fbe36db537f9ff311af5d2caee25daf", size = 494502, upload-time = "2025-09-08T10:07:23.2Z" }, +] + +[[package]] +name = "termcolor" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/6c/3d75c196ac07ac8749600b60b03f4f6094d54e132c4d94ebac6ee0e0add0/termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970", size = 14324, upload-time = "2025-04-30T11:37:53.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/bd/de8d508070629b6d84a30d01d57e4a65c69aa7f5abe7560b8fad3b50ea59/termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa", size = 7684, upload-time = "2025-04-30T11:37:52.382Z" }, +] + +[[package]] +name = "text-generation" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "huggingface-hub" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/53/1b2dc20686079464ae381f230a9fc412984a4255cea73c21afb6a46bc21f/text_generation-0.7.0.tar.gz", hash = "sha256:689200cd1f0d4141562af2515393c2c21cdbd9fac21c8398bf3043cdcc14184e", size = 10373, upload-time = "2024-03-22T16:09:22.167Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/79/8fc351fd919a41287243c998a47692c7eb0fa5acded13db0080f2c6f1852/text_generation-0.7.0-py3-none-any.whl", hash = "sha256:02ab337a0ee0e7c70e04a607b311c261caae74bde46a7d837c6fdd150108f4d8", size = 12718, upload-time = "2024-03-22T16:09:20.874Z" }, +] + +[[package]] +name = "tifffile" +version = "2025.9.30" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/6e/82f9e07290e1c5270f295f22e32ffd0d86bef31494b004b2a247530a449a/tifffile-2025.9.30.tar.gz", hash = "sha256:1a259f11e94489a9ab599e4e9f40a0e72b17cad206587097209f630768dfcdf3", size = 369276, upload-time = "2025-09-29T20:36:50.356Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/af/14e1f7dd76a735495293a82fb2dea3f3769812ca95cb1383c929173d0884/tifffile-2025.9.30-py3-none-any.whl", hash = "sha256:0b2c42b6821583335407a8c48686358fcfee6e9e94f38895cbf9b111a6186c86", size = 230238, upload-time = "2025-09-29T20:36:48.475Z" }, +] + +[[package]] +name = "tiktoken" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/86/ad0155a37c4f310935d5ac0b1ccf9bdb635dcb906e0a9a26b616dd55825a/tiktoken-0.11.0.tar.gz", hash = "sha256:3c518641aee1c52247c2b97e74d8d07d780092af79d5911a6ab5e79359d9b06a", size = 37648, upload-time = "2025-08-08T23:58:08.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/9e/eceddeffc169fc75fe0fd4f38471309f11cb1906f9b8aa39be4f5817df65/tiktoken-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fd9e6b23e860973cf9526544e220b223c60badf5b62e80a33509d6d40e6c8f5d", size = 1055199, upload-time = "2025-08-08T23:57:45.076Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cf/5f02bfefffdc6b54e5094d2897bc80efd43050e5b09b576fd85936ee54bf/tiktoken-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a76d53cee2da71ee2731c9caa747398762bda19d7f92665e882fef229cb0b5b", size = 996655, upload-time = "2025-08-08T23:57:46.304Z" }, + { url = "https://files.pythonhosted.org/packages/65/8e/c769b45ef379bc360c9978c4f6914c79fd432400a6733a8afc7ed7b0726a/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef72aab3ea240646e642413cb363b73869fed4e604dcfd69eec63dc54d603e8", size = 1128867, upload-time = "2025-08-08T23:57:47.438Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2d/4d77f6feb9292bfdd23d5813e442b3bba883f42d0ac78ef5fdc56873f756/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f929255c705efec7a28bf515e29dc74220b2f07544a8c81b8d69e8efc4578bd", size = 1183308, upload-time = "2025-08-08T23:57:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/7a/65/7ff0a65d3bb0fc5a1fb6cc71b03e0f6e71a68c5eea230d1ff1ba3fd6df49/tiktoken-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61f1d15822e4404953d499fd1dcc62817a12ae9fb1e4898033ec8fe3915fdf8e", size = 1244301, upload-time = "2025-08-08T23:57:49.642Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6e/5b71578799b72e5bdcef206a214c3ce860d999d579a3b56e74a6c8989ee2/tiktoken-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:45927a71ab6643dfd3ef57d515a5db3d199137adf551f66453be098502838b0f", size = 884282, upload-time = "2025-08-08T23:57:50.759Z" }, +] + +[[package]] +name = "together" +version = "1.5.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "click" }, + { name = "eval-type-backport" }, + { name = "filelock" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "rich" }, + { name = "tabulate" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/96/77d36c1aad4ca22623a097e22cb03b05a05637878334fe558e70af96af3d/together-1.5.26.tar.gz", hash = "sha256:1f23591aa5297480949028472fa4cbca9e8042cc9a706e004da06f00e9512f54", size = 79427, upload-time = "2025-09-23T07:35:15.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/cf/3915e34873c410bdf383074019315bb02341472ff691a74c69243159d2ba/together-1.5.26-py3-none-any.whl", hash = "sha256:e28d2607732e24178f727e13866773fc52447efaea9ba82700839db5476f43b7", size = 107519, upload-time = "2025-09-23T07:35:14.694Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/46/fb6854cec3278fbfa4a75b50232c77622bc517ac886156e6afbfa4d8fc6e/tokenizers-0.22.1.tar.gz", hash = "sha256:61de6522785310a309b3407bac22d99c4db5dba349935e99e4d15ea2226af2d9", size = 363123, upload-time = "2025-09-19T09:49:23.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/33/f4b2d94ada7ab297328fc671fed209368ddb82f965ec2224eb1892674c3a/tokenizers-0.22.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59fdb013df17455e5f950b4b834a7b3ee2e0271e6378ccb33aa74d178b513c73", size = 3069318, upload-time = "2025-09-19T09:49:11.848Z" }, + { url = "https://files.pythonhosted.org/packages/1c/58/2aa8c874d02b974990e89ff95826a4852a8b2a273c7d1b4411cdd45a4565/tokenizers-0.22.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d4e484f7b0827021ac5f9f71d4794aaef62b979ab7608593da22b1d2e3c4edc", size = 2926478, upload-time = "2025-09-19T09:49:09.759Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3b/55e64befa1e7bfea963cf4b787b2cea1011362c4193f5477047532ce127e/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d2962dd28bc67c1f205ab180578a78eef89ac60ca7ef7cbe9635a46a56422a", size = 3256994, upload-time = "2025-09-19T09:48:56.701Z" }, + { url = "https://files.pythonhosted.org/packages/71/0b/fbfecf42f67d9b7b80fde4aabb2b3110a97fac6585c9470b5bff103a80cb/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38201f15cdb1f8a6843e6563e6e79f4abd053394992b9bbdf5213ea3469b4ae7", size = 3153141, upload-time = "2025-09-19T09:48:59.749Z" }, + { url = "https://files.pythonhosted.org/packages/17/a9/b38f4e74e0817af8f8ef925507c63c6ae8171e3c4cb2d5d4624bf58fca69/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1cbe5454c9a15df1b3443c726063d930c16f047a3cc724b9e6e1a91140e5a21", size = 3508049, upload-time = "2025-09-19T09:49:05.868Z" }, + { url = "https://files.pythonhosted.org/packages/d2/48/dd2b3dac46bb9134a88e35d72e1aa4869579eacc1a27238f1577270773ff/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7d094ae6312d69cc2a872b54b91b309f4f6fbce871ef28eb27b52a98e4d0214", size = 3710730, upload-time = "2025-09-19T09:49:01.832Z" }, + { url = "https://files.pythonhosted.org/packages/93/0e/ccabc8d16ae4ba84a55d41345207c1e2ea88784651a5a487547d80851398/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afd7594a56656ace95cdd6df4cca2e4059d294c5cfb1679c57824b605556cb2f", size = 3412560, upload-time = "2025-09-19T09:49:03.867Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c6/dc3a0db5a6766416c32c034286d7c2d406da1f498e4de04ab1b8959edd00/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ef6063d7a84994129732b47e7915e8710f27f99f3a3260b8a38fc7ccd083f4", size = 3250221, upload-time = "2025-09-19T09:49:07.664Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a6/2c8486eef79671601ff57b093889a345dd3d576713ef047776015dc66de7/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ba0a64f450b9ef412c98f6bcd2a50c6df6e2443b560024a09fa6a03189726879", size = 9345569, upload-time = "2025-09-19T09:49:14.214Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/32ce667f14c35537f5f605fe9bea3e415ea1b0a646389d2295ec348d5657/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:331d6d149fa9c7d632cde4490fb8bbb12337fa3a0232e77892be656464f4b446", size = 9271599, upload-time = "2025-09-19T09:49:16.639Z" }, + { url = "https://files.pythonhosted.org/packages/51/7c/a5f7898a3f6baa3fc2685c705e04c98c1094c523051c805cdd9306b8f87e/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:607989f2ea68a46cb1dfbaf3e3aabdf3f21d8748312dbeb6263d1b3b66c5010a", size = 9533862, upload-time = "2025-09-19T09:49:19.146Z" }, + { url = "https://files.pythonhosted.org/packages/36/65/7e75caea90bc73c1dd8d40438adf1a7bc26af3b8d0a6705ea190462506e1/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a0f307d490295717726598ef6fa4f24af9d484809223bbc253b201c740a06390", size = 9681250, upload-time = "2025-09-19T09:49:21.501Z" }, + { url = "https://files.pythonhosted.org/packages/30/2c/959dddef581b46e6209da82df3b78471e96260e2bc463f89d23b1bf0e52a/tokenizers-0.22.1-cp39-abi3-win32.whl", hash = "sha256:b5120eed1442765cd90b903bb6cfef781fd8fe64e34ccaecbae4c619b7b12a82", size = 2472003, upload-time = "2025-09-19T09:49:27.089Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/e33a8c93907b631a99377ef4c5f817ab453d0b34f93529421f42ff559671/tokenizers-0.22.1-cp39-abi3-win_amd64.whl", hash = "sha256:65fd6e3fb11ca1e78a6a93602490f134d1fdeb13bcef99389d5102ea318ed138", size = 2674684, upload-time = "2025-09-19T09:49:24.953Z" }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" }, +] + +[[package]] +name = "torch" +version = "2.6.0+cu124" +source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "filelock", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "fsspec", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "jinja2", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "networkx", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "nvidia-cublas-cu12", version = "12.4.5.8", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-cupti-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-nvrtc-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-runtime-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cudnn-cu12", version = "9.1.0.70", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cufft-cu12", version = "11.2.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-curand-cu12", version = "10.3.5.147", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusolver-cu12", version = "11.6.1.9", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparse-cu12", version = "12.3.1.170", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparselt-cu12", version = "0.6.2", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nccl-cu12", version = "2.21.5", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvtx-cu12", version = "12.4.127", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "sympy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "typing-extensions", marker = "extra == 'extra-8-skyagent-verl'" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl", hash = "sha256:a393b506844035c0dac2f30ea8478c343b8e95a429f06f3b3cadfc7f53adb597" }, +] + +[package.metadata] +requires-dist = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==12.4.5.8" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==12.4.127" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==12.4.127" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==12.4.127" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==9.1.0.70" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==11.2.1.3" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==10.3.5.147" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==11.6.1.9" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==12.3.1.170" }, + { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==0.6.2" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==2.21.5" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==12.4.127" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==12.4.127" }, + { name = "opt-einsum", marker = "extra == 'opt-einsum'", specifier = ">=3.3" }, + { name = "optree", marker = "extra == 'optree'", specifier = ">=0.13.0" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "sympy", marker = "python_full_version >= '3.9'", specifier = "==1.13.1" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==3.2.0" }, + { name = "typing-extensions", specifier = ">=4.10.0" }, +] +provides-extras = ["opt-einsum", "optree"] + +[[package]] +name = "torch" +version = "2.7.1+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "filelock", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "fsspec", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "jinja2", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "networkx", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "nvidia-cublas-cu12", version = "12.8.3.14", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-cupti-cu12", version = "12.8.57", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-nvrtc-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-runtime-cu12", version = "12.8.57", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cudnn-cu12", version = "9.7.1.26", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cufft-cu12", version = "11.3.3.41", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cufile-cu12", version = "1.13.0.11", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-curand-cu12", version = "10.3.9.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusolver-cu12", version = "11.7.2.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.7.53", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparselt-cu12", version = "0.6.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nccl-cu12", version = "2.26.2", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvtx-cu12", version = "12.8.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "sympy", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "triton", version = "3.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "typing-extensions", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:268e54db9f0bc2b7b9eb089852d3e592c2dea2facc3db494100c3d3b796549fa" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0b64f7d0a6f2a739ed052ba959f7b67c677028c9566ce51997f9f90fe573ddaa" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:2bb8c05d48ba815b316879a18195d53a6472a03e297d971e916753f8e1053d30" }, +] + +[[package]] +name = "torch" +version = "2.8.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "filelock", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "fsspec", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "jinja2", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "networkx", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-cupti-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-nvrtc-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cuda-runtime-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cudnn-cu12", version = "9.10.2.21", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cufft-cu12", version = "11.3.3.83", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cufile-cu12", version = "1.13.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-curand-cu12", version = "10.3.9.90", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusolver-cu12", version = "11.7.3.90", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-cusparselt-cu12", version = "0.7.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nccl-cu12", version = "2.27.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "nvidia-nvtx-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "sympy", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "triton", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "typing-extensions", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.8.0%2Bcu128-cp312-cp312-win_amd64.whl" }, +] + +[[package]] +name = "torchaudio" +version = "2.6.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/4a/d71b932bda4171970bdf4997541b5c778daa0e2967ed5009d207fca86ded/torchaudio-2.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d0e4b08c42325bf4b887de9a25c44ed882997001740e1bd7d901f65581cf1ab", size = 1812899, upload-time = "2025-01-29T16:29:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/ed/aa/9082e715a673dd8e22b6a60cec7f301e897406023672b2090f8bcd8a5959/torchaudio-2.6.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:715aa21f6bdbd085454c313ae3a2c7cc07bf2e8cf05752f819afb5b4c57f4e6f", size = 3379510, upload-time = "2025-01-29T16:29:14.127Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e7/0bcb2e33f4bdec69477344eccfe25c515b90496888095e99f837ea422089/torchaudio-2.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6291d9507dc1d6b4ffe8843fbfb201e6c8270dd8c42ad70bb76226c0ebdcad56", size = 1653523, upload-time = "2025-01-29T16:29:32.803Z" }, + { url = "https://files.pythonhosted.org/packages/80/95/29e917905328337c7b104ce81f3bb5e2ad8dc70af2edf1d43f67eb621513/torchaudio-2.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:86d6239792bf94741a41acd6fe3d549faaf0d50e7275d17d076a190bd007e2f9", size = 2449191, upload-time = "2025-01-29T16:29:06.485Z" }, +] + +[[package]] +name = "torchaudio" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d1/eb8bc3b3502dddb1b789567b7b19668b1d32817266887b9f381494cfe463/torchaudio-2.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9306dcfc4586cebd7647a93fe9a448e791c4f83934da616b9433b75597a1f978", size = 1846897, upload-time = "2025-06-04T17:44:07.79Z" }, + { url = "https://files.pythonhosted.org/packages/62/7d/6c15f15d3edc5271abc808f70713644b50f0f7bfb85a09dba8b5735fbad3/torchaudio-2.7.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d66bd76b226fdd4135c97650e1b7eb63fb7659b4ed0e3a778898e41dbba21b61", size = 1686680, upload-time = "2025-06-04T17:43:58.986Z" }, + { url = "https://files.pythonhosted.org/packages/48/65/0f46ba74cdc67ea9a8c37c8acfb5194d81639e481e85903c076bcd97188c/torchaudio-2.7.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9cbcdaab77ad9a73711acffee58f4eebc8a0685289a938a3fa6f660af9489aee", size = 3506966, upload-time = "2025-06-04T17:44:06.537Z" }, + { url = "https://files.pythonhosted.org/packages/52/29/06f887baf22cbba85ae331b71b110b115bf11b3968f5914a50c17dde5ab7/torchaudio-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:9cfb8f6ace8e01e2b89de74eb893ba5ce936b88b415383605b0a4d974009dec7", size = 2484265, upload-time = "2025-06-04T17:44:00.277Z" }, +] + +[[package]] +name = "torchaudio" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/cc/c2e2a3eb6ee956f73c68541e439916f8146170ea9cc61e72adea5c995312/torchaudio-2.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ddef94bf181e6447cbb05f38beaca8f6c5bb8d2b9ddced1aa3452025b9fc70d3", size = 1856736, upload-time = "2025-08-06T14:58:36.3Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0d/24dad878784f1edd62862f27173781669f0c71eb46368636787d1e364188/torchaudio-2.8.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:862e2e40bf09d865e5df080a84c1a39bbcef40e43140f4b1737eb3a389d3b38f", size = 1692930, upload-time = "2025-08-06T14:58:41.312Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a6/84d80f34472503e9eb82245d7df501c59602d75d7360e717fb9b84f91c5e/torchaudio-2.8.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:93a8583f280fe83ba021aa713319381ea71362cc87b67ee38e97a43cb2254aee", size = 4014607, upload-time = "2025-08-06T14:58:47.234Z" }, + { url = "https://files.pythonhosted.org/packages/43/ab/96ad33afa320738a7cfb4b51ba97e2f3cfb1e04ae3115d5057655103ba4f/torchaudio-2.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:4b82cacd1b8ccd543b1149d8cab257a40dfda8119023d2e3a96c66349c84bffb", size = 2499890, upload-time = "2025-08-06T14:58:55.066Z" }, +] + +[[package]] +name = "torchdata" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "urllib3" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/d4/af694ef718aedbe95a72760ab9ff7a6a7a44ace2d7f70c27bfeb67c5c503/torchdata-0.11.0-py3-none-any.whl", hash = "sha256:52b940fbbe0e00fb21cabddf528449d1bec5bfb0d0823b7487b15f951658ee33", size = 61968, upload-time = "2025-02-20T22:26:30.666Z" }, +] + +[[package]] +name = "torchvision" +version = "0.21.0+cu124" +source = { url = "https://download.pytorch.org/whl/cu124/torchvision-0.21.0%2Bcu124-cp312-cp312-linux_x86_64.whl" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "pillow", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu124/torchvision-0.21.0%2Bcu124-cp312-cp312-linux_x86_64.whl", hash = "sha256:efb53ea0af7bf09b7b53e2a18b9be6d245f7d46a90b51d5cf97f37e9b929a991" }, +] + +[package.metadata] +requires-dist = [ + { name = "gdown", marker = "extra == 'gdown'", specifier = ">=4.7.3" }, + { name = "numpy" }, + { name = "pillow", specifier = ">=5.3.0,!=8.3.*" }, + { name = "scipy", marker = "extra == 'scipy'" }, + { name = "torch", specifier = "==2.6.0" }, +] +provides-extras = ["gdown", "scipy"] + +[[package]] +name = "torchvision" +version = "0.22.1" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "pillow", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2ad7fe412b821333fc05b4046263d77c14ba86f3965366adbada8dc397ea45b4" }, +] + +[[package]] +name = "torchvision" +version = "0.22.1+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "pillow", marker = "(platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f64ef9bb91d71ab35d8384912a19f7419e35928685bc67544d58f45148334373" }, + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:650561ba326d21021243f5e064133dc62dc64d52f79623db5cd76637a9665f96" }, +] + +[[package]] +name = "torchvision" +version = "0.23.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pillow", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.23.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9cb3c13997afcb44057ca10d943c6c4cba3068afde0f370965abce9c89fcffa9" }, + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.23.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:20fa9c7362a006776630b00b8a01919fedcf504a202b81358d32c5aef39956fe" }, +] + +[[package]] +name = "tornado" +version = "6.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/ce/1eb500eae19f4648281bb2186927bb062d2438c2e5093d1360391afd2f90/tornado-6.5.2.tar.gz", hash = "sha256:ab53c8f9a0fa351e2c0741284e06c7a45da86afb544133201c5cc8578eb076a0", size = 510821, upload-time = "2025-08-08T18:27:00.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/48/6a7529df2c9cc12efd2e8f5dd219516184d703b34c06786809670df5b3bd/tornado-6.5.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:2436822940d37cde62771cff8774f4f00b3c8024fe482e16ca8387b8a2724db6", size = 442563, upload-time = "2025-08-08T18:26:42.945Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b5/9b575a0ed3e50b00c40b08cbce82eb618229091d09f6d14bce80fc01cb0b/tornado-6.5.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:583a52c7aa94ee046854ba81d9ebb6c81ec0fd30386d96f7640c96dad45a03ef", size = 440729, upload-time = "2025-08-08T18:26:44.473Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4e/619174f52b120efcf23633c817fd3fed867c30bff785e2cd5a53a70e483c/tornado-6.5.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0fe179f28d597deab2842b86ed4060deec7388f1fd9c1b4a41adf8af058907e", size = 444295, upload-time = "2025-08-08T18:26:46.021Z" }, + { url = "https://files.pythonhosted.org/packages/95/fa/87b41709552bbd393c85dd18e4e3499dcd8983f66e7972926db8d96aa065/tornado-6.5.2-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b186e85d1e3536d69583d2298423744740986018e393d0321df7340e71898882", size = 443644, upload-time = "2025-08-08T18:26:47.625Z" }, + { url = "https://files.pythonhosted.org/packages/f9/41/fb15f06e33d7430ca89420283a8762a4e6b8025b800ea51796ab5e6d9559/tornado-6.5.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e792706668c87709709c18b353da1f7662317b563ff69f00bab83595940c7108", size = 443878, upload-time = "2025-08-08T18:26:50.599Z" }, + { url = "https://files.pythonhosted.org/packages/11/92/fe6d57da897776ad2e01e279170ea8ae726755b045fe5ac73b75357a5a3f/tornado-6.5.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:06ceb1300fd70cb20e43b1ad8aaee0266e69e7ced38fa910ad2e03285009ce7c", size = 444549, upload-time = "2025-08-08T18:26:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/c8f4f6c9204526daf3d760f4aa555a7a33ad0e60843eac025ccfd6ff4a93/tornado-6.5.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:74db443e0f5251be86cbf37929f84d8c20c27a355dd452a5cfa2aada0d001ec4", size = 443973, upload-time = "2025-08-08T18:26:53.625Z" }, + { url = "https://files.pythonhosted.org/packages/ae/2d/f5f5707b655ce2317190183868cd0f6822a1121b4baeae509ceb9590d0bd/tornado-6.5.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b5e735ab2889d7ed33b32a459cac490eda71a1ba6857b0118de476ab6c366c04", size = 443954, upload-time = "2025-08-08T18:26:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/e8/59/593bd0f40f7355806bf6573b47b8c22f8e1374c9b6fd03114bd6b7a3dcfd/tornado-6.5.2-cp39-abi3-win32.whl", hash = "sha256:c6f29e94d9b37a95013bb669616352ddb82e3bfe8326fccee50583caebc8a5f0", size = 445023, upload-time = "2025-08-08T18:26:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/f609b420c2f564a748a2d80ebfb2ee02a73ca80223af712fca591386cafb/tornado-6.5.2-cp39-abi3-win_amd64.whl", hash = "sha256:e56a5af51cc30dd2cae649429af65ca2f6571da29504a07995175df14c18f35f", size = 445427, upload-time = "2025-08-08T18:26:57.91Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e1f65e8f8c76d73658b33d33b81eed4322fb5085350e4328d5c956f0c8f9/tornado-6.5.2-cp39-abi3-win_arm64.whl", hash = "sha256:d6c33dc3672e3a1f3618eb63b7ef4683a7688e7b9e6e8f0d9aa5726360a004af", size = 444456, upload-time = "2025-08-08T18:26:59.207Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + +[[package]] +name = "traitlets" +version = "5.14.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/79/72064e6a701c2183016abbbfedaba506d81e30e232a68c9f0d6f6fcd1574/traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", size = 161621, upload-time = "2024-04-19T11:11:49.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, +] + +[[package]] +name = "transformers" +version = "4.56.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "huggingface-hub" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e5/82/0bcfddd134cdf53440becb5e738257cc3cf34cf229d63b57bfd288e6579f/transformers-4.56.2.tar.gz", hash = "sha256:5e7c623e2d7494105c726dd10f6f90c2c99a55ebe86eef7233765abd0cb1c529", size = 9844296, upload-time = "2025-09-19T15:16:26.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/26/2591b48412bde75e33bfd292034103ffe41743cacd03120e3242516cd143/transformers-4.56.2-py3-none-any.whl", hash = "sha256:79c03d0e85b26cb573c109ff9eafa96f3c8d4febfd8a0774e8bba32702dd6dde", size = 11608055, upload-time = "2025-09-19T15:16:23.736Z" }, +] + +[[package]] +name = "tree-sitter" +version = "0.24.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/a2/698b9d31d08ad5558f8bfbfe3a0781bd4b1f284e89bde3ad18e05101a892/tree-sitter-0.24.0.tar.gz", hash = "sha256:abd95af65ca2f4f7eca356343391ed669e764f37748b5352946f00f7fc78e734", size = 168304, upload-time = "2025-01-17T05:06:38.115Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/57/3a590f287b5aa60c07d5545953912be3d252481bf5e178f750db75572bff/tree_sitter-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14beeff5f11e223c37be7d5d119819880601a80d0399abe8c738ae2288804afc", size = 140788, upload-time = "2025-01-17T05:06:08.492Z" }, + { url = "https://files.pythonhosted.org/packages/61/0b/fc289e0cba7dbe77c6655a4dd949cd23c663fd62a8b4d8f02f97e28d7fe5/tree_sitter-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26a5b130f70d5925d67b47db314da209063664585a2fd36fa69e0717738efaf4", size = 133945, upload-time = "2025-01-17T05:06:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/86/d7/80767238308a137e0b5b5c947aa243e3c1e3e430e6d0d5ae94b9a9ffd1a2/tree_sitter-0.24.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fc5c3c26d83c9d0ecb4fc4304fba35f034b7761d35286b936c1db1217558b4e", size = 564819, upload-time = "2025-01-17T05:06:13.549Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b3/6c5574f4b937b836601f5fb556b24804b0a6341f2eb42f40c0e6464339f4/tree_sitter-0.24.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:772e1bd8c0931c866b848d0369b32218ac97c24b04790ec4b0e409901945dd8e", size = 579303, upload-time = "2025-01-17T05:06:16.685Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f4/bd0ddf9abe242ea67cca18a64810f8af230fc1ea74b28bb702e838ccd874/tree_sitter-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:24a8dd03b0d6b8812425f3b84d2f4763322684e38baf74e5bb766128b5633dc7", size = 581054, upload-time = "2025-01-17T05:06:19.439Z" }, + { url = "https://files.pythonhosted.org/packages/8c/1c/ff23fa4931b6ef1bbeac461b904ca7e49eaec7e7e5398584e3eef836ec96/tree_sitter-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9e8b1605ab60ed43803100f067eed71b0b0e6c1fb9860a262727dbfbbb74751", size = 120221, upload-time = "2025-01-17T05:06:20.654Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2a/9979c626f303177b7612a802237d0533155bf1e425ff6f73cc40f25453e2/tree_sitter-0.24.0-cp312-cp312-win_arm64.whl", hash = "sha256:f733a83d8355fc95561582b66bbea92ffd365c5d7a665bc9ebd25e049c2b2abb", size = 108234, upload-time = "2025-01-17T05:06:21.713Z" }, +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/dc/1c55c33cc6bbe754359b330534cf9f261c1b9b2c26ddf23aef3c5fa67759/tree_sitter_javascript-0.23.1.tar.gz", hash = "sha256:b2059ce8b150162cda05a457ca3920450adbf915119c04b8c67b5241cd7fcfed", size = 110058, upload-time = "2024-11-10T05:40:42.357Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/d3/c67d7d49967344b51208ad19f105233be1afdf07d3dcb35b471900265227/tree_sitter_javascript-0.23.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6ca583dad4bd79d3053c310b9f7208cd597fd85f9947e4ab2294658bb5c11e35", size = 59333, upload-time = "2024-11-10T05:40:31.988Z" }, + { url = "https://files.pythonhosted.org/packages/a5/db/ea0ee1547679d1750e80a0c4bc60b3520b166eeaf048764cfdd1ba3fd5e5/tree_sitter_javascript-0.23.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:94100e491a6a247aa4d14caf61230c171b6376c863039b6d9cd71255c2d815ec", size = 61071, upload-time = "2024-11-10T05:40:33.458Z" }, + { url = "https://files.pythonhosted.org/packages/67/6e/07c4857e08be37bfb55bfb269863df8ec908b2f6a3f1893cd852b893ecab/tree_sitter_javascript-0.23.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a6bc1055b061c5055ec58f39ee9b2e9efb8e6e0ae970838af74da0afb811f0a", size = 96999, upload-time = "2024-11-10T05:40:34.869Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f5/4de730afe8b9422845bc2064020a8a8f49ebd1695c04261c38d1b3e3edec/tree_sitter_javascript-0.23.1-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:056dc04fb6b24293f8c5fec43c14e7e16ba2075b3009c643abf8c85edc4c7c3c", size = 94020, upload-time = "2024-11-10T05:40:35.735Z" }, + { url = "https://files.pythonhosted.org/packages/77/0a/f980520da86c4eff8392867840a945578ef43372c9d4a37922baa6b121fe/tree_sitter_javascript-0.23.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a11ca1c0f736da42967586b568dff8a465ee148a986c15ebdc9382806e0ce871", size = 92927, upload-time = "2024-11-10T05:40:37.92Z" }, + { url = "https://files.pythonhosted.org/packages/ff/5c/36a98d512aa1d1082409d6b7eda5d26b820bd4477a54100ad9f62212bc55/tree_sitter_javascript-0.23.1-cp39-abi3-win_amd64.whl", hash = "sha256:041fa22b34250ea6eb313d33104d5303f79504cb259d374d691e38bbdc49145b", size = 58824, upload-time = "2024-11-10T05:40:39.903Z" }, + { url = "https://files.pythonhosted.org/packages/dc/79/ceb21988e6de615355a63eebcf806cd2a0fe875bec27b429d58b63e7fb5f/tree_sitter_javascript-0.23.1-cp39-abi3-win_arm64.whl", hash = "sha256:eb28130cd2fb30d702d614cbf61ef44d1c7f6869e7d864a9cc17111e370be8f7", size = 57027, upload-time = "2024-11-10T05:40:40.841Z" }, +] + +[[package]] +name = "tree-sitter-languages" +version = "1.10.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tree-sitter" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/bf/a9bd2d6ecbd053de0a5a50c150105b69c90eb49089f9e1d4fc4937e86adc/tree_sitter_languages-1.10.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c0f4c8b2734c45859edc7fcaaeaab97a074114111b5ba51ab4ec7ed52104763c", size = 8884771, upload-time = "2024-02-04T10:28:39.655Z" }, + { url = "https://files.pythonhosted.org/packages/14/fb/1f6fe5903aeb7435cc66d4b56621e9a30a4de64420555b999de65b31fcae/tree_sitter_languages-1.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eecd3c1244ac3425b7a82ba9125b4ddb45d953bbe61de114c0334fd89b7fe782", size = 9724562, upload-time = "2024-02-04T10:28:42.275Z" }, + { url = "https://files.pythonhosted.org/packages/20/6c/1855a65c9d6b50600f7a68e0182153db7cb12ff81fdebd93e87851dfdd8f/tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15db3c8510bc39a80147ee7421bf4782c15c09581c1dc2237ea89cefbd95b846", size = 8678682, upload-time = "2024-02-04T10:28:44.642Z" }, + { url = "https://files.pythonhosted.org/packages/d0/75/eff180f187ce4dc3e5177b3f8508e0061ea786ac44f409cf69cf24bf31a6/tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92c6487a6feea683154d3e06e6db68c30e0ae749a7ce4ce90b9e4e46b78c85c7", size = 8595099, upload-time = "2024-02-04T10:28:47.767Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e6/eddc76ad899d77adcb5fca6cdf651eb1d33b4a799456bf303540f6cf8204/tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2f1cd1d1bdd65332f9c2b67d49dcf148cf1ded752851d159ac3e5ee4f4d260", size = 8433569, upload-time = "2024-02-04T10:28:50.404Z" }, + { url = "https://files.pythonhosted.org/packages/06/95/a13da048c33a876d0475974484bf66b1fae07226e8654b1365ab549309cd/tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:976c8039165b8e12f17a01ddee9f4e23ec6e352b165ad29b44d2bf04e2fbe77e", size = 9196003, upload-time = "2024-02-04T10:28:52.466Z" }, + { url = "https://files.pythonhosted.org/packages/ec/13/9e5cb03914d60dd51047ecbfab5400309fbab14bb25014af388f492da044/tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:dafbbdf16bf668a580902e1620f4baa1913e79438abcce721a50647564c687b9", size = 9175560, upload-time = "2024-02-04T10:28:55.064Z" }, + { url = "https://files.pythonhosted.org/packages/19/76/25bb32a9be1c476e388835d5c8de5af2920af055e295770003683896cfe2/tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1aeabd3d60d6d276b73cd8f3739d595b1299d123cc079a317f1a5b3c5461e2ca", size = 8956249, upload-time = "2024-02-04T10:28:57.094Z" }, + { url = "https://files.pythonhosted.org/packages/52/01/8e2f97a444d25dde1380ec20b338722f733b6cc290524357b1be3dd452ab/tree_sitter_languages-1.10.2-cp312-cp312-win32.whl", hash = "sha256:fab8ee641914098e8933b87ea3d657bea4dd00723c1ee7038b847b12eeeef4f5", size = 8363094, upload-time = "2024-02-04T10:28:59.156Z" }, + { url = "https://files.pythonhosted.org/packages/47/58/0262e875dd899447476a8ffde7829df3716ffa772990095c65d6de1f053c/tree_sitter_languages-1.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e606430d736367e5787fa5a7a0c5a1ec9b85eded0b3596bbc0d83532a40810b", size = 8268983, upload-time = "2024-02-04T10:29:00.987Z" }, +] + +[[package]] +name = "tree-sitter-python" +version = "0.23.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/30/6766433b31be476fda6569a3a374c2220e45ffee0bff75460038a57bf23b/tree_sitter_python-0.23.6.tar.gz", hash = "sha256:354bfa0a2f9217431764a631516f85173e9711af2c13dbd796a8815acfe505d9", size = 155868, upload-time = "2024-12-22T23:09:55.918Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/67/577a02acae5f776007c924ca86ef14c19c12e71de0aa9d2a036f3c248e7b/tree_sitter_python-0.23.6-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:28fbec8f74eeb2b30292d97715e60fac9ccf8a8091ce19b9d93e9b580ed280fb", size = 74361, upload-time = "2024-12-22T23:09:42.37Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a6/194b3625a7245c532ad418130d63077ce6cd241152524152f533e4d6edb0/tree_sitter_python-0.23.6-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:680b710051b144fedf61c95197db0094f2245e82551bf7f0c501356333571f7a", size = 76436, upload-time = "2024-12-22T23:09:43.566Z" }, + { url = "https://files.pythonhosted.org/packages/d0/62/1da112689d6d282920e62c40e67ab39ea56463b0e7167bfc5e81818a770e/tree_sitter_python-0.23.6-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a9dcef55507b6567207e8ee0a6b053d0688019b47ff7f26edc1764b7f4dc0a4", size = 112060, upload-time = "2024-12-22T23:09:44.721Z" }, + { url = "https://files.pythonhosted.org/packages/5d/62/c9358584c96e38318d69b6704653684fd8467601f7b74e88aa44f4e6903f/tree_sitter_python-0.23.6-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29dacdc0cd2f64e55e61d96c6906533ebb2791972bec988450c46cce60092f5d", size = 112338, upload-time = "2024-12-22T23:09:48.323Z" }, + { url = "https://files.pythonhosted.org/packages/1a/58/c5e61add45e34fb8ecbf057c500bae9d96ed7c9ca36edb7985da8ae45526/tree_sitter_python-0.23.6-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7e048733c36f564b379831689006801feb267d8194f9e793fbb395ef1723335d", size = 109382, upload-time = "2024-12-22T23:09:49.49Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f3/9b30893cae9b3811fe652dc6f90aaadfda12ae0b2757f5722fc7266f423c/tree_sitter_python-0.23.6-cp39-abi3-win_amd64.whl", hash = "sha256:a24027248399fb41594b696f929f9956828ae7cc85596d9f775e6c239cd0c2be", size = 75904, upload-time = "2024-12-22T23:09:51.597Z" }, + { url = "https://files.pythonhosted.org/packages/87/cb/ce35a65f83a47b510d8a2f1eddf3bdbb0d57aabc87351c8788caf3309f76/tree_sitter_python-0.23.6-cp39-abi3-win_arm64.whl", hash = "sha256:71334371bd73d5fe080aed39fbff49ed8efb9506edebe16795b0c7567ed6a272", size = 73649, upload-time = "2024-12-22T23:09:53.71Z" }, +] + +[[package]] +name = "tree-sitter-ruby" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/5b/6d24be4fde4743481bd8e3fd24b434870cb6612238c8544b71fe129ed850/tree_sitter_ruby-0.23.1.tar.gz", hash = "sha256:886ed200bfd1f3ca7628bf1c9fefd42421bbdba70c627363abda67f662caa21e", size = 489602, upload-time = "2024-11-11T04:51:30.328Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/2e/2717b9451c712b60f833827a696baf29d8e50a0f7dccbf22a8d7006cc19e/tree_sitter_ruby-0.23.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:39f391322d2210843f07081182dbf00f8f69cfbfa4687b9575cac6d324bae443", size = 177959, upload-time = "2024-11-11T04:51:19.958Z" }, + { url = "https://files.pythonhosted.org/packages/e7/38/c41ecf7692b8ecccd26861d3293a88150a4a52fc081abe60f837030d7315/tree_sitter_ruby-0.23.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:aa4ee7433bd42fac22e2dad4a3c0f332292ecf482e610316828c711a0bb7f794", size = 195069, upload-time = "2024-11-11T04:51:21.82Z" }, + { url = "https://files.pythonhosted.org/packages/d8/01/14ef2d5107e6f42b64a400c3bbc3dd3b8fd24c3cef5306004ae03668f231/tree_sitter_ruby-0.23.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62b36813a56006b7569db7868f6b762caa3f4e419bd0f8cf9ccbb4abb1b6254c", size = 226761, upload-time = "2024-11-11T04:51:23.021Z" }, + { url = "https://files.pythonhosted.org/packages/23/dd/1171b5dd25da10f768732a20fb62d2e3ae66e3b42329351f2ce5bf723abb/tree_sitter_ruby-0.23.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7bcd93972b4ca2803856d4fe0fbd04123ff29c4592bbb9f12a27528bd252341", size = 214427, upload-time = "2024-11-11T04:51:24.854Z" }, + { url = "https://files.pythonhosted.org/packages/60/bc/de76c877a90fd8a62cd60f496d7832efddc1b18a148593d9aa9b4a9ce5e0/tree_sitter_ruby-0.23.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66c65d6c2a629783ca4ab2bab539bd6f271ce6f77cacb62845831e11665b5bd3", size = 210409, upload-time = "2024-11-11T04:51:26.093Z" }, + { url = "https://files.pythonhosted.org/packages/dd/4a/f5bcca350b84cdf75a53e918b8efa06c46ed650d99d3ef22195e9d8020cc/tree_sitter_ruby-0.23.1-cp39-abi3-win_amd64.whl", hash = "sha256:02e2c19ebefe29226c14aa63e11e291d990f5b5c20a99940ab6e7eda44e744e5", size = 179843, upload-time = "2024-11-11T04:51:27.265Z" }, + { url = "https://files.pythonhosted.org/packages/71/5c/a2e068ad4b2c4ba9b774a88b24149168d3bcd94f58b964e49dcabfe5fd24/tree_sitter_ruby-0.23.1-cp39-abi3-win_arm64.whl", hash = "sha256:ed042007e89f2cceeb1cbdd8b0caa68af1e2ce54c7eb2053ace760f90657ac9f", size = 178025, upload-time = "2024-11-11T04:51:29.051Z" }, +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/fc/bb52958f7e399250aee093751e9373a6311cadbe76b6e0d109b853757f35/tree_sitter_typescript-0.23.2.tar.gz", hash = "sha256:7b167b5827c882261cb7a50dfa0fb567975f9b315e87ed87ad0a0a3aedb3834d", size = 773053, upload-time = "2024-11-11T02:36:11.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/95/4c00680866280e008e81dd621fd4d3f54aa3dad1b76b857a19da1b2cc426/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478", size = 286677, upload-time = "2024-11-11T02:35:58.839Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2f/1f36fda564518d84593f2740d5905ac127d590baf5c5753cef2a88a89c15/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8", size = 302008, upload-time = "2024-11-11T02:36:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/96/2d/975c2dad292aa9994f982eb0b69cc6fda0223e4b6c4ea714550477d8ec3a/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31", size = 351987, upload-time = "2024-11-11T02:36:02.669Z" }, + { url = "https://files.pythonhosted.org/packages/49/d1/a71c36da6e2b8a4ed5e2970819b86ef13ba77ac40d9e333cb17df6a2c5db/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c", size = 344960, upload-time = "2024-11-11T02:36:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cb/f57b149d7beed1a85b8266d0c60ebe4c46e79c9ba56bc17b898e17daf88e/tree_sitter_typescript-0.23.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0", size = 340245, upload-time = "2024-11-11T02:36:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ab/dd84f0e2337296a5f09749f7b5483215d75c8fa9e33738522e5ed81f7254/tree_sitter_typescript-0.23.2-cp39-abi3-win_amd64.whl", hash = "sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9", size = 278015, upload-time = "2024-11-11T02:36:07.631Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e4/81f9a935789233cf412a0ed5fe04c883841d2c8fb0b7e075958a35c65032/tree_sitter_typescript-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7", size = 274052, upload-time = "2024-11-11T02:36:09.514Z" }, +] + +[[package]] +name = "triton" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/00/59500052cb1cf8cf5316be93598946bc451f14072c6ff256904428eaf03c/triton-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d9b215efc1c26fa7eefb9a157915c92d52e000d2bf83e5f69704047e63f125c", size = 253159365, upload-time = "2025-01-22T19:13:24.648Z" }, +] + +[[package]] +name = "triton" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'darwin' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", + "sys_platform != 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +dependencies = [ + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'linux' and extra != 'extra-8-skyagent-verl') or extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'linux' and extra == 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/5f/950fb373bf9c01ad4eb5a8cd5eaf32cdf9e238c02f9293557a2129b9c4ac/triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43", size = 155669138, upload-time = "2025-05-29T23:39:51.771Z" }, +] + +[[package]] +name = "triton" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl'", + "sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl'", +] +dependencies = [ + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/66/b1eb52839f563623d185f0927eb3530ee4d5ffe9d377cdaf5346b306689e/triton-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c1d84a5c0ec2c0f8e8a072d7fd150cab84a9c239eaddc6706c081bfae4eb04", size = 155560068, upload-time = "2025-07-30T19:58:37.081Z" }, +] + +[[package]] +name = "trove-classifiers" +version = "2025.9.11.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/9a/778622bc06632529817c3c524c82749a112603ae2bbcf72ee3eb33a2c4f1/trove_classifiers-2025.9.11.17.tar.gz", hash = "sha256:931ca9841a5e9c9408bc2ae67b50d28acf85bef56219b56860876dd1f2d024dd", size = 16975, upload-time = "2025-09-11T17:07:50.97Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/85/a4ff8758c66f1fc32aa5e9a145908394bf9cf1c79ffd1113cfdeb77e74e4/trove_classifiers-2025.9.11.17-py3-none-any.whl", hash = "sha256:5d392f2d244deb1866556457d6f3516792124a23d1c3a463a2e8668a5d1c15dd", size = 14158, upload-time = "2025-09-11T17:07:49.886Z" }, +] + +[[package]] +name = "typer" +version = "0.15.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/89/c527e6c848739be8ceb5c44eb8208c52ea3515c6cf6406aa61932887bf58/typer-0.15.4.tar.gz", hash = "sha256:89507b104f9b6a0730354f27c39fae5b63ccd0c95b1ce1f1a6ba0cfd329997c3", size = 101559, upload-time = "2025-05-14T16:34:57.704Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/62/d4ba7afe2096d5659ec3db8b15d8665bdcb92a3c6ff0b95e99895b335a9c/typer-0.15.4-py3-none-any.whl", hash = "sha256:eb0651654dcdea706780c466cf06d8f174405a659ffff8f163cfbfee98c0e173", size = 45258, upload-time = "2025-05-14T16:34:55.583Z" }, +] + +[[package]] +name = "types-certifi" +version = "2021.10.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/68/943c3aeaf14624712a0357c4a67814dba5cea36d194f5c764dad7959a00c/types-certifi-2021.10.8.3.tar.gz", hash = "sha256:72cf7798d165bc0b76e1c10dd1ea3097c7063c42c21d664523b928e88b554a4f", size = 2095, upload-time = "2022-06-09T15:19:05.244Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/63/2463d89481e811f007b0e1cd0a91e52e141b47f9de724d20db7b861dcfec/types_certifi-2021.10.8.3-py3-none-any.whl", hash = "sha256:b2d1e325e69f71f7c78e5943d410e650b4707bb0ef32e4ddf3da37f54176e88a", size = 2136, upload-time = "2022-06-09T15:19:03.127Z" }, +] + +[[package]] +name = "types-requests" +version = "2.32.4.20250913" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/27/489922f4505975b11de2b5ad07b4fe1dca0bca9be81a703f26c5f3acfce5/types_requests-2.32.4.20250913.tar.gz", hash = "sha256:abd6d4f9ce3a9383f269775a9835a4c24e5cd6b9f647d64f88aa4613c33def5d", size = 23113, upload-time = "2025-09-13T02:40:02.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/20/9a227ea57c1285986c4cf78400d0a91615d25b24e257fd9e2969606bdfae/types_requests-2.32.4.20250913-py3-none-any.whl", hash = "sha256:78c9c1fffebbe0fa487a418e0fa5252017e9c60d1a2da394077f1780f655d7e1", size = 20658, upload-time = "2025-09-13T02:40:01.115Z" }, +] + +[[package]] +name = "types-toml" +version = "0.10.8.20240310" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/47/3e4c75042792bff8e90d7991aa5c51812cc668828cc6cce711e97f63a607/types-toml-0.10.8.20240310.tar.gz", hash = "sha256:3d41501302972436a6b8b239c850b26689657e25281b48ff0ec06345b8830331", size = 4392, upload-time = "2024-03-10T02:18:37.518Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/a2/d32ab58c0b216912638b140ab2170ee4b8644067c293b170e19fba340ccc/types_toml-0.10.8.20240310-py3-none-any.whl", hash = "sha256:627b47775d25fa29977d9c70dc0cbab3f314f32c8d8d0c012f2ef5de7aaec05d", size = 4777, upload-time = "2024-03-10T02:18:36.568Z" }, +] + +[[package]] +name = "types-tqdm" +version = "4.67.0.20250809" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/d0/cf498fc630d9fdaf2428b93e60b0e67b08008fec22b78716b8323cf644dc/types_tqdm-4.67.0.20250809.tar.gz", hash = "sha256:02bf7ab91256080b9c4c63f9f11b519c27baaf52718e5fdab9e9606da168d500", size = 17200, upload-time = "2025-08-09T03:17:43.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/13/3ff0781445d7c12730befce0fddbbc7a76e56eb0e7029446f2853238360a/types_tqdm-4.67.0.20250809-py3-none-any.whl", hash = "sha256:1a73053b31fcabf3c1f3e2a9d5ecdba0f301bde47a418cd0e0bdf774827c5c57", size = 24020, upload-time = "2025-08-09T03:17:42.453Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspect" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7609f121aaa6b609744687f1d158b3c3a5bf4cc94238/typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28", size = 75726, upload-time = "2025-05-21T18:55:23.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "unidiff" +version = "0.7.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/48/81be0ac96e423a877754153699731ef439fd7b80b4c8b5425c94ed079ebd/unidiff-0.7.5.tar.gz", hash = "sha256:2e5f0162052248946b9f0970a40e9e124236bf86c82b70821143a6fc1dea2574", size = 20931, upload-time = "2023-03-10T01:05:39.185Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/54/57c411a6e8f7bd7848c8b66e4dcaffa586bf4c02e63f2280db0327a4e6eb/unidiff-0.7.5-py2.py3-none-any.whl", hash = "sha256:c93bf2265cc1ba2a520e415ab05da587370bc2a3ae9e0414329f54f0c2fc09e8", size = 14386, upload-time = "2023-03-10T01:05:36.594Z" }, +] + +[[package]] +name = "uritemplate" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/57/1616c8274c3442d802621abf5deb230771c7a0fec9414cb6763900eb3868/uvicorn-0.37.0.tar.gz", hash = "sha256:4115c8add6d3fd536c8ee77f0e14a7fd2ebba939fed9b02583a97f80648f9e13", size = 80367, upload-time = "2025-09-23T13:33:47.486Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/cd/584a2ceb5532af99dd09e50919e3615ba99aa127e9850eafe5f31ddfdb9a/uvicorn-0.37.0-py3-none-any.whl", hash = "sha256:913b2b88672343739927ce381ff9e2ad62541f9f8289664fa1d1d3803fa2ce6c", size = 67976, upload-time = "2025-09-23T13:33:45.842Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/c0/854216d09d33c543f12a44b393c402e89a920b1a0a7dc634c42de91b9cf6/uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3", size = 2492741, upload-time = "2024-10-14T23:38:35.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/4c/03f93178830dc7ce8b4cdee1d36770d2f5ebb6f3d37d354e061eefc73545/uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c", size = 1471284, upload-time = "2024-10-14T23:37:47.833Z" }, + { url = "https://files.pythonhosted.org/packages/43/3e/92c03f4d05e50f09251bd8b2b2b584a2a7f8fe600008bcc4523337abe676/uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2", size = 821349, upload-time = "2024-10-14T23:37:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ef/a02ec5da49909dbbfb1fd205a9a1ac4e88ea92dcae885e7c961847cd51e2/uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d", size = 4580089, upload-time = "2024-10-14T23:37:51.703Z" }, + { url = "https://files.pythonhosted.org/packages/06/a7/b4e6a19925c900be9f98bec0a75e6e8f79bb53bdeb891916609ab3958967/uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc", size = 4693770, upload-time = "2024-10-14T23:37:54.122Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0c/f07435a18a4b94ce6bd0677d8319cd3de61f3a9eeb1e5f8ab4e8b5edfcb3/uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb", size = 4451321, upload-time = "2024-10-14T23:37:55.766Z" }, + { url = "https://files.pythonhosted.org/packages/8f/eb/f7032be105877bcf924709c97b1bf3b90255b4ec251f9340cef912559f28/uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f", size = 4659022, upload-time = "2024-10-14T23:37:58.195Z" }, +] + +[[package]] +name = "verl" +source = { editable = "verl" } +dependencies = [ + { name = "accelerate" }, + { name = "codetiming" }, + { name = "datasets" }, + { name = "dill" }, + { name = "hydra-core" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pandas" }, + { name = "peft" }, + { name = "pyarrow" }, + { name = "pybind11" }, + { name = "pylatexenc" }, + { name = "ray" }, + { name = "tensordict", version = "0.9.1", source = { registry = "https://pypi.org/simple" } }, + { name = "torchdata" }, + { name = "transformers" }, + { name = "wandb" }, +] + +[package.optional-dependencies] +gpu = [ + { name = "flash-attn", version = "2.7.4.post1", source = { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl" } }, + { name = "liger-kernel" }, +] +vllm = [ + { name = "tensordict", version = "0.9.1", source = { registry = "https://pypi.org/simple" } }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" } }, + { name = "torchvision", version = "0.21.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torchvision-0.21.0%2Bcu124-cp312-cp312-linux_x86_64.whl" } }, + { name = "vllm", version = "0.8.5", source = { registry = "https://pypi.org/simple" } }, +] + +[package.metadata] +requires-dist = [ + { name = "accelerate" }, + { name = "codetiming" }, + { name = "datasets" }, + { name = "dill" }, + { name = "flash-attn", marker = "extra == 'gpu'", url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl" }, + { name = "hydra-core" }, + { name = "liger-kernel", marker = "extra == 'gpu'" }, + { name = "math-verify", marker = "extra == 'math'" }, + { name = "mathruler", marker = "extra == 'geo'" }, + { name = "mbridge", marker = "extra == 'mcore'" }, + { name = "numpy", specifier = "<2.0.0" }, + { name = "packaging", specifier = ">=20.0" }, + { name = "pandas" }, + { name = "peft" }, + { name = "pre-commit", marker = "extra == 'test'" }, + { name = "py-spy", marker = "extra == 'test'" }, + { name = "pyarrow", specifier = ">=19.0.0" }, + { name = "pybind11" }, + { name = "pyext", marker = "extra == 'prime'" }, + { name = "pylatexenc" }, + { name = "pytest", marker = "extra == 'test'" }, + { name = "pytest-asyncio", marker = "extra == 'test'" }, + { name = "qwen-vl-utils", marker = "extra == 'geo'" }, + { name = "ray", extras = ["default"], specifier = ">=2.41.0" }, + { name = "sglang", extras = ["openai", "srt"], marker = "extra == 'sglang'", specifier = "==0.4.6.post5" }, + { name = "tensordict", specifier = ">=0.8.0,!=0.9.0,<=0.9.1" }, + { name = "tensordict", marker = "extra == 'sglang'", specifier = ">=0.8.0,!=0.9.0,<=0.9.1" }, + { name = "tensordict", marker = "extra == 'vllm'", specifier = ">=0.8.0,!=0.9.0,<=0.9.1" }, + { name = "torch", marker = "extra == 'sglang'", specifier = "==2.6.0" }, + { name = "torch", marker = "extra == 'vllm'", url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, + { name = "torch-memory-saver", marker = "extra == 'sglang'", specifier = ">=0.0.5" }, + { name = "torchdata" }, + { name = "torchvision", marker = "extra == 'geo'" }, + { name = "torchvision", marker = "extra == 'vllm'", url = "https://download.pytorch.org/whl/cu124/torchvision-0.21.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, + { name = "transformers" }, + { name = "trl", marker = "extra == 'trl'", specifier = "<=0.9.6" }, + { name = "vllm", marker = "extra == 'vllm'", specifier = ">=0.7.3,<=0.8.5" }, + { name = "wandb" }, +] +provides-extras = ["test", "prime", "geo", "gpu", "math", "vllm", "sglang", "trl", "mcore"] + +[[package]] +name = "virtualenv" +version = "20.34.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/14/37fcdba2808a6c615681cd216fecae00413c9dab44fb2e57805ecf3eaee3/virtualenv-20.34.0.tar.gz", hash = "sha256:44815b2c9dee7ed86e387b842a84f20b93f7f417f95886ca1996a72a4138eb1a", size = 6003808, upload-time = "2025-08-13T14:24:07.464Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/06/04c8e804f813cf972e3262f3f8584c232de64f0cde9f703b46cf53a45090/virtualenv-20.34.0-py3-none-any.whl", hash = "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026", size = 5983279, upload-time = "2025-08-13T14:24:05.111Z" }, +] + +[[package]] +name = "vllm" +version = "0.8.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "aiohttp", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "blake3", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "cachetools", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "cloudpickle", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "compressed-tensors", version = "0.9.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "depyf", version = "0.18.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "einops", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "fastapi", extra = ["standard"], marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "filelock", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "gguf", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "huggingface-hub", extra = ["hf-xet"], marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "importlib-metadata", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "lark", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "llguidance", marker = "(platform_machine == 'aarch64' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'arm64' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'x86_64' and extra == 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "lm-format-enforcer", version = "0.10.12", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "mistral-common", extra = ["opencv"], marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "msgspec", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "ninja", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numba", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "openai", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opencv-python-headless", version = "4.11.0.86", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-api", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-exporter-otlp", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-sdk", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "opentelemetry-semantic-conventions-ai", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "outlines", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "partial-json-parser", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "pillow", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "prometheus-client", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "prometheus-fastapi-instrumentator", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "protobuf", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "psutil", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "py-cpuinfo", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "pydantic", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "python-json-logger", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "pyyaml", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "pyzmq", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "ray", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "requests", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "scipy", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "sentencepiece", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "six", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "tiktoken", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "tokenizers", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torchaudio", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torchvision", version = "0.21.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torchvision-0.21.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "tqdm", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "transformers", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "typing-extensions", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "watchfiles", marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "xformers", version = "0.0.29.post2", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "xgrammar", marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/37/7846c244e6637c15f6d5ed6d3f572eedd02ed8ab80aec8c3eb228f35b23c/vllm-0.8.5.tar.gz", hash = "sha256:c7e04d1046304397b4580334038b558fe491af155fdea508224f140172cf9a82", size = 7340498, upload-time = "2025-04-28T23:59:48.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/7e/08d7a75c47792fd07d02b2be6fe6adbb29dfcbd31e94784844d7f730d0bc/vllm-0.8.5-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:74bfe92953bee1269c1e1c27827bc156777751cdd6a3457ee8e27dd8ebf1e247", size = 326421025, upload-time = "2025-04-28T23:59:40.401Z" }, +] + +[[package]] +name = "vllm" +version = "0.10.1.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation != 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "aiohttp", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "blake3", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "cachetools", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "cbor2", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "cloudpickle", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "compressed-tensors", version = "0.10.2", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "depyf", version = "0.19.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "diskcache", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "einops", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "fastapi", extra = ["standard"], marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "filelock", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "gguf", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "lark", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "llguidance", marker = "(platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine == 'arm64' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine == 'x86_64' and extra == 'extra-8-skyagent-skyrl-train') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "lm-format-enforcer", version = "0.10.12", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "mistral-common", extra = ["audio", "image"], marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "msgspec", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "ninja", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "numba", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "openai", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "openai-harmony", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "opencv-python-headless", version = "4.12.0.88", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "outlines-core", version = "0.2.10", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "partial-json-parser", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "pillow", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "prometheus-client", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "prometheus-fastapi-instrumentator", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "protobuf", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "psutil", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "py-cpuinfo", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "pybase64", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "pydantic", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "python-json-logger", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "pyyaml", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "pyzmq", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "ray", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "regex", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "requests", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "scipy", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "sentencepiece", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "setproctitle", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "six", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "tiktoken", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "tokenizers", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "torchaudio", version = "2.7.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "torchvision", version = "0.22.1", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "tqdm", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "transformers", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "typing-extensions", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "watchfiles", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "xformers", version = "0.0.31", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "xgrammar", marker = "extra == 'extra-8-skyagent-skyrl-train'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/a8/b53e7950c05bf8e6ef4438603a70c8992e7bc90fcad5a7a4f360c7d748a6/vllm-0.10.1.1.tar.gz", hash = "sha256:3099824ee4bdaa14c4c4f7178a092101a0ec206d4c9371edf295849b2b730a39", size = 10510801, upload-time = "2025-08-20T23:17:16.776Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/14/6fac6c3f6be5657a70911c8c48062c4749925dc5e24d254b1ec493add7a0/vllm-0.10.1.1-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:8ca0dd985e1ceac8540e7719c654f1553b3ba8a43c685ac8d3fa1366ffb6443a", size = 414400912, upload-time = "2025-08-20T23:17:09.179Z" }, +] + +[[package]] +name = "vllm" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "aiohttp", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "blake3", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "cachetools", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "cbor2", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "cloudpickle", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "compressed-tensors", version = "0.11.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "depyf", version = "0.19.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "diskcache", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "einops", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "fastapi", extra = ["standard"], marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "filelock", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "gguf", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "lark", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "llguidance", marker = "(platform_machine == 'aarch64' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (platform_machine == 'arm64' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (platform_machine == 'x86_64' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "lm-format-enforcer", version = "0.11.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "mistral-common", extra = ["audio", "image"], marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "msgspec", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "ninja", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "numba", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "openai", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "openai-harmony", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "opencv-python-headless", version = "4.12.0.88", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "outlines-core", version = "0.2.11", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "partial-json-parser", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pillow", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "prometheus-client", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "prometheus-fastapi-instrumentator", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "protobuf", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "psutil", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "py-cpuinfo", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pybase64", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pydantic", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "python-json-logger", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pyyaml", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "pyzmq", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "ray", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "regex", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "requests", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "scipy", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "sentencepiece", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "setproctitle", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "six", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "tiktoken", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "tokenizers", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "torchaudio", version = "2.8.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "torchvision", version = "0.23.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "tqdm", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "transformers", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "typing-extensions", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "watchfiles", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "xformers", version = "0.0.32.post1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "xgrammar", marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0a/278d7bbf454f7de5322a5007427eed3e8b34ed6c2802491b56bbdfd7bbb4/vllm-0.10.2.tar.gz", hash = "sha256:57608f44cf61f5d80fb182c98e06e524cb2925bb528258a7b247c8e43a52d13e", size = 10908356, upload-time = "2025-09-13T23:00:34.918Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/1a/365479f413e7408b314c0237d6c929569874d5c002bc7c8b5a7fbf40c7d9/vllm-0.10.2-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:e0cba6110483d9bf25c4402d8655cf78d366dd13e4155210980cc3480ed98b7b", size = 436413211, upload-time = "2025-09-13T23:00:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/2f/29/b02687cb515be113a9fe188271b119a37178e9feefeb9c9b23dac87144e4/vllm-0.10.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:b011a55544c0e7a8b6d247da1585b1d073bbb8dc4981d576559784ea2d3fafa1", size = 399379773, upload-time = "2025-09-16T00:29:34.059Z" }, +] + +[[package]] +name = "wadler-lindig" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/67/cbae4bf7683a64755c2c1778c418fea96d00e34395bb91743f08bd951571/wadler_lindig-0.1.7.tar.gz", hash = "sha256:81d14d3fe77d441acf3ebd7f4aefac20c74128bf460e84b512806dccf7b2cd55", size = 15842, upload-time = "2025-06-18T07:00:42.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/96/04e7b441807b26b794da5b11e59ed7f83b2cf8af202bd7eba8ad2fa6046e/wadler_lindig-0.1.7-py3-none-any.whl", hash = "sha256:e3ec83835570fd0a9509f969162aeb9c65618f998b1f42918cfc8d45122fe953", size = 20516, upload-time = "2025-06-18T07:00:41.684Z" }, +] + +[[package]] +name = "wandb" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/6f/be255b13157cab6c6670594171f59f67bd8a89f20d1978dc4eb892e2de27/wandb-0.22.1.tar.gz", hash = "sha256:6a1d668ecd6bd6531a73f6f7cfec0a93a08ef578c16ccf7167168c52cbf8cb12", size = 40246806, upload-time = "2025-09-29T17:15:55.207Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/73/de1d62301ef5d084160221637f34a821b7ed90b0769698b7b420686608ab/wandb-0.22.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:d862d0d28919556a5c32977138449812a2d127853e20b0deb39a5ab17700230f", size = 18370574, upload-time = "2025-09-29T17:15:18.236Z" }, + { url = "https://files.pythonhosted.org/packages/42/0e/4f60d9c7f1fa9d249dcbe70c6bad1573a9cfc070d00c3d8dbd62f715938d/wandb-0.22.1-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:ce57213de331717270020f7e07b098a0ea37646550b63758eabf8cb05eeb066f", size = 19392851, upload-time = "2025-09-29T17:15:22.093Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8b/757ede4a581eece5e72ade51fd4b43cfedbd3e39b85fe29d0198bc98131b/wandb-0.22.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5a0ac652c23bf88e12bf0c04e911ff4f95696ac60a3612d81e54f1f8d89f3c5", size = 18171463, upload-time = "2025-09-29T17:15:24.588Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e6/275d60292183d4de89fc9053887192f978fd8612e55c8f7719aa5c81bbd1/wandb-0.22.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dc14dd06938c282900dd9f72cbaaed45368b0c6b9bc2ffd1f45d07eeb13095b", size = 19585538, upload-time = "2025-09-29T17:15:28.432Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5c/4199abb92d06de6ebd63ee33551ba0de6d91a814ac42e372dec6d8009ea0/wandb-0.22.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1b2d2b9d8d1ea8aea3c2cbf9de7696105432886ba9845c50e7cc71613aa6c8ef", size = 18210525, upload-time = "2025-09-29T17:15:33.459Z" }, + { url = "https://files.pythonhosted.org/packages/0d/00/a7719c048115825861a31435fa911887c9949b20096dbc7307e11b3c981b/wandb-0.22.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:051906d7f22bdf8c07c8837ffc6d9ae357d61dcd74cfb7d29fd0243e03da8f4a", size = 19680055, upload-time = "2025-09-29T17:15:38.521Z" }, + { url = "https://files.pythonhosted.org/packages/6f/5d/e6270557a315211e880c1efa9c1cab577f945d81168bc1c1187fd483f1bb/wandb-0.22.1-py3-none-win32.whl", hash = "sha256:b2df59bd70771329f27171f55d25d5557731bb0674d60db4735c173a8fb8076d", size = 18769036, upload-time = "2025-09-29T17:15:43.19Z" }, + { url = "https://files.pythonhosted.org/packages/92/fe/34cdfd491ea6c89495794f361102b727b922adcc4f3eedb47c8aa16984c3/wandb-0.22.1-py3-none-win_amd64.whl", hash = "sha256:c1b442e6de805d78743321200a27099517509f9e4aa2e6d330211a4809f932d7", size = 18769038, upload-time = "2025-09-29T17:15:46.977Z" }, + { url = "https://files.pythonhosted.org/packages/1e/9e/fe95f5d48ff10215b7d7e67dc998cba3f660027829fac2a67c79ce89e985/wandb-0.22.1-py3-none-win_arm64.whl", hash = "sha256:52758008c9ef9e7201113af08d6015322a699ebe3497a6e6fc885b39f5652b4d", size = 17077774, upload-time = "2025-09-29T17:15:50.588Z" }, +] + +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, + { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/9a/d451fcc97d029f5812e898fd30a53fd8c15c7bbd058fd75cfc6beb9bd761/watchfiles-1.1.0.tar.gz", hash = "sha256:693ed7ec72cbfcee399e92c895362b6e66d63dac6b91e2c11ae03d10d503e575", size = 94406, upload-time = "2025-06-15T19:06:59.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/b8/858957045a38a4079203a33aaa7d23ea9269ca7761c8a074af3524fbb240/watchfiles-1.1.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9dc001c3e10de4725c749d4c2f2bdc6ae24de5a88a339c4bce32300a31ede179", size = 402339, upload-time = "2025-06-15T19:05:24.516Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/98b222cca751ba68e88521fabd79a4fab64005fc5976ea49b53fa205d1fa/watchfiles-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9ba68ec283153dead62cbe81872d28e053745f12335d037de9cbd14bd1877f5", size = 394409, upload-time = "2025-06-15T19:05:25.469Z" }, + { url = "https://files.pythonhosted.org/packages/86/50/dee79968566c03190677c26f7f47960aff738d32087087bdf63a5473e7df/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:130fc497b8ee68dce163e4254d9b0356411d1490e868bd8790028bc46c5cc297", size = 450939, upload-time = "2025-06-15T19:05:26.494Z" }, + { url = "https://files.pythonhosted.org/packages/40/45/a7b56fb129700f3cfe2594a01aa38d033b92a33dddce86c8dfdfc1247b72/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50a51a90610d0845a5931a780d8e51d7bd7f309ebc25132ba975aca016b576a0", size = 457270, upload-time = "2025-06-15T19:05:27.466Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c8/fa5ef9476b1d02dc6b5e258f515fcaaecf559037edf8b6feffcbc097c4b8/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc44678a72ac0910bac46fa6a0de6af9ba1355669b3dfaf1ce5f05ca7a74364e", size = 483370, upload-time = "2025-06-15T19:05:28.548Z" }, + { url = "https://files.pythonhosted.org/packages/98/68/42cfcdd6533ec94f0a7aab83f759ec11280f70b11bfba0b0f885e298f9bd/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a543492513a93b001975ae283a51f4b67973662a375a403ae82f420d2c7205ee", size = 598654, upload-time = "2025-06-15T19:05:29.997Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/b2a1544224118cc28df7e59008a929e711f9c68ce7d554e171b2dc531352/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ac164e20d17cc285f2b94dc31c384bc3aa3dd5e7490473b3db043dd70fbccfd", size = 478667, upload-time = "2025-06-15T19:05:31.172Z" }, + { url = "https://files.pythonhosted.org/packages/8c/77/e3362fe308358dc9f8588102481e599c83e1b91c2ae843780a7ded939a35/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7590d5a455321e53857892ab8879dce62d1f4b04748769f5adf2e707afb9d4f", size = 452213, upload-time = "2025-06-15T19:05:32.299Z" }, + { url = "https://files.pythonhosted.org/packages/6e/17/c8f1a36540c9a1558d4faf08e909399e8133599fa359bf52ec8fcee5be6f/watchfiles-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:37d3d3f7defb13f62ece99e9be912afe9dd8a0077b7c45ee5a57c74811d581a4", size = 626718, upload-time = "2025-06-15T19:05:33.415Z" }, + { url = "https://files.pythonhosted.org/packages/26/45/fb599be38b4bd38032643783d7496a26a6f9ae05dea1a42e58229a20ac13/watchfiles-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7080c4bb3efd70a07b1cc2df99a7aa51d98685be56be6038c3169199d0a1c69f", size = 623098, upload-time = "2025-06-15T19:05:34.534Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/fdf40e038475498e160cd167333c946e45d8563ae4dd65caf757e9ffe6b4/watchfiles-1.1.0-cp312-cp312-win32.whl", hash = "sha256:cbcf8630ef4afb05dc30107bfa17f16c0896bb30ee48fc24bf64c1f970f3b1fd", size = 279209, upload-time = "2025-06-15T19:05:35.577Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d3/3ae9d5124ec75143bdf088d436cba39812122edc47709cd2caafeac3266f/watchfiles-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:cbd949bdd87567b0ad183d7676feb98136cde5bb9025403794a4c0db28ed3a47", size = 292786, upload-time = "2025-06-15T19:05:36.559Z" }, + { url = "https://files.pythonhosted.org/packages/26/2f/7dd4fc8b5f2b34b545e19629b4a018bfb1de23b3a496766a2c1165ca890d/watchfiles-1.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:0a7d40b77f07be87c6faa93d0951a0fcd8cbca1ddff60a1b65d741bac6f3a9f6", size = 284343, upload-time = "2025-06-15T19:05:37.5Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.2.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/30/6b0809f4510673dc723187aeaf24c7f5459922d01e2f794277a3dfb90345/wcwidth-0.2.14.tar.gz", hash = "sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605", size = 102293, upload-time = "2025-09-22T16:29:53.023Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1", size = 37286, upload-time = "2025-09-22T16:29:51.641Z" }, +] + +[[package]] +name = "weblinx" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/c6/5d65086c948f8b0b874f5d94d33e87d7c698049c6aba37124a67aac5d7f2/weblinx-0.3.2.tar.gz", hash = "sha256:259946c2b08cf50b48929fdd1c17f09ff5808b7d53528d728d25b583b8a06c85", size = 80931, upload-time = "2024-10-01T23:55:58.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/3c/940850b54ea8b5927e0b794fda6839a7cf2cc542c03af2388a55f0483d7b/weblinx-0.3.2-py3-none-any.whl", hash = "sha256:9ab8de1c631617827955debaeb76864b8b3122d230185af1f9c30f5c793e7213", size = 82905, upload-time = "2024-10-01T23:55:57.15Z" }, +] + +[package.optional-dependencies] +eval = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pandas" }, + { name = "sacrebleu" }, + { name = "tqdm" }, +] + +[[package]] +name = "weblinx-browsergym" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "browsergym-core" }, + { name = "datasets" }, + { name = "gymnasium" }, + { name = "huggingface-hub" }, + { name = "lxml" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "pillow" }, + { name = "playwright" }, + { name = "tqdm" }, + { name = "weblinx", extra = ["eval"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/c1/5ebe2c84ca123b0e83458fe4110dec88723dc17b7f2834967a0eeb357623/weblinx_browsergym-0.0.3.tar.gz", hash = "sha256:b4c8ed834207b65334147c3dd538d4177654e71f99704a189d7241829aa41356", size = 14765, upload-time = "2025-04-25T19:08:21.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/ee/83fe48c6e4e5cc9ecdfa044ca0c3e6a515a9ccb28924234daa869d43786d/weblinx_browsergym-0.0.3-py3-none-any.whl", hash = "sha256:02b5a40cec543b0b2558a0c891fcb1cd7af929cfec26929b2da3078526d47adf", size = 11394, upload-time = "2025-04-25T19:08:20.385Z" }, +] + +[[package]] +name = "websockets" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, + { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, + { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, + { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, + { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" }, +] + +[[package]] +name = "whatthepatch" +version = "1.0.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/28/55bc3e107a56fdcf7d5022cb32b8c21d98a9cc2df5cd9f3b93e10419099e/whatthepatch-1.0.7.tar.gz", hash = "sha256:9eefb4ebea5200408e02d413d2b4bc28daea6b78bb4b4d53431af7245f7d7edf", size = 34612, upload-time = "2024-11-16T17:21:22.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/93/af1d6ccb69ab6b5a00e03fa0cefa563f9862412667776ea15dd4eece3a90/whatthepatch-1.0.7-py3-none-any.whl", hash = "sha256:1b6f655fd31091c001c209529dfaabbabdbad438f5de14e3951266ea0fc6e7ed", size = 11964, upload-time = "2024-11-16T17:21:20.761Z" }, +] + +[[package]] +name = "widgetsnbextension" +version = "4.0.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/53/2e0253c5efd69c9656b1843892052a31c36d37ad42812b5da45c62191f7e/widgetsnbextension-4.0.14.tar.gz", hash = "sha256:a3629b04e3edb893212df862038c7232f62973373869db5084aed739b437b5af", size = 1097428, upload-time = "2025-04-10T13:01:25.628Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/51/5447876806d1088a0f8f71e16542bf350918128d0a69437df26047c8e46f/widgetsnbextension-4.0.14-py3-none-any.whl", hash = "sha256:4875a9eaf72fbf5079dc372a51a9f268fc38d46f767cbf85c43a36da5cb9b575", size = 2196503, upload-time = "2025-04-10T13:01:23.086Z" }, +] + +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] + +[[package]] +name = "wrapt" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +] + +[[package]] +name = "wsproto" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/4a/44d3c295350d776427904d73c189e10aeae66d7f555bb2feee16d1e4ba5a/wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", size = 53425, upload-time = "2022-08-23T19:58:21.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/58/e860788190eba3bcce367f74d29c4675466ce8dddfba85f7827588416f01/wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736", size = 24226, upload-time = "2022-08-23T19:58:19.96Z" }, +] + +[[package]] +name = "xattr" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/65/14438ae55acf7f8fc396ee8340d740a3e1d6ef382bf25bf24156cfb83563/xattr-1.2.0.tar.gz", hash = "sha256:a64c8e21eff1be143accf80fd3b8fde3e28a478c37da298742af647ac3e5e0a7", size = 17293, upload-time = "2025-07-14T03:15:44.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/e5/b048a5f6c5a489915026b70b9133242a2a368383ddab24e4e3a5bdba7ebd/xattr-1.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:daff0c1f5c5e4eaf758c56259c4f72631fa9619875e7a25554b6077dc73da964", size = 24240, upload-time = "2025-07-14T03:14:46.173Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f5/d795774f719a0be6137041d4833ca00b178f816e538948548dff79530f34/xattr-1.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:109b11fb3f73a0d4e199962f11230ab5f462e85a8021874f96c1732aa61148d5", size = 19218, upload-time = "2025-07-14T03:14:47.412Z" }, + { url = "https://files.pythonhosted.org/packages/cb/8b/65f3bed09ca9ced27bbba8d4a3326f14a58b98ac102163d85b545f81d9c2/xattr-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7c7c12968ce0bf798d8ba90194cef65de768bee9f51a684e022c74cab4218305", size = 19539, upload-time = "2025-07-14T03:14:48.413Z" }, + { url = "https://files.pythonhosted.org/packages/96/2d/01ecfdf41ce70f7e29c8a21e730de3c157fb1cb84391923581af81a44c45/xattr-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d37989dabf25ff18773e4aaeebcb65604b9528f8645f43e02bebaa363e3ae958", size = 39631, upload-time = "2025-07-14T03:14:49.665Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e9/15cbf9c59cf1117e3c45dd429c52f9dab25d95e65ac245c5ad9532986bec/xattr-1.2.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:165de92b0f2adafb336f936931d044619b9840e35ba01079f4dd288747b73714", size = 37552, upload-time = "2025-07-14T03:14:50.718Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f5/cb4dad87843fe79d605cf5d10caad80e2c338a06f0363f1443449185f489/xattr-1.2.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82191c006ae4c609b22b9aea5f38f68fff022dc6884c4c0e1dba329effd4b288", size = 39472, upload-time = "2025-07-14T03:14:51.74Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d9/012df7b814cc4a0ae41afb59ac31d0469227397b29f58c1377e8db0f34ba/xattr-1.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2b2e9c87dc643b09d86befad218e921f6e65b59a4668d6262b85308de5dbd1dd", size = 38802, upload-time = "2025-07-14T03:14:52.801Z" }, + { url = "https://files.pythonhosted.org/packages/d8/08/e107a5d294a816586f274c33aea480fe740fd446276efc84c067e6c82de2/xattr-1.2.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:14edd5d47d0bb92b23222c0bb6379abbddab01fb776b2170758e666035ecf3aa", size = 37125, upload-time = "2025-07-14T03:14:54.313Z" }, + { url = "https://files.pythonhosted.org/packages/3e/6c/a6f9152e10543af67ea277caae7c5a6400a581e407c42156ffce71dd8242/xattr-1.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:12183d5eb104d4da787638c7dadf63b718472d92fec6dbe12994ea5d094d7863", size = 38456, upload-time = "2025-07-14T03:14:55.383Z" }, +] + +[[package]] +name = "xformers" +version = "0.0.29.post2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine == 'aarch64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/ed/04ec7ef97a7e1c836add41ef5a2aef8cbdd45c0190ca42cc08f3c21e2b7b/xformers-0.0.29.post2.tar.gz", hash = "sha256:6ca3d1a6db6f2abff25c1154adee96987f77f4dfd5141771805afa5fc13e9395", size = 8468494, upload-time = "2025-02-01T02:33:48.209Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/ac/1aca7e44c93876dbda00e80f79c0bda78bc65e236c68ceb2fc6b26f77df5/xformers-0.0.29.post2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0d0eb14db56cf08ec3fb9cb36ed5e98de1303411571539ca4dc080c5861e2744", size = 44289739, upload-time = "2025-02-01T02:32:54.559Z" }, + { url = "https://files.pythonhosted.org/packages/bf/40/1753e0f01f200673f1e02c0ac8ed6f38bac4df02620903daafa9a5676ebe/xformers-0.0.29.post2-cp312-cp312-win_amd64.whl", hash = "sha256:a3ddb47abce3810d3928e8f48b290c0423c7939764a217c2b35ac8124a3cf641", size = 167776625, upload-time = "2025-02-01T02:33:10.82Z" }, +] + +[[package]] +name = "xformers" +version = "0.0.31" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/35/91c172a57681e1c03de5ad1ca654dc87c282279b941052ed04e616ae5bcd/xformers-0.0.31.tar.gz", hash = "sha256:3fccb159c6327c13fc1b08f8b963c2779ca526e2e50755dee9bcc1bac67d20c6", size = 12102740, upload-time = "2025-06-25T15:12:10.241Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/e3/ee90c62a3235152d4ea8e983a5eb7ac00b10582fee86aaadb11571c1ecba/xformers-0.0.31-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:50aedaea82a38d7d28631f77617d1ed1f6f37c60bdc4bf167a69cbc0e39cee76", size = 117057673, upload-time = "2025-06-25T15:11:59.775Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/7b60ec52da94a0da3383ba385b43e42cee8854c001b5cb6cf98a909665c8/xformers-0.0.31-cp39-abi3-win_amd64.whl", hash = "sha256:23331bdb9831ba0df96f55258537ca0df7ad888efc75cea97a0de79b5e2291c4", size = 100228305, upload-time = "2025-06-25T15:12:05.924Z" }, +] + +[[package]] +name = "xformers" +version = "0.0.32.post1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/33/3b9c4d3d5b2da453d27de891df4ad653ac5795324961aa3a5c15b0353fe6/xformers-0.0.32.post1.tar.gz", hash = "sha256:1de84a45c497c8d92326986508d81f4b0a8c6be4d3d62a29b8ad6048a6ab51e1", size = 12106196, upload-time = "2025-08-14T18:07:45.486Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/df/6817346f1a77278315d5fe1fc9f239ba3282ba36e8ab3256babd448dde62/xformers-0.0.32.post1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5f245b5555188da112070d8fefb6b7ae1ae47422856521d66c837e9d2352fbe4", size = 117199943, upload-time = "2025-08-14T18:07:34.78Z" }, + { url = "https://files.pythonhosted.org/packages/91/70/7dee5a786d77a63cf20dac60c38086bd2202d59ae89c8acef0ef6331c374/xformers-0.0.32.post1-cp39-abi3-win_amd64.whl", hash = "sha256:feb452bc2c8731da1c5d0e2e4536ba95bb214f77b41e91f24443c74d6f98a126", size = 100221531, upload-time = "2025-08-14T18:07:41.112Z" }, +] + +[[package]] +name = "xgrammar" +version = "0.1.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nanobind" }, + { name = "ninja" }, + { name = "pydantic" }, + { name = "sentencepiece" }, + { name = "tiktoken" }, + { name = "torch", version = "2.6.0+cu124", source = { url = "https://download.pytorch.org/whl/cu124/torch-2.6.0%2Bcu124-cp312-cp312-linux_x86_64.whl" }, marker = "extra == 'extra-8-skyagent-verl'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-8-skyagent-skyrl-train'" }, + { name = "torch", version = "2.8.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl')" }, + { name = "transformers" }, + { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and platform_system == 'linux' and sys_platform == 'linux' and extra == 'extra-8-skyagent-verl') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_system != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (sys_platform != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "triton", version = "3.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and platform_system == 'linux' and sys_platform != 'linux') or (platform_machine == 'x86_64' and platform_system == 'linux' and extra == 'extra-8-skyagent-skyrl-train') or (platform_machine != 'x86_64' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl') or (platform_system != 'linux' and extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, + { name = "triton", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and platform_system == 'linux' and sys_platform == 'linux' and extra != 'extra-8-skyagent-skyrl-train' and extra != 'extra-8-skyagent-verl') or (extra == 'extra-8-skyagent-skyrl-train' and extra == 'extra-8-skyagent-verl')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/f9/6d530ce703cf5aae65d594a5ab984b9c0c4956e6fdbcc3279e8b1eaa358e/xgrammar-0.1.17.tar.gz", hash = "sha256:8f6cd7b3436482ad8c94b6cc93892a7f36381315c443e8e7f256f8d71c3efdee", size = 1679977, upload-time = "2025-03-25T11:08:13.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/32/deaee8f04d24bc2ed38c14fb01d6faa2319fb361353bbbebac4bdf801ac6/xgrammar-0.1.17-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:cc8e1e4a3298aae9856416e1366ccd07d4c6b5556921ecd108c579b1184522d2", size = 371412, upload-time = "2025-03-25T11:07:54.23Z" }, + { url = "https://files.pythonhosted.org/packages/35/ed/59a89ef003235f746fa989bf82e8425e6b046d65349feacd1b57b4763141/xgrammar-0.1.17-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a7712942793727f0c490f6f2388d5995632cc0c8258a7aff33577ff0f47bc513", size = 340973, upload-time = "2025-03-25T11:07:55.76Z" }, + { url = "https://files.pythonhosted.org/packages/48/bc/f6f5f16d9cb57684f23a62d3f51deed410da6c9708bf3d5eb679dd867dc0/xgrammar-0.1.17-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b3e998ab30662b5f090978d04928f20467df973116c17624f868fa7717ff683", size = 4236280, upload-time = "2025-03-25T11:07:57.085Z" }, + { url = "https://files.pythonhosted.org/packages/8a/89/8d4b7a8bf5af80564081555f1734d668e5496e90171280de9153d0696065/xgrammar-0.1.17-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1687ce767c5ca0fe101f699c2691762a037a6b0159608f6c4a720bccdb57ee8a", size = 4310624, upload-time = "2025-03-25T11:07:58.554Z" }, + { url = "https://files.pythonhosted.org/packages/27/37/8e31a5a44b21e89755795103df04fadb390db395c9fe65179acc9bf067b4/xgrammar-0.1.17-cp312-cp312-win_amd64.whl", hash = "sha256:9572b4c571cf39f6ffd29915b73d3cc13303c72aa86043660f46f66746b5b947", size = 421404, upload-time = "2025-03-25T11:07:59.954Z" }, +] + +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + +[[package]] +name = "xxhash" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/5e/d6e5258d69df8b4ed8c83b6664f2b47d30d2dec551a29ad72a6c69eafd31/xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f", size = 84241, upload-time = "2024-08-17T09:20:38.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/0e/1bfce2502c57d7e2e787600b31c83535af83746885aa1a5f153d8c8059d6/xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00", size = 31969, upload-time = "2024-08-17T09:18:24.025Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d6/8ca450d6fe5b71ce521b4e5db69622383d039e2b253e9b2f24f93265b52c/xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9", size = 30787, upload-time = "2024-08-17T09:18:25.318Z" }, + { url = "https://files.pythonhosted.org/packages/5b/84/de7c89bc6ef63d750159086a6ada6416cc4349eab23f76ab870407178b93/xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84", size = 220959, upload-time = "2024-08-17T09:18:26.518Z" }, + { url = "https://files.pythonhosted.org/packages/fe/86/51258d3e8a8545ff26468c977101964c14d56a8a37f5835bc0082426c672/xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793", size = 200006, upload-time = "2024-08-17T09:18:27.905Z" }, + { url = "https://files.pythonhosted.org/packages/02/0a/96973bd325412feccf23cf3680fd2246aebf4b789122f938d5557c54a6b2/xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be", size = 428326, upload-time = "2024-08-17T09:18:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/11/a7/81dba5010f7e733de88af9555725146fc133be97ce36533867f4c7e75066/xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6", size = 194380, upload-time = "2024-08-17T09:18:30.706Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7d/f29006ab398a173f4501c0e4977ba288f1c621d878ec217b4ff516810c04/xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90", size = 207934, upload-time = "2024-08-17T09:18:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6e/6e88b8f24612510e73d4d70d9b0c7dff62a2e78451b9f0d042a5462c8d03/xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27", size = 216301, upload-time = "2024-08-17T09:18:33.474Z" }, + { url = "https://files.pythonhosted.org/packages/af/51/7862f4fa4b75a25c3b4163c8a873f070532fe5f2d3f9b3fc869c8337a398/xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2", size = 203351, upload-time = "2024-08-17T09:18:34.889Z" }, + { url = "https://files.pythonhosted.org/packages/22/61/8d6a40f288f791cf79ed5bb113159abf0c81d6efb86e734334f698eb4c59/xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d", size = 210294, upload-time = "2024-08-17T09:18:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/17/02/215c4698955762d45a8158117190261b2dbefe9ae7e5b906768c09d8bc74/xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab", size = 414674, upload-time = "2024-08-17T09:18:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/31/5c/b7a8db8a3237cff3d535261325d95de509f6a8ae439a5a7a4ffcff478189/xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e", size = 192022, upload-time = "2024-08-17T09:18:40.138Z" }, + { url = "https://files.pythonhosted.org/packages/78/e3/dd76659b2811b3fd06892a8beb850e1996b63e9235af5a86ea348f053e9e/xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8", size = 30170, upload-time = "2024-08-17T09:18:42.163Z" }, + { url = "https://files.pythonhosted.org/packages/d9/6b/1c443fe6cfeb4ad1dcf231cdec96eb94fb43d6498b4469ed8b51f8b59a37/xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e", size = 30040, upload-time = "2024-08-17T09:18:43.699Z" }, + { url = "https://files.pythonhosted.org/packages/0f/eb/04405305f290173acc0350eba6d2f1a794b57925df0398861a20fbafa415/xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2", size = 26796, upload-time = "2024-08-17T09:18:45.29Z" }, +] + +[[package]] +name = "yarl" +version = "1.20.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" }, + { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" }, + { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" }, + { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" }, + { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" }, + { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" }, + { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" }, + { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" }, + { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" }, + { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" }, + { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" }, + { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" }, + { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" }, + { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" }, + { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] + +[[package]] +name = "zope-interface" +version = "7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools", version = "79.0.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-skyrl-train' or extra != 'extra-8-skyagent-verl'" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-8-skyagent-verl'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/0b/c7516bc3bad144c2496f355e35bd699443b82e9437aa02d9867653203b4a/zope.interface-7.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:086ee2f51eaef1e4a52bd7d3111a0404081dadae87f84c0ad4ce2649d4f708b7", size = 208959, upload-time = "2024-11-28T08:47:47.788Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e9/1463036df1f78ff8c45a02642a7bf6931ae4a38a4acd6a8e07c128e387a7/zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21328fcc9d5b80768bf051faa35ab98fb979080c18e6f84ab3f27ce703bce465", size = 209357, upload-time = "2024-11-28T08:47:50.897Z" }, + { url = "https://files.pythonhosted.org/packages/07/a8/106ca4c2add440728e382f1b16c7d886563602487bdd90004788d45eb310/zope.interface-7.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6dd02ec01f4468da0f234da9d9c8545c5412fef80bc590cc51d8dd084138a89", size = 264235, upload-time = "2024-11-28T09:18:15.56Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ca/57286866285f4b8a4634c12ca1957c24bdac06eae28fd4a3a578e30cf906/zope.interface-7.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e7da17f53e25d1a3bde5da4601e026adc9e8071f9f6f936d0fe3fe84ace6d54", size = 259253, upload-time = "2024-11-28T08:48:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702, upload-time = "2024-11-28T08:48:37.363Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466, upload-time = "2024-11-28T08:49:14.397Z" }, +] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, +] diff --git a/skyrl-gym/README.md b/skyrl-gym/README.md new file mode 100644 index 0000000000..9a3a310ebc --- /dev/null +++ b/skyrl-gym/README.md @@ -0,0 +1,32 @@ +# SkyRL-Gym + +A library of RL environments for LLMs implemented with the Gymnasium API. + +## Key Features + +- Simple `Environment` interface following the Gymnasium API. +- Library of ready-built environments for math, code, search, and text-to-SQL. +- A reusable `tool` interface. Developers can implement a tool once, and use it across any environment. +- Supports multi-tool environments + +## Installation + +You can install the latest release from PyPI: + +```bash +pip install skyrl-gym +``` + +or install from source: + +```bash +git clone https://github.com/NovaSky-AI/SkyRL.git +cd SkyRL/skyrl-gym +pip install -e . +``` + +## Documentation + +To build your first environment, see our [Walkthrough Docs](https://skyrl.readthedocs.io/en/latest/tutorials/new_env.html). + +All docs are available at [https://skyrl.readthedocs.io/en/latest/](https://skyrl.readthedocs.io/en/latest/). diff --git a/skyrl-gym/pyproject.toml b/skyrl-gym/pyproject.toml new file mode 100644 index 0000000000..ab70c22b82 --- /dev/null +++ b/skyrl-gym/pyproject.toml @@ -0,0 +1,81 @@ +# basic pyproject.toml for skyrl-gym +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "skyrl-gym" +version = "0.1.1" +description = "RL environments for LLMs implemented with the Gymnasium API." +authors = [ + { name = "NovaSkyAI AI Team"} +] +license = { text = "MIT" } +readme = "README.md" +requires-python = ">=3.10" + +dependencies = [ + "func_timeout", + "pandas", + "requests", + "omegaconf", +] + +[project.urls] +Repository = "https://github.com/NovaSky-AI/SkyRL" + +[tool.setuptools.packages.find] +include = ["skyrl_gym*"] + +[project.optional-dependencies] +dev = [ + "pytest" +] + +[tool.black] +line-length = 120 +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.flake8] +max-line-length = 120 +max-doc-length = 120 +extend-ignore = [ + # Default ignored errors by flake8 + "E121", "E123", "E126", "E226", "E24", "E704", + # F401 module imported but unused + "F401", + # E203 whitespace before ':' (conflict with black) + "E203", + # E231 missing whitespace after ',' (conflict with black) + "E231", + # E501 line too long (conflict with black) + "E501", + # E741 do not use variables named 'l', 'O', or 'I' + "E741", + # W503 line break before binary operator (conflict with black) + "W503", + # W504 line break after binary operator (conflict with black) + "W504", + # W505 doc line too long (conflict with black) + "W505", + # W605 invalid escape sequence 'x' (conflict with latex within docs) + "W605", +] + +[tool.ruff.lint] +ignore = [ + "F722" # Syntax error in annotation - ignored because this doesn't play well with jaxtyping +] \ No newline at end of file diff --git a/skyrl-gym/skyrl_gym/__init__.py b/skyrl-gym/skyrl_gym/__init__.py new file mode 100644 index 0000000000..d26bfc8491 --- /dev/null +++ b/skyrl-gym/skyrl_gym/__init__.py @@ -0,0 +1,30 @@ +"""Root `__init__` of the gym module setting the `__all__` of skyrl modules.""" + +from skyrl_gym.core import Env +from skyrl_gym import error + +from skyrl_gym.envs.registration import ( + make, + spec, + register, + registry, + pprint_registry, +) +from skyrl_gym import tools, envs + +# Define __all__ to control what's exposed +__all__ = [ + # core classes + "Env", + # registration + "make", + "spec", + "register", + "registry", + "pprint_registry", + # module folders + "envs", + "tools", + "error", +] +__version__ = "0.0.0" diff --git a/skyrl-gym/skyrl_gym/core.py b/skyrl-gym/skyrl_gym/core.py new file mode 100644 index 0000000000..a8e58d66a2 --- /dev/null +++ b/skyrl-gym/skyrl_gym/core.py @@ -0,0 +1,97 @@ +"""Core API for Environment""" + +from __future__ import annotations + +from typing import Generic, Any, SupportsFloat, TypeVar, TypedDict, Optional +from typing import Tuple, Dict + +ObsType = TypeVar("ObsType") +ActType = TypeVar("ActType") + + +class EnvStepOutput(TypedDict): + observations: ObsType + reward: SupportsFloat + done: bool + metadata: Optional[Dict[str, Any]] = None + + +class Env(Generic[ObsType, ActType]): + """ + The main SkyRL Gym class for implementing Reinforcement Learning Agents environments. + + The main API methods that users of this class need to know are: + + - `step` - Perform actions (e.g. tool calls) in the environment. + Return the observations, the reward for taking that actions, and a boolean value `done`. + + - `init` - Initializes the environment to an initial state, required before calling step. + Returns the first observations for a turn and information, i.e. metrics, debug info. + + - `close` - Closes the environment. + Important when external software is used, i.e. pygame for rendering, databases + """ + + def step(self, action: ActType) -> EnvStepOutput: + """ + Parse and run one step of action in the environment. + + Args: + action (ActType): An action provided to the environment. + For example, in our case, the action can be a [str] response generated by an LLM, + which must be parsed and executed accordingly. + + Returns: + observations (ObsType): The resulting observations after executing the action. + For example, this could involve executing a SQL query derived from the LLM response + and observing {'role': 'user', 'content': 'str(observations)'} output or any error messages from database. + + reward (SupportsFloat): The reward obtained by taking the action. + + done (bool): A boolean value for if the episode has ended, in which case further `step` calls will + return undefined results. + + info (Dict): Contains auxiliary diagnostic information (helpful for debugging, learning, and logging). + This might, for instance, contain: metrics that describe the performance state, variables that are + hidden from observations, or individual reward terms that are combined to produce the total reward. + """ + raise NotImplementedError + + def init(self, *kwargs) -> Tuple[ObsType, Dict[str, Any]]: + """ + Initialize the environment, returning initial observation and optional metadata. + + Returns: + observations (ObsType): Observations of the initial state. This is analogous to the observations returned by `step`. + info (Dict): This dictionary contains auxiliary information complementing ``observation``. It should be analogous to + the ``info`` returned by `step`. + """ + raise NotImplementedError + + def close(self): + """ + After the user has finished using the environment, close contains the code necessary to "clean up" the environment. + + This is critical for closing rendering windows, database or HTTP connections. + Calling ``close`` on an already closed environment has no effect and won't raise an error. + """ + pass + + def __str__(self): + """ + Returns a string of the environment. + + Returns: + A string identifying the environment + """ + return f"Env({type(self).__name__})" + + def __enter__(self): + """Support with-statement for the environment.""" + return self + + def __exit__(self, *args: Any): + """Support with-statement for the environment and closes the environment.""" + self.close() + # propagate exception + return False diff --git a/skyrl-gym/skyrl_gym/envs/README.md b/skyrl-gym/skyrl_gym/envs/README.md new file mode 100644 index 0000000000..5f12a95b99 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/README.md @@ -0,0 +1,16 @@ +# Supported Environments + +## Basic Environments +Search, SQL, Math, + Simple Code (LCB)? + +## Multi-Tool Environment +The model decides which tool to call in each step, showcase Search + Python Code example scripts +- New environment, tool parsing logic different + +## Mix Dataset Environment +Datasets consist of multi-domain, switch between environment initialization +- Reuse environment, agent loop different + +## Replicate results +- Sky-SQL-RL-7B +- Search-r1 3B \ No newline at end of file diff --git a/skyrl-gym/skyrl_gym/envs/__init__.py b/skyrl-gym/skyrl_gym/envs/__init__.py new file mode 100644 index 0000000000..770b65e1e8 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/__init__.py @@ -0,0 +1,38 @@ +"""Registers the internal gym envs.""" + +from skyrl_gym.envs.registration import register + +register( + id="aime", + entry_point="skyrl_gym.envs.aime.env:AIMEEnv", +) + +register( + id="gsm8k", + entry_point="skyrl_gym.envs.gsm8k.env:GSM8kEnv", +) + +register( + id="gsm8k_multi_turn", + entry_point="skyrl_gym.envs.gsm8k.multi_turn_env:GSM8kMultiTurnEnv", +) + +register( + id="text2sql", + entry_point="skyrl_gym.envs.sql.env:SQLEnv", +) + +register( + id="search", + entry_point="skyrl_gym.envs.search.env:SearchEnv", +) + +register( + id="lcb", + entry_point="skyrl_gym.envs.lcb.env:LCBEnv", +) + +register( + id="searchcode", + entry_point="skyrl_gym.envs.searchcode.env:SearchCodeEnv", +) diff --git a/skyrl-gym/skyrl_gym/envs/aime/env.py b/skyrl-gym/skyrl_gym/envs/aime/env.py new file mode 100644 index 0000000000..33a30c19b1 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/aime/env.py @@ -0,0 +1,27 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from skyrl_gym.envs.aime import utils +from typing import Dict, Any +from omegaconf import DictConfig + + +class AIMEEnv(BaseTextEnv): + """ + Environment for Math execution tasks. + """ + + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + + assert "reward_model" in extras, "reward_model field is required" + assert "ground_truth" in extras["reward_model"], "ground_truth is required in reward_model field" + self.ground_truth = extras["reward_model"]["ground_truth"] + + def step(self, action: str) -> BaseTextEnvStepOutput: + done = True # always done after one step + + score_info = utils.compute_score(action, self.ground_truth) + reward = score_info["score"] + metadata = {"acc": score_info["acc"], "pred": score_info["pred"]} + + # No observation in aime, and no tool call + return BaseTextEnvStepOutput(observations=[], reward=reward, done=done, metadata=metadata) diff --git a/skyrl-gym/skyrl_gym/envs/aime/utils.py b/skyrl-gym/skyrl_gym/envs/aime/utils.py new file mode 100644 index 0000000000..29225ec50f --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/aime/utils.py @@ -0,0 +1,277 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/hendrycks_math/utils.py +# https://github.com/volcengine/verl/blob/1a62568f801ba35ac1f5387e27232a2df7eac488/verl/utils/reward_score/math_dapo.py + +import re +from typing import Optional, Dict, Any + + +def last_boxed_only_string(string: str) -> Optional[str]: + """Extract the last LaTeX boxed expression from a string. + + Args: + string: Input string containing LaTeX code + + Returns: + The last boxed expression or None if not found + """ + idx = string.rfind("\\boxed{") + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + return string[idx : right_brace_idx + 1] if right_brace_idx is not None else None + + +def remove_boxed(s: str) -> str: + """Remove the LaTeX boxed command from a string. + + Args: + s: String with format "\\boxed{content}" + + Returns: + The content inside the boxed command + """ + left = "\\boxed{" + assert s[: len(left)] == left, f"box error: {s}" + assert s[-1] == "}", f"box error: {s}" + return s[len(left) : -1] + + +# Constants for normalization +SUBSTITUTIONS = [ + ("an ", ""), + ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), +] + +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """Normalize a final answer to a quantitative reasoning question. + + Args: + final_answer: The answer string to normalize + + Returns: + Normalized answer string + """ + final_answer = final_answer.split("=")[-1] + + # Apply substitutions and removals + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + + # Extract and normalize LaTeX math + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + + # Normalize numbers + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + + return final_answer.strip() + + +def is_correct_minerva( + solution_str: str, gt: str, gt_need_extract: bool = False, answer_pattern: str = r"(?i)Answer\s*:\s*([^\n<]+)" +) -> tuple[bool, str]: + """Check if the solution is correct according to Minerva criteria. + + Args: + solution_str: The solution string to check + gt: The ground truth answer + gt_need_extract: Whether the ground truth needs extraction + answer_pattern: Regex pattern to extract the answer + + Returns: + Tuple of (is_correct, normalized_prediction) + + NOTE(shu): parsing the answer before the last character "<" + Later refactor this into the tokenizer + """ + # Extract answer from solution + match = re.findall(answer_pattern, solution_str) + extracted_answer = match[-1] if match else "[INVALID]" + pred = normalize_final_answer(extracted_answer) + + # Process ground truth + if gt_need_extract: + gt = normalize_final_answer(remove_boxed(last_boxed_only_string(gt))) + else: + gt = normalize_final_answer(gt) + + return (pred == gt), pred + + +def is_correct_strict_box( + pred: str, gt: str, pause_tokens_index: Optional[list[int]] = None +) -> tuple[int, Optional[str]]: + """Check if the prediction is correct using strict boxed answer criteria. + + Args: + pred: The prediction string + gt: The ground truth answer + pause_tokens_index: Indices of pause tokens + + Returns: + Tuple of (score, extracted_prediction) + """ + # Extract the relevant part of the prediction + if pause_tokens_index is not None: + assert len(pause_tokens_index) == 4 + pred = pred[pause_tokens_index[-1] - 100 :] + else: + pred = pred[-100:] + + # Extract and check the boxed answer + boxed_pred = last_boxed_only_string(pred) + extracted_pred = remove_boxed(boxed_pred) if boxed_pred is not None else None + + return 1 if (extracted_pred == gt) else -1, extracted_pred + + +def verify( + solution_str: str, answer: str, strict_box_verify: bool = False, pause_tokens_index: Optional[list[int]] = None +) -> bool: + """Verify if the solution is correct. + + Args: + solution_str: The solution string to verify + answer: The ground truth answer + strict_box_verify: Whether to use strict box verification + pause_tokens_index: Indices of pause tokens + + Returns: + True if the solution is correct, False otherwise + """ + if strict_box_verify: + correct, pred = is_correct_strict_box(solution_str, answer, pause_tokens_index) + return correct == 1, pred + + correct, pred = is_correct_minerva(solution_str, answer) + return correct, pred + + +def compute_score( + solution_str: str, + ground_truth: str, + strict_box_verify: bool = False, + pause_tokens_index: Optional[list[int]] = None, +) -> Dict[str, Any]: + """Compute the reward score for a solution. + + Args: + solution_str: The solution string + ground_truth: The ground truth answer + strict_box_verify: Whether to use strict box verification + pause_tokens_index: Indices of pause tokens + + Returns: + Reward score (1.0 for correct, -1.0 for incorrect) + and other information + """ + # Limit solution length for efficiency + solution_str = solution_str[-300:] # The longest answer in MATH-500 has 159 characters + + # Verify the solution + correct, pred = verify(solution_str, ground_truth, strict_box_verify, pause_tokens_index) + + reward = 1.0 if correct else -1.0 + acc = correct + + return { + "score": reward, + "acc": acc, + "pred": pred, + } diff --git a/skyrl-gym/skyrl_gym/envs/base_text_env.py b/skyrl-gym/skyrl_gym/envs/base_text_env.py new file mode 100644 index 0000000000..b2376e20ba --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/base_text_env.py @@ -0,0 +1,100 @@ +from typing import Any, Dict, List, Optional, TypedDict +from skyrl_gym import Env +from typing import Tuple + +MessageType = Dict[str, str] +ConversationType = List[MessageType] + + +class BaseTextEnvStepOutput(TypedDict): + observations: ConversationType # OpenAI API Messages Format + reward: float + done: bool + metadata: Dict[str, Any] + postprocessed_action: Optional[str] = None + + +class BaseTextEnv(Env[ConversationType, str]): + """ + Base environment class for all text-in / text-out environments. + Supports tool-calling and multi-turn trajectories. + + Exposes only `step`, `init` and `close`. + + Input Types: + - ObsType: ConversationType (tool output, LLM input) + - ActType: str (LLM output) + """ + + def __init__(self): + super().__init__() + + # Metadata + self.turns = 0 + self.max_turns = 1 + + # Tool groups + self.tool_groups = [] + self.tool_to_toolgroup = {} + + def init_tool_groups(self, tool_groups: List = []) -> None: + """ + Initialize the tool groups for the environment. + """ + # Find ToolGroup for a given tool + self.tool_groups = tool_groups + self.tool_to_toolgroup = {} + for tool_group in self.tool_groups: + self.tool_to_toolgroup.update(tool_group.get_tool_to_group_mapping()) + + def _execute_tool(self, tool_group_name: str, tool_name: str, tool_input: Any) -> str: + """ + Find the right ToolGroup and Tool and execute it. + """ + for group in self.tool_groups: + if group.name == tool_group_name: + return group.execute_tool(tool_name, *tool_input) # tool_input must be tuple or list + + raise ValueError(f"ToolGroup '{tool_group_name}' not found.") + + def step(self, action: str) -> BaseTextEnvStepOutput: + """ + Runs one environment step. + + Return: + - observations: [{"role": "user", "content": observation}] + - reward: float + - done: bool + - postprocessed_action: Optional[str] + - metadata: Dict[str, Any] any metadata + """ + pass + + def init(self, prompt: ConversationType) -> Tuple[ConversationType, Dict[str, Any]]: + """ + Return the first prompt to be given to the model and optional metadata. + """ + return prompt, {} + + def close(self): + """ + Closes the environment, override if needed by subclasses. + """ + pass + + def get_metrics(self) -> Dict[str, Any]: + """ + Return environment-specific metrics for the episode. + Default is empty dict (no metrics). + """ + return {} + + @staticmethod + def aggregate_metrics(metrics: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + Static method to aggregate metrics across many episodes of this env class. + Default behavior: average the numerics, drop the non-numerics. + """ + from skyrl_gym.metrics import default_aggregate_metrics + + return default_aggregate_metrics(metrics) diff --git a/skyrl-gym/skyrl_gym/envs/gsm8k/env.py b/skyrl-gym/skyrl_gym/envs/gsm8k/env.py new file mode 100644 index 0000000000..f7f0abdc25 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/gsm8k/env.py @@ -0,0 +1,26 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from skyrl_gym.envs.gsm8k import utils +from typing import Dict, Any +from omegaconf import DictConfig + + +class GSM8kEnv(BaseTextEnv): + """ + Environment for Math execution tasks. + """ + + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + + assert "reward_spec" in extras, "reward_spec field is required" + assert "ground_truth" in extras["reward_spec"], "ground_truth is required in reward_spec field" + self.ground_truth = extras["reward_spec"]["ground_truth"] + + def _get_reward(self, action: str) -> float: + return utils.compute_score(action, self.ground_truth) + + def step(self, action: str) -> BaseTextEnvStepOutput: + done = True # always done after one step + reward = self._get_reward(action) + # No observation in gsm8k, and no tool call + return BaseTextEnvStepOutput(observations=[], reward=reward, done=done, metadata={}) diff --git a/skyrl-gym/skyrl_gym/envs/gsm8k/multi_turn_env.py b/skyrl-gym/skyrl_gym/envs/gsm8k/multi_turn_env.py new file mode 100644 index 0000000000..05f5086b18 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/gsm8k/multi_turn_env.py @@ -0,0 +1,80 @@ +from typing import Dict, Any +from omegaconf import DictConfig + +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from skyrl_gym.envs.gsm8k import utils + + +class GSM8kMultiTurnEnv(BaseTextEnv): + """ + Multi-turn GSM8k environment with turn-level rewards. + """ + + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + reward_spec = extras.get("reward_spec", {}) + assert "ground_truth" in reward_spec, "reward_spec.ground_truth is required" + + self.ground_truth: str = reward_spec["ground_truth"] + self.max_turns = 5 + if "max_turns" in extras: + self.max_turns = int(extras["max_turns"]) + elif "max_turns" in extras["extra_info"]: + self.max_turns = int(extras["extra_info"]["max_turns"]) + + format_score = 0.2 + self.format_score_per_turn: float = format_score / self.max_turns + + def init(self, prompt): + # No special pre-processing; return prompt and empty metadata + return prompt, {} + + def _make_observation(self) -> list[dict]: + remaining = self.max_turns - self.turns + if remaining <= 0: + return [] + + if remaining > 1: + msg = ( + "Please provide your step-by-step reasoning, " + "and also include a tentative numeric answer at the end in the exact format: '#### ANSWER'." + ) + else: + msg = "Now provide only the final numeric answer in the exact format: '#### ANSWER'." + + return [{"role": "user", "content": msg}] + + def step(self, action: str) -> BaseTextEnvStepOutput: + self.turns += 1 + + # Per-turn reward: 1.0 if correct, 0.2/max_turns if well-formatted but incorrect, 0.0 otherwise. + reward = utils.compute_score( + solution_str=action, + ground_truth=self.ground_truth, + method="strict", + format_score=self.format_score_per_turn, + score=1.0, + ) + done = self.turns >= self.max_turns or reward == 1.0 + + observations = [] if done else self._make_observation() + + return BaseTextEnvStepOutput( + observations=observations, + reward=reward, + done=done, + metadata={}, + ) + + def get_metrics(self) -> Dict[str, Any]: + return { + "steps": self.turns, + } + + @staticmethod + def aggregate_metrics(metrics: list[Dict[str, Any]]) -> Dict[str, Any]: + if not metrics: + return {} + n = len(metrics) + avg_steps = sum(float(m.get("steps", 0)) for m in metrics) / n + return {"avg_steps": avg_steps} diff --git a/skyrl-gym/skyrl_gym/envs/gsm8k/utils.py b/skyrl-gym/skyrl_gym/envs/gsm8k/utils.py new file mode 100644 index 0000000000..f5d4c1585f --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/gsm8k/utils.py @@ -0,0 +1,63 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + + +def extract_solution(solution_str, method="strict"): + assert method in ["strict", "flexible"] + + if method == "strict": + # this also tests the formatting of the model + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + if solution is None: + final_answer = None + else: + final_answer = solution.group(0) + final_answer = final_answer.split("#### ")[1].replace(",", "").replace("$", "") + elif method == "flexible": + answer = re.findall("(\\-?[0-9\\.\\,]+)", solution_str) + final_answer = None + if len(answer) == 0: + # no reward is there is no answer + pass + else: + invalid_str = ["", "."] + # find the last number that is not '.' + for final_answer in reversed(answer): + if final_answer not in invalid_str: + break + return final_answer + + +def compute_score(solution_str, ground_truth, method="strict", format_score=0.0, score=1.0): + """The scoring function for GSM8k. + + Reference: Trung, Luong, et al. "Reft: Reasoning with reinforced fine-tuning." Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). 2024. + + Args: + solution_str: the solution text + ground_truth: the ground truth + method: the method to extract the solution, choices are 'strict' and 'flexible' + format_score: the score for the format + score: the score for the correct answer + """ + answer = extract_solution(solution_str=solution_str, method=method) + if answer is None: + return 0 + else: + if answer == ground_truth: + return score + else: + return format_score diff --git a/skyrl-gym/skyrl_gym/envs/lcb/env.py b/skyrl-gym/skyrl_gym/envs/lcb/env.py new file mode 100644 index 0000000000..19955e46d5 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/lcb/env.py @@ -0,0 +1,32 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from typing import Any, Dict +from skyrl_gym.envs.lcb.livecodebench import compute_score +import json +from omegaconf import DictConfig + + +class LCBEnv(BaseTextEnv): + """ + Environment for LiveCodeBench execution environment. + """ + + def __init__( + self, + env_config: DictConfig, + extras: Dict[str, Any] = {}, + ): + super().__init__() + + assert "reward_spec" in extras, "reward_spec field is required" + assert "ground_truth" in extras["reward_spec"], "ground_truth is required in reward_spec field" + self.tests = json.loads(extras["reward_spec"]["ground_truth"]) + + def _get_reward(self, action: str) -> float: + return compute_score(action, self.tests) + + def step(self, action: str) -> BaseTextEnvStepOutput: + done = True + parsed_code, reward = compute_score(action, self.tests) + + # RL on LCB w/ single-turn + return BaseTextEnvStepOutput(observations=[], reward=reward, done=done, metadata={"parsed_code": parsed_code}) diff --git a/skyrl-gym/skyrl_gym/envs/lcb/livecodebench.py b/skyrl-gym/skyrl_gym/envs/lcb/livecodebench.py new file mode 100644 index 0000000000..2246ad6839 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/lcb/livecodebench.py @@ -0,0 +1,664 @@ +# Taken from https://github.com/LiveCodeBench/LiveCodeBench/blob/998c52d394b836f15fff3b9a29866191108ff81b/lcb_runner/evaluation/testing_util.py +# Adapt from rllm: https://github.com/agentica-project/rllm/blob/main/rllm/rewards/reward_types.py#L24 +import ast +import json +import sys +import faulthandler +import platform +import re +from datetime import datetime +import signal + +from io import StringIO +from unittest.mock import patch, mock_open +from types import ModuleType +from enum import Enum +from decimal import Decimal +import time + +import multiprocessing + + +BASE_IMPORTS = """from itertools import accumulate, chain, combinations, count, permutations, product, groupby, islice, repeat +from copy import deepcopy +from string import ascii_lowercase, ascii_uppercase +from math import floor, log2, log10, sqrt, comb, gcd, ceil, inf, isqrt, factorial, atan2, pi +from collections import defaultdict, deque, Counter +from bisect import bisect, bisect_left, bisect_right, insort +from heapq import heappush, heappop, heapify, merge, nlargest, nsmallest, heapreplace +from functools import reduce, cache, lru_cache, cmp_to_key, reduce +from random import randrange, shuffle +from operator import itemgetter, sub, xor, or_ +from re import search as re_search # Assuming 're' refers to a regex search +from os.path import commonprefix +from typing import List, Tuple, Dict, Set, Optional, Union, Any, Callable, Iterable, Iterator, Generator, Deque +import copy +import string +import math +import collections +import bisect +import heapq +import functools +import random +import itertools +import operator +import re +import datetime +from time import time +import numpy as np +import pandas as pd +from math import log, prod # 'log' and 'prod' are functions in the math module +from collections import deque, defaultdict, Counter, OrderedDict +from itertools import accumulate, permutations, combinations, product, groupby, islice, chain, repeat, zip_longest, cycle, pairwise +from functools import lru_cache, reduce, partial +from operator import iand +import sys +import io, os +""" + +import_string = BASE_IMPORTS + +# "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(50000)\n" + + +def truncatefn(s, length=300): + if isinstance(s, str): + pass + else: + s = str(s) + if len(s) <= length: + return s + + return s[: length // 2] + "...(truncated) ..." + s[-length // 2 :] + + +class CODE_TYPE(Enum): + call_based = 0 + standard_input = 1 + + +# stuff for setting up signal timer +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + print("timeout occured: alarm went off") + raise TimeoutException + + +# used to capture stdout as a list +# from https://stackoverflow.com/a/16571630/6416660 +# alternative use redirect_stdout() from contextlib +class Capturing(list): + def __enter__(self): + self._stdout = sys.stdout + sys.stdout = self._stringio = StringIO() + # Make closing the StringIO a no-op + self._stringio.close = lambda x: 1 + return self + + def __exit__(self, *args): + self.append(self._stringio.getvalue()) + del self._stringio # free up some memory + sys.stdout = self._stdout + + +def clean_if_name(code: str) -> str: + try: + astree = ast.parse(code) + last_block = astree.body[-1] + if isinstance(last_block, ast.If): + condition = last_block.test + if ast.unparse(condition).strip() == "__name__ == '__main__'": + code = ast.unparse(astree.body[:-1]) + "\n" + ast.unparse(last_block.body) # type: ignore + except Exception: + pass + + return code + + +def make_function(code: str) -> str: + try: + import_stmts = [] + all_other_stmts = [] + astree = ast.parse(code) + for stmt in astree.body: + if isinstance(stmt, (ast.Import, ast.ImportFrom)): + import_stmts.append(stmt) + else: + all_other_stmts.append(stmt) + + function_ast = ast.FunctionDef( + name="wrapped_function", + args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), + body=all_other_stmts, + decorator_list=[], + lineno=-1, + ) + main_code = ( + import_string + + "\n" + + ast.unparse(import_stmts) # type: ignore + + "\n" + + ast.unparse(function_ast) # type: ignore + ) + return main_code + except Exception: + return code + + +def call_method(method, inputs): + + if isinstance(inputs, list): + inputs = "\n".join(inputs) + + inputs_line_iterator = iter(inputs.split("\n")) + + # sys.setrecursionlimit(10000) + + # @patch('builtins.input', side_effect=inputs.split("\n")) + @patch("builtins.open", mock_open(read_data=inputs)) + @patch("sys.stdin", StringIO(inputs)) + @patch("sys.stdin.readline", lambda *args: next(inputs_line_iterator)) + @patch("sys.stdin.readlines", lambda *args: inputs.split("\n")) + @patch("sys.stdin.read", lambda *args: inputs) + # @patch('sys.stdout.write', print) + def _inner_call_method(_method): + try: + return _method() + except SystemExit: + pass + finally: + pass + + return _inner_call_method(method) + + +def get_function(compiled_sol, fn_name: str): # type: ignore + try: + assert hasattr(compiled_sol, fn_name) + return getattr(compiled_sol, fn_name) + except Exception: + return + + +def compile_code(code: str, timeout: int): + signal.alarm(timeout) + try: + tmp_sol = ModuleType("tmp_sol", "") + exec(code, tmp_sol.__dict__) + if "class Solution" in code: + # leetcode wraps solutions in `Solution` + # this is a hack to check if it is leetcode solution or not + # currently livecodebench only supports LeetCode but + # else condition allows future extensibility to other platforms + compiled_sol = tmp_sol.Solution() + else: + # do nothing in the other case since function is accesible + compiled_sol = tmp_sol + + assert compiled_sol is not None + finally: + signal.alarm(0) + + return compiled_sol + + +def convert_line_to_decimals(line: str) -> tuple[bool, list[Decimal]]: + try: + decimal_line = [Decimal(elem) for elem in line.split()] + except Exception: + return False, [] + return True, decimal_line + + +def get_stripped_lines(val: str): + ## you don't want empty lines to add empty list after splitlines! + val = val.strip() + + return [val_line.strip() for val_line in val.split("\n")] + + +def grade_call_based(code: str, all_inputs: list, all_outputs: list, fn_name: str, timeout: int): + # call-based clean up logic + # need to wrap in try-catch logic after to catch the correct errors, but for now this is fine. + code = import_string + "\n\n" + code + compiled_sol = compile_code(code, timeout) + + if compiled_sol is None: + return + + method = get_function(compiled_sol, fn_name) + + if method is None: + return + + all_inputs = [[json.loads(line) for line in inputs.split("\n")] for inputs in all_inputs] + + all_outputs = [json.loads(output) for output in all_outputs] + + total_execution = 0 + all_results = [] + for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)): + signal.alarm(timeout) + faulthandler.enable() + try: + # can lock here so time is useful + start = time.time() + prediction = method(*gt_inp) + total_execution += time.time() - start + signal.alarm(0) + + # don't penalize model if it produces tuples instead of lists + # ground truth sequences are not tuples + if isinstance(prediction, tuple): + prediction = list(prediction) + + tmp_result = prediction == gt_out + + # handle floating point comparisons + + all_results.append(tmp_result) + + if not tmp_result: + return all_results, { + "output": truncatefn(prediction), + "inputs": truncatefn(gt_inp), + "expected": truncatefn(gt_out), + "error_code": -2, + "error_message": "Wrong Answer", + } + except Exception as e: + signal.alarm(0) + if "timeoutexception" in repr(e).lower(): + all_results.append(-3) + return all_results, { + "error": repr(e), + "error_code": -3, + "error_message": "Time Limit Exceeded", + "inputs": truncatefn(gt_inp), + "expected": truncatefn(gt_out), + } + else: + all_results.append(-4) + return all_results, { + "error": repr(e), + "error_code": -4, + "error_message": "Runtime Error", + "inputs": truncatefn(gt_inp), + "expected": truncatefn(gt_out), + } + + finally: + signal.alarm(0) + faulthandler.disable() + + return all_results, {"execution time": total_execution} + + +def grade_stdio( + code: str, + all_inputs: list, + all_outputs: list, + timeout: int, +): + ## runtime doesn't interact well with __name__ == '__main__' + code = clean_if_name(code) + + ## we wrap the given code inside another function + code = make_function(code) + + compiled_sol = compile_code(code, timeout) + + if compiled_sol is None: + return + + method = get_function(compiled_sol, "wrapped_function") + + if method is None: + return + + all_results = [] + total_execution_time = 0 + for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)): + signal.alarm(timeout) + faulthandler.enable() + + signal.alarm(timeout) + with Capturing() as captured_output: + try: + start = time.time() + call_method(method, gt_inp) + total_execution_time += time.time() - start + # reset the alarm + signal.alarm(0) + except Exception as e: + signal.alarm(0) + if "timeoutexception" in repr(e).lower(): + all_results.append(-3) + return all_results, { + "error": repr(e), + "error_code": -3, + "error_message": "Time Limit Exceeded", + "inputs": truncatefn(gt_inp), + "expected": truncatefn(gt_out), + } + else: + all_results.append(-4) + return all_results, { + "error": repr(e), + "error_code": -4, + "error_message": "Runtime Error", + "inputs": truncatefn(gt_inp), + "expected": truncatefn(gt_out), + } + + finally: + signal.alarm(0) + faulthandler.disable() + + prediction = captured_output[0] + + stripped_prediction_lines = get_stripped_lines(prediction) + stripped_gt_out_lines = get_stripped_lines(gt_out) + + ## WA happens in multiple circumstances + ## so cache the return to make it clean! + WA_send_args = { + "output": truncatefn(prediction), + "inputs": truncatefn(gt_inp), + "expected": truncatefn(gt_out), + "error_code": -2, + } + + if len(stripped_prediction_lines) != len(stripped_gt_out_lines): + all_results.append(-2) + WA_send_args["error_message"] = "Wrong answer: mismatched output length" + return all_results, WA_send_args + + for output_line_idx, ( + stripped_prediction_line, + stripped_gt_out_line, + ) in enumerate(zip(stripped_prediction_lines, stripped_gt_out_lines)): + WA_send_args["error_message"] = ( + f"Wrong answer at {output_line_idx=}: {truncatefn(stripped_prediction_line)} != {truncatefn(stripped_gt_out_line)}" + ) + + ## CASE 1: exact match + if stripped_prediction_line == stripped_gt_out_line: + continue + + ## CASE 2: element-wise comparision + ## if there are floating elements + ## use `decimal` library for good floating point comparision + ## otherwise gotcha: np.isclose(50000000000000000, 50000000000000001) = True + ## note that we should always be able to convert to decimals + + success, decimal_prediction_line = convert_line_to_decimals(stripped_prediction_line) + if not success: + all_results.append(-2) + return all_results, WA_send_args + success, decimal_gtout_line = convert_line_to_decimals(stripped_gt_out_line) + if not success: + all_results.append(-2) + return all_results, WA_send_args + + if decimal_prediction_line == decimal_gtout_line: + continue + + all_results.append(-2) + return all_results, WA_send_args + all_results.append(True) + + return all_results, {"execution time": total_execution_time} + + +def run_test(sample, test=None, debug=False, timeout=6): + """ + if test(generated_code) is not None it'll try to run the code. + otherwise it'll just return an input and output pair. + """ + signal.signal(signal.SIGALRM, timeout_handler) + + # Disable functionalities that can make destructive changes to the test. + # max memory is set to 4GB + reliability_guard() + + if debug: + print(f"start = {datetime.now().time()}") + + try: + in_outs = json.loads(sample["input_output"]) + except ValueError as e: + raise e + in_outs = None + + if in_outs: + if in_outs.get("fn_name") is None: + which_type = CODE_TYPE.standard_input # Standard input + method_name = None + else: + which_type = CODE_TYPE.call_based # Call-based + method_name = in_outs["fn_name"] + + if debug: + print(f"loaded input_output = {datetime.now().time()}") + + if test is None: + assert False, "should not happen: test code is none" + return in_outs, {"error": "no test code provided"} + elif test is not None: + results = [] + # sol = import_string + if debug: + print(f"loading test code = {datetime.now().time()}") + + if which_type == CODE_TYPE.call_based: + signal.alarm(timeout) + try: + results, metadata = grade_call_based( + code=test, + all_inputs=in_outs["inputs"], + all_outputs=in_outs["outputs"], + fn_name=method_name, + timeout=timeout, + ) + return results, metadata + except Exception as e: + return [-4], { + "error_code": -4, + "error_message": f"Error during testing: {e}", + } + finally: + signal.alarm(0) + elif which_type == CODE_TYPE.standard_input: + # sol + # if code has if __name__ == "__main__": then remove it + + signal.alarm(timeout) + try: + results, metadata = grade_stdio( + code=test, + all_inputs=in_outs["inputs"], + all_outputs=in_outs["outputs"], + timeout=timeout, + ) + return results, metadata + except Exception as e: + return [-4], { + "error_code": -4, + "error_message": f"Error during testing: {e}", + } + finally: + signal.alarm(0) + + +def reliability_guard(maximum_memory_bytes=None): + """ + This disables various destructive functions and prevents the generated code + from interfering with the test (e.g. fork bomb, killing other processes, + removing filesystem files, etc.) + WARNING + This function is NOT a security sandbox. Untrusted code, including, model- + generated code, should not be blindly executed outside of one. See the + Codex paper for more information about OpenAI's code sandbox, and proceed + with caution. + """ + + if maximum_memory_bytes is not None: + import resource + + resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) + if not platform.uname().system == "Darwin": + resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) + + faulthandler.disable() + + import builtins + + # builtins.exit = None + builtins.quit = None + + import os + + os.environ["OMP_NUM_THREADS"] = "1" + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.fchdir = None + os.lchflags = None + os.lchmod = None + os.lchown = None + os.getcwd = None + os.chdir = None + + import shutil + + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + import subprocess + + subprocess.Popen = None # type: ignore + + if isinstance(__builtins__, dict): + __builtins__["help"] = None + elif hasattr(__builtins__, "help"): + setattr(__builtins__, "help", None) + + import sys + + sys.modules["ipdb"] = None + sys.modules["joblib"] = None + sys.modules["resource"] = None + sys.modules["psutil"] = None + sys.modules["tkinter"] = None + + +def postprocess_lcb_sample(sample): + sample_inputs = [sample["input"] for sample in sample] + sample_outputs = [sample["output"] for sample in sample] + + sample_dict = { + "inputs": sample_inputs, + "outputs": sample_outputs, + } + + if sample[0].get("testtype") == "functional": + metadata = sample[0].get("metadata", {}) + fn_name = metadata.get("func_name", None) + assert ( + fn_name is not None + ), f"Function name is not found, check if your LCB data is preprocessed correctly: {metadata}" + # Fill in the blank + sample_dict["fn_name"] = fn_name + + sample = { + "input_output": json.dumps(sample_dict), + } + return sample + + +def lcb_check_correctness(sample, generation, timeout=6, debug=False): + """Check correctness of code generation with a global timeout. + The global timeout is to catch some extreme/rare cases not handled by the timeouts + inside `run_test`""" + assert len(sample) >= 1, "Sample must contain at least one test case" + sample = postprocess_lcb_sample(sample) + + manager = multiprocessing.Manager() + result = manager.list() + metadata_list = manager.list() + + def _temp_run(sample, generation, debug, result, metadata_list, timeout): + res, metadata = run_test(sample, test=generation, debug=debug, timeout=timeout) + result.append(res) + metadata_list.append(metadata) + + p = multiprocessing.Process( + target=_temp_run, + args=(sample, generation, debug, result, metadata_list, timeout), + ) + p.start() + p.join(timeout=(timeout + 1) * len(json.loads(sample["input_output"])["inputs"]) + 5) + if p.is_alive(): + p.kill() + if not result: + in_outs = json.loads(sample["input_output"]) + # consider that all tests failed + result = [[-1 for i in range(len(in_outs["inputs"]))]] + if debug: + print("global timeout") + if not result: + return False + # print(result[0], metadata_list) + # Check if all elements in result[0] are True + return all(x is True for x in result[0]) + + +def extract_code_from_model(model_response: str): + """ + Extracts the code from a Markdown-style code block in an LLM output. + + Parameters: + model_response (str): The text output from the LLM. + + Returns: + str: The extracted code, or an empty string if no code block is found. + """ + code_blocks = re.findall(r"```(?:\w+)?\n(.*?)```", model_response, re.DOTALL) + if not code_blocks: + return None + return code_blocks[-1].strip() + + +def compute_score(model_response, tests): + output_code = extract_code_from_model(model_response) + if output_code is None: + return output_code, 0.0 + is_correct = lcb_check_correctness(tests, output_code, debug=False) + + reward = 1.0 if is_correct else 0.0 + return output_code, reward diff --git a/skyrl-gym/skyrl_gym/envs/registration.py b/skyrl-gym/skyrl_gym/envs/registration.py new file mode 100644 index 0000000000..9cc6f8450d --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/registration.py @@ -0,0 +1,345 @@ +"""Functions for registering environments within skyrl_gym using public functions ``make``, ``register`` and ``spec``.""" + +from __future__ import annotations + +import copy +import dataclasses +import importlib +import json +from dataclasses import dataclass, field +from skyrl_gym import Env, error + +from typing import Protocol, Dict, Any + +__all__ = [ + "registry", + "EnvSpec", + # Functions + "make", + "spec", + "register", + "pprint_registry", +] + + +class EnvCreator(Protocol): + """Function type expected for an environment.""" + + def __call__(self, **kwargs: Any) -> Env: ... + + +@dataclass +class EnvSpec: + """A specification for creating environments with `skyrl_gym.make`. + + * **id**: The string used to create the environment with `skyrl_gym.make` + + * **entry_point**: A string for the environment location, ``(import path):(environment name)`` or a function that creates the environment. + NOTE[shu]: example is like a database path? or local RAG environment? + + * **kwargs**: Additional keyword arguments passed to the environment during initialisation + + """ + + id: str + entry_point: EnvCreator | str | None = field(default=None) + + # Environment arguments + kwargs: Dict[str, Any] = field(default_factory=dict) + + # post-init attributes + name: str = field(init=False) + + def __post_init__(self): + """Calls after the spec is created to extract name from the environment id.""" + self.name = self.id + + def make(self, **kwargs: Any) -> Env: + """Calls ``make`` using the environment spec and any keyword arguments.""" + return make(self, **kwargs) + + def to_json(self) -> str: + """Converts the environment spec into a json compatible string. + + Returns: + A jsonified string for the environment spec + """ + env_spec_dict = dataclasses.asdict(self) + env_spec_dict.pop("name") + + # To check that the environment spec can be transformed to a json compatible type + self._check_can_jsonify(env_spec_dict) + + return json.dumps(env_spec_dict) + + @staticmethod + def _check_can_jsonify(env_spec: Dict[str, Any]): + """Warns the user about serialisation failing if the spec contains a callable. + + Args: + env_spec: An environment. + + Returns: The specification with lambda functions converted to strings. + + """ + spec_name = env_spec["name"] if "name" in env_spec else env_spec["id"] + + for key, value in env_spec.items(): + if callable(value): + raise ValueError( + f"Callable found in {spec_name} for {key} attribute with value={value}. Currently, skyrl_gym does not support serialising callables." + ) + + @staticmethod + def from_json(json_env_spec: str) -> EnvSpec: + """Converts a JSON string into a specification stack. + + Args: + json_env_spec: A JSON string representing the env specification. + + Returns: + An environment spec + """ + parsed_env_spec = json.loads(json_env_spec) + + try: + env_spec = EnvSpec(**parsed_env_spec) + except Exception as e: + raise ValueError(f"An issue occurred when trying to make {parsed_env_spec} an EnvSpec") from e + + return env_spec + + def pprint( + self, + disable_print: bool = False, + include_entry_points: bool = False, + ) -> str | None: + """Pretty prints the environment spec. + + Args: + disable_print: If to disable print and return the output + include_entry_points: If to include the entry_points in the output + print_all: If to print all information, including variables with default values + + Returns: + If ``disable_print is True`` a string otherwise ``None`` + """ + output = f"id={self.id}" + if include_entry_points: + output += f"\nentry_point={self.entry_point}" + + if disable_print: + return output + else: + print(output) + + +# Global registry of environments. Meant to be accessed through `register` and `make` +registry: Dict[str, EnvSpec] = {} + + +def _find_spec(env_id: str) -> EnvSpec: + # For string id's, load the environment spec from the registry then make the environment spec + assert isinstance(env_id, str) + + # load the env spec from the registry + env_name = env_id + env_spec = registry.get(env_name) + + if env_spec is None: + raise error.Error( + f"No registered env with id: {env_name}. Did you register it, or import the package that registers it? Use `skyrl_gym.pprint_registry()` to see all of the registered environments." + ) + + return env_spec + + +def load_env_creator(name: str) -> EnvCreator: + """Loads an environment with name of style ``"(import path):(environment name)"`` and returns the environment creation function, normally the environment class type. + + Args: + name: The environment name + + Returns: + The environment constructor for the given environment name. + """ + mod_name, attr_name = name.split(":") + mod = importlib.import_module(mod_name) + fn = getattr(mod, attr_name) + return fn + + +def _check_spec_register(testing_spec: EnvSpec): + """NOTE[shu]: Checks that no environment with the same name already exists in the registry.""" + for env_spec in registry.values(): + if env_spec.name == testing_spec.name: + raise error.RegistrationError( + f"An environment with name `{testing_spec.name}` is already registered (id=`{env_spec.id}`). " + "Environment names must be unique." + ) + + +def register( + id: str, + entry_point: EnvCreator | str | None = None, + kwargs: Dict[str, Any] | None = None, +): + """ + Registers an environment in skyrl_gym with an ``id`` to use with `skyrl_gym.make` with the ``entry_point`` + being a string or callable for creating the environment. + + The ``id`` parameter corresponds to the name of the environment. + + It takes arbitrary keyword arguments, which are passed to the :class:`EnvSpec` ``kwargs`` parameter. + + Args: + id: The environment id + entry_point: The entry point for creating the environment + kwargs: arbitrary keyword arguments which are passed to the environment constructor on initialisation. + """ + assert entry_point is not None, "`entry_point` must be provided" + global registry + + if kwargs is None: + kwargs = dict() + + new_spec = EnvSpec( + id=id, + entry_point=entry_point, + kwargs=kwargs, + ) + _check_spec_register(new_spec) + registry[new_spec.id] = new_spec + + +def make( + id: str | EnvSpec, + **kwargs: Any, +) -> Env: + """Creates an environment previously registered with `skyrl_gym.register` or a :class:`EnvSpec`. + + To find all available environments use ``skyrl_gym.envs.registry.keys()`` for all valid ids. + + Args: + id: A string for the environment id or a :class:`EnvSpec`. + kwargs: Additional arguments to pass to the environment constructor. + + Returns: + An instance of the environment. + + Raises: + Error: If the ``id`` doesn't exist in the `registry` + """ + if isinstance(id, EnvSpec): + env_spec = id + else: + # For string id's, load the environment spec from the registry then make the environment spec + assert isinstance(id, str) + + # The environment name can include an unloaded module in "module:env_name" style + env_spec = _find_spec(id) + + assert isinstance(env_spec, EnvSpec) + + # Update the env spec kwargs with the `make` kwargs + env_spec_kwargs = copy.deepcopy(env_spec.kwargs) + env_spec_kwargs.update(kwargs) + + # Load the environment creator + if env_spec.entry_point is None: + raise error.Error(f"{env_spec.id} registered but entry_point is not specified") + elif callable(env_spec.entry_point): + env_creator = env_spec.entry_point + else: + # Assume it's a string + env_creator = load_env_creator(env_spec.entry_point) + + try: + env = env_creator(**env_spec_kwargs) + except TypeError as e: + raise type(e)(f"{e} was raised from the environment creator for {env_spec.id} with kwargs ({env_spec_kwargs})") + + if not isinstance(env, Env): + if str(env.__class__.__base__) == "": + raise TypeError( + "Gym is incompatible with skyrl_gym, please update the environment class to `skyrl_gym.Env`. " + ) + else: + raise TypeError(f"The environment must inherit from the skyrl_gym.Env class, actual class: {type(env)}. ") + + # Set the minimal env spec for the environment. + env.spec = EnvSpec( + id=env_spec.id, + entry_point=env_spec.entry_point, + kwargs=env_spec_kwargs, + ) + + return env + + +def spec(env_id: str) -> EnvSpec: + """Retrieve the :class:`EnvSpec` for the environment id from the `registry`. + + Args: + env_id: The environment id with the expected format of ``[(namespace)/]id[-v(version)]`` + + Returns: + The environment spec if it exists + + Raises: + Error: If the environment id doesn't exist + """ + env_spec = registry.get(env_id) + if env_spec is None: + raise error.Error(f"No registered env with id: {env_id}") + else: + assert isinstance( + env_spec, EnvSpec + ), f"Expected the registry for {env_id} to be an `EnvSpec`, actual type is {type(env_spec)}" + return env_spec + + +def pprint_registry( + print_registry: Dict[str, EnvSpec] = registry, + *, + num_cols: int = 3, + disable_print: bool = False, +) -> str | None: + """Pretty prints all environments in the registry without grouping by namespace. + + Args: + print_registry: Environment registry to be printed. By default, uses the global `registry`. + num_cols: Number of columns to arrange environments in. + disable_print: Whether to return a string instead of printing it. + """ + # Get all environment ids + env_ids = sorted(print_registry.keys()) + + if not env_ids: + output = "No environments registered." + if disable_print: + return output + else: + print(output) + return + + # Find the max width for nice column alignment + max_justify = max(len(env_id) for env_id in env_ids) + + # Build the output + output_lines = [] + current_line = "" + + for count, env_id in enumerate(env_ids, 1): + current_line += env_id.ljust(max_justify + 2) + + if count % num_cols == 0 or count == len(env_ids): + output_lines.append(current_line.rstrip()) + current_line = "" + + final_output = "\n".join(output_lines) + + if disable_print: + return final_output + else: + print(final_output) diff --git a/skyrl-gym/skyrl_gym/envs/search/env.py b/skyrl-gym/skyrl_gym/envs/search/env.py new file mode 100644 index 0000000000..32fd752097 --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/search/env.py @@ -0,0 +1,116 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput, ConversationType +from typing import Any +from skyrl_gym.envs.search.utils import compute_score +from skyrl_gym.tools import SearchToolGroup +import re +from typing import Dict, Optional, List +from omegaconf import DictConfig + + +class SearchEnv(BaseTextEnv): + """ + Environment for Search execution tasks. + + Based on Verl + Search-R1 integration + """ + + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + + assert "reward_spec" in extras, "reward_spec field is required" + assert "ground_truth" in extras["reward_spec"], "ground_truth is required in reward_spec field" + self.ground_truth = extras["reward_spec"]["ground_truth"] + self.max_turns = extras["max_turns"] if "max_turns" in extras else 2 + + # Initialize the tools + # name is hardcoded to "SearchToolGroup", with tool name "search" + self.tool_group = SearchToolGroup( + search_url=env_config.search_url, + topk=env_config.topk, + timeout=env_config.timeout, + log_requests=env_config.log_requests, + ) + self.init_tool_groups([self.tool_group]) + + # Chat history + # role (user, assistant), content (tool observation or LLM response) + self.chat_history: ConversationType = [] + + def _parse_action(self, action: str) -> List[Optional[str]]: + match = None + if "" in action and "" in action: + match = re.search(r"(.*?)", action, re.DOTALL) + return [match.group(1)] if match else [None] + + def _get_reward(self, action: str, done: bool) -> float: + if done: + # Concat all chat history into a single string and compute reward + chat_history_str = "".join([item["content"] for item in self.chat_history]) + return compute_score(chat_history_str, self.ground_truth) + else: + # No reward for intermediate steps for Search tasks + return 0 + + def _is_done(self, action: str) -> bool: + if self.turns >= self.max_turns: + return True + return "" in action and "" in action + + def _validate_action(self, action: str): + stop_tags = ["", ""] + for tag in stop_tags: + if tag in action: + assert action.split(tag, 1)[1] == "", ( + f"{tag} detected in the response but it is not the last string generated. " + f"Use {stop_tags} as stop strings in the configuration." + ) + + def _execute_tool(self, tool_group_name: str, tool_name: str, tool_input: Any) -> str: + tool_output = super()._execute_tool(tool_group_name, tool_name, tool_input) + + return "\n" + tool_output + "\n" + + def step(self, action: str) -> BaseTextEnvStepOutput: + self.turns += 1 + self._validate_action(action) + self.chat_history.append({"role": "assistant", "content": action}) + + error = None + done = self._is_done(action) + reward = self._get_reward(action, done) + + if done: + return BaseTextEnvStepOutput(observations=[], reward=reward, done=done, metadata={}) + + try: + query = self._parse_action(action) + observation = self._execute_tool("SearchToolGroup", "search", query) + except Exception as e: + error = str(e) + observation = None + + # Wrap the observation properly as a message + if observation: + new_obs = {"role": "user", "content": observation} + elif error: + # Give error as observation if any + new_obs = {"role": "user", "content": error} + else: + new_obs = None + + info = { + "tool_group": "SearchToolGroup", + "tool_name": "search", + "tool_input": query, + } + + # Update chat history + if new_obs: + self.chat_history.append(new_obs) + + return BaseTextEnvStepOutput( + observations=[new_obs] if new_obs else [], + reward=reward, + done=done, + metadata=info, + ) diff --git a/skyrl-gym/skyrl_gym/envs/search/utils.py b/skyrl-gym/skyrl_gym/envs/search/utils.py new file mode 100644 index 0000000000..916e4ccbed --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/search/utils.py @@ -0,0 +1,118 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# Copyright 2025 Search-R1 Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/PeterGriffinJin/Search-R1/blob/main/verl/utils/reward_score/qa_em.py + +import re +import string + + +def normalize_answer(s): + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def em_check(prediction, golden_answers): + if isinstance(golden_answers, str): + golden_answers = [golden_answers] + normalized_prediction = normalize_answer(prediction) + score = 0 + for golden_answer in golden_answers: + golden_answer = normalize_answer(golden_answer) + if golden_answer == normalized_prediction: + score = 1 + break + return score + + +def subem_check(prediction, golden_answers): + if isinstance(golden_answers, str): + golden_answers = [golden_answers] + normalized_prediction = normalize_answer(prediction) + score = 0 + for golden_answer in golden_answers: + golden_answer = normalize_answer(golden_answer) + if golden_answer in normalized_prediction: + score = 1 + break + return score + + +def extract_solution(solution_str): + """Extract the equation from the solution string.""" + answer_pattern = r"(.*?)" + match = re.finditer(answer_pattern, solution_str, re.DOTALL) + matches = list(match) + + # If there are 0 matches, return None + if len(matches) < 1: + return None + + # If there are 2 or more matches, return the last one + return matches[-1].group(1).strip() + + +def compute_score(solution_str, ground_truth, method="strict", format_score=0.0, score=1.0): + """The scoring function for exact match (EM). + + Args: + solution_str: the solution text + ground_truth: the ground truth + method: the method to extract the solution, choices are 'strict' and 'flexible' + format_score: the score for the format + score: the score for the correct answer + """ + answer = extract_solution(solution_str=solution_str) + + if answer is None: + return 0 + else: + if em_check(answer, ground_truth["target"]): + return score + else: + return format_score + + +def compute_score_subem(solution_str, ground_truth, method="strict", format_score=0.0, score=1.0): + """The scoring function for substring exact match (EM). + + Args: + solution_str: the solution text + ground_truth: the ground truth + method: the method to extract the solution, choices are 'strict' and 'flexible' + format_score: the score for the format + score: the score for the correct answer + """ + answer = extract_solution(solution_str=solution_str) + + if answer is None: + return 0 + else: + if subem_check(answer, ground_truth["target"]): + return score + else: + return format_score diff --git a/skyrl-gym/skyrl_gym/envs/searchcode/env.py b/skyrl-gym/skyrl_gym/envs/searchcode/env.py new file mode 100644 index 0000000000..c307c052dc --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/searchcode/env.py @@ -0,0 +1,116 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput, ConversationType +from typing import Tuple, Any +from skyrl_gym.tools import SearchToolGroup, PythonCodeExecutorToolGroup +from skyrl_gym.envs.gsm8k import utils +import re +from typing import Dict +from omegaconf import DictConfig + + +class SearchCodeEnv(BaseTextEnv): + """ + Environment that calls multiple tools + """ + + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + + assert "reward_spec" in extras, "reward_spec field is required" + assert "ground_truth" in extras["reward_spec"], "ground_truth is required in reward_spec field" + self.ground_truth = extras["reward_spec"]["ground_truth"] + self.max_turns = extras["max_turns"] if "max_turns" in extras else 2 + + # Initialize the tools + self.init_tool_groups([SearchToolGroup(), PythonCodeExecutorToolGroup()]) + + # Chat history + # role (user, assistant), content (tool observation or LLM response) + self.chat_history: ConversationType = [] + + def _parse_action(self, action: str) -> Tuple[str, str, Any]: + """ + Parses the action string to detect a tool call in the format: + ..., where tool_name can be one of the + predefined tool names in the tool group (e.g., search, python). + Returns the tool group name, tool name, and corresponding tool input. + """ + tool_block_match = re.search(r"(.*?)", action, re.DOTALL) + if not tool_block_match: + raise ValueError("No ... block found in action string.") + + tool_content = tool_block_match.group(1).strip() + inner_tag_match = re.search(r"<(\w+)>(.*?)", tool_content, re.DOTALL) + if not inner_tag_match: + raise ValueError("No valid inner tool tag found inside block.") + + tool_name = inner_tag_match.group(1) + tool_input = inner_tag_match.group(2).strip() + + # Lookup the tool by name from tool group + if tool_name not in self.tool_to_toolgroup: + raise ValueError(f"Tool '{tool_name}' not found in any registered tool group.") + tool_group_name = self.tool_to_toolgroup[tool_name] + + return tool_group_name, tool_name, [tool_input] + + def _get_reward(self, action: str, done: bool) -> float: + if done: + # Concat all chat history into a single string and compute reward + chat_history_str = "\n".join([item["content"] for item in self.chat_history]) + return utils.compute_score(chat_history_str, self.ground_truth) + else: + return 0 + + def _is_done(self, action: str) -> bool: + if self.turns >= self.max_turns: + return True + return "" in action and "" in action + + def step(self, action: str) -> BaseTextEnvStepOutput: + """ + Step 1: parse action to get the corresponding tool group, tool name, and tool input + Step 2: execute the tool and get observation (as a string) + Step 3: get reward based on the action and observation + Step 4: determine `done` based on the action and observation + """ + self.turns += 1 + self.chat_history.append({"role": "assistant", "content": action}) + + error = None + done = self._is_done(action) + reward = self._get_reward(action, done) + + if done: + return BaseTextEnvStepOutput(observations=[], reward=reward, done=done, metadata={}) + + try: + tool_group_name, tool_name, tool_input = self._parse_action(action) + observation = self._execute_tool(tool_group_name, tool_name, tool_input) + except Exception as e: + error = str(e) + observation = None + tool_group_name = None + tool_name = None + tool_input = "" + + # Wrap the observation properly as a message + if observation: + new_obs = {"role": "user", "content": observation} + elif error: + # Give error as observation if any + new_obs = {"role": "user", "content": error} + else: + new_obs = None + + info = { + "tool_group": tool_group_name, + "tool_name": tool_name, + "tool_input": tool_input, + "tools": [tg.get_tool_names() for tg in self.tool_groups], + } + + # Update chat history + if new_obs: + self.chat_history.append(new_obs) + + return BaseTextEnvStepOutput(observations=[new_obs] if new_obs else [], reward=reward, done=done, metadata=info) diff --git a/skyrl-gym/skyrl_gym/envs/sql/env.py b/skyrl-gym/skyrl_gym/envs/sql/env.py new file mode 100644 index 0000000000..8a5e5d67cf --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/sql/env.py @@ -0,0 +1,142 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput, ConversationType +from typing import Tuple, Any +from skyrl_gym.tools import SQLCodeExecutorToolGroup +import re +from skyrl_gym.envs.sql.utils import compute_score_single +import os +from typing import Dict +from omegaconf import DictConfig + + +class SQLEnv(BaseTextEnv): + """ + Environment for one SQL execution task. + """ + + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + + # Initialize the environment + assert "db_id" in extras, "db_id field is required" + assert "reward_spec" in extras, "reward_spec field is required" + assert "data" in extras, "data field is required" + + self.db_path = env_config.db_path + self.db_id = extras["db_id"] + self.gold_sql = extras["reward_spec"]["ground_truth"] + self.task = extras["data"] + + if self.task == "synsql": + self.db_path = os.path.join( + self.db_path, + "SynSQL-2.5M/databases", + ) + elif self.task == "spider": + self.db_path = os.path.join( + self.db_path, + "spider/database", + ) + elif self.task == "bird": + self.db_path = os.path.join( + self.db_path, + "bird/train/train_databases", + ) + else: + raise NotImplementedError + + self.db_file = os.path.join(self.db_path, self.db_id, self.db_id + ".sqlite") + # Check for DB file existence + if not os.path.exists(self.db_file): + raise FileNotFoundError(f"Database file not found at: {self.db_file}") + + # override parent + self.max_turns = extras["max_turns"] if "max_turns" in extras else 5 + + # Initialize the tools + self.tool_group = SQLCodeExecutorToolGroup(db_file_path=self.db_path) + self.init_tool_groups([self.tool_group]) + + # Chat history + # Dict[str, str]: role (user, assistant), content (tool observation or LLM response) + self.chat_history: ConversationType = [] + + def _parse_action(self, action: str) -> Tuple[str, str, Any]: + """ + Parse action string to return tool name and corresponding arguments. + + Expected: ... + """ + match = re.search(r"(.*?)", action, re.DOTALL) + tool_input = match.group(1) if match else None + # NOTE: hard code + # NOTE (shu): in the future imagine can use different tools here + # Format tool_nametool_input + tool_group_name = self.tool_group.get_name() + tool_name = self.tool_group.get_tool_names()[0] + return tool_group_name, tool_name, (self.db_id, tool_input, self.max_turns - self.turns) + + def _get_reward(self, action: str, done: bool) -> float: + if done: + # Concat all chat history into a single string and compute reward + chat_history_str = "".join([item["content"] for item in self.chat_history]) + return compute_score_single(chat_history_str, self.gold_sql, self.db_file) + else: + # No reward for intermediate steps for SQL tasks + return 0 + + def _is_done(self, action: str) -> bool: + if self.turns >= self.max_turns: + return True + return "" in action and "" in action + + def _validate_action(self, action: str): + stop_tags = ["", ""] + for tag in stop_tags: + if tag in action: + assert action.split(tag, 1)[1] == "", ( + f"{tag} detected in the response but it is not the last string generated. " + f"Use {stop_tags} as stop strings in the configuration." + ) + + def step(self, action: str) -> BaseTextEnvStepOutput: + self.turns += 1 + self._validate_action(action) + self.chat_history.append({"role": "assistant", "content": action}) + + error = None + done = self._is_done(action) + reward = self._get_reward(action, done) + + if done: + return BaseTextEnvStepOutput(observations=[], reward=reward, done=done, metadata={}) + + try: + tool_group_name, tool_name, tool_input = self._parse_action(action) + observation = self._execute_tool(tool_group_name, tool_name, tool_input) + except Exception as e: + error = str(e) + observation = None + tool_group_name = None + tool_name = None + tool_input = "" + + # Wrap the observation properly as a message + if observation: + new_obs = {"role": "user", "content": observation} + elif error: + # Give error as observation if any + new_obs = {"role": "user", "content": error} + else: + new_obs = None + + info = {"tool_group": tool_group_name, "tool_name": tool_name, "tool_input": tool_input} + # Update chat history + if new_obs: + self.chat_history.append(new_obs) + + return BaseTextEnvStepOutput( + observations=[new_obs] if new_obs else [], + reward=reward, + done=done, + metadata=info, + ) diff --git a/skyrl-gym/skyrl_gym/envs/sql/utils.py b/skyrl-gym/skyrl_gym/envs/sql/utils.py new file mode 100644 index 0000000000..7113189a3b --- /dev/null +++ b/skyrl-gym/skyrl_gym/envs/sql/utils.py @@ -0,0 +1,133 @@ +""" +SynSQL reward calculation. + +- Format reward: ... ... +- Outcome reward: check ground truth and predicted patch similarity +""" + +import re +import sqlite3 +import sys +import threading + + +THINK_START, THINK_END = "", "" +SQL_START, SQL_END = "", "" +SOLUTION_START, SOLUTION_END = "", "" +OBS_START, OBS_END = "", "" + + +# NOTE: bring back reward +def verify_format_and_extract(output: str): + if output.count(SOLUTION_START) != 1: + return False, None, None, None + pre_solution, tail = output.split(SOLUTION_START, 1) + + if tail.count(SOLUTION_END) != 1: + return False, None, None, None + + solution_text, _ = tail.split(SOLUTION_END, 1) + + if re.search(r"(.*?)", output, re.S) + if not thoughts: + return False, None, None, None + + for m in re.finditer(r"", pre_solution, re.I): + rest = pre_solution[m.end() :].lstrip() + if not rest.lower().startswith(THINK_START): + return False, None, None, None + + return True, thoughts, solution_text.strip(), None + + +def execute_sql_wrapper_single(db_file, sql, timeout, output_str): + """ + Execute SQL with a timeout using a worker thread and conn.interrupt(). + Ensures long-running queries are interrupted and connections are closed. + """ + res = (db_file, sql, None, 0) + done = threading.Event() + conn_holder = {"conn": None} + + def worker(): + nonlocal res + conn = None + try: + conn = sqlite3.connect(db_file, check_same_thread=False) + conn_holder["conn"] = conn + cur = conn.cursor() + conn.execute("BEGIN TRANSACTION;") + cur.execute(sql) + rows = frozenset(cur.fetchall()) + conn.rollback() + res = (db_file, sql, rows, 1) + except Exception: + res = (db_file, sql, None, 0) + finally: + if conn is not None: + try: + conn.rollback() + except Exception: + pass + try: + conn.close() + except Exception: + pass + done.set() + + threading.Thread(target=worker, daemon=True).start() + + try: + if not done.wait(timeout): + conn = conn_holder.get("conn") + if conn is not None: + try: + conn.interrupt() + except Exception: + pass + done.wait() + except KeyboardInterrupt: + sys.exit(0) + except Exception: + res = (db_file, sql, None, 0) + + if isinstance(res, tuple): + res = res + (output_str,) + + return res + + +def calculate_reward_single(completion, reference, db_file, timeout=30): + reward = 0.0 + num_comparisons = 0 + + is_valid, _, pred_sql, _ = verify_format_and_extract(completion) + if not is_valid: + reward = -1.0 + return reward + else: + num_comparisons += 1 + + pred = execute_sql_wrapper_single(db_file, pred_sql, timeout, completion) + ref = execute_sql_wrapper_single(db_file, reference, timeout, completion) + + _, _, pred_results, _, _ = pred + _, _, gt_results, _, _ = ref + + if pred_results is not None and gt_results is not None and pred_results == gt_results: + reward = 1.0 + else: + reward = 0.0 + return reward + + +def compute_score_single(completion, reference, db_file): + try: + res = calculate_reward_single(completion, reference, db_file) + return res + except Exception as e: + print(f"Unexpected error: {e}; Setting reward as 0") + return 0 diff --git a/skyrl-gym/skyrl_gym/error.py b/skyrl-gym/skyrl_gym/error.py new file mode 100644 index 0000000000..6fdd7fe791 --- /dev/null +++ b/skyrl-gym/skyrl_gym/error.py @@ -0,0 +1,9 @@ +"""Set of Error classes for skyrl_gym.""" + + +class Error(Exception): + """Error superclass.""" + + +class RegistrationError(Error): + """Raised when the user attempts to register an invalid env.""" diff --git a/skyrl-gym/skyrl_gym/metrics.py b/skyrl-gym/skyrl_gym/metrics.py new file mode 100644 index 0000000000..6de0b74f1a --- /dev/null +++ b/skyrl-gym/skyrl_gym/metrics.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List +from skyrl_gym.envs.registration import registry, load_env_creator + + +def default_aggregate_metrics(metrics: List[Dict[str, Any]]) -> Dict[str, float]: + """ + A minimal default metric aggregator: average numeric fields across episode-level metric dicts. + """ + if not metrics: + return {} + aggregated_metrics: Dict[str, list[float]] = {} + for m in metrics: + for k, v in m.items(): + if isinstance(v, bool): + v = float(v) + elif isinstance(v, (int, float)): + v = float(v) + else: + continue + aggregated_metrics.setdefault(k, []).append(v) + return {k: sum(vals) / len(vals) for k, vals in aggregated_metrics.items()} + + +def aggregate_for_environment(env_name: str, metrics: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + Call aggregate_metrics() for the class specified by env_name. + + Args: + env_name: The registered environment name (e.g., "gsm8k") + metrics: List of metric dictionaries to aggregate + + Returns: + Aggregated metrics for the environment class + """ + # Look up the environment spec in the registry + env_spec = registry.get(env_name) + if env_spec is None: + raise ValueError(f"No registered env with id: {env_name}") + + # Get the environment class from the entry_point + entry_point = env_spec.entry_point + if callable(entry_point): + env_cls = entry_point + else: + # Load the class from the string entry point + env_cls = load_env_creator(entry_point) + + return env_cls.aggregate_metrics(metrics) diff --git a/skyrl-gym/skyrl_gym/tools/__init__.py b/skyrl-gym/skyrl_gym/tools/__init__.py new file mode 100644 index 0000000000..58f675635b --- /dev/null +++ b/skyrl-gym/skyrl_gym/tools/__init__.py @@ -0,0 +1,5 @@ +from .sql import SQLCodeExecutorToolGroup +from .search import SearchToolGroup +from .python import PythonCodeExecutorToolGroup + +__all__ = ["SQLCodeExecutorToolGroup", "SearchToolGroup", "PythonCodeExecutorToolGroup"] diff --git a/skyrl-gym/skyrl_gym/tools/core.py b/skyrl-gym/skyrl_gym/tools/core.py new file mode 100644 index 0000000000..12d9a98c4c --- /dev/null +++ b/skyrl-gym/skyrl_gym/tools/core.py @@ -0,0 +1,60 @@ +from typing import Dict, Callable, Any, Optional, List + + +class tool: + """ + A tool that can be used to execute a function. + """ + + def __init__(self, func: Callable): + self.func = func + self.name = func.__name__ + + def __get__(self, instance, owner): + if instance is None: + return self # Return the descriptor itself when accessed from the class + return lambda *args, **kwargs: self.func(instance, *args, **kwargs) + + +class ToolGroup: + """ + A group of tools that can be used together. + """ + + def __init__(self, name: str): + self.name = name + self._tool_registry: Dict[str, Callable] = {} + self._register_tools() + + def get_name(self): + return self.name + + def _register_tools(self): + # Register all methods decorated with @tool + + # Tool names must be unique across tool groups. + # TODO: Support duplicate tool names across tool groups via namespacing + for attr_name in dir(self): + # Look for the descriptor on the class, not the instance + raw = getattr(type(self), attr_name, None) + if isinstance(raw, tool): + self._tool_registry[raw.name] = getattr(self, attr_name) + + def get_tool(self, name: str) -> Optional[Callable]: + # Get a tool by name, returns None if not found + return self._tool_registry.get(name) + + def get_tool_names(self) -> List[str]: + # Get all available tool names + return list(self._tool_registry.keys()) + + def execute_tool(self, name: str, *args, **kwargs) -> Any: + # Execute a tool by name with given arguments + tool_func = self.get_tool(name) + if tool_func: + return tool_func(*args, **kwargs) + raise ValueError(f"Tool '{name}' not found in group '{self.name}'.") + + def get_tool_to_group_mapping(self) -> Dict[str, str]: + # Get mapping of tool names to group name + return {name: self.name for name in self._tool_registry} diff --git a/skyrl-gym/skyrl_gym/tools/python.py b/skyrl-gym/skyrl_gym/tools/python.py new file mode 100644 index 0000000000..419393110d --- /dev/null +++ b/skyrl-gym/skyrl_gym/tools/python.py @@ -0,0 +1,20 @@ +from skyrl_gym.tools.core import tool, ToolGroup +import subprocess + + +class PythonCodeExecutorToolGroup(ToolGroup): + def __init__(self, timeout: float = 10.0): + self.timeout = timeout + super().__init__(name="PythonCodeExecutorToolGroup") + + @tool + def python(self, code: str) -> str: + try: + result = subprocess.run( + ["python", "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=self.timeout, text=True + ) + if result.stderr: + return f"Error executing Python code: {result.stderr.strip()}" + return result.stdout.strip() if result.stdout else "" + except subprocess.TimeoutExpired: + return f"Error executing Python code: Execution timed out after {self.timeout} seconds." diff --git a/skyrl-gym/skyrl_gym/tools/search.py b/skyrl-gym/skyrl_gym/tools/search.py new file mode 100644 index 0000000000..4c9e23b17e --- /dev/null +++ b/skyrl-gym/skyrl_gym/tools/search.py @@ -0,0 +1,254 @@ +import json +import logging +import requests +import uuid +import time +import threading +from typing import Tuple, Optional, Any, Dict +from urllib.parse import urlparse + +from skyrl_gym.tools.core import tool, ToolGroup + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +DEFAULT_TIMEOUT = 30 +MAX_RETRIES = 10 +INITIAL_RETRY_DELAY = 1 + + +def call_search_api( + retrieval_service_url: str, + query: str, + topk: int = 3, + return_scores: bool = True, + timeout: int = DEFAULT_TIMEOUT, + log_requests: bool = True, + session: Optional[requests.Session] = None, +) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """ + Calls the search API with a single query. + + Args: + retrieval_service_url: The URL of the search API. + query: The query to search for. + topk: The number of results to return. + return_scores: Whether to return scores for the results. + timeout: The timeout for the request. + log_requests: Whether to log requests. + session: The session to use for the request. If none is provided, a new session will be created. + + Returns: + response: The response from the search API (json if successful, None otherwise) + error_msg: The error message if the request failed. + """ + request_id = str(uuid.uuid4()) + log_prefix = f"[Search Request ID: {request_id}] " + + payload = {"query": query, "topk": topk, "return_scores": return_scores} + headers = {"Content-Type": "application/json", "Accept": "application/json"} + + # Use provided session or create a new one for this request + if session is None: + session = requests.Session() + should_close_session = True + else: + should_close_session = False + + last_error = None + for attempt in range(MAX_RETRIES): + try: + if log_requests: + logger.info( + f"{log_prefix}Attempt {attempt + 1}/{MAX_RETRIES}: Calling search API at {retrieval_service_url}" + ) + response = session.post( + retrieval_service_url, + headers=headers, + json=payload, + timeout=timeout, + ) + + # Check for Gateway Timeout (504) and other server errors for retrying + if response.status_code in [500, 502, 503, 504]: + last_error = f"{log_prefix}API Request Error: Server Error ({response.status_code}) on attempt {attempt + 1}/{MAX_RETRIES}" + logger.warning(last_error) + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + logger.info(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + + # Check for other HTTP errors (e.g., 4xx) + response.raise_for_status() + + # If successful (status code 2xx) + if log_requests: + logger.info(f"{log_prefix}Search API call successful on attempt {attempt + 1}") + + # Close session if we created it + if should_close_session: + session.close() + + return response.json(), None + + except requests.exceptions.ConnectionError as e: + last_error = f"{log_prefix}Connection Error: {e}" + logger.warning(last_error) + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + logger.info(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + except requests.exceptions.Timeout as e: + last_error = f"{log_prefix}Timeout Error: {e}" + logger.warning(last_error) + if attempt < MAX_RETRIES - 1: + delay = INITIAL_RETRY_DELAY * (attempt + 1) + logger.info(f"{log_prefix}Retrying after {delay} seconds...") + time.sleep(delay) + continue + except requests.exceptions.RequestException as e: + last_error = f"{log_prefix}API Request Error: {e}" + break # Exit retry loop on other request errors + except json.JSONDecodeError as e: + raw_response_text = response.text if "response" in locals() else "N/A" + last_error = f"{log_prefix}API Response JSON Decode Error: {e}, Response: {raw_response_text[:200]}" + break # Exit retry loop on JSON decode errors + except Exception as e: + last_error = f"{log_prefix}Unexpected Error: {e}" + break # Exit retry loop on other unexpected errors + + # If we reach here, all attempts failed + logger.error(f"{log_prefix}API Request Failed after {MAX_RETRIES} attempts: {last_error}") + + # Close session if we created it + if should_close_session: + session.close() + + return None, last_error + + +def _passages2string(retrieval_result): + format_reference = "" + for idx, doc_item in enumerate(retrieval_result): + content = doc_item["document"]["contents"].strip() + format_reference += f"Doc {idx+1}: {content}\n" + return format_reference + + +class SearchToolGroup(ToolGroup): + # Class-level session pool shared across all instances + _session_pool = {} + _session_lock = threading.Lock() + + @classmethod + def _get_shared_session(cls, base_url: str) -> requests.Session: + """Get or create a shared session for the given base URL""" + with cls._session_lock: + if base_url not in cls._session_pool: + session = requests.Session() + # Configure connection pooling + adapter = requests.adapters.HTTPAdapter( + pool_connections=20, # Number of connection pools + pool_maxsize=20, # Max connections per pool + max_retries=0, # We handle retries ourselves + pool_block=False, # Don't block if pool is full + ) + session.mount("http://", adapter) + session.mount("https://", adapter) + cls._session_pool[base_url] = session + logger.info(f"Created shared session pool for {base_url}") + return cls._session_pool[base_url] + + def __init__(self, search_url="http://127.0.0.1:8000/retrieve", topk=3, timeout=DEFAULT_TIMEOUT, log_requests=True): + self.search_url = search_url + self.topk = topk + self.timeout = timeout + self.log_requests = log_requests + + # Extract base URL for session sharing + parsed_url = urlparse(self.search_url) + self.base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + + # Get shared session for this base URL + self.session = self._get_shared_session(self.base_url) + if self.log_requests: + logger.info(f"SearchToolGroup initialized using shared session pool for {self.base_url}") + + super().__init__(name="SearchToolGroup") + + @tool + def search(self, query: str) -> str: + # NOTE(shu): add warning messages here? + if query is None: + return "" + + query = query.strip() + + try: + api_response, error_msg = call_search_api( + retrieval_service_url=self.search_url, + query=query, + topk=self.topk, + timeout=self.timeout, + log_requests=self.log_requests, + session=self.session, # Pass our shared session for connection reuse + ) + except Exception as e: + error_msg = f"API Request Exception during batch search: {e}" + logger.error(f"Batch search: {error_msg}") + + metadata = { + "query": query, + "api_request_error": error_msg, + "api_response": None, + "status": "unknown", + "total_results": 0, + "formatted_result": None, + } + + result_text = json.dumps({"result": "Search request failed or timed out after retries."}) + + if error_msg: + metadata["status"] = "api_error" + result_text = json.dumps({"result": f"Search error: {error_msg}"}) + logger.error(f"Batch search: API error occurred: {error_msg}") + elif api_response: + logger.debug(f"Batch search: API Response: {api_response}") + metadata["api_response"] = api_response + + try: + raw_results = api_response.get("result", []) + if raw_results: + pretty_results = [] + total_results = 0 + for retrieval in raw_results: + formatted = _passages2string(retrieval) + pretty_results.append(formatted) + total_results += len(retrieval) if isinstance(retrieval, list) else 1 + + final_result = "\n---\n".join(pretty_results) + result_text = json.dumps({"result": final_result}) + metadata["status"] = "success" + metadata["total_results"] = total_results + metadata["formatted_result"] = final_result + if self.log_requests: + logger.info(f"Batch search: Successful, got {total_results} total results") + else: + result_text = json.dumps({"result": "No search results found."}) + metadata["status"] = "no_results" + metadata["total_results"] = 0 + if self.log_requests: + logger.info("Batch search: No results found") + except Exception as e: + error_msg = f"Error processing search results: {e}" + result_text = json.dumps({"result": error_msg}) + metadata["status"] = "processing_error" + logger.error(f"Batch search: {error_msg}") + else: + metadata["status"] = "unknown_api_state" + result_text = json.dumps({"result": "Unknown API state (no response and no error message)."}) + logger.error("Batch search: Unknown API state.") + + return result_text diff --git a/skyrl-gym/skyrl_gym/tools/sql.py b/skyrl-gym/skyrl_gym/tools/sql.py new file mode 100644 index 0000000000..06f1e7a92f --- /dev/null +++ b/skyrl-gym/skyrl_gym/tools/sql.py @@ -0,0 +1,97 @@ +from skyrl_gym.tools.core import tool, ToolGroup +import pandas as pd +import sqlite3 +import sys +import os +import threading + + +class SQLCodeExecutorToolGroup(ToolGroup): + def __init__(self, db_file_path: str): + self.db_path = db_file_path + super().__init__(name="SQLCodeExecutorToolGroup") + + @tool + def sql(self, db_id, sql, turns_left, timeout=5) -> str: + def _execute_sql_wrapper(db_file, sql, timeout=5) -> str: + """ + Execute SQL with a timeout using a worker thread and conn.interrupt(). + Ensures long-running queries are interrupted and connections are closed. + """ + res_holder = {"value": None} + done = threading.Event() + conn_holder = {"conn": None} + + def worker(): + conn = None + result = None + try: + conn = sqlite3.connect(db_file, check_same_thread=False) + conn_holder["conn"] = conn + cursor = conn.cursor() + conn.execute("BEGIN TRANSACTION;") + cursor.execute(sql) + execution_res = frozenset(cursor.fetchall()) + conn.rollback() + result = execution_res + except Exception as e: + result = f"Error executing SQL: {str(e)}, db file: {db_file}" + finally: + if conn is not None: + try: + conn.rollback() + except Exception: + pass + try: + conn.close() + except Exception: + pass + res_holder["value"] = result + done.set() + + threading.Thread(target=worker, daemon=True).start() + + timeout_occurred = False + try: + if not done.wait(timeout): + timeout_occurred = True + conn = conn_holder.get("conn") + if conn is not None: + try: + conn.interrupt() + except Exception: + pass + # Wait for worker to clean up its connection + done.wait() + except KeyboardInterrupt: + sys.exit(0) + except Exception as e: + res_holder["value"] = str(e) + + if timeout_occurred: + return f"SQL Timeout:\n{sql}" + + res = res_holder["value"] + if isinstance(res, frozenset): + df = pd.DataFrame(res) + res = df.to_string(index=False) + # NOTE: observation too long, just truncate + if len(res) > 9000: + truncated_df = df.head(50) + res = "Truncated to 50 lines since returned response too long: " + truncated_df.to_string( + index=False + ) # or index=True if you want row numbers + else: + res = str(res) + + return res + + # TODO (erictang000): move this logic up into the text2sql env, since this is more specific logic + reminder_text = f"You have {turns_left} turns left to complete the task." + if sql is None: + obs = "Your previous action is invalid. Follow the format of outputting thinking process and sql tool, and try again." + else: + db_file = os.path.join(self.db_path, db_id, db_id + ".sqlite") + obs = _execute_sql_wrapper(db_file, sql, timeout) + + return f"\n\n{obs}\n{reminder_text}\n\n" diff --git a/skyrl-gym/tests/test_aime.py b/skyrl-gym/tests/test_aime.py new file mode 100644 index 0000000000..74fa0f87e3 --- /dev/null +++ b/skyrl-gym/tests/test_aime.py @@ -0,0 +1,29 @@ +import skyrl_gym +import pytest +from omegaconf import DictConfig + + +@pytest.mark.parametrize( + "output, ground_truth, expected", + [ + ("Answer: \\boxed{42}", "42", 1.0), + ("Answer: 42", "42", 1.0), + ("Answer: \\boxed{43}", "42", -1.0), + ("Answer: \\boxed{\\frac{1}{2}}", "\\frac{1}{2}", 1.0), + ("Answer: \\boxed{0.5}", "\\frac{1}{2}", -1.0), + ("Answer: \\boxed{\\text{forty-two}}", "42", -1.0), + # test EOS tokens + ("<|im_start|>Answer: \\boxed{42}<|im_end|>", "42", 1.0), + ("<|im_start|>Answer: \\boxed{42}im_end|>", "42", -1.0), + ("Answer: \\boxed{42}<|eot_id|>", "42", 1.0), + ("Answer: \\boxed{42}|eot_id|>", "42", -1.0), + ], +) +def test_compute_score(output, ground_truth, expected): + env = skyrl_gym.make( + "aime", + env_config=DictConfig({"env_class": "aime"}), + extras={"reward_model": {"method": "rule", "ground_truth": ground_truth}}, + ) + step_output = env.step(output) + assert step_output["reward"] == expected diff --git a/skyrl-gym/tests/test_gsm8k.py b/skyrl-gym/tests/test_gsm8k.py new file mode 100644 index 0000000000..3199ce2b8b --- /dev/null +++ b/skyrl-gym/tests/test_gsm8k.py @@ -0,0 +1,23 @@ +import skyrl_gym +import pytest +from omegaconf import DictConfig + + +@pytest.mark.parametrize( + "output, ground_truth, expected", + [ + ("The answer is #### 42", "42", 1.0), + ("The answer is #### 42", "43", 0.0), + # answer is not in the expected format + ("The answer is 42", "42", 0.0), + ], +) +def test_compute_score(output, ground_truth, expected): + env = skyrl_gym.make( + "gsm8k", + env_config=DictConfig({"env_class": "gsm8k"}), + extras={"reward_spec": {"method": "rule", "ground_truth": ground_truth}}, + ) + # Skip init() since it's not used in this test + step_output = env.step(output) + assert step_output["reward"] == expected diff --git a/skyrl-gym/tests/test_lcb.py b/skyrl-gym/tests/test_lcb.py new file mode 100644 index 0000000000..796b555b76 --- /dev/null +++ b/skyrl-gym/tests/test_lcb.py @@ -0,0 +1,77 @@ +import pytest +import skyrl_gym +import json +from omegaconf import DictConfig + + +@pytest.mark.parametrize( + "model_response, tests, expected_reward", + [ + # Correct code: second largest index + ( + """```python +def main(): + N = int(input()) + A = list(map(int, input().split())) + B = sorted(A, reverse=True) + second = B[1] + print(A.index(second) + 1) + +if __name__ == "__main__": + main() +```""", + json.dumps( + [ + {"input": "4\n8 2 5 1\n", "output": "3\n", "testtype": "stdin"}, + {"input": "3\n3 2 1\n", "output": "2\n", "testtype": "stdin"}, + ] + ), + 1.0, + ), + # Wrong logic: returns index of largest + ( + """```python +def main(): + N = int(input()) + A = list(map(int, input().split())) + print(A.index(max(A)) + 1) + +if __name__ == "__main__": + main() +```""", + json.dumps( + [ + {"input": "4\n8 2 5 1\n", "output": "3\n", "testtype": "stdin"}, + ] + ), + 0.0, + ), + # Missing main() call — runtime error + ( + """```python +def main(): + A = list(map(int, input().split())) + B = sorted(A, reverse=True) + second = B[1] + print(A.index(second) + 1) + +# forgot to call main() +```""", + json.dumps( + [ + {"input": "4\n8 2 5 1\n", "output": "3\n", "testtype": "stdin"}, + ] + ), + 0.0, + ), + ], +) +def test_compute_score(model_response, tests, expected_reward): + env = skyrl_gym.make( + "lcb", + env_config=DictConfig({"env_class": "lcb"}), + extras={"reward_spec": {"method": "rule", "ground_truth": tests}}, + ) + # Skip init() since it's not used in this test + step_output = env.step(model_response) + assert step_output["reward"] == expected_reward diff --git a/skyrl-gym/tests/test_search.py b/skyrl-gym/tests/test_search.py new file mode 100644 index 0000000000..841383f6a4 --- /dev/null +++ b/skyrl-gym/tests/test_search.py @@ -0,0 +1,476 @@ +""" +uv run --isolated --extra dev pytest tests/test_search.py +""" + +import skyrl_gym +import pytest +from unittest.mock import patch +from omegaconf import DictConfig +from skyrl_gym.envs.search.utils import extract_solution, em_check + + +# Mock search API responses for testing +MOCK_SEARCH_RESULTS = { + "Who is the president of France?": { + "result": [ + [ + {"document": {"contents": "Emmanuel Macron is the current President of France, serving since 2017."}}, + {"document": {"contents": "France is a republic with a president as head of state."}}, + ] + ] + }, + "What is the capital of Italy?": { + "result": [[{"document": {"contents": "Rome is the capital and largest city of Italy."}}]] + }, + "Invalid query": {"result": []}, +} + + +@pytest.fixture +def mock_search_api(): + """Mock the search API requests.""" + with patch("skyrl_gym.tools.search.call_search_api") as mock_call: + + def mock_search_response( + retrieval_service_url, query, topk=3, return_scores=True, timeout=30, log_requests=True, session=None + ): + if query in MOCK_SEARCH_RESULTS: + return MOCK_SEARCH_RESULTS[query], None + elif query == "timeout_query": + return None, "Timeout Error: Request timed out" + elif query == "server_error": + return None, "Server Error: 500 Internal Server Error" + else: + return {"result": []}, None + + mock_call.side_effect = mock_search_response + yield mock_call + + +@pytest.fixture +def search_env(): + """Create a search environment for testing.""" + return skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": False, "search_url": "http://127.0.0.1:8000/retrieve", "topk": 3, "timeout": 30} + ), + extras={"reward_spec": {"method": "rule", "ground_truth": {"target": "Emmanuel Macron"}}, "max_turns": 3}, + ) + + +# ============================================================================= +# BASIC SEARCH ENVIRONMENT FUNCTIONALITY TESTS +# ============================================================================= + + +def test_env_initialization(search_env): + """Test that the environment initializes correctly.""" + assert search_env.ground_truth == {"target": "Emmanuel Macron"} + assert search_env.max_turns == 3 + assert search_env.turns == 0 + assert search_env.chat_history == [] + assert search_env.tool_group.get_name() == "SearchToolGroup" + assert search_env.tool_group.get_tool_names() == ["search"] + + +def test_env_with_custom_config(): + """Test environment with custom configuration.""" + env = skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": True, "search_url": "http://custom.example.com/search", "topk": 5, "timeout": 60} + ), + extras={"reward_spec": {"method": "rule", "ground_truth": {"target": "Custom Answer"}}, "max_turns": 5}, + ) + + assert env.ground_truth == {"target": "Custom Answer"} + assert env.max_turns == 5 + assert env.tool_group.search_url == "http://custom.example.com/search" + assert env.tool_group.topk == 5 + assert env.tool_group.timeout == 60 + + +# ============================================================================= +# ACTION PARSING FUNCTIONALITY TESTS +# ============================================================================= + + +@pytest.mark.parametrize( + "action, expected_input", + [ + # Valid search query + ( + "Who is the president of France?", + ["Who is the president of France?"], + ), + # Search with extra whitespace + ( + " Who is the president of France? ", + [" Who is the president of France? "], + ), + # Search with multiline content + ( + "Who is the president\nof France?", + ["Who is the president\nof France?"], + ), + # No search tags + ("Just plain text", [None]), + # Malformed search tag (no closing tag) + ("Who is the president of France?", [None]), + # Empty search tag + ("", [""]), + # Multiple search tags (should match first) + ("Query 1 Query 2", ["Query 1"]), + # Search with answer tag (should still parse search) + ("Query Answer", ["Query"]), + ], +) +def test_parse_action(search_env, action, expected_input): + """Test action parsing with various input formats.""" + query = search_env._parse_action(action) + assert query == expected_input + + +# ============================================================================= +# EPISODE TERMINATION CONDITIONS TESTS +# ============================================================================= + + +@pytest.mark.parametrize( + "action, current_turns, max_turns, expected", + [ + # Answer tag should trigger done + ("Emmanuel Macron", 1, 3, True), + # Search tag should not trigger done + ("Who is the president?", 1, 3, False), + # Max turns reached + ("Query", 3, 3, True), + # Max turns not reached, no answer tag + ("Just text", 2, 3, False), + # Answer tag with max turns reached + ("Answer", 3, 3, True), + # Mixed content with answer tag + ("Query Answer", 1, 3, True), + ], +) +def test_is_done(search_env, action, current_turns, max_turns, expected): + """Test episode termination conditions.""" + search_env.turns = current_turns + search_env.max_turns = max_turns + + result = search_env._is_done(action) + assert result == expected + + +# ============================================================================= +# TOOL EXECUTION WITH MOCKED SEARCH API TESTS +# ============================================================================= + + +def test_successful_search(search_env, mock_search_api): + """Test successful search execution.""" + action = "Who is the president of France?" + + result = search_env.step(action) + + # Verify API was called correctly + mock_search_api.assert_called_once() + call_args = mock_search_api.call_args + assert call_args[1]["query"] == "Who is the president of France?" + assert call_args[1]["topk"] == 3 + + # Verify result structure + assert not result["done"] + assert result["reward"] == 0.0 # No reward for intermediate steps + assert len(result["observations"]) == 1 + assert result["observations"][0]["role"] == "user" + + # Verify observation contains search results + observation_content = result["observations"][0]["content"] + assert "" in observation_content + assert "" in observation_content + assert "Emmanuel Macron" in observation_content + + # Verify chat history is updated + assert len(search_env.chat_history) == 2 + assert search_env.chat_history[0]["role"] == "assistant" + assert search_env.chat_history[0]["content"] == action + assert search_env.chat_history[1]["role"] == "user" + + +def test_search_with_no_results(search_env, mock_search_api): + """Test search with no results.""" + action = "Invalid query" + + result = search_env.step(action) + + # Verify result structure + assert not result["done"] + assert result["reward"] == 0.0 + assert len(result["observations"]) == 1 + + # Verify observation contains no results message + observation_content = result["observations"][0]["content"] + assert "No search results found" in observation_content + + +def test_search_timeout_error(search_env, mock_search_api): + """Test search with timeout error.""" + action = "timeout_query" + + result = search_env.step(action) + + # Verify result structure + assert not result["done"] + assert result["reward"] == 0.0 + assert len(result["observations"]) == 1 + + # Verify observation contains error message + observation_content = result["observations"][0]["content"] + assert "Search error" in observation_content + assert "Timeout Error" in observation_content + + +def test_search_server_error(search_env, mock_search_api): + """Test search with server error.""" + action = "server_error" + + result = search_env.step(action) + + # Verify result structure + assert not result["done"] + assert result["reward"] == 0.0 + assert len(result["observations"]) == 1 + + # Verify observation contains error message + observation_content = result["observations"][0]["content"] + assert "Search error" in observation_content + assert "Server Error" in observation_content + + +def test_invalid_search_parsing(search_env, mock_search_api): + """Test search with invalid parsing (None query).""" + action = "Invalid query" # No closing tag + + result = search_env.step(action) + + # Should not call the API with None query + mock_search_api.assert_not_called() + + # Should return empty result + observation_content = result["observations"][0]["content"] + assert "" in observation_content + + +# ============================================================================= +# REWARD COMPUTATION AND SCORING TESTS +# ============================================================================= + + +@pytest.mark.parametrize( + "action, ground_truth, expected_reward, expected_done", + [ + # Correct answer + ("Emmanuel Macron", {"target": "Emmanuel Macron"}, 1.0, True), + # Incorrect answer + ("Nicolas Sarkozy", {"target": "Emmanuel Macron"}, 0.0, True), + # Search action (not done) + ("Who is the president?", {"target": "Emmanuel Macron"}, 0.0, False), + # Answer with extra whitespace + (" Emmanuel Macron ", {"target": "Emmanuel Macron"}, 1.0, True), + # Case insensitive match + ("emmanuel macron", {"target": "Emmanuel Macron"}, 1.0, True), + # Answer without articles + ("Emmanuel Macron", {"target": "The Emmanuel Macron"}, 1.0, True), + # No answer tag + ("Just text without answer tag", {"target": "Emmanuel Macron"}, 0.0, False), + ], +) +def test_reward_computation(action, ground_truth, expected_reward, expected_done): + """Test reward computation for different scenarios.""" + env = skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": False, "search_url": "http://127.0.0.1:8000/retrieve", "topk": 3, "timeout": 30} + ), + extras={"reward_spec": {"method": "rule", "ground_truth": ground_truth}, "max_turns": 3}, + ) + + result = env.step(action) + + assert result["done"] == expected_done + assert result["reward"] == expected_reward + + +# ============================================================================= +# COMPLETE EPISODE SCENARIOS TESTS +# ============================================================================= + + +def test_successful_search_and_answer(mock_search_api): + """Test a complete successful episode.""" + env = skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": False, "search_url": "http://127.0.0.1:8000/retrieve", "topk": 3, "timeout": 30} + ), + extras={"reward_spec": {"method": "rule", "ground_truth": {"target": "Emmanuel Macron"}}, "max_turns": 3}, + ) + + # Step 1: Search + result1 = env.step("Who is the president of France?") + assert not result1["done"] + assert result1["reward"] == 0.0 + assert len(result1["observations"]) == 1 + assert "Emmanuel Macron" in result1["observations"][0]["content"] + + # Step 2: Answer + result2 = env.step("Emmanuel Macron") + assert result2["done"] + assert result2["reward"] == 1.0 + assert len(result2["observations"]) == 0 # No observations when done + + # Verify chat history + assert len(env.chat_history) == 3 + assert env.chat_history[0]["content"] == "Who is the president of France?" + assert env.chat_history[2]["content"] == "Emmanuel Macron" + + +def test_max_turns_reached(mock_search_api): + """Test episode termination when max turns reached.""" + env = skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": False, "search_url": "http://127.0.0.1:8000/retrieve", "topk": 3, "timeout": 30} + ), + extras={"reward_spec": {"method": "rule", "ground_truth": {"target": "Emmanuel Macron"}}, "max_turns": 2}, + ) + + # Step 1: Search + result1 = env.step("Who is the president of France?") + assert not result1["done"] + assert result1["reward"] == 0.0 + + # Step 2: Another search (should terminate due to max turns) + result2 = env.step("More info about France?") + assert result2["done"] + assert result2["reward"] == 0.0 # No correct answer, so no reward + + +def test_search_then_wrong_answer(mock_search_api): + """Test episode with search followed by wrong answer.""" + env = skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": False, "search_url": "http://127.0.0.1:8000/retrieve", "topk": 3, "timeout": 30} + ), + extras={"reward_spec": {"method": "rule", "ground_truth": {"target": "Emmanuel Macron"}}, "max_turns": 3}, + ) + + # Step 1: Search + result1 = env.step("Who is the president of France?") + assert not result1["done"] + assert result1["reward"] == 0.0 + + # Step 2: Wrong answer + result2 = env.step("Nicolas Sarkozy") + assert result2["done"] + assert result2["reward"] == 0.0 + + +# ============================================================================= +# UTILITY FUNCTIONS USED BY SEARCH ENVIRONMENT TESTS +# ============================================================================= + + +@pytest.mark.parametrize( + "solution_str, expected", + [ + # Basic answer extraction + ("Emmanuel Macron", "Emmanuel Macron"), + # Answer with whitespace + (" Emmanuel Macron ", "Emmanuel Macron"), + # Multiple answer tags (should return last one) + ("Wrong Emmanuel Macron", "Emmanuel Macron"), + # No answer tags + ("No answer here", None), + # Empty answer + ("", ""), + # Answer with newlines + ("Emmanuel\nMacron", "Emmanuel\nMacron"), + # Malformed answer tag + ("Emmanuel Macron", None), + ], +) +def test_extract_solution(solution_str, expected): + """Test solution extraction from text.""" + result = extract_solution(solution_str) + assert result == expected + + +@pytest.mark.parametrize( + "prediction, golden_answers, expected", + [ + # Exact match + ("Emmanuel Macron", "Emmanuel Macron", 1), + # Case insensitive + ("emmanuel macron", "Emmanuel Macron", 1), + # With articles + ("The Emmanuel Macron", "Emmanuel Macron", 1), + # With punctuation + ("Emmanuel Macron!", "Emmanuel Macron", 1), + # No match + ("Nicolas Sarkozy", "Emmanuel Macron", 0), + # List of golden answers + ("Emmanuel Macron", ["Emmanuel Macron", "Macron"], 1), + # No match in list + ("Sarkozy", ["Emmanuel Macron", "Macron"], 0), + ], +) +def test_em_check(prediction, golden_answers, expected): + """Test exact match checking.""" + result = em_check(prediction, golden_answers) + assert result == expected + + +# ============================================================================= +# ERROR HANDLING SCENARIOS TESTS +# ============================================================================= + + +def test_missing_reward_spec(): + """Test that missing reward_spec raises an error.""" + with pytest.raises(AssertionError, match="reward_spec field is required"): + skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": False, "search_url": "http://127.0.0.1:8000/retrieve", "topk": 3, "timeout": 30} + ), + extras={"max_turns": 3}, # Missing reward_spec + ) + + +def test_missing_ground_truth(): + """Test that missing ground_truth raises an error.""" + with pytest.raises(AssertionError, match="ground_truth is required in reward_spec field"): + skyrl_gym.make( + "search", + env_config=DictConfig( + {"log_requests": False, "search_url": "http://127.0.0.1:8000/retrieve", "topk": 3, "timeout": 30} + ), + extras={"reward_spec": {"method": "rule"}, "max_turns": 3}, # Missing ground_truth + ) + + +def test_tool_execution_exception(search_env): + """Test tool execution when an exception occurs.""" + # Mock the tool execution to raise an exception + with patch.object(search_env.tool_group, "execute_tool", side_effect=Exception("Tool execution failed")): + result = search_env.step("Test query") + + # Should handle the exception gracefully + assert not result["done"] + assert result["reward"] == 0.0 + assert len(result["observations"]) == 1 + assert "Tool execution failed" in result["observations"][0]["content"] diff --git a/skyrl-gym/tests/test_searchcode.py b/skyrl-gym/tests/test_searchcode.py new file mode 100644 index 0000000000..02610deb57 --- /dev/null +++ b/skyrl-gym/tests/test_searchcode.py @@ -0,0 +1,61 @@ +import skyrl_gym +import pytest +from omegaconf import DictConfig + + +@pytest.mark.parametrize( + "action, expected_tool_name, expected_tool_input", + [ + ( + "how to reverse a string in Python", + "search", + ["how to reverse a string in Python"], + ), + ("print('hello')", "python", ["print('hello')"]), + ("binary search in Java", "search", ["binary search in Java"]), + ], +) +def test_tool_parsing(action, expected_tool_name, expected_tool_input): + # Test the parsing logic and reward logic with dummy input + env = skyrl_gym.make( + "searchcode", + env_config=DictConfig({"env_class": "searchcode"}), + extras={"reward_spec": {"method": "rule", "ground_truth": "random"}, "max_turns": 2}, + ) + # Skip init() since it's not used in this test + + _, tool_name, tool_input = env._parse_action(action) + + assert tool_name == expected_tool_name + assert tool_input == expected_tool_input + + +@pytest.mark.parametrize( + "action, expected_output", + [ + ( + "print('hello world')", + "hello world", + ), + ( + "print(1 + 1)", + "2", + ), + ( + "raise ValueError('fail')", + 'Error executing Python code: Traceback (most recent call last):\n File "", line 1, in \nValueError: fail', + ), + ], +) +def test_python_code_execution(action, expected_output): + env = skyrl_gym.make( + "searchcode", + env_config=DictConfig({"env_class": "searchcode"}), + extras={"reward_spec": {"method": "rule", "ground_truth": "random"}, "max_turns": 2}, + ) + env.init([]) # Initialize the environment + + output = env.step(action) + observation_content = output["observations"][0]["content"] + + assert expected_output == observation_content diff --git a/skyrl-gym/tests/test_sql.py b/skyrl-gym/tests/test_sql.py new file mode 100644 index 0000000000..9a62d8b85c --- /dev/null +++ b/skyrl-gym/tests/test_sql.py @@ -0,0 +1,272 @@ +""" +uv run --isolated --extra dev pytest tests/test_sql.py +""" + +import skyrl_gym +import pytest +from unittest.mock import patch, MagicMock +from omegaconf import DictConfig +from skyrl_gym.envs.sql.utils import verify_format_and_extract + +# Mock data for testing +MOCK_DB_RESULTS = { + "SELECT name FROM employees WHERE id = 1;": [("John Doe",)], + "SELECT name FROM employees WHERE id = 2;": [("Jane Smith",)], + "GOLDEN_SQL;": [("John Doe",)], +} + + +@pytest.fixture +def mock_db_file(): + with patch("os.path.exists") as mock_exists: + # Make the mock return True for any database file path + mock_exists.return_value = True + yield mock_exists + + +@pytest.fixture +def mock_sqlite_connection(): + with patch("sqlite3.connect") as mock_connect: + # Create a mock connection + mock_conn = MagicMock() + mock_cursor = MagicMock() + + # Setup the mock cursor's execute and fetchall methods + def mock_execute(sql, *args, **kwargs): + if sql in MOCK_DB_RESULTS: + mock_cursor.fetchall.return_value = MOCK_DB_RESULTS[sql] + else: + mock_cursor.fetchall.return_value = [] + + mock_cursor.execute = mock_execute + mock_conn.cursor.return_value = mock_cursor + mock_connect.return_value = mock_conn + + yield mock_connect + + +@pytest.mark.parametrize( + "step_1_output, step_2_output, ground_truth, expected", + [ + # Test case 1: matching sql and solution + ( + "a\n\n\ + SELECT name FROM employees WHERE id = 1;", + "b\n\n\ + SELECT name FROM employees WHERE id = 1;", + "SELECT name FROM employees WHERE id = 1;", + 1.0, + ), + # Test case 2: identical sql but different solution + ( + "a\n\n\ + SELECT name FROM employees WHERE id = 1;", + "b\n\n\ + SELECT name FROM employees WHERE id = 1;", + "GOLDEN_SQL;", + 1.0, + ), + # Test case 3: solution is wrong + ( + "a\n\n\ + SELECT name FROM employees WHERE id = 1;", + "b\n\n\ + SELECT name FROM employees WHERE id = 2;", + "SELECT name FROM employees WHERE id = 1;", + 0.0, + ), + # Test case 4: includes invalid sql msg + ( + "a", # no sql tag here + "b\n\n\ + SELECT name FROM employees WHERE id = 1;", + "SELECT name FROM employees WHERE id = 1;", + 1.0, + ), + ], + ids=["correct_sql_and_solution", "golden_sql_same_result", "wrong_solution", "invalid_sql_msg"], +) +def test_compute_score(mock_db_file, mock_sqlite_connection, step_1_output, step_2_output, ground_truth, expected): + extras = { + "reward_spec": {"method": "rule", "ground_truth": "SELECT name FROM employees WHERE id = 1;"}, + "max_turns": 3, + "db_id": "test_db", + "data": "spider", + } + env = skyrl_gym.make("text2sql", env_config=DictConfig({"db_path": "/home/ray/default/sql_data"}), extras=extras) + # Skip init() since it's not used in this test + + reminder_text = "You have 2 turns left to complete the task." + invalid_sql_message = ( + "Your previous action is invalid. Follow the format of outputting thinking process and sql tool, and try again." + ) + + output = env.step(step_1_output) + obs1 = output["observations"] + reward = output["reward"] + + # intermediate step reward is 0 + assert reward == 0.0 + # check reminder message + assert reminder_text in obs1[0]["content"] + if "" not in step_1_output: + assert invalid_sql_message in obs1[0]["content"] + output = env.step(step_2_output) + reward = output["reward"] + + # Only assert reward when done is True + if output["done"]: + assert reward == expected + else: + assert reward == 0.0 + + +@pytest.mark.parametrize( + "output, parse_ground_truth", + [ + # Test case 1: Valid SQL query + ("SELECT name FROM employees WHERE id = 1;", "SELECT name FROM employees WHERE id = 1;"), + # Test case 3: Invalid formatting + ("SELECT name FROM employees", None), + ], +) +def test_tool_parsing(mock_db_file, mock_sqlite_connection, output, parse_ground_truth): + extras = { + "reward_spec": {"method": "rule", "ground_truth": "SELECT name FROM employees WHERE id = 1;"}, + "max_turns": 2, + "db_id": "test_db", + "data": "spider", + } + env = skyrl_gym.make("text2sql", env_config=DictConfig({"db_path": "/home/ray/default/sql_data"}), extras=extras) + # Skip init() since it's not used in this test + + # Step once and get the tool input in `info` + output = env.step(output) + info = output["metadata"] + + sql_query = info["tool_input"][1] # SQL query is the second element in tool_input + + # assert it matches the parsed ground truth + assert sql_query == parse_ground_truth + + +# Additional test for SQL execution +def test_sql_execution(mock_db_file, mock_sqlite_connection): + extras = { + "reward_spec": {"method": "rule", "ground_truth": "SELECT name FROM employees WHERE id = 1;"}, + "max_turns": 2, + "db_id": "test_db", + "data": "spider", + } + env = skyrl_gym.make("text2sql", env_config=DictConfig({"db_path": "/home/ray/default/sql_data"}), extras=extras) + # Skip init() since it's not used in this test + + # Test a valid SQL query + output = "SELECT name FROM employees WHERE id = 1;" + output = env.step(output) + observation = output["observations"] + + # Verify the SQL was executed + mock_sqlite_connection.assert_called_once() + + # Verify the observation contains the expected result + assert "John Doe" in observation[0]["content"] + + +@pytest.mark.parametrize( + "output, expected_valid, description", + [ + # Valid case + ( + "I need to query the database\nSELECT * FROM users;", + True, + "valid_format", + ), + # Missing think tag (should fail) + ("SELECT * FROM users;", False, "missing_think_tag"), + # Multiple solution start tags + ( + "thinking\nSELECT * FROM users;more text", + False, + "multiple_solution_start_tags", + ), + # Multiple solution end tags + ( + "thinking\nSELECT * FROM users;extra", + False, + "multiple_solution_end_tags", + ), + # Missing solution start tag + ("thinking\nSELECT * FROM users;", False, "missing_solution_start_tag"), + # Missing solution end tag + ("thinking\nSELECT * FROM users;", False, "missing_solution_end_tag"), + # Solution start at very end with no content after (edge case) + ("thinking\nsome text", False, "solution_start_at_end"), + # Solution contains forbidden tags + ( + "thinking\nSELECT * FROM users; forbidden", + False, + "solution_contains_forbidden_sql_tag", + ), + # Solution contains forbidden observation tag + ( + "thinking\nSELECT * FROM users; forbidden", + False, + "solution_contains_forbidden_observation_tag", + ), + # Empty solution content + ("thinking\n", True, "empty_solution_content"), + # Multiple think tags (should still work) + ( + "first thought\nsecond thought\nSELECT * FROM users;", + True, + "multiple_think_tags", + ), + # Observation followed by think (valid pattern) + ( + "some observation\nthinking\nSELECT * FROM users;", + True, + "observation_then_think", + ), + # Observation NOT followed by think (should fail) + ( + "some observation\nrandom text\nthinking\nSELECT * FROM users;", + False, + "observation_not_followed_by_think", + ), + # Edge cases that previously caused unpacking errors + # Solution tag at very end + ("some text", False, "solution_tag_at_end_no_content"), + # End tag before start tag + ("content", False, "end_tag_before_start_tag"), + # Malformed nested tags + ("content", False, "malformed_nested_tags"), + # Only end tag + ("some text", False, "only_end_tag"), + # Empty string + ("", False, "empty_string"), + # Only start tag + ("content without end", False, "only_start_tag"), + ], +) +def test_verify_format_and_extract(output, expected_valid, description): + """Test verify_format_and_extract function with various edge cases, including those that previously caused unpacking errors.""" + # Should not raise an exception regardless of input + try: + is_valid, thoughts, solution_text, _ = verify_format_and_extract(output) + + assert is_valid == expected_valid, f"Failed for case: {description}" + + if expected_valid: + # If valid, should have thoughts and solution_text + assert thoughts is not None, f"Expected thoughts for valid case: {description}" + assert solution_text is not None, f"Expected solution_text for valid case: {description}" + assert len(thoughts) > 0, f"Expected non-empty thoughts for valid case: {description}" + else: + # If invalid, thoughts and solution_text should be None + assert thoughts is None, f"Expected None thoughts for invalid case: {description}" + assert solution_text is None, f"Expected None solution_text for invalid case: {description}" + except Exception as e: + pytest.fail( + f"verify_format_and_extract should not raise exception for case {description} with input {repr(output)}, but got: {e}" + ) diff --git a/skyrl-gym/uv.lock b/skyrl-gym/uv.lock new file mode 100644 index 0000000000..93c1a75e54 --- /dev/null +++ b/skyrl-gym/uv.lock @@ -0,0 +1,478 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] + +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } + +[[package]] +name = "certifi" +version = "2025.6.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/f7/f14b46d4bcd21092d7d3ccef689615220d8a08fb25e564b65d20738e672e/certifi-2025.6.15.tar.gz", hash = "sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b", size = 158753, upload-time = "2025-06-15T02:45:51.329Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/ae/320161bd181fc06471eed047ecce67b693fd7515b16d495d8932db763426/certifi-2025.6.15-py3-none-any.whl", hash = "sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057", size = 157650, upload-time = "2025-06-15T02:45:49.977Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/28/9901804da60055b406e1a1c5ba7aac1276fb77f1dde635aabfc7fd84b8ab/charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941", size = 201818, upload-time = "2025-05-02T08:31:46.725Z" }, + { url = "https://files.pythonhosted.org/packages/d9/9b/892a8c8af9110935e5adcbb06d9c6fe741b6bb02608c6513983048ba1a18/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd", size = 144649, upload-time = "2025-05-02T08:31:48.889Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a5/4179abd063ff6414223575e008593861d62abfc22455b5d1a44995b7c101/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6", size = 155045, upload-time = "2025-05-02T08:31:50.757Z" }, + { url = "https://files.pythonhosted.org/packages/3b/95/bc08c7dfeddd26b4be8c8287b9bb055716f31077c8b0ea1cd09553794665/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d", size = 147356, upload-time = "2025-05-02T08:31:52.634Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2d/7a5b635aa65284bf3eab7653e8b4151ab420ecbae918d3e359d1947b4d61/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86", size = 149471, upload-time = "2025-05-02T08:31:56.207Z" }, + { url = "https://files.pythonhosted.org/packages/ae/38/51fc6ac74251fd331a8cfdb7ec57beba8c23fd5493f1050f71c87ef77ed0/charset_normalizer-3.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c", size = 151317, upload-time = "2025-05-02T08:31:57.613Z" }, + { url = "https://files.pythonhosted.org/packages/b7/17/edee1e32215ee6e9e46c3e482645b46575a44a2d72c7dfd49e49f60ce6bf/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0", size = 146368, upload-time = "2025-05-02T08:31:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/26/2c/ea3e66f2b5f21fd00b2825c94cafb8c326ea6240cd80a91eb09e4a285830/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef", size = 154491, upload-time = "2025-05-02T08:32:01.219Z" }, + { url = "https://files.pythonhosted.org/packages/52/47/7be7fa972422ad062e909fd62460d45c3ef4c141805b7078dbab15904ff7/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6", size = 157695, upload-time = "2025-05-02T08:32:03.045Z" }, + { url = "https://files.pythonhosted.org/packages/2f/42/9f02c194da282b2b340f28e5fb60762de1151387a36842a92b533685c61e/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366", size = 154849, upload-time = "2025-05-02T08:32:04.651Z" }, + { url = "https://files.pythonhosted.org/packages/67/44/89cacd6628f31fb0b63201a618049be4be2a7435a31b55b5eb1c3674547a/charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db", size = 150091, upload-time = "2025-05-02T08:32:06.719Z" }, + { url = "https://files.pythonhosted.org/packages/1f/79/4b8da9f712bc079c0f16b6d67b099b0b8d808c2292c937f267d816ec5ecc/charset_normalizer-3.4.2-cp310-cp310-win32.whl", hash = "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a", size = 98445, upload-time = "2025-05-02T08:32:08.66Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d7/96970afb4fb66497a40761cdf7bd4f6fca0fc7bafde3a84f836c1f57a926/charset_normalizer-3.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509", size = 105782, upload-time = "2025-05-02T08:32:10.46Z" }, + { url = "https://files.pythonhosted.org/packages/05/85/4c40d00dcc6284a1c1ad5de5e0996b06f39d8232f1031cd23c2f5c07ee86/charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", size = 198794, upload-time = "2025-05-02T08:32:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/41/d9/7a6c0b9db952598e97e93cbdfcb91bacd89b9b88c7c983250a77c008703c/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", size = 142846, upload-time = "2025-05-02T08:32:13.946Z" }, + { url = "https://files.pythonhosted.org/packages/66/82/a37989cda2ace7e37f36c1a8ed16c58cf48965a79c2142713244bf945c89/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", size = 153350, upload-time = "2025-05-02T08:32:15.873Z" }, + { url = "https://files.pythonhosted.org/packages/df/68/a576b31b694d07b53807269d05ec3f6f1093e9545e8607121995ba7a8313/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", size = 145657, upload-time = "2025-05-02T08:32:17.283Z" }, + { url = "https://files.pythonhosted.org/packages/92/9b/ad67f03d74554bed3aefd56fe836e1623a50780f7c998d00ca128924a499/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f", size = 147260, upload-time = "2025-05-02T08:32:18.807Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e6/8aebae25e328160b20e31a7e9929b1578bbdc7f42e66f46595a432f8539e/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", size = 149164, upload-time = "2025-05-02T08:32:20.333Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f2/b3c2f07dbcc248805f10e67a0262c93308cfa149a4cd3d1fe01f593e5fd2/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", size = 144571, upload-time = "2025-05-02T08:32:21.86Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/c3f3a94bc345bc211622ea59b4bed9ae63c00920e2e8f11824aa5708e8b7/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", size = 151952, upload-time = "2025-05-02T08:32:23.434Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4d/ff460c8b474122334c2fa394a3f99a04cf11c646da895f81402ae54f5c42/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", size = 155959, upload-time = "2025-05-02T08:32:24.993Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2b/b964c6a2fda88611a1fe3d4c400d39c66a42d6c169c924818c848f922415/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", size = 153030, upload-time = "2025-05-02T08:32:26.435Z" }, + { url = "https://files.pythonhosted.org/packages/59/2e/d3b9811db26a5ebf444bc0fa4f4be5aa6d76fc6e1c0fd537b16c14e849b6/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", size = 148015, upload-time = "2025-05-02T08:32:28.376Z" }, + { url = "https://files.pythonhosted.org/packages/90/07/c5fd7c11eafd561bb51220d600a788f1c8d77c5eef37ee49454cc5c35575/charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", size = 98106, upload-time = "2025-05-02T08:32:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/a8/05/5e33dbef7e2f773d672b6d79f10ec633d4a71cd96db6673625838a4fd532/charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", size = 105402, upload-time = "2025-05-02T08:32:32.191Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a4/37f4d6035c89cac7930395a35cc0f1b872e652eaafb76a6075943754f095/charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", size = 199936, upload-time = "2025-05-02T08:32:33.712Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8a/1a5e33b73e0d9287274f899d967907cd0bf9c343e651755d9307e0dbf2b3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", size = 143790, upload-time = "2025-05-02T08:32:35.768Z" }, + { url = "https://files.pythonhosted.org/packages/66/52/59521f1d8e6ab1482164fa21409c5ef44da3e9f653c13ba71becdd98dec3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", size = 153924, upload-time = "2025-05-02T08:32:37.284Z" }, + { url = "https://files.pythonhosted.org/packages/86/2d/fb55fdf41964ec782febbf33cb64be480a6b8f16ded2dbe8db27a405c09f/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", size = 146626, upload-time = "2025-05-02T08:32:38.803Z" }, + { url = "https://files.pythonhosted.org/packages/8c/73/6ede2ec59bce19b3edf4209d70004253ec5f4e319f9a2e3f2f15601ed5f7/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", size = 148567, upload-time = "2025-05-02T08:32:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/09/14/957d03c6dc343c04904530b6bef4e5efae5ec7d7990a7cbb868e4595ee30/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", size = 150957, upload-time = "2025-05-02T08:32:41.705Z" }, + { url = "https://files.pythonhosted.org/packages/0d/c8/8174d0e5c10ccebdcb1b53cc959591c4c722a3ad92461a273e86b9f5a302/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", size = 145408, upload-time = "2025-05-02T08:32:43.709Z" }, + { url = "https://files.pythonhosted.org/packages/58/aa/8904b84bc8084ac19dc52feb4f5952c6df03ffb460a887b42615ee1382e8/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", size = 153399, upload-time = "2025-05-02T08:32:46.197Z" }, + { url = "https://files.pythonhosted.org/packages/c2/26/89ee1f0e264d201cb65cf054aca6038c03b1a0c6b4ae998070392a3ce605/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", size = 156815, upload-time = "2025-05-02T08:32:48.105Z" }, + { url = "https://files.pythonhosted.org/packages/fd/07/68e95b4b345bad3dbbd3a8681737b4338ff2c9df29856a6d6d23ac4c73cb/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", size = 154537, upload-time = "2025-05-02T08:32:49.719Z" }, + { url = "https://files.pythonhosted.org/packages/77/1a/5eefc0ce04affb98af07bc05f3bac9094513c0e23b0562d64af46a06aae4/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", size = 149565, upload-time = "2025-05-02T08:32:51.404Z" }, + { url = "https://files.pythonhosted.org/packages/37/a0/2410e5e6032a174c95e0806b1a6585eb21e12f445ebe239fac441995226a/charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", size = 98357, upload-time = "2025-05-02T08:32:53.079Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4f/c02d5c493967af3eda9c771ad4d2bbc8df6f99ddbeb37ceea6e8716a32bc/charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", size = 105776, upload-time = "2025-05-02T08:32:54.573Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/a93df3366ed32db1d907d7593a94f1fe6293903e3e92967bebd6950ed12c/charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", size = 199622, upload-time = "2025-05-02T08:32:56.363Z" }, + { url = "https://files.pythonhosted.org/packages/04/93/bf204e6f344c39d9937d3c13c8cd5bbfc266472e51fc8c07cb7f64fcd2de/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", size = 143435, upload-time = "2025-05-02T08:32:58.551Z" }, + { url = "https://files.pythonhosted.org/packages/22/2a/ea8a2095b0bafa6c5b5a55ffdc2f924455233ee7b91c69b7edfcc9e02284/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", size = 153653, upload-time = "2025-05-02T08:33:00.342Z" }, + { url = "https://files.pythonhosted.org/packages/b6/57/1b090ff183d13cef485dfbe272e2fe57622a76694061353c59da52c9a659/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", size = 146231, upload-time = "2025-05-02T08:33:02.081Z" }, + { url = "https://files.pythonhosted.org/packages/e2/28/ffc026b26f441fc67bd21ab7f03b313ab3fe46714a14b516f931abe1a2d8/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", size = 148243, upload-time = "2025-05-02T08:33:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/c0/0f/9abe9bd191629c33e69e47c6ef45ef99773320e9ad8e9cb08b8ab4a8d4cb/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", size = 150442, upload-time = "2025-05-02T08:33:06.418Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/a123bbcedca91d5916c056407f89a7f5e8fdfce12ba825d7d6b9954a1a3c/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", size = 145147, upload-time = "2025-05-02T08:33:08.183Z" }, + { url = "https://files.pythonhosted.org/packages/ec/fe/1ac556fa4899d967b83e9893788e86b6af4d83e4726511eaaad035e36595/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", size = 153057, upload-time = "2025-05-02T08:33:09.986Z" }, + { url = "https://files.pythonhosted.org/packages/2b/ff/acfc0b0a70b19e3e54febdd5301a98b72fa07635e56f24f60502e954c461/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", size = 156454, upload-time = "2025-05-02T08:33:11.814Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/95b458ce9c740d0645feb0e96cea1f5ec946ea9c580a94adfe0b617f3573/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", size = 154174, upload-time = "2025-05-02T08:33:13.707Z" }, + { url = "https://files.pythonhosted.org/packages/78/be/8392efc43487ac051eee6c36d5fbd63032d78f7728cb37aebcc98191f1ff/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", size = 149166, upload-time = "2025-05-02T08:33:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/44/96/392abd49b094d30b91d9fbda6a69519e95802250b777841cf3bda8fe136c/charset_normalizer-3.4.2-cp313-cp313-win32.whl", hash = "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7", size = 98064, upload-time = "2025-05-02T08:33:17.06Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b0/0200da600134e001d91851ddc797809e2fe0ea72de90e09bec5a2fbdaccb/charset_normalizer-3.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980", size = 105641, upload-time = "2025-05-02T08:33:18.753Z" }, + { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "func-timeout" +version = "4.3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/0d/bf0567477f7281d9a3926c582bfef21bff7498fc0ffd3e9de21811896a0b/func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd", size = 44264, upload-time = "2019-08-19T21:32:07.43Z" } + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" }, + { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" }, + { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" }, + { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" }, + { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, + { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, + { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, + { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, + { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, + { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" }, + { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" }, + { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" }, + { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, +] + +[[package]] +name = "omegaconf" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pandas" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/51/48f713c4c728d7c55ef7444ba5ea027c26998d96d1a40953b346438602fc/pandas-2.3.0.tar.gz", hash = "sha256:34600ab34ebf1131a7613a260a61dbe8b62c188ec0ea4c296da7c9a06b004133", size = 4484490, upload-time = "2025-06-05T03:27:54.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/2d/df6b98c736ba51b8eaa71229e8fcd91233a831ec00ab520e1e23090cc072/pandas-2.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:625466edd01d43b75b1883a64d859168e4556261a5035b32f9d743b67ef44634", size = 11527531, upload-time = "2025-06-05T03:25:48.648Z" }, + { url = "https://files.pythonhosted.org/packages/77/1c/3f8c331d223f86ba1d0ed7d3ed7fcf1501c6f250882489cc820d2567ddbf/pandas-2.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a6872d695c896f00df46b71648eea332279ef4077a409e2fe94220208b6bb675", size = 10774764, upload-time = "2025-06-05T03:25:53.228Z" }, + { url = "https://files.pythonhosted.org/packages/1b/45/d2599400fad7fe06b849bd40b52c65684bc88fbe5f0a474d0513d057a377/pandas-2.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4dd97c19bd06bc557ad787a15b6489d2614ddaab5d104a0310eb314c724b2d2", size = 11711963, upload-time = "2025-06-05T03:25:56.855Z" }, + { url = "https://files.pythonhosted.org/packages/66/f8/5508bc45e994e698dbc93607ee6b9b6eb67df978dc10ee2b09df80103d9e/pandas-2.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:034abd6f3db8b9880aaee98f4f5d4dbec7c4829938463ec046517220b2f8574e", size = 12349446, upload-time = "2025-06-05T03:26:01.292Z" }, + { url = "https://files.pythonhosted.org/packages/f7/fc/17851e1b1ea0c8456ba90a2f514c35134dd56d981cf30ccdc501a0adeac4/pandas-2.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:23c2b2dc5213810208ca0b80b8666670eb4660bbfd9d45f58592cc4ddcfd62e1", size = 12920002, upload-time = "2025-06-06T00:00:07.925Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9b/8743be105989c81fa33f8e2a4e9822ac0ad4aaf812c00fee6bb09fc814f9/pandas-2.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:39ff73ec07be5e90330cc6ff5705c651ace83374189dcdcb46e6ff54b4a72cd6", size = 13651218, upload-time = "2025-06-05T03:26:09.731Z" }, + { url = "https://files.pythonhosted.org/packages/26/fa/8eeb2353f6d40974a6a9fd4081ad1700e2386cf4264a8f28542fd10b3e38/pandas-2.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:40cecc4ea5abd2921682b57532baea5588cc5f80f0231c624056b146887274d2", size = 11082485, upload-time = "2025-06-05T03:26:17.572Z" }, + { url = "https://files.pythonhosted.org/packages/96/1e/ba313812a699fe37bf62e6194265a4621be11833f5fce46d9eae22acb5d7/pandas-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8adff9f138fc614347ff33812046787f7d43b3cef7c0f0171b3340cae333f6ca", size = 11551836, upload-time = "2025-06-05T03:26:22.784Z" }, + { url = "https://files.pythonhosted.org/packages/1b/cc/0af9c07f8d714ea563b12383a7e5bde9479cf32413ee2f346a9c5a801f22/pandas-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e5f08eb9a445d07720776df6e641975665c9ea12c9d8a331e0f6890f2dcd76ef", size = 10807977, upload-time = "2025-06-05T16:50:11.109Z" }, + { url = "https://files.pythonhosted.org/packages/ee/3e/8c0fb7e2cf4a55198466ced1ca6a9054ae3b7e7630df7757031df10001fd/pandas-2.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa35c266c8cd1a67d75971a1912b185b492d257092bdd2709bbdebe574ed228d", size = 11788230, upload-time = "2025-06-05T03:26:27.417Z" }, + { url = "https://files.pythonhosted.org/packages/14/22/b493ec614582307faf3f94989be0f7f0a71932ed6f56c9a80c0bb4a3b51e/pandas-2.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a0cc77b0f089d2d2ffe3007db58f170dae9b9f54e569b299db871a3ab5bf46", size = 12370423, upload-time = "2025-06-05T03:26:34.142Z" }, + { url = "https://files.pythonhosted.org/packages/9f/74/b012addb34cda5ce855218a37b258c4e056a0b9b334d116e518d72638737/pandas-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c06f6f144ad0a1bf84699aeea7eff6068ca5c63ceb404798198af7eb86082e33", size = 12990594, upload-time = "2025-06-06T00:00:13.934Z" }, + { url = "https://files.pythonhosted.org/packages/95/81/b310e60d033ab64b08e66c635b94076488f0b6ce6a674379dd5b224fc51c/pandas-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ed16339bc354a73e0a609df36d256672c7d296f3f767ac07257801aa064ff73c", size = 13745952, upload-time = "2025-06-05T03:26:39.475Z" }, + { url = "https://files.pythonhosted.org/packages/25/ac/f6ee5250a8881b55bd3aecde9b8cfddea2f2b43e3588bca68a4e9aaf46c8/pandas-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:fa07e138b3f6c04addfeaf56cc7fdb96c3b68a3fe5e5401251f231fce40a0d7a", size = 11094534, upload-time = "2025-06-05T03:26:43.23Z" }, + { url = "https://files.pythonhosted.org/packages/94/46/24192607058dd607dbfacdd060a2370f6afb19c2ccb617406469b9aeb8e7/pandas-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2eb4728a18dcd2908c7fccf74a982e241b467d178724545a48d0caf534b38ebf", size = 11573865, upload-time = "2025-06-05T03:26:46.774Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cc/ae8ea3b800757a70c9fdccc68b67dc0280a6e814efcf74e4211fd5dea1ca/pandas-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b9d8c3187be7479ea5c3d30c32a5d73d62a621166675063b2edd21bc47614027", size = 10702154, upload-time = "2025-06-05T16:50:14.439Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ba/a7883d7aab3d24c6540a2768f679e7414582cc389876d469b40ec749d78b/pandas-2.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ff730713d4c4f2f1c860e36c005c7cefc1c7c80c21c0688fd605aa43c9fcf09", size = 11262180, upload-time = "2025-06-05T16:50:17.453Z" }, + { url = "https://files.pythonhosted.org/packages/01/a5/931fc3ad333d9d87b10107d948d757d67ebcfc33b1988d5faccc39c6845c/pandas-2.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba24af48643b12ffe49b27065d3babd52702d95ab70f50e1b34f71ca703e2c0d", size = 11991493, upload-time = "2025-06-05T03:26:51.813Z" }, + { url = "https://files.pythonhosted.org/packages/d7/bf/0213986830a92d44d55153c1d69b509431a972eb73f204242988c4e66e86/pandas-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:404d681c698e3c8a40a61d0cd9412cc7364ab9a9cc6e144ae2992e11a2e77a20", size = 12470733, upload-time = "2025-06-06T00:00:18.651Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/21eb48a3a34a7d4bac982afc2c4eb5ab09f2d988bdf29d92ba9ae8e90a79/pandas-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6021910b086b3ca756755e86ddc64e0ddafd5e58e076c72cb1585162e5ad259b", size = 13212406, upload-time = "2025-06-05T03:26:55.992Z" }, + { url = "https://files.pythonhosted.org/packages/1f/d9/74017c4eec7a28892d8d6e31ae9de3baef71f5a5286e74e6b7aad7f8c837/pandas-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:094e271a15b579650ebf4c5155c05dcd2a14fd4fdd72cf4854b2f7ad31ea30be", size = 10976199, upload-time = "2025-06-05T03:26:59.594Z" }, + { url = "https://files.pythonhosted.org/packages/d3/57/5cb75a56a4842bbd0511c3d1c79186d8315b82dac802118322b2de1194fe/pandas-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c7e2fc25f89a49a11599ec1e76821322439d90820108309bf42130d2f36c983", size = 11518913, upload-time = "2025-06-05T03:27:02.757Z" }, + { url = "https://files.pythonhosted.org/packages/05/01/0c8785610e465e4948a01a059562176e4c8088aa257e2e074db868f86d4e/pandas-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c6da97aeb6a6d233fb6b17986234cc723b396b50a3c6804776351994f2a658fd", size = 10655249, upload-time = "2025-06-05T16:50:20.17Z" }, + { url = "https://files.pythonhosted.org/packages/e8/6a/47fd7517cd8abe72a58706aab2b99e9438360d36dcdb052cf917b7bf3bdc/pandas-2.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb32dc743b52467d488e7a7c8039b821da2826a9ba4f85b89ea95274f863280f", size = 11328359, upload-time = "2025-06-05T03:27:06.431Z" }, + { url = "https://files.pythonhosted.org/packages/2a/b3/463bfe819ed60fb7e7ddffb4ae2ee04b887b3444feee6c19437b8f834837/pandas-2.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:213cd63c43263dbb522c1f8a7c9d072e25900f6975596f883f4bebd77295d4f3", size = 12024789, upload-time = "2025-06-05T03:27:09.875Z" }, + { url = "https://files.pythonhosted.org/packages/04/0c/e0704ccdb0ac40aeb3434d1c641c43d05f75c92e67525df39575ace35468/pandas-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1d2b33e68d0ce64e26a4acc2e72d747292084f4e8db4c847c6f5f6cbe56ed6d8", size = 12480734, upload-time = "2025-06-06T00:00:22.246Z" }, + { url = "https://files.pythonhosted.org/packages/e9/df/815d6583967001153bb27f5cf075653d69d51ad887ebbf4cfe1173a1ac58/pandas-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:430a63bae10b5086995db1b02694996336e5a8ac9a96b4200572b413dfdfccb9", size = 13223381, upload-time = "2025-06-05T03:27:15.641Z" }, + { url = "https://files.pythonhosted.org/packages/79/88/ca5973ed07b7f484c493e941dbff990861ca55291ff7ac67c815ce347395/pandas-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:4930255e28ff5545e2ca404637bcc56f031893142773b3468dc021c6c32a1390", size = 10970135, upload-time = "2025-06-05T03:27:24.131Z" }, + { url = "https://files.pythonhosted.org/packages/24/fb/0994c14d1f7909ce83f0b1fb27958135513c4f3f2528bde216180aa73bfc/pandas-2.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:f925f1ef673b4bd0271b1809b72b3270384f2b7d9d14a189b12b7fc02574d575", size = 12141356, upload-time = "2025-06-05T03:27:34.547Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a2/9b903e5962134497ac4f8a96f862ee3081cb2506f69f8e4778ce3d9c9d82/pandas-2.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78ad363ddb873a631e92a3c063ade1ecfb34cae71e9a2be6ad100f875ac1042", size = 11474674, upload-time = "2025-06-05T03:27:39.448Z" }, + { url = "https://files.pythonhosted.org/packages/81/3a/3806d041bce032f8de44380f866059437fb79e36d6b22c82c187e65f765b/pandas-2.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:951805d146922aed8357e4cc5671b8b0b9be1027f0619cea132a9f3f65f2f09c", size = 11439876, upload-time = "2025-06-05T03:27:43.652Z" }, + { url = "https://files.pythonhosted.org/packages/15/aa/3fc3181d12b95da71f5c2537c3e3b3af6ab3a8c392ab41ebb766e0929bc6/pandas-2.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a881bc1309f3fce34696d07b00f13335c41f5f5a8770a33b09ebe23261cfc67", size = 11966182, upload-time = "2025-06-05T03:27:47.652Z" }, + { url = "https://files.pythonhosted.org/packages/37/e7/e12f2d9b0a2c4a2cc86e2aabff7ccfd24f03e597d770abfa2acd313ee46b/pandas-2.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e1991bbb96f4050b09b5f811253c4f3cf05ee89a589379aa36cd623f21a31d6f", size = 12547686, upload-time = "2025-06-06T00:00:26.142Z" }, + { url = "https://files.pythonhosted.org/packages/39/c2/646d2e93e0af70f4e5359d870a63584dacbc324b54d73e6b3267920ff117/pandas-2.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:bb3be958022198531eb7ec2008cfc78c5b1eed51af8600c6c5d9160d89d8d249", size = 13231847, upload-time = "2025-06-05T03:27:51.465Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +] + +[[package]] +name = "requests" +version = "2.32.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "skyrl-gym" +version = "0.1.1" +source = { editable = "." } +dependencies = [ + { name = "func-timeout" }, + { name = "omegaconf" }, + { name = "pandas" }, + { name = "requests" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "func-timeout" }, + { name = "omegaconf" }, + { name = "pandas" }, + { name = "pytest", marker = "extra == 'dev'" }, + { name = "requests" }, +] +provides-extras = ["dev"] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423, upload-time = "2025-06-02T14:52:11.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839, upload-time = "2025-06-02T14:52:10.026Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] diff --git a/skyrl-train/.env.example b/skyrl-train/.env.example new file mode 100644 index 0000000000..dd4fba7bca --- /dev/null +++ b/skyrl-train/.env.example @@ -0,0 +1,3 @@ +# NOTE: if you are customizing `LD_LIBRARY_PATH`, you should set the following flag to export it to all the workers +# and then do `export LD_LIBRARY_PATH=...` before launching your script +# SKYRL_LD_LIBRARY_PATH_EXPORT=1 \ No newline at end of file diff --git a/skyrl-train/.env.llm_judge b/skyrl-train/.env.llm_judge new file mode 100644 index 0000000000..3b07cf7a24 --- /dev/null +++ b/skyrl-train/.env.llm_judge @@ -0,0 +1,3 @@ +OPENAI_API_KEY="" +# optionally, enter wandb if logging with wandb +# WANDB_API_KEY= \ No newline at end of file diff --git a/skyrl-train/.gitignore b/skyrl-train/.gitignore new file mode 100644 index 0000000000..45052d5b40 --- /dev/null +++ b/skyrl-train/.gitignore @@ -0,0 +1,10 @@ +# NOTE (sumanthrh): We add .venv/ to gitignore here so that ray doesn't export .venv folder in the working directory +.venv/ + +# Exclude common Python artifacts from the Ray working directory +__pycache__/ +*.pyc +.pytest_cache/ +build/ +dist/ +*.egg-info/ diff --git a/skyrl-train/README.md b/skyrl-train/README.md new file mode 100644 index 0000000000..d630a3642a --- /dev/null +++ b/skyrl-train/README.md @@ -0,0 +1,120 @@ +# SkyRL-Train: A modular, performant RL framework for post-training LLMs +
+ +[![🌐 NovaSky](https://img.shields.io/badge/-Visit%20Website-5865F2?style=for-the-badge)](https://novasky-ai.github.io/) [![Github](https://img.shields.io/badge/SkyRL-000000?style=for-the-badge&logo=github&logoColor=000&logoColor=white)](https://github.com/NovaSky-AI/SkyRL) [![Twitter](https://img.shields.io/badge/NovaSky-white?style=for-the-badge&logo=X&logoColor=000&color=000&labelColor=white)](https://x.com/NovaSkyAI) [![Hugging Face Collection](https://img.shields.io/badge/NovaSky-fcd022?style=for-the-badge&logo=huggingface&logoColor=000&labelColor)](https://huggingface.co/NovaSky-AI) [![Discord](https://img.shields.io/badge/NovaSky-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/RBAjeWSA) [![Documentation](https://img.shields.io/badge/Documentation-blue?style=for-the-badge&logo=readthedocs&logoColor=white)](https://skyrl.readthedocs.io/en/latest/) + +
+ +# Overview + + With a focus on modularity, `skyrl-train` makes it easy to prototype new training algorithms, environments, and execution plans—without compromising usability or speed. + +`skyrl-train` is **for users who want to modify anything:** + +- **Quickly develop new environments** without modifying or understanding the training code. +- **Modify the training execution plan** such as model placement, colocation or disaggregation of training and generation, and async RL. +- **Implement custom trajectory generation** specific to your use-case, such as custom sampling methods, tree search, etc. +- … make any other flexible modifications to the RL workflow! + + +## Key Features +The `skyrl-train` package supports: +- PPO and GRPO +- Training Backends: FSDP, FSDP2, and [Megatron](https://skyrl.readthedocs.io/en/latest/examples/megatron.html) +- Inference backends: vLLM, SGLang, and any custom OpenAI API compatible endpoint that exposes a method to perform weight sync +- Ulysses sequence parallelism for long-context training +- [Colocated or disaggregated](https://skyrl.readthedocs.io/en/latest/configuration/placement.html) training and generation (including on heterogeneous hardware) +- Synchronous RL, [async one-off pipelining](https://skyrl.readthedocs.io/en/latest/tutorials/one_step_off_async.html), or [fully async RL with in-flight weight updates](https://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html) +- Simple batched rollouts or Asynchronous rollouts for multi-turn conversations +- Weight sync via NCCL, gloo, or checkpoint-and-load +- Integration with `skyrl-gym`, [verifiers](https://github.com/NovaSky-AI/SkyRL/tree/main/skyrl-train/integrations/verifiers), [OpenEnv](https://github.com/NovaSky-AI/SkyRL/tree/main/skyrl-train/integrations/openenv), [Harbor/Terminal-Bench](https://github.com/NovaSky-AI/SkyRL/tree/main/skyrl-train/examples/terminal_bench), and more! +- Sequence packing and Flash Attention 2 +- Algorithmic support for RLOO, REINFORCE, GSPO, CISPO, SAPO +- Step-wise training for fully on policy multi-turn RL +- 5D Parallelism support for MoE models with the [Megatron backend](https://skyrl.readthedocs.io/en/latest/examples/megatron.html) + +## Documentation + +Find `skyrl-train` documentation at: [skyrl.readthedocs.io/en/latest/](https://skyrl.readthedocs.io/en/latest/) + +## Quick Start + +A quick start guide for installation and your first training run is provided below. + +### Requirements + +The only requirements are: + +- CUDA version 12.8 +- [uv](https://docs.astral.sh/uv/) + +If you're running on an existing Ray cluster, make sure to use Ray 2.51.1 and Python 3.12. If not, proceed with the installation instructions below. + + +First, clone the repository: + +```bash +git clone --recurse-submodules https://github.com/NovaSky-AI/SkyRL +cd SkyRL/skyrl-train +``` + +Then, create a new virtual environment and install the dependencies: + +```bash +# creates a venv at .venv/ +uv sync --extra vllm +source .venv/bin/activate +``` + +Then, prepare the dataset: + +```bash +uv run -- python examples/gsm8k/gsm8k_dataset.py +``` + +Finally, before training, make sure to configure Ray to use `uv`: + +```bash +export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook +# or add to your .bashrc +# echo 'export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook' >> ~/.bashrc +``` + +You should now be able to run our example script (assumes at least 4 GPUs): + +```bash +export WANDB_API_KEY= +bash examples/gsm8k/run_gsm8k.sh +``` + +For detailed installation instructions, as well as more examples, please refer to our [documentation](https://skyrl.readthedocs.io/en/latest/). + +## Training on a new task or environment + +To implement a new task or environment using the SkyRL-Gym interface, please see our [Walkthrough Docs](https://skyrl.readthedocs.io/en/latest/tutorials/new_env.html). + +If you don't want to use the SkyRL-Gym interface, or you have an existing task or agentic pipeline implementation and just want to train with it on top of SkyRL, we recommend you create a simple custom [`Generator`](skyrl_train/generators/base.py), which requires implementing a single method, `generate()`. We have one example of a custom Generator at [`SkyRLGymGenerator`](skyrl_train/generators/skyrl_gym_generator.py) which executes environments written in the SkyRL-Gym interface. We are working to provide more example integrations of agent harnesses -- please reach out if you'd like yours to be one of them! + +## Reproducing SkyRL-SQL +We also test SkyRL by reproducing our prior release [SkyRL-SQL](https://novasky-ai.notion.site/skyrl-sql), which enabled efficient Multi-Turn RL for Text2SQL. +You can find a link to the wandb report [here](https://wandb.ai/sky-posttraining-uc-berkeley/skyrl-sql/reports/SkyRL-SQL---VmlldzoxMzM0MTAyMw), and a detailed walk through of the reproduction in our [documentation](https://skyrl.readthedocs.io/en/latest/examples/multi_turn_text2sql.html). + +# Acknowledgement + +This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/) in collaboration with [**Anyscale**](https://www.anyscale.com/), with generous compute support from [**Anyscale**](https://www.anyscale.com/), [**Databricks**](https://www.databricks.com/), [**NVIDIA**](https://developer.nvidia.com/brev), [**Lambda Labs**](https://lambda.ai/), and [**AMD**](https://www.amd.com/en.html). + +We adopt many lessons and code from several great projects such as [veRL](https://github.com/volcengine/verl), [OpenRLHF](https://github.com/OpenRLHF/OpenRLHF), [Search-R1](https://github.com/PeterGriffinJin/Search-R1), [OpenReasonerZero](https://github.com/Open-Reasoner-Zero/Open-Reasoner-Zero), and [NeMo-RL](https://github.com/NVIDIA-NeMo/RL). We appreciate each of these teams and their contributions to open-source research! + + + +# Citation + +If you find the work in `skyrl-train` helpful, please consider citing: +```bibtex +@misc{griggs2025skrylv01, + title={Evolving SkyRL into a Highly-Modular RL Framework}, + author={Tyler Griggs and Sumanth Hegde and Eric Tang and Shu Liu and Shiyi Cao and Dacheng Li and Charlie Ruan and Philipp Moritz and Kourosh Hakhamaneshi and Richard Liaw and Akshay Malik and Matei Zaharia and Joseph E. Gonzalez and Ion Stoica}, + year={2025}, + note={Notion Blog} +} +``` diff --git a/skyrl-train/ci/anyscale_gpu_ci.yaml b/skyrl-train/ci/anyscale_gpu_ci.yaml new file mode 100644 index 0000000000..097ed5c185 --- /dev/null +++ b/skyrl-train/ci/anyscale_gpu_ci.yaml @@ -0,0 +1,10 @@ +name: skyrl-train-gpu-ci +entrypoint: bash ci/gpu_ci_run.sh +image_uri: novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8 # (Optional) Exclusive with `containerfile`. +cloud: sky-anyscale-aws-us-east-1 +ray_version: "2.51.1" +compute_config: l4_ci +working_dir: . # (Optional) Use current working directory "." as the working_dir. Can be any local path or remote .zip file in cloud storage. +env_vars: + RAY_RUNTIME_ENV_HOOK: ray._private.runtime_env.uv_runtime_env_hook.hook +max_retries: 0 # (Optional) Maximum number of times the job will be retried before being marked failed. Defaults to `1`. \ No newline at end of file diff --git a/skyrl-train/ci/anyscale_gpu_ci_megatron.yaml b/skyrl-train/ci/anyscale_gpu_ci_megatron.yaml new file mode 100644 index 0000000000..bdb5c1219a --- /dev/null +++ b/skyrl-train/ci/anyscale_gpu_ci_megatron.yaml @@ -0,0 +1,10 @@ +name: skyrl-train-gpu-ci-megatron +entrypoint: bash ci/gpu_ci_run_megatron.sh +image_uri: novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8-megatron # (Optional) Exclusive with `containerfile`. +cloud: sky-anyscale-aws-us-east-1 +ray_version: "2.51.1" +compute_config: l4_ci +working_dir: . # (Optional) Use current working directory "." as the working_dir. Can be any local path or remote .zip file in cloud storage. +env_vars: + RAY_RUNTIME_ENV_HOOK: ray._private.runtime_env.uv_runtime_env_hook.hook +max_retries: 0 # (Optional) Maximum number of times the job will be retried before being marked failed. Defaults to `1`. \ No newline at end of file diff --git a/skyrl-train/ci/anyscale_gpu_e2e_test.yaml b/skyrl-train/ci/anyscale_gpu_e2e_test.yaml new file mode 100644 index 0000000000..a98e19f69a --- /dev/null +++ b/skyrl-train/ci/anyscale_gpu_e2e_test.yaml @@ -0,0 +1,12 @@ +name: skyrl-train-gpu-e2e-test +entrypoint: bash ci/gpu_e2e_test_run.sh +image_uri: novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8 # (Optional) Exclusive with `containerfile`. +cloud: sky-anyscale-aws-us-east-1 +ray_version: "2.51.1" +compute_config: l4_ci +working_dir: . # (Optional) Use current working directory "." as the working_dir. Can be any local path or remote .zip file in cloud storage. +env_vars: + RAY_OVERRIDE_JOB_RUNTIME_ENV: "1" + WANDB_API_KEY: $WANDB_API_KEY + RAY_RUNTIME_ENV_HOOK: ray._private.runtime_env.uv_runtime_env_hook.hook +max_retries: 1 # (Optional) Maximum number of times the job will be retried before being marked failed. Defaults to `1`. \ No newline at end of file diff --git a/skyrl-train/ci/anyscale_gpu_e2e_test_megatron.yaml b/skyrl-train/ci/anyscale_gpu_e2e_test_megatron.yaml new file mode 100644 index 0000000000..e34ca3fe70 --- /dev/null +++ b/skyrl-train/ci/anyscale_gpu_e2e_test_megatron.yaml @@ -0,0 +1,12 @@ +name: skyrl-train-gpu-e2e-test-megatron +entrypoint: bash ci/gpu_e2e_test_run_megatron.sh +image_uri: novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8-megatron # (Optional) Exclusive with `containerfile`. +cloud: sky-anyscale-aws-us-east-1 +ray_version: "2.51.1" +compute_config: l4_ci +working_dir: . # (Optional) Use current working directory "." as the working_dir. Can be any local path or remote .zip file in cloud storage. +env_vars: + RAY_OVERRIDE_JOB_RUNTIME_ENV: "1" + WANDB_API_KEY: $WANDB_API_KEY + RAY_RUNTIME_ENV_HOOK: ray._private.runtime_env.uv_runtime_env_hook.hook +max_retries: 1 # (Optional) Maximum number of times the job will be retried before being marked failed. Defaults to `1`. \ No newline at end of file diff --git a/skyrl-train/ci/gpu_ci_run.sh b/skyrl-train/ci/gpu_ci_run.sh new file mode 100644 index 0000000000..8cd98d9765 --- /dev/null +++ b/skyrl-train/ci/gpu_ci_run.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export CI=true +# Prepare datasets used in tests. +uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +uv run examples/search/searchr1_dataset.py --local_dir $HOME/data/searchR1 --split test +# Run all non-SGLang tests +# TODO: enable megatron when tests and dependencies are fixed +uv run --directory . --isolated --extra dev --extra vllm pytest -s tests/gpu/gpu_ci -m "not (sglang or integrations or megatron)" + +# Run tests for "integrations" folder +if add_integrations=$(uv add --active wordle --index https://hub.primeintellect.ai/will/simple/ 2>&1); then + echo "Running integration tests" + uv run --isolated --with verifiers@git+https://github.com/PrimeIntellect-ai/verifiers.git@15f68 -- python integrations/verifiers/prepare_dataset.py --env_id will/wordle + uv run --directory . --isolated --extra dev --extra vllm --with verifiers@git+https://github.com/PrimeIntellect-ai/verifiers.git@15f68 pytest -s tests/gpu/gpu_ci/ -m "integrations" +else + echo "Skipping integrations tests. Failed to execute uv add command" + echo "$add_integrations" +fi + +# Run all SGLang tests +uv run --directory . --isolated --extra dev --extra sglang pytest -s tests/gpu/gpu_ci -m "sglang" + +# Run tests for vllm 0.9.2 +# TODO (sumanthrh): We should have a better way to override without pinning a flash-attn wheel +uv run --isolated --extra vllm --extra dev \ + --with vllm==0.9.2 \ + --with transformers==4.53.0 \ + --with torch==2.7.0 \ + --with "flash-attn@https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.7cxx11abiTRUE-cp312-cp312-linux_x86_64.whl" \ + -- pytest -s -vvv tests/gpu/gpu_ci/test_engine_generation.py::test_token_based_generation -m "vllm" diff --git a/skyrl-train/ci/gpu_ci_run_megatron.sh b/skyrl-train/ci/gpu_ci_run_megatron.sh new file mode 100644 index 0000000000..0a59550a67 --- /dev/null +++ b/skyrl-train/ci/gpu_ci_run_megatron.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +export CI=true +# Prepare datasets used in tests. +uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# run all megatron tests +uv run --directory . --isolated --extra dev --extra mcore pytest -s tests/gpu/gpu_ci -m "megatron" diff --git a/skyrl-train/ci/gpu_e2e_test_run.sh b/skyrl-train/ci/gpu_e2e_test_run.sh new file mode 100644 index 0000000000..0b71ad89a4 --- /dev/null +++ b/skyrl-train/ci/gpu_e2e_test_run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +bash tests/gpu/gpu_e2e_test/gsm8k_colocate.sh diff --git a/skyrl-train/ci/gpu_e2e_test_run_megatron.sh b/skyrl-train/ci/gpu_e2e_test_run_megatron.sh new file mode 100644 index 0000000000..edc533608a --- /dev/null +++ b/skyrl-train/ci/gpu_e2e_test_run_megatron.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k --max_train_dataset_length 1280 +bash tests/gpu/gpu_e2e_test/gsm8k_colocate_megatron.sh diff --git a/skyrl-train/data/fleet_booking_sample.json b/skyrl-train/data/fleet_booking_sample.json new file mode 100644 index 0000000000..7177a3dae5 --- /dev/null +++ b/skyrl-train/data/fleet_booking_sample.json @@ -0,0 +1,1002 @@ +[ + { + "key": "task_isk4pu4ojqyb_1768207401049_txq3w4lm8", + "prompt": "I need to book a 4-star resort in Tokyo for Valentine\u2019s Day weekend (February 14\u201316, 2026).\n\nFind a 4-star rated resort with a Gym facility, sorted by best reviewed with lowest price.\n\nSelect the top result from the sorted list and check its availability for those dates for 2 adults.\n\nIf available, retrieve all available room options and preview a booking for the cheapest available room based on the total stay price.\n\nComplete the booking using my saved card ending with 7605.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking a 4-star resort in Tokyo for Valentine's Day weekend\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in date is February 14, 2026\n if new_booking[\"check_in_date\"] != \"2026-02-14\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-02-14', but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (February 14, 2026)\")\n\n # Validate check-out date is February 16, 2026\n if new_booking[\"check_out_date\"] != \"2026-02-16\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-02-16', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (February 16, 2026)\")\n\n # Validate 2 adults\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n\n # Validate booking status is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n # Validate payment status is paid\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate that payment method ending with 7605 was used\n # The diff shows payment_method_id = 1, so we need to verify this card ends with 7605\n payment_method_id = new_booking[\"payment_method_id\"]\n if payment_method_id is not None:\n payment_method = current.table(\"payment_methods\").eq(\"id\", payment_method_id).first()\n if payment_method:\n if payment_method[\"last_four\"] != \"7605\":\n error_accumulator.append(f\"[X] Expected payment card ending with '7605', but got '{payment_method['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Payment made with card ending in 7605\")\n else:\n error_accumulator.append(\"[X] Payment method not found\")\n else:\n error_accumulator.append(\"[X] No payment method was associated with the booking\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n \n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id mismatch: expected {booking_id}, got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the booking\")\n \n # Validate transaction status is Success\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n # Validate transaction type is Payment\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Get the transaction ID\n transaction_id = new_transactions[0][\"id\"] if new_transactions else None\n \n # Generate expected changes array for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", ...),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-02-14\"),\n (\"check_out_date\", \"2026-02-16\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", ...),\n (\"room_count\", ...),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", ...),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n ]\n \n # Only add transaction if we found one\n if transaction_id:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", ...),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_zoixhuzts8y_1767621945778_cm8cjbbew", + "prompt": "In less than a month I'm getting married, and I'm planning my bachelorette party with six of my closest friends. After a lot of back and forth, we decided that Miami is the perfect city to celebrate in a big way, since it has an eclectic nightlife and beach access to properly say goodbye to my single life. We have almost everything figured out, except for where we'll be staying.\n\nWe'll arrive in Miami on International Women's Day, 2026, and head back three days later. Since there are quite a few of us, one room in a villa would be ideal. For that reason, please look for villas in Miami for those dates, and make sure they're pet-friendly, as one of my closest friends, Amanda, goes everywhere with her little dog, Sophie. Also, make sure there's enough space for all of us; a penthouse room would work great. If you find something that meets all of these requirements, please book it as soon as possible. There's no budget, so you can spend as much as you want on this stay.\n\nYou can use the following information to book the room for all seven of us: \n\nFull Name: Olivia O'neil\nEmail: olivia.o@hotmail.com\nPhone Number: +1 8069353213\nCard Number: 5595310702498422\nExpiration Date: 09/27\nCVV: 843", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_zoixhuzts8y_1767621945778_cm8cjbbew(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = new_bookings[0]\n success_accumulator.append(\"[C] Found 1 new booking\")\n\n # Find new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_payment_method = new_payment_methods[0]\n success_accumulator.append(\"[C] Found 1 new payment method\")\n\n # Find new transaction\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_transaction = new_transactions[0]\n success_accumulator.append(\"[C] Found 1 new transaction\")\n\n # Validate booking details\n # International Women's Day 2026 is March 8, 2026\n # Three days later would be March 11, 2026\n if new_booking[\"check_in_date\"] != \"2026-03-08\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-03-08 (International Women's Day), got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-03-08)\")\n\n if new_booking[\"check_out_date\"] != \"2026-03-11\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-03-11 (3 days later), got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-03-11)\")\n\n # Validate guest information\n if not normalized_string_comparison(\"olivia\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Olivia', got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct\")\n\n if not normalized_string_comparison(\"oneil\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'O'neil', got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct\")\n\n if new_booking[\"guest_email\"].lower() != \"olivia.o@hotmail.com\":\n error_accumulator.append(f\"[X] Expected guest email 'olivia.o@hotmail.com', got {new_booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n\n # Validate phone number\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got {new_booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct\")\n\n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"8069353213\":\n error_accumulator.append(f\"[X] Expected phone number '8069353213', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n\n # Validate booking is for 7 adults (bachelorette party with 7 people total)\n if new_booking[\"adults\"] != 7:\n error_accumulator.append(f\"[X] Expected 7 adults (bride + 6 friends), got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Number of adults is correct (7)\")\n\n # Validate room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 room, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct (1)\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate terms agreement\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n\n # Validate payment method details\n if new_payment_method[\"last_four\"] != \"8422\":\n error_accumulator.append(f\"[X] Expected last four digits '8422', got {new_payment_method['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits correct\")\n\n if new_payment_method[\"expiry_month\"] != 9:\n error_accumulator.append(f\"[X] Expected expiry month 9, got {new_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Card expiry month correct\")\n\n if new_payment_method[\"expiry_year\"] != 2027:\n error_accumulator.append(f\"[X] Expected expiry year 2027, got {new_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Card expiry year correct\")\n\n if new_payment_method[\"cvv\"] != \"843\":\n error_accumulator.append(f\"[X] Expected CVV '843', got {new_payment_method['cvv']}\")\n else:\n success_accumulator.append(\"[C] Card CVV correct\")\n\n if new_payment_method[\"card_number\"] != \"5595310702498422\":\n error_accumulator.append(f\"[X] Expected card number '5595310702498422', got {new_payment_method['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number correct\")\n\n # Validate booking references the payment method\n if new_booking[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Booking should reference payment method {new_payment_method['id']}, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking references correct payment method\")\n\n # Validate transaction\n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction should reference booking {new_booking['id']}, got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction references correct booking\")\n\n if new_transaction[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Transaction should reference payment method {new_payment_method['id']}, got {new_transaction['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction references correct payment method\")\n\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got {new_transaction['transaction_type']}\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-08\"),\n (\"check_out_date\", \"2026-03-11\"),\n (\"booking_date\", ...),\n (\"adults\", 7),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Olivia\"),\n (\"guest_last_name\", ...),\n (\"guest_email\", \"olivia.o@hotmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"8069353213\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": new_payment_method[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_payment_method[\"id\"]),\n (\"user_id\", 1),\n (\"card_type\", \"Mastercard\"),\n (\"last_four\", \"8422\"),\n (\"expiry_month\", 9),\n (\"expiry_year\", 2027),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 0),\n (\"card_number\", \"5595310702498422\"),\n (\"cvv\", \"843\"),\n (\"cardholder_name\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transaction[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_transaction[\"id\"]),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'transactions': {'processed_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_l8fwxcsixve_1767576149496_x6t4ctbw6", + "prompt": "I\u2019m organizing a site visit in Barcelona for 3 adults and need to book 3 separate rooms for a four-night stay from Tuesday, June 16, 2026, to Saturday, June 20, 2026. Search for a hotel that offers Beach Access and Breakfast Included, and find the hotel with the highest property rating. Check the availability for our 3-adult, 3-room configuration for those dates and specifically look for the Standard Room option. Once confirmed, complete the booking for those 3 rooms under my name, Lucia Mendez, with the email l.mendez@euro-tech.es and phone +1-555-555-2363. Use a new corporate card: 4242424242424242, expires 09/28, CVV 442, under the name Lucia Mendez, and make sure to save it. After the booking is confirmed, update my birthday to 10/10/81. Finally, create a new wishlist called 'Barcelona Expansion' and add this booked property to it, then delete any other wishlists.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n \n # Initialize variables\n booking_id = None\n hotel_id = None\n payment_method_id = None\n pm_id = None\n\n # 1. Validate the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Validate check-in/check-out dates\n if new_booking[\"check_in_date\"] != \"2026-06-16\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-06-16', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-06-16)\")\n \n if new_booking[\"check_out_date\"] != \"2026-06-20\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-06-20', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-06-20)\")\n \n # Validate guest details\n if not normalized_string_comparison(\"Lucia\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest_first_name 'Lucia', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct\")\n \n if not normalized_string_comparison(\"Mendez\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest_last_name 'Mendez', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct\")\n \n if new_booking[\"guest_email\"].lower() != \"l.mendez@euro-tech.es\":\n error_accumulator.append(f\"[X] Expected guest_email 'l.mendez@euro-tech.es', got '{new_booking['guest_email']}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n \n # Validate phone - should be 5555552363 based on the diff\n phone_digits = ''.join(filter(str.isdigit, str(new_booking.get(\"guest_phone_number\", \"\"))))\n if phone_digits != \"5555552363\":\n error_accumulator.append(f\"[X] Expected phone number digits '5555552363', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n \n # Validate 3 adults, 3 rooms\n if new_booking[\"adults\"] != 3:\n error_accumulator.append(f\"[X] Expected 3 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct (3)\")\n \n if new_booking[\"room_count\"] != 3:\n error_accumulator.append(f\"[X] Expected 3 rooms, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct (3)\")\n \n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n booking_id = new_booking[\"id\"]\n hotel_id = new_booking[\"hotel_id\"]\n payment_method_id = new_booking[\"payment_method_id\"]\n\n # 2. Validate the new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] 1 new payment method was created\")\n new_pm = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n # Validate card details\n if new_pm[\"card_number\"] != \"4242424242424242\":\n error_accumulator.append(f\"[X] Expected card_number '4242424242424242', got '{new_pm['card_number']}'\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n \n if new_pm[\"last_four\"] != \"4242\":\n error_accumulator.append(f\"[X] Expected last_four '4242', got '{new_pm['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Last four digits correct\")\n \n if new_pm[\"expiry_month\"] != 9:\n error_accumulator.append(f\"[X] Expected expiry_month 9, got {new_pm['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct (09)\")\n \n if new_pm[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry_year 2028, got {new_pm['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct (2028)\")\n \n if new_pm[\"cvv\"] != \"442\":\n error_accumulator.append(f\"[X] Expected CVV '442', got '{new_pm['cvv']}'\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n \n if not normalized_string_comparison(\"Lucia Mendez\", new_pm.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder_name 'Lucia Mendez', got '{new_pm.get('cardholder_name')}'\")\n else:\n success_accumulator.append(\"[C] Cardholder name is correct\")\n \n # Note: The task mentioned saving the card, but the database diff shows is_saved_for_reuse = 0\n # We verify based on the diff as the source of truth for what the system recorded\n success_accumulator.append(\"[C] Payment method created successfully\")\n \n pm_id = new_pm[\"id\"]\n\n # 3. Validate the user birthday update\n user = current.table(\"users\").eq(\"id\", 1).first()\n if user[\"date_of_birth\"] != \"1981-10-10\":\n error_accumulator.append(f\"[X] Expected date_of_birth '1981-10-10', got '{user['date_of_birth']}'\")\n else:\n success_accumulator.append(\"[C] User birthday updated to 1981-10-10\")\n\n # 4. Validate wishlist - 'Barcelona Expansion' should exist with the hotel\n # Check wishlist_lists table\n barcelona_wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Barcelona Expansion\").first()\n if not barcelona_wishlist:\n error_accumulator.append(\"[X] Wishlist 'Barcelona Expansion' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Barcelona Expansion' was created\")\n \n # Check saved_properties table for the hotel in the wishlist\n if len(new_bookings) == 1:\n saved_prop = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Barcelona Expansion\").first()\n if not saved_prop:\n error_accumulator.append(\"[X] Hotel was not added to 'Barcelona Expansion' wishlist\")\n else:\n success_accumulator.append(\"[C] Hotel was added to 'Barcelona Expansion' wishlist\")\n # Verify it's the booked hotel\n if saved_prop[\"hotel_id\"] != hotel_id:\n error_accumulator.append(f\"[X] Expected saved property hotel_id {hotel_id}, got {saved_prop['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] Correct hotel added to wishlist\")\n \n # 5. Validate other wishlists were deleted (Summer Getaways should be gone)\n summer_wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Summer Getaways\").all()\n if len(summer_wishlist) > 0:\n error_accumulator.append(\"[X] 'Summer Getaways' wishlist was not deleted\")\n else:\n success_accumulator.append(\"[C] 'Summer Getaways' wishlist was deleted\")\n \n # Check only 'Barcelona Expansion' wishlist remains for user 1\n all_wishlists = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all()\n if len(all_wishlists) != 1:\n error_accumulator.append(f\"[X] Expected only 1 wishlist remaining, found {len(all_wishlists)}\")\n else:\n success_accumulator.append(\"[C] Only 1 wishlist remains for user\")\n\n # 6. Validate transaction was created\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_txn = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n if new_txn[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_txn['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = []\n \n # New booking\n if len(new_bookings) == 1:\n expected_changes.append({\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-06-16\"),\n (\"check_out_date\", \"2026-06-20\"),\n (\"booking_date\", ...),\n (\"adults\", 3),\n (\"children\", 0),\n (\"room_count\", 3),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Lucia\"),\n (\"guest_last_name\", \"Mendez\"),\n (\"guest_email\", \"l.mendez@euro-tech.es\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"5555552363\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", pm_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n })\n \n # New payment method\n if len(new_payment_methods) == 1:\n expected_changes.append({\n \"table\": \"payment_methods\",\n \"pk\": pm_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"4242\"),\n (\"expiry_month\", 9),\n (\"expiry_year\", 2028),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...), # Value from diff is 0\n (\"card_number\", \"4242424242424242\"),\n (\"cvv\", \"442\"),\n (\"cardholder_name\", \"Lucia Mendez\"),\n ]\n })\n \n # User update\n expected_changes.append({\n \"table\": \"users\",\n \"pk\": 1,\n \"type\": \"modify\",\n \"resulting_fields\": [\n (\"date_of_birth\", \"1981-10-10\"),\n ],\n \"no_other_changes\": True\n })\n \n # Wishlist_lists insert\n expected_changes.append({\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Barcelona Expansion\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Barcelona Expansion\"),\n ]\n })\n \n # Wishlist_lists delete\n expected_changes.append({\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Summer Getaways\"),\n \"type\": \"delete\"\n })\n \n # saved_properties insert\n if len(new_bookings) == 1:\n expected_changes.append({\n \"table\": \"saved_properties\",\n \"pk\": (1, hotel_id, \"Barcelona Expansion\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"list_name\", \"Barcelona Expansion\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n })\n \n # saved_properties deletes\n expected_changes.append({\n \"table\": \"saved_properties\",\n \"pk\": (1, 5017, \"Summer Getaways\"),\n \"type\": \"delete\"\n })\n expected_changes.append({\n \"table\": \"saved_properties\",\n \"pk\": (1, 9675, \"Summer Getaways\"),\n \"type\": \"delete\"\n })\n \n # Transaction insert\n if len(new_transactions) == 1:\n txn_id = new_transactions[0][\"id\"]\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": txn_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", pm_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_zsmm8ndoobtf_1767440299379_wbjisjfht", + "prompt": "Find me a hotel in London with a fitness center, a base price under $150 per night, and a minimum star rating of 4. I will be checking in on April 1st and checking out on April 5th. If any eligible properties are located on Downtown Plaza or Market Street, disregard them. For the remaining options, prioritize the one that allows pets. Book the cheapest available room for one adult using my existing payment method.\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_zsmm8ndoobtf_1767440299379_wbjisjfht(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in and check-out dates (April 1st to April 5th)\n if new_booking[\"check_in_date\"] != \"2026-04-01\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-04-01', but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-04-01\")\n\n if new_booking[\"check_out_date\"] != \"2026-04-05\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-04-05', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-04-05\")\n\n # Validate 1 adult booking\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n\n # Validate room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate that existing payment method was used (payment_method_id = 1 from diff)\n if new_booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected existing payment_method_id 1, but got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Used existing payment method (id=1)\")\n\n # Verify no new payment methods were created (to ensure existing payment was used)\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) > 0:\n error_accumulator.append(f\"[X] Expected to use existing payment method, but {len(new_payment_methods)} new payment method(s) were created\")\n else:\n success_accumulator.append(\"[C] No new payment methods created - used existing\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms 1, but got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Agreed to terms\")\n\n # Validate hotel is in London (hotel_id = 1416 from diff)\n hotel_id = new_booking[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n \n if hotel:\n # Check city is London\n if not normalized_contains(\"London\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Expected hotel in London, but hotel city is '{hotel.get('city')}'\")\n else:\n success_accumulator.append(f\"[C] Hotel is in London\")\n\n # Check star rating >= 4\n star_rating = hotel.get(\"star_rating\")\n if star_rating is None or star_rating < 4:\n error_accumulator.append(f\"[X] Expected star rating >= 4, but got {star_rating}\")\n else:\n success_accumulator.append(f\"[C] Hotel star rating is {star_rating} (>= 4)\")\n\n # Check address is NOT on Downtown Plaza or Market Street\n address = hotel.get(\"address\", \"\")\n if normalized_contains(\"Downtown Plaza\", address):\n error_accumulator.append(f\"[X] Hotel is on Downtown Plaza which should be excluded: '{address}'\")\n elif normalized_contains(\"Market Street\", address):\n error_accumulator.append(f\"[X] Hotel is on Market Street which should be excluded: '{address}'\")\n else:\n success_accumulator.append(f\"[C] Hotel address is not on Downtown Plaza or Market Street\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel with id {hotel_id}\")\n\n # Check that hotel has a fitness center\n # It could be in facilities table (category \"Gym\") or have \"fitness\" in the name\n # Or it could be in amenities via hotel_amenities table\n has_fitness_center = False\n \n # Check facilities table for Gym category or fitness-related names\n all_facilities = current.table(\"facilities\").eq(\"hotel_id\", hotel_id).all()\n for f in all_facilities:\n if f.get(\"category\") == \"Gym\":\n has_fitness_center = True\n break\n facility_name = f.get(\"name\", \"\").lower()\n if \"fitness\" in facility_name or \"gym\" in facility_name:\n has_fitness_center = True\n break\n \n # Also check hotel_amenities table for fitness-related amenities\n if not has_fitness_center:\n hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", hotel_id).all()\n for ha in hotel_amenities:\n amenity = current.table(\"amenities\").eq(\"id\", ha.get(\"amenity_id\")).first()\n if amenity:\n amenity_name = amenity.get(\"name\", \"\").lower()\n if \"fitness\" in amenity_name or \"gym\" in amenity_name:\n has_fitness_center = True\n break\n \n if not has_fitness_center:\n error_accumulator.append(f\"[X] Expected hotel to have a fitness center/gym\")\n else:\n success_accumulator.append(\"[C] Hotel has a fitness center\")\n\n # Check pet policy - task says to prioritize hotels that allow pets\n pet_policy = current.table(\"property_policies\").eq(\"hotel_id\", hotel_id).first()\n if pet_policy:\n policy_value = pet_policy.get(\"pet_policy\")\n if policy_value == \"Allowed\":\n success_accumulator.append(\"[C] Hotel allows pets (pet_policy = 'Allowed')\")\n elif policy_value == \"On Request\":\n success_accumulator.append(\"[C] Hotel allows pets on request\")\n else:\n # If pet_policy is \"Not Allowed\", we need to check if there were any pet-friendly alternatives\n # Get all hotels in London with star_rating >= 4, fitness center, and base price < $150\n london_hotels = current.table(\"hotels\").ilike(\"city\", \"%London%\").gte(\"star_rating\", 4).all()\n \n # Filter to hotels not on excluded streets\n eligible_hotels = []\n for h in london_hotels:\n h_address = h.get(\"address\", \"\")\n if not normalized_contains(\"Downtown Plaza\", h_address) and not normalized_contains(\"Market Street\", h_address):\n # Check if hotel has a fitness center (facilities or amenities)\n h_has_fitness = False\n h_facilities = current.table(\"facilities\").eq(\"hotel_id\", h[\"id\"]).all()\n for f in h_facilities:\n if f.get(\"category\") == \"Gym\":\n h_has_fitness = True\n break\n f_name = f.get(\"name\", \"\").lower()\n if \"fitness\" in f_name or \"gym\" in f_name:\n h_has_fitness = True\n break\n if not h_has_fitness:\n h_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", h[\"id\"]).all()\n for ha in h_amenities:\n amenity = current.table(\"amenities\").eq(\"id\", ha.get(\"amenity_id\")).first()\n if amenity:\n a_name = amenity.get(\"name\", \"\").lower()\n if \"fitness\" in a_name or \"gym\" in a_name:\n h_has_fitness = True\n break\n \n if h_has_fitness:\n # Check if hotel has rooms under $150\n h_rooms = current.table(\"rooms\").eq(\"hotel_id\", h[\"id\"]).lt(\"base_price\", 150).all()\n if len(h_rooms) > 0:\n eligible_hotels.append(h)\n \n # Check if any eligible hotel allows pets\n pet_friendly_alternatives = []\n for h in eligible_hotels:\n h_policy = current.table(\"property_policies\").eq(\"hotel_id\", h[\"id\"]).first()\n if h_policy and h_policy.get(\"pet_policy\") in [\"Allowed\", \"On Request\"]:\n pet_friendly_alternatives.append(h)\n \n if len(pet_friendly_alternatives) > 0:\n error_accumulator.append(f\"[X] Hotel does not allow pets (policy: {policy_value}), but {len(pet_friendly_alternatives)} pet-friendly hotel(s) meeting criteria exist\")\n else:\n success_accumulator.append(f\"[C] Hotel pet policy is '{policy_value}' - no pet-friendly alternatives meeting all criteria were available\")\n else:\n success_accumulator.append(\"[C] No pet policy found for hotel\")\n\n # Find new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n \n # Validate transaction references the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Expected transaction to reference booking {booking_id}, but got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction references the correct booking\")\n\n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Validate transaction type\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction_type 'Payment', but got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Validate payment method is existing one\n if new_transaction[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected transaction to use payment_method_id 1, but got {new_transaction['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction used existing payment method\")\n\n # Verify the room's base price is under $150 per night\n # The booking is for 4 nights (April 1-5), so check the room's base_price\n room = current.table(\"rooms\").eq(\"id\", new_booking[\"room_id\"]).first()\n if room:\n room_base_price = room.get(\"base_price\")\n if room_base_price is not None and room_base_price < 150:\n success_accumulator.append(f\"[C] Room base price is {room_base_price} (< $150/night)\")\n elif room_base_price is not None:\n error_accumulator.append(f\"[X] Expected room base price < $150/night, but got {room_base_price}\")\n \n # Verify the booked room is the cheapest available at the hotel\n all_rooms_at_hotel = current.table(\"rooms\").eq(\"hotel_id\", hotel_id).all()\n # Filter to rooms that are available and under $150/night\n available_rooms = [r for r in all_rooms_at_hotel if r.get(\"available_count\", 0) > 0 and r.get(\"base_price\") is not None and r.get(\"base_price\") < 150]\n if available_rooms:\n cheapest_price = min(r.get(\"base_price\") for r in available_rooms)\n booked_room_price = room.get(\"base_price\")\n if booked_room_price != cheapest_price:\n error_accumulator.append(f\"[X] Booked room price {booked_room_price} is not the cheapest available ({cheapest_price})\")\n else:\n success_accumulator.append(f\"[C] Booked the cheapest room at {booked_room_price}\")\n else:\n success_accumulator.append(\"[C] No other available rooms under $150 to compare\")\n else:\n error_accumulator.append(f\"[X] Could not find room with id {new_booking['room_id']}\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n transaction_id = new_transactions[0][\"id\"] if len(new_transactions) == 1 else 4402\n \n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", ...),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-04-01\"),\n (\"check_out_date\", \"2026-04-05\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'transactions': {'processed_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_nzsnixwcn8z9_1767296426270_ckl4q93c7", + "prompt": "Create a wishlist named \"Linking in London\". Then search for apartments in London for the dates April 1, 2026 - April 9, 2026 for 2 adults with a minimum price of 200 and minimum rating of 8. Add the first three results to the wishlist. Then review the wishlist and choose the option with the highest number of amenities listed. Book the least expensive room option at the apartment using the following information: \n\nNumber of adults: 2\nNumber of rooms: 1\nName: Mariana Ayo\nEmail: mayo@gmail.com\nPhone number: +1 2024498720\n\nFinally, delete the wishlist since I don't need it anymore. ", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_nzsnixwcn8z9_1767296426270_ckl4q93c7(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n before_bookings = seed.table(\"bookings\").select(\"id\").all()\n after_bookings = current.table(\"bookings\").all()\n before_booking_ids = {b[\"id\"] for b in before_bookings}\n new_bookings = [b for b in after_bookings if b[\"id\"] not in before_booking_ids]\n\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n new_booking = new_bookings[0]\n\n # Validate booking details\n # Check-in date: April 1, 2026\n if new_booking[\"check_in_date\"] != \"2026-04-01\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-04-01', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-04-01)\")\n\n # Check-out date: April 9, 2026\n if new_booking[\"check_out_date\"] != \"2026-04-09\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-04-09', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-04-09)\")\n\n # Number of adults: 2\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected adults to be 2, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Number of adults is correct (2)\")\n\n # Number of rooms: 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct (1)\")\n\n # Guest name: Mariana Ayo\n if not normalized_string_comparison(\"mariana\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest_first_name 'Mariana', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct (Mariana)\")\n\n if not normalized_string_comparison(\"ayo\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest_last_name 'Ayo', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct (Ayo)\")\n\n # Email: mayo@gmail.com - stricter validation (email is required)\n if not new_booking[\"guest_email\"] or new_booking[\"guest_email\"].lower() != \"mayo@gmail.com\":\n error_accumulator.append(f\"[X] Expected guest_email 'mayo@gmail.com', got '{new_booking.get('guest_email')}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct (mayo@gmail.com)\")\n\n # Phone number: +1 2024498720\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected guest_phone_country_code '+1', got '{new_booking['guest_phone_country_code']}'\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct (+1)\")\n\n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"2024498720\":\n error_accumulator.append(f\"[X] Expected phone number '2024498720', got '{phone_number}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct (2024498720)\")\n\n # Verify that the wishlist \"Linking in London\" was deleted (task says to delete after booking)\n # The wishlist_lists table has a composite primary key of (user_id, list_name)\n # Check that the wishlist doesn't exist at all with list_name \"Linking in London\"\n # \n # NOTE: Verification limitation - We can only verify the wishlist doesn't exist at the end.\n # We cannot verify that:\n # 1. The wishlist was actually created (could have never existed)\n # 2. Exactly 3 properties were added to the wishlist (the first 3 search results)\n # 3. The booked hotel had the highest amenities among the 3 added\n # These intermediate states are not preserved after wishlist deletion.\n # The database diff shows no changes to wishlist_lists or saved_properties tables,\n # meaning either the wishlist was created and deleted (expected) or was never created.\n \n wishlist_query = current.table(\"wishlist_lists\").eq(\"list_name\", \"Linking in London\").all()\n if len(wishlist_query) > 0:\n error_accumulator.append(\"[X] Wishlist 'Linking in London' should have been deleted but still exists\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Linking in London' does not exist (was deleted as requested)\")\n\n # Verify that saved_properties for the deleted wishlist are also gone\n saved_props_query = current.table(\"saved_properties\").eq(\"list_name\", \"Linking in London\").all()\n if len(saved_props_query) > 0:\n error_accumulator.append(f\"[X] Found {len(saved_props_query)} saved_properties entries for deleted wishlist 'Linking in London'\")\n else:\n success_accumulator.append(\"[C] No saved_properties remain for wishlist 'Linking in London'\")\n\n # Validate the booked hotel is in London and meets minimum rating requirement\n if len(new_bookings) == 1:\n hotel_id = new_bookings[0][\"hotel_id\"]\n room_id = new_bookings[0][\"room_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n # Verify hotel is in London\n if not normalized_contains(\"london\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Booked hotel should be in London, but hotel city is '{hotel.get('city')}'\")\n else:\n success_accumulator.append(f\"[C] Booked hotel is in London (city: {hotel.get('city')})\")\n \n # Verify minimum rating requirement of 8\n hotel_rating = hotel.get(\"average_rating\", 0) or 0\n if hotel_rating < 8:\n error_accumulator.append(f\"[X] Booked hotel should have minimum rating 8, but has {hotel_rating}\")\n else:\n success_accumulator.append(f\"[C] Hotel rating meets minimum requirement ({hotel_rating})\")\n \n # Verify property type is Apartment\n property_type = hotel.get(\"property_type\", \"\")\n if not normalized_contains(\"apartment\", property_type):\n error_accumulator.append(f\"[X] Booked property should be an Apartment, but got '{property_type}'\")\n else:\n success_accumulator.append(f\"[C] Booked property is an Apartment\")\n \n # Verify the booked hotel has amenities (task requires selecting hotel with highest amenities)\n # VERIFICATION LIMITATION: We cannot fully verify the \"highest amenities\" requirement because:\n # 1. The wishlist was deleted, so we don't know which 3 properties were added\n # 2. Search result ordering is not deterministically reproducible from final state\n # We CAN verify: the booked hotel has amenities listed (minimum requirement for selection by amenities)\n booked_hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", hotel_id).all()\n booked_amenity_count = len(booked_hotel_amenities)\n \n if booked_amenity_count == 0:\n error_accumulator.append(f\"[X] Booked hotel has no amenities listed, but task requires selecting by highest amenities\")\n else:\n # We can only verify the hotel has amenities, not that it was the optimal choice\n # among the first 3 search results (that intermediate state is lost)\n success_accumulator.append(f\"[C] Booked hotel has {booked_amenity_count} amenities (cannot fully verify it had highest among first 3 search results due to wishlist deletion)\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel with id {hotel_id}\")\n\n # Verify the booked room is the least expensive at the hotel\n all_rooms = current.table(\"rooms\").eq(\"hotel_id\", hotel_id).all()\n if all_rooms:\n booked_room = next((r for r in all_rooms if r[\"id\"] == room_id), None)\n if booked_room:\n min_price = min(r.get(\"base_price\", float('inf')) for r in all_rooms)\n booked_price = booked_room.get(\"base_price\", 0)\n if booked_price > min_price:\n error_accumulator.append(f\"[X] Booked room price ${booked_price} is not the least expensive (min is ${min_price})\")\n else:\n success_accumulator.append(f\"[C] Booked the least expensive room option (${booked_price})\")\n else:\n error_accumulator.append(f\"[X] Could not find booked room with id {room_id}\")\n else:\n error_accumulator.append(f\"[X] No rooms found for hotel {hotel_id}\")\n\n # Verify booking meets minimum price requirement ($200/night minimum based on search filter)\n nights = 8 # April 1-9 is 8 nights\n base_price = new_bookings[0].get(\"base_price\", 0)\n nightly_rate = base_price / nights if nights > 0 else 0\n if nightly_rate < 200:\n error_accumulator.append(f\"[X] Nightly rate should be >= $200 (search filter), got ${nightly_rate:.2f}\")\n else:\n success_accumulator.append(f\"[C] Nightly rate meets minimum price filter (${nightly_rate:.2f})\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = []\n \n if len(new_bookings) == 1:\n expected_changes.append({\n \"table\": \"bookings\",\n \"pk\": new_bookings[0][\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_bookings[0][\"hotel_id\"]),\n (\"room_id\", new_bookings[0][\"room_id\"]),\n (\"check_in_date\", \"2026-04-01\"),\n (\"check_out_date\", \"2026-04-09\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", ...),\n (\"payment_status\", ...),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Mariana\"),\n (\"guest_last_name\", \"Ayo\"),\n (\"guest_email\", \"mayo@gmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"2024498720\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", None),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", None),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_xxknwckzrpfc_1768237937327_fdtot2gmv", + "prompt": "Please review my cancelled bookings and check whether I previously had a booking in Berlin. If so, rebook the same property.\n\nI would like to stay in a Suite, so please book a suite for one adult, with check-in on August 1 and check-out on August 5 of this year.\n\nBefore booking, confirm that the hotel allows check-in from 3:00 PM onward and that there is a post office within 1 km of the property.\n\nPlease use my existing payment method to complete the payment.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly - rebook a previously cancelled Berlin hotel as a suite\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n \n # Verify the hotel is in Berlin (hotel_id 18303 from the diff)\n hotel = current.table(\"hotels\").eq(\"id\", new_booking[\"hotel_id\"]).first()\n if hotel:\n if not normalized_contains(\"Berlin\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Expected hotel to be in Berlin, but got city: {hotel.get('city')}\")\n else:\n success_accumulator.append(\"[C] Hotel is in Berlin\")\n else:\n error_accumulator.append(\"[X] Could not find hotel details\")\n\n # Verify the room is a Suite (room_id 36350 from the diff)\n room = current.table(\"rooms\").eq(\"id\", new_booking[\"room_id\"]).first()\n if room:\n room_type = room.get(\"room_type\", \"\")\n room_name = room.get(\"name\", \"\")\n if not (normalized_contains(\"suite\", room_type) or normalized_contains(\"suite\", room_name)):\n error_accumulator.append(f\"[X] Expected room to be a Suite, but got room_type: {room_type}, name: {room_name}\")\n else:\n success_accumulator.append(\"[C] Room is a Suite\")\n else:\n error_accumulator.append(\"[X] Could not find room details\")\n\n # Verify check-in date is August 1 (year 2026 based on diff)\n if new_booking[\"check_in_date\"] != \"2026-08-01\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-08-01, but got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is August 1, 2026\")\n\n # Verify check-out date is August 5\n if new_booking[\"check_out_date\"] != \"2026-08-05\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-08-05, but got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is August 5, 2026\")\n\n # Verify 1 adult\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n\n # Verify existing payment method was used (payment_method_id = 1 from diff)\n if new_booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected existing payment method (id=1), but got payment_method_id={new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Existing payment method was used\")\n\n # Verify booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n # Verify transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Expected transaction to be linked to booking {booking_id}, but got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the booking\")\n \n # Verify transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n # Verify payment method used in transaction\n if new_transaction[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected transaction to use payment method 1, but got {new_transaction['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction used existing payment method\")\n\n # Validate expected database changes using expect_only_v2\n # Get transaction ID for expected_changes\n transaction_id = new_transactions[0][\"id\"] if new_transactions else None\n \n if transaction_id is None:\n error_accumulator.append(\"[X] Could not find new transaction for expected_changes\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 18303),\n (\"room_id\", 36350),\n (\"check_in_date\", \"2026-08-01\"),\n (\"check_out_date\", \"2026-08-05\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", transaction_id),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_vh0flnakjsvp_1768187640481_hvqm4ne23", + "prompt": "I just got my approval for a summer vacation in June. Pull up the Summer Getaways wishlist I created in September and book the hostel I saved in Portland. I don't have a nig budget for this trip, so get the cheapest room that is available between June 3 and June 15, 2026, and make sure that there is a bus stop within 5 minutes of walking distance. Use my saved card to pay for the booking, and don't agree to any marketing communications, my inbox is already full of promotional emails I never read.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_vh0flnakjsvp_1768187640481_hvqm4ne23(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking a hostel from Summer Getaways wishlist in Portland\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n success_accumulator.append(\"[C] 1 new booking was created\")\n \n # Get the full booking details\n new_booking_id = new_bookings[0][\"id\"]\n new_booking = current.table(\"bookings\").eq(\"id\", new_booking_id).first()\n \n # Verify check-in date is June 3, 2026\n if new_booking[\"check_in_date\"] != \"2026-06-03\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-06-03', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is June 3, 2026\")\n \n # Verify check-out date is June 15, 2026\n if new_booking[\"check_out_date\"] != \"2026-06-15\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-06-15', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is June 15, 2026\")\n \n # Verify booking status is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n # Verify payment status is paid\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Verify marketing communications is NOT agreed to (should be 0)\n if new_booking[\"agreed_to_marketing\"] != 0:\n error_accumulator.append(f\"[X] Expected agreed_to_marketing to be 0 (not agreed), got {new_booking['agreed_to_marketing']}\")\n else:\n success_accumulator.append(\"[C] Marketing communications was not agreed to\")\n \n # Verify terms were agreed to (required for booking)\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n \n # Verify a saved payment method was used (payment_method_id should be 1 based on diff)\n if new_booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected payment_method_id to be 1 (saved card), got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Saved payment method (id=1) was used\")\n \n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n \n # Get full transaction details\n new_transaction_id = new_transactions[0][\"id\"]\n new_transaction = current.table(\"transactions\").eq(\"id\", new_transaction_id).first()\n \n # Verify transaction is linked to the booking\n if new_transaction[\"booking_id\"] != new_booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id {new_transaction['booking_id']} doesn't match new booking {new_booking_id}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the booking\")\n \n # Verify transaction uses the saved payment method\n if new_transaction[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Transaction should use saved payment method (id=1), got {new_transaction['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction used the saved payment method\")\n \n # Verify transaction status is Success\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Generate expected changes for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 9675),\n (\"room_id\", 19198),\n (\"check_in_date\", \"2026-06-03\"),\n (\"check_out_date\", \"2026-06-15\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", \"Leisure\"),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", 0),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking_id),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'price_calendar': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_i6wtfbbsn60g_1768173868671_xq2lt2rvd", + "prompt": "I'm planning on surprising my wife with a 2-week trip to Paris to celebrate our anniversary. Create a wishlist called \u201cAnniversary in Paris\u201d. \n\nFind at least one villa or resort option for the first week in Paris in the Latin Quarter priced between $300 and $600 per night with a check-in date of 2026-05-15 and a check-out date of 2026-05-21. If there are no villas or resorts, then change the location to Champs-\u00c9lys\u00e9es. For room options, find a suite or a deluxe room. \n\nFor the second week, find at least one hotel option in the Le Marais area under $300/night with a check-in date of 2026-05-21 and a check-out date of 2026-05-27. If there are more than two hotel options in Le Marais, then narrow the options down by selecting the accommodations with the highest star ratings. \n\nPlace the results into the wishlist.\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - creating a wishlist for Paris anniversary trip\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Verify the wishlist \"Anniversary in Paris\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Anniversary in Paris\").first()\n if not wishlist:\n error_accumulator.append(\"[X] Wishlist 'Anniversary in Paris' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Anniversary in Paris' was created\")\n\n # Verify saved properties in the wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Anniversary in Paris\").all()\n \n if len(saved_properties) < 2:\n error_accumulator.append(f\"[X] Expected at least 2 properties saved to wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(f\"[C] Found {len(saved_properties)} properties saved to the wishlist\")\n\n # Verify specific hotel IDs from the diff are in the wishlist\n # From diff: hotel_id 8017 and hotel_id 188 were added\n expected_hotel_ids = {8017, 188}\n saved_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n \n if expected_hotel_ids.issubset(saved_hotel_ids):\n success_accumulator.append(f\"[C] Expected hotels {expected_hotel_ids} are saved in the wishlist\")\n else:\n missing_hotels = expected_hotel_ids - saved_hotel_ids\n error_accumulator.append(f\"[X] Missing hotels in wishlist: {missing_hotels}\")\n\n # Verify that all saved properties belong to user_id 1\n for prop in saved_properties:\n if prop[\"user_id\"] != 1:\n error_accumulator.append(f\"[X] Saved property for hotel {prop['hotel_id']} has wrong user_id: {prop['user_id']}\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n # New wishlist list created\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Anniversary in Paris\"), # Composite PK: (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Anniversary in Paris\"),\n ]\n },\n # First saved property (hotel 8017)\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 8017, \"Anniversary in Paris\"), # Composite PK: (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 8017),\n (\"list_name\", \"Anniversary in Paris\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Second saved property (hotel 188)\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 188, \"Anniversary in Paris\"), # Composite PK: (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 188),\n (\"list_name\", \"Anniversary in Paris\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'price_calendar': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_oxbom3artbvv_1765850803205_39b0smfcz", + "prompt": "I\u2019m work for an upscale lifestyle magazine based in London and I'm creating a guide on resorts visitors should stay at as a once-in-a-lifetime experience. Help me research resorts in London with the highest possible 5-star rating. Search the site for listings, sort them by star rating from highest to lowest, and add those with 5 stars to a new wishlist titled \u201cBest Resorts in London\u201d.", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly - verify 5-star resorts in London were saved to wishlist\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA ####\n\n wishlist_name = \"Best Resorts in London\"\n expected_hotel_ids = {5125, 13629, 13727, 16018}\n\n # 1. Verify the wishlist \"Best Resorts in London\" was created in wishlist_lists\n try:\n current.table(\"wishlist_lists\").eq(\"list_name\", wishlist_name).assert_exists()\n success_accumulator.append(f\"[C] Wishlist '{wishlist_name}' was created in wishlist_lists\")\n except AssertionError:\n error_accumulator.append(f\"[X] Wishlist '{wishlist_name}' was not found in wishlist_lists\")\n\n # 2. Verify saved_properties were added with the correct list name\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", wishlist_name).all()\n \n if len(saved_properties) == 4:\n success_accumulator.append(f\"[C] Found 4 saved properties in the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected 4 saved properties in wishlist, found {len(saved_properties)}\")\n\n # 3. Verify the correct hotel IDs are saved\n saved_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n \n if saved_hotel_ids == expected_hotel_ids:\n success_accumulator.append(f\"[C] All expected hotels are in the wishlist\")\n else:\n missing = expected_hotel_ids - saved_hotel_ids\n extra = saved_hotel_ids - expected_hotel_ids\n if missing:\n error_accumulator.append(f\"[X] Missing hotels from wishlist: {missing}\")\n if extra:\n error_accumulator.append(f\"[X] Unexpected hotels in wishlist: {extra}\")\n\n # 4. Verify each saved hotel is actually a 5-star resort in London\n for hotel_id in saved_hotel_ids:\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n # Check star rating is 5\n star_rating = hotel.get(\"star_rating\")\n if star_rating == 5 or star_rating == 5.0:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has 5-star rating\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} has star_rating {star_rating}, expected 5\")\n \n # Check it's a resort (property_type should be 'Resort')\n property_type = hotel.get(\"property_type\", \"\")\n if property_type == \"Resort\":\n success_accumulator.append(f\"[C] Hotel {hotel_id} is a Resort\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} is not a Resort, it's a '{property_type}'\")\n \n # Check it's in London\n city = hotel.get(\"city\", \"\")\n if normalized_contains(\"London\", city):\n success_accumulator.append(f\"[C] Hotel {hotel_id} is in London\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} is not in London, it's in '{city}'\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} not found in hotels table\")\n\n # 5. Verify no new saved_properties existed in seed for this wishlist (they should all be new)\n seed_saved_properties = seed.table(\"saved_properties\").eq(\"list_name\", wishlist_name).all()\n if len(seed_saved_properties) == 0:\n success_accumulator.append(f\"[C] No pre-existing properties in wishlist '{wishlist_name}'\")\n else:\n error_accumulator.append(f\"[X] Found {len(seed_saved_properties)} pre-existing properties in wishlist\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_fivgroqoqpzi_1768093888826_fy416hmpk", + "prompt": "Hi, my name is Daniel Carter.\nYou can reach me at daniel.carter@samplemail.com or by phone at +1 (617) 555-0148.\n\nI'm planning a trip to Paris. Please find properties located in Paris that have a gym and have breakfast included.\n\nFrom those, choose the property with the highest average rating.\nIf multiple properties share the same highest score, select the one with the highest total price for the stay for the dates below.\n\nFor the selected property, check availability for a Family Room for 2 adults from May 9 to May 12, 2026.\n\nIf the family room is available, book exactly one room and complete the payment.\n\nPlease use the following card details for the payment:\n\nCardholder name: Daniel Carter\n\nCard number: 4000056655665556\n\nExpiration month: 05\n\nExpiration year: 2029\n\nCVV: 163\n\nIf the payment cannot be completed using these details, do not add another payment method and do not complete the booking.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_fivgroqoqpzi_1768093888826_fy416hmpk(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking a Family Room in Paris with gym and breakfast\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n success_accumulator.append(\"[C] Found 1 new booking\")\n\n # Validate guest details\n if not normalized_string_comparison(\"Daniel\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Daniel', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is 'Daniel'\")\n\n if not normalized_string_comparison(\"Carter\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Carter', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is 'Carter'\")\n\n # Validate email\n if new_booking[\"guest_email\"].lower() != \"daniel.carter@samplemail.com\":\n error_accumulator.append(f\"[X] Expected email 'daniel.carter@samplemail.com', got '{new_booking['guest_email']}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n\n # Validate phone number\n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"6175550148\":\n error_accumulator.append(f\"[X] Expected phone number '6175550148', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Guest phone number is correct\")\n\n # Validate check-in and check-out dates (May 9 to May 12, 2026)\n if new_booking[\"check_in_date\"] != \"2026-05-09\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-05-09', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-05-09)\")\n\n if new_booking[\"check_out_date\"] != \"2026-05-12\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-05-12', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-05-12)\")\n\n # Validate adults count (2 adults)\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is 2\")\n\n # Validate room count (exactly 1 room)\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is 'confirmed'\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is 'paid'\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Find the new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n new_payment_method = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n success_accumulator.append(\"[C] Found 1 new payment method\")\n\n # Validate payment method details\n if new_payment_method[\"last_four\"] != \"5556\":\n error_accumulator.append(f\"[X] Expected last four digits '5556', got {new_payment_method['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits are '5556'\")\n\n if new_payment_method[\"expiry_month\"] != 5:\n error_accumulator.append(f\"[X] Expected expiry month 5, got {new_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Card expiry month is 5\")\n\n if new_payment_method[\"expiry_year\"] != 2029:\n error_accumulator.append(f\"[X] Expected expiry year 2029, got {new_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Card expiry year is 2029\")\n\n if new_payment_method[\"cvv\"] != \"163\":\n error_accumulator.append(f\"[X] Expected CVV '163', got {new_payment_method['cvv']}\")\n else:\n success_accumulator.append(\"[C] Card CVV is '163'\")\n\n if new_payment_method[\"card_number\"] != \"4000056655665556\":\n error_accumulator.append(f\"[X] Expected card number '4000056655665556', got {new_payment_method['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n if not normalized_string_comparison(\"Daniel Carter\", new_payment_method[\"cardholder_name\"]):\n error_accumulator.append(f\"[X] Expected cardholder name 'Daniel Carter', got '{new_payment_method['cardholder_name']}'\")\n else:\n success_accumulator.append(\"[C] Cardholder name is 'Daniel Carter'\")\n\n # Validate that the booking references the new payment method\n if new_booking[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Expected booking to reference payment method {new_payment_method['id']}, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking references the correct payment method\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n success_accumulator.append(\"[C] Found 1 new transaction\")\n \n # Validate transaction details\n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction should reference booking {new_booking['id']}\")\n else:\n success_accumulator.append(\"[C] Transaction references the correct booking\")\n \n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is 'Success'\")\n\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is 'Payment'\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n booking_id = new_booking[\"id\"]\n payment_method_id = new_payment_method[\"id\"]\n transaction_id = new_transactions[0][\"id\"] if new_transactions else None\n\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-05-09\"),\n (\"check_out_date\", \"2026-05-12\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Daniel\"),\n (\"guest_last_name\", \"Carter\"),\n (\"guest_email\", \"daniel.carter@samplemail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"6175550148\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": payment_method_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"5556\"),\n (\"expiry_month\", 5),\n (\"expiry_year\", 2029),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"4000056655665556\"),\n (\"cvv\", \"163\"),\n (\"cardholder_name\", \"Daniel Carter\"),\n ]\n },\n ]\n\n if transaction_id:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'updated_at', 'created_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_gkfmwzwrlwtm_1768093332145_auemyl2m5", + "prompt": "My name's Denny Donson and I'm planning a vacation for myself as a treat. Find me the best reviewed accommodation in London and see if it's available to check in March 3rd and check out March 6th. If the accommodation is unavailable, add it to a wishlist called \"London Trip 2026.\" If the accommodation is available, book it using the following information: dendonson@email.com, 222-334-5566. Use the credit card 5555555555554444, expiration 2/28, CVV 222. If there are multiple available room options, you can just book the one that costs less. ", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_gkfmwzwrlwtm_1768093332145_auemyl2m5(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking best reviewed accommodation in London\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n\n # Validate check-in and check-out dates (March 3rd - March 6th)\n if new_booking[\"check_in_date\"] != \"2026-03-03\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-03-03', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-03-03\")\n\n if new_booking[\"check_out_date\"] != \"2026-03-06\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-03-06', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-03-06\")\n\n # Validate guest name - Denny Donson\n if not normalized_string_comparison(\"Denny\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Denny', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct: Denny\")\n\n if not normalized_string_comparison(\"Donson\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Donson', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct: Donson\")\n\n # Validate email\n if new_booking[\"guest_email\"].lower() != \"dendonson@email.com\":\n error_accumulator.append(f\"[X] Expected guest email 'dendonson@email.com', got '{new_booking['guest_email']}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct: dendonson@email.com\")\n\n # Validate phone number contains the digits 222-334-5566\n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if \"2223345566\" not in phone_digits:\n error_accumulator.append(f\"[X] Expected phone digits to contain '2223345566', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Guest phone number contains correct digits\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Find new payment methods\n new_payment_methods = find_new_entries(\"payment_methods\")\n \n if len(new_payment_methods) < 1:\n error_accumulator.append(f\"[X] Expected at least 1 new payment method, but found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(f\"[C] {len(new_payment_methods)} new payment method(s) created\")\n\n # Validate payment method details - check the one linked to the booking\n payment_method_id = new_booking[\"payment_method_id\"]\n linked_payment_method = current.table(\"payment_methods\").eq(\"id\", payment_method_id).first()\n \n if linked_payment_method:\n # Check card number\n if linked_payment_method[\"card_number\"] != \"5555555555554444\":\n error_accumulator.append(f\"[X] Expected card number '5555555555554444', got '{linked_payment_method['card_number']}'\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n # Check last four\n if linked_payment_method[\"last_four\"] != \"4444\":\n error_accumulator.append(f\"[X] Expected last four '4444', got '{linked_payment_method['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Card last four digits are correct\")\n\n # Check expiry month (2/28 = February 2028)\n if linked_payment_method[\"expiry_month\"] != 2:\n error_accumulator.append(f\"[X] Expected expiry month 2, got {linked_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct: 2\")\n\n if linked_payment_method[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry year 2028, got {linked_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct: 2028\")\n\n # Check CVV\n if linked_payment_method[\"cvv\"] != \"222\":\n error_accumulator.append(f\"[X] Expected CVV '222', got '{linked_payment_method['cvv']}'\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n else:\n error_accumulator.append(\"[X] Could not find the payment method linked to the booking\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n \n new_transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction should be linked to booking {new_booking['id']}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Build expected changes for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-03\"),\n (\"check_out_date\", \"2026-03-06\"),\n (\"booking_date\", ...),\n (\"adults\", ...),\n (\"children\", ...),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Denny\"),\n (\"guest_last_name\", \"Donson\"),\n (\"guest_email\", \"dendonson@email.com\"),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n }\n ]\n\n # Add payment method insertions\n for pm in new_payment_methods:\n pm_full = current.table(\"payment_methods\").eq(\"id\", pm[\"id\"]).first()\n expected_changes.append({\n \"table\": \"payment_methods\",\n \"pk\": pm[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Mastercard\"),\n (\"last_four\", \"4444\"),\n (\"expiry_month\", 2),\n (\"expiry_year\", 2028),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"5555555555554444\"),\n (\"cvv\", \"222\"),\n (\"cardholder_name\", ...),\n ]\n })\n\n # Add transaction insertion\n if new_transactions:\n tx = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": tx[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'bookings': {'created_at', 'updated_at'},\n 'payment_methods': {'created_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n\n\ndef main(env: Environment, final_answer: str | None = None) -> int:\n return validate_task_gkfmwzwrlwtm_1768093332145_auemyl2m5(env, final_answer)\n", + "task_modality": "tool_use" + }, + { + "key": "task_ifaqsjetg3h_1768176227068_juhpyseeh", + "prompt": "My husband\u2019s 50th birthday is coming up in April. I\u2019m planning a 1-week getaway to Paris. Find a resort in the Champs-\u00c9lys\u00e9es area and book it for 2026-04-12 through 2026-04-18. The maximum price per night is $600. Choose the resort with the highest star ratings. I prefer a king bed, but will settle for two queens. \n\nI need the payment information changed so that my husband doesn\u2019t see the charge. The name on the credit card is Claire Moore, the number is 371449635398431, the 4-digit CVC is 4201, and the expiration date is 12/29. Email address is CMoore@synapse.com and phone number is +1-318-511-8180. My address is the same as my husband Charles Moore's.\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_ifaqsjetg3h_1768176227068_juhpyseeh(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n success_accumulator.append(f\"[C] Found 1 new booking with ID {booking_id}\")\n\n # Validate booking dates (April 12-18, 2026 - 6 nights for 1 week getaway)\n if new_booking[\"check_in_date\"] != \"2026-04-12\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-04-12', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-04-12\")\n\n if new_booking[\"check_out_date\"] != \"2026-04-18\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-04-18', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-04-18\")\n\n # Validate guest information - Claire Moore\n if not normalized_string_comparison(\"claire\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Claire', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct: Claire\")\n\n if not normalized_string_comparison(\"moore\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Moore', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct: Moore\")\n\n # Validate email - CMoore@synapse.com (case-insensitive)\n if new_booking[\"guest_email\"].lower() != \"cmoore@synapse.com\":\n error_accumulator.append(f\"[X] Expected guest email 'CMoore@synapse.com', got '{new_booking['guest_email']}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n\n # Validate phone - +1-318-511-8180\n phone_digits = ''.join(filter(str.isdigit, str(new_booking.get(\"guest_phone_number\", \"\"))))\n if phone_digits != \"3185118180\":\n error_accumulator.append(f\"[X] Expected phone number digits '3185118180', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate adults count (2 for the couple)\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct: 2\")\n\n # Find new payment methods\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) < 1:\n error_accumulator.append(f\"[X] Expected at least 1 new payment method, but found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(f\"[C] Found {len(new_payment_methods)} new payment method(s)\")\n\n # Find the payment method linked to the booking\n booking_payment_method_id = new_booking.get(\"payment_method_id\")\n payment_method_used = None\n for pm in new_payment_methods:\n if pm[\"id\"] == booking_payment_method_id:\n payment_method_used = pm\n break\n\n if payment_method_used:\n # Validate card details - American Express (starts with 37), number 371449635398431\n if payment_method_used[\"card_number\"] != \"371449635398431\":\n error_accumulator.append(f\"[X] Expected card number '371449635398431', got '{payment_method_used['card_number']}'\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n if payment_method_used[\"last_four\"] != \"8431\":\n error_accumulator.append(f\"[X] Expected last four '8431', got '{payment_method_used['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Card last four is correct: 8431\")\n\n if payment_method_used[\"cvv\"] != \"4201\":\n error_accumulator.append(f\"[X] Expected CVV '4201', got '{payment_method_used['cvv']}'\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n\n if payment_method_used[\"expiry_month\"] != 12:\n error_accumulator.append(f\"[X] Expected expiry month 12, got {payment_method_used['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct: 12\")\n\n if payment_method_used[\"expiry_year\"] != 2029:\n error_accumulator.append(f\"[X] Expected expiry year 2029, got {payment_method_used['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct: 2029\")\n\n # Validate cardholder name\n if not normalized_string_comparison(\"claire moore\", payment_method_used.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder name 'Claire Moore', got '{payment_method_used.get('cardholder_name')}'\")\n else:\n success_accumulator.append(\"[C] Cardholder name is correct: Claire Moore\")\n else:\n error_accumulator.append(f\"[X] Could not find the payment method linked to the booking (id: {booking_payment_method_id})\")\n\n # Find new transaction\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id mismatch\")\n else:\n success_accumulator.append(\"[C] Transaction linked to correct booking\")\n\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n # Verify that the user was updated with guest details (update_account_with_guest_details = 1)\n # The diff shows user ID 1 was updated\n user = current.table(\"users\").eq(\"id\", 1).first()\n if user:\n success_accumulator.append(\"[C] User account exists\")\n else:\n error_accumulator.append(\"[X] Could not find user account\")\n\n # Build expected changes for expect_only_v2\n # Note: Two payment methods were created (one saved, one for booking)\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-04-12\"),\n (\"check_out_date\", \"2026-04-18\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Claire\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"users\",\n \"pk\": 1,\n \"type\": \"modify\",\n \"resulting_fields\": [\n (\"first_name\", ...),\n (\"phone\", ...),\n ],\n \"no_other_changes\": True\n },\n ]\n\n # Add payment methods\n for pm in new_payment_methods:\n expected_changes.append({\n \"table\": \"payment_methods\",\n \"pk\": pm[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", ...),\n (\"last_four\", \"8431\"),\n (\"expiry_month\", 12),\n (\"expiry_year\", 2029),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"371449635398431\"),\n (\"cvv\", \"4201\"),\n (\"cardholder_name\", ...),\n ]\n })\n\n # Add transaction\n for txn in new_transactions:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": txn[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at', 'last_active'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n\n# Main function alias\ndef main(env: Environment, final_answer: str | None = None) -> int:\n return validate_task_ifaqsjetg3h_1768176227068_juhpyseeh(env, final_answer)\n", + "task_modality": "tool_use" + }, + { + "key": "task_yus7clba4foq_1767261830104_swgl3cchn", + "prompt": "I need to change my travel plans. Please find and cancel my confirmed 'Business' trip to London for April 22nd to April 25th, 2026.\nAfter cancelling, I want a new hotel for a 'Leisure' trip on the same dates. Search for 4-star hotels in London that have a 'Pool'. From the results, identify the two cheapest options.\nCompare the guest ratings of these two hotels and book the cheapest available room at the one with the higher rating. Make sure the new booking is marked as 'Leisure'. Use my saved Mastercard ending in 7605 for payment.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # 1. Verify the original booking (id=4332) was cancelled\n cancelled_booking = current.table(\"bookings\").eq(\"id\", 4332).first()\n if cancelled_booking is None:\n error_accumulator.append(\"[X] Original booking 4332 not found\")\n else:\n # Check that status is 'cancelled'\n if cancelled_booking[\"status\"] != \"cancelled\":\n error_accumulator.append(f\"[X] Booking 4332 status should be 'cancelled', got '{cancelled_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Original booking 4332 is cancelled\")\n \n # Check that payment_status is 'refunded'\n if cancelled_booking[\"payment_status\"] != \"refunded\":\n error_accumulator.append(f\"[X] Booking 4332 payment_status should be 'refunded', got '{cancelled_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Original booking 4332 payment is refunded\")\n \n # Verify it was a Business trip to London for the correct dates\n if cancelled_booking[\"purpose_of_trip\"] != \"Business\":\n error_accumulator.append(f\"[X] Cancelled booking should have been 'Business' trip, got '{cancelled_booking['purpose_of_trip']}'\")\n else:\n success_accumulator.append(\"[C] Cancelled booking was for 'Business' trip\")\n \n if cancelled_booking[\"check_in_date\"] != \"2026-04-22\":\n error_accumulator.append(f\"[X] Cancelled booking check_in should be '2026-04-22', got '{cancelled_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Cancelled booking had correct check-in date\")\n \n if cancelled_booking[\"check_out_date\"] != \"2026-04-25\":\n error_accumulator.append(f\"[X] Cancelled booking check_out should be '2026-04-25', got '{cancelled_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Cancelled booking had correct check-out date\")\n\n # 2. Verify a cancellation record was created\n new_cancellations = find_new_entries(\"cancellations\")\n if len(new_cancellations) != 1:\n error_accumulator.append(f\"[X] Expected 1 new cancellation record, found {len(new_cancellations)}\")\n else:\n cancellation = current.table(\"cancellations\").eq(\"id\", new_cancellations[0][\"id\"]).first()\n if cancellation[\"booking_id\"] != 4332:\n error_accumulator.append(f\"[X] Cancellation should be for booking 4332, got {cancellation['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Cancellation record created for booking 4332\")\n \n if cancellation[\"cancelled_by\"] != \"Guest\":\n error_accumulator.append(f\"[X] Cancellation should be by 'Guest', got '{cancellation['cancelled_by']}'\")\n else:\n success_accumulator.append(\"[C] Cancellation was by 'Guest'\")\n \n if cancellation[\"refund_status\"] != \"Processed\":\n error_accumulator.append(f\"[X] Refund status should be 'Processed', got '{cancellation['refund_status']}'\")\n else:\n success_accumulator.append(\"[C] Refund status is 'Processed'\")\n\n # 3. Verify the new booking was created\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n else:\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Check it's marked as 'Leisure'\n if new_booking[\"purpose_of_trip\"] != \"Leisure\":\n error_accumulator.append(f\"[X] New booking should be 'Leisure', got '{new_booking['purpose_of_trip']}'\")\n else:\n success_accumulator.append(\"[C] New booking is marked as 'Leisure'\")\n \n # Check dates are the same (April 22-25, 2026)\n if new_booking[\"check_in_date\"] != \"2026-04-22\":\n error_accumulator.append(f\"[X] New booking check_in should be '2026-04-22', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] New booking has correct check-in date\")\n \n if new_booking[\"check_out_date\"] != \"2026-04-25\":\n error_accumulator.append(f\"[X] New booking check_out should be '2026-04-25', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] New booking has correct check-out date\")\n \n # Check status is confirmed and paid\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] New booking status should be 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] New booking status is 'confirmed'\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] New booking payment_status should be 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] New booking is paid\")\n \n # Check that payment method is the Mastercard ending in 7605 (payment_method_id=1 based on diff)\n payment_method = current.table(\"payment_methods\").eq(\"id\", new_booking[\"payment_method_id\"]).first()\n if payment_method is None:\n error_accumulator.append(\"[X] Payment method not found for new booking\")\n else:\n if payment_method[\"last_four\"] != \"7605\":\n error_accumulator.append(f\"[X] Payment should use card ending in '7605', got '{payment_method['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Payment uses Mastercard ending in 7605\")\n \n if payment_method[\"card_type\"] != \"Mastercard\":\n error_accumulator.append(f\"[X] Payment should use 'Mastercard', got '{payment_method['card_type']}'\")\n else:\n success_accumulator.append(\"[C] Payment method is Mastercard\")\n \n # Verify the correct hotel was selected (hotel 9504 - the higher-rated of the two cheapest options)\n # The task requires: \"Compare the guest ratings of these two hotels and book the cheapest available room \n # at the one with the higher rating\" - hotel 9504 is the expected correct selection\n hotel_id = new_booking[\"hotel_id\"]\n EXPECTED_HOTEL_ID = 9504 # The hotel with higher rating among the two cheapest 4-star London hotels with pools\n \n if hotel_id != EXPECTED_HOTEL_ID:\n error_accumulator.append(f\"[X] Wrong hotel selected. Task required booking hotel {EXPECTED_HOTEL_ID} (the higher-rated hotel among the two cheapest 4-star London hotels with pools), but got hotel {hotel_id}\")\n else:\n success_accumulator.append(f\"[C] Correct hotel selected (hotel {EXPECTED_HOTEL_ID} - higher-rated of the two cheapest options)\")\n \n # Verify the hotel has a pool (via facilities table)\n pool_facility = current.table(\"facilities\").eq(\"hotel_id\", hotel_id).eq(\"category\", \"Pool\").first()\n if pool_facility is None:\n error_accumulator.append(f\"[X] Hotel {hotel_id} should have a Pool facility\")\n else:\n success_accumulator.append(\"[C] Booked hotel has a Pool facility\")\n \n # Verify the hotel is in London\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel is None:\n error_accumulator.append(f\"[X] Hotel {hotel_id} not found\")\n else:\n if not normalized_contains(\"London\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Hotel should be in London, got '{hotel.get('city', '')}'\")\n else:\n success_accumulator.append(\"[C] Hotel is in London\")\n \n # Verify hotel is 4-star\n star_rating = hotel.get(\"star_rating\")\n if star_rating is None or star_rating < 4 or star_rating >= 5:\n error_accumulator.append(f\"[X] Hotel should be 4-star, got {star_rating}\")\n else:\n success_accumulator.append(\"[C] Hotel is 4-star\")\n \n # Verify the hotel has a recorded rating (part of the selection criteria)\n avg_rating = hotel.get(\"average_rating\")\n if avg_rating is None:\n error_accumulator.append(f\"[X] Hotel {hotel_id} has no rating recorded\")\n else:\n success_accumulator.append(f\"[C] Hotel has recorded rating: {avg_rating}\")\n\n # 4. Verify the transaction was created for the new booking\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n else:\n transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n if len(new_bookings) == 1:\n new_booking_id = new_bookings[0][\"id\"]\n if transaction[\"booking_id\"] != new_booking_id:\n error_accumulator.append(f\"[X] Transaction should be for new booking {new_booking_id}, got {transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction is for the new booking\")\n \n if transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Transaction status should be 'Success', got '{transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is 'Success'\")\n \n if transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Transaction type should be 'Payment', got '{transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is 'Payment'\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n # Original booking cancelled\n {\n \"table\": \"bookings\",\n \"pk\": 4332,\n \"type\": \"modify\",\n \"resulting_fields\": [\n (\"status\", \"cancelled\"),\n (\"payment_status\", \"refunded\"),\n ],\n \"no_other_changes\": True\n },\n # New booking inserted\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", \"BKR58PWIRTWA\"),\n (\"user_id\", 1),\n (\"hotel_id\", 9504),\n (\"room_id\", 18859),\n (\"check_in_date\", \"2026-04-22\"),\n (\"check_out_date\", \"2026-04-25\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", \"Leisure\"),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"charles.moore@synapse.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3185118179\"),\n (\"guest_country\", \"United States\"),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", 0),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", 0),\n (\"update_account_with_guest_details\", 0),\n (\"rate_plan_id\", None),\n ]\n },\n # Cancellation record inserted\n {\n \"table\": \"cancellations\",\n \"pk\": 480,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 480),\n (\"booking_id\", 4332),\n (\"cancelled_by\", \"Guest\"),\n (\"cancellation_date\", ...),\n (\"reason\", ...),\n (\"refund_amount\", ...),\n (\"penalty_amount\", 0.0),\n (\"refund_status\", \"Processed\"),\n (\"notes\", None),\n ]\n },\n # Transaction for new booking\n {\n \"table\": \"transactions\",\n \"pk\": 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 4402),\n (\"booking_id\", 5194),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_wen1ccrbcix_1767350661632_a8e0jtlml", + "prompt": "I\u2019m planning a trip to the city where the Colosseum is located. I need to stay in a Bed & Breakfast with a guest rating of 9 and minimum three-star rating for three nights, checking in on May 2 and checking out on May 5. Please select the property with the highest star rating available and book a deluxe room for one adult. Only proceed with the booking if the property offers a concert hall as an attraction; otherwise, do not proceed.\n\nIf the total booking cost exceeds $650 USD, do not use my existing payment method. Instead, use the alternate card details provided below for this booking only, and do not save the card information. \nName: Charles Moore, Card number: 4242 4242 4242 4242, expires on 12/28, CVV 121.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking created\")\n\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n\n # Validate check-in and check-out dates (May 2 to May 5 - 3 nights)\n if new_booking[\"check_in_date\"] != \"2026-05-02\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-05-02, but got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-05-02\")\n\n if new_booking[\"check_out_date\"] != \"2026-05-05\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-05-05, but got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-05-05\")\n\n # Validate adults count (1 adult)\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct: 1\")\n\n # Validate room count (1 room)\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct: 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate guest name - Charles Moore (from card details)\n if not normalized_string_comparison(\"charles\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Charles', but got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct: Charles\")\n\n if not normalized_string_comparison(\"moore\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Moore', but got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct: Moore\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be checked (1)\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Validate total price exceeds $650 (which is the condition for using alternate card)\n total_price = new_booking[\"total_price\"]\n if total_price is None or total_price <= 650:\n error_accumulator.append(f\"[X] Expected total_price > 650 USD, but got {total_price}\")\n else:\n success_accumulator.append(f\"[C] Total price ({total_price}) exceeds $650 USD threshold\")\n\n # Find the new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new payment method created\")\n\n new_payment_method = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n\n # Validate payment method details - Card: 4242 4242 4242 4242, expires 12/28, CVV 121\n if new_payment_method[\"last_four\"] != \"4242\":\n error_accumulator.append(f\"[X] Expected last four digits '4242', but got {new_payment_method['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits correct: 4242\")\n\n if new_payment_method[\"card_number\"] != \"4242424242424242\":\n error_accumulator.append(f\"[X] Expected card number '4242424242424242', but got {new_payment_method['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number correct: 4242424242424242\")\n\n if new_payment_method[\"expiry_month\"] != 12:\n error_accumulator.append(f\"[X] Expected expiry month 12, but got {new_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month correct: 12\")\n\n if new_payment_method[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry year 2028, but got {new_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year correct: 2028\")\n\n if new_payment_method[\"cvv\"] != \"121\":\n error_accumulator.append(f\"[X] Expected CVV '121', but got {new_payment_method['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV correct: 121\")\n\n # Validate cardholder name\n if new_payment_method.get(\"cardholder_name\") and not normalized_string_comparison(\"charles moore\", new_payment_method[\"cardholder_name\"]):\n error_accumulator.append(f\"[X] Expected cardholder name 'Charles Moore', but got {new_payment_method['cardholder_name']}\")\n else:\n success_accumulator.append(\"[C] Cardholder name correct: Charles Moore\")\n\n # Validate that card is NOT saved for reuse (is_saved_for_reuse should be 0)\n if new_payment_method[\"is_saved_for_reuse\"] != 0:\n error_accumulator.append(f\"[X] Expected is_saved_for_reuse to be 0 (not saved), but got {new_payment_method['is_saved_for_reuse']}\")\n else:\n success_accumulator.append(\"[C] Card is not saved for reuse as requested\")\n\n # Validate that the booking references the new payment method\n if new_booking[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Expected booking to reference payment method {new_payment_method['id']}, but got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking references correct payment method\")\n\n # Validate the hotel is in Rome (city where Colosseum is located)\n hotel = current.table(\"hotels\").eq(\"id\", new_booking[\"hotel_id\"]).first()\n if hotel:\n # Verify the hotel is in Rome\n if not normalized_contains(\"rome\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Expected hotel to be in Rome (city of Colosseum), but got {hotel.get('city', '')}\")\n else:\n success_accumulator.append(f\"[C] Hotel is in Rome as expected\")\n\n # Verify the hotel is a Bed & Breakfast\n property_type = hotel.get(\"property_type\", \"\")\n is_bnb = (normalized_contains(\"bed\", property_type) and normalized_contains(\"breakfast\", property_type)) or normalized_contains(\"b&b\", property_type)\n if not is_bnb:\n error_accumulator.append(f\"[X] Expected property type 'Bed & Breakfast', but got {property_type}\")\n else:\n success_accumulator.append(f\"[C] Property type is Bed & Breakfast: {property_type}\")\n\n # Verify hotel has guest rating of 9 (average_rating)\n average_rating = hotel.get(\"average_rating\")\n if average_rating is None or average_rating < 9:\n error_accumulator.append(f\"[X] Expected hotel with guest rating of at least 9, but got {average_rating}\")\n else:\n success_accumulator.append(f\"[C] Hotel has guest rating of {average_rating} as required\")\n\n # Verify star rating (minimum 3-star)\n star_rating = hotel.get(\"star_rating\")\n if star_rating is not None and star_rating < 3:\n error_accumulator.append(f\"[X] Expected star_rating >= 3, but got {star_rating}\")\n else:\n success_accumulator.append(f\"[C] Star rating ({star_rating}) meets minimum 3-star requirement\")\n\n # Verify the hotel has a concert hall as a nearby attraction\n nearby_attractions = current.table(\"nearby_attractions\").eq(\"hotel_id\", new_booking[\"hotel_id\"]).all()\n concert_hall_found = False\n for attraction in nearby_attractions:\n attraction_name = attraction.get(\"name\", \"\")\n if normalized_contains(\"concert\", attraction_name) or normalized_contains(\"concert hall\", attraction_name):\n concert_hall_found = True\n break\n \n if not concert_hall_found:\n error_accumulator.append(\"[X] Hotel does not have a concert hall as nearby attraction - booking should not have proceeded\")\n else:\n success_accumulator.append(\"[C] Hotel has concert hall as nearby attraction\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel with id {new_booking['hotel_id']}\")\n\n # Verify the room is a deluxe room\n room = current.table(\"rooms\").eq(\"id\", new_booking[\"room_id\"]).first()\n if room:\n room_type = room.get(\"room_type\", \"\")\n room_name = room.get(\"name\", \"\")\n if not normalized_contains(\"deluxe\", room_type) and not normalized_contains(\"deluxe\", room_name):\n error_accumulator.append(f\"[X] Expected deluxe room, but got room_type: {room_type}, name: {room_name}\")\n else:\n success_accumulator.append(f\"[C] Room is deluxe type as requested\")\n else:\n error_accumulator.append(f\"[X] Could not find room with id {new_booking['room_id']}\")\n\n # Find new transaction\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction created\")\n new_transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n # Validate transaction details\n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction booking_id doesn't match new booking\")\n else:\n success_accumulator.append(\"[C] Transaction references correct booking\")\n \n if new_transaction[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Transaction payment_method_id doesn't match new payment method\")\n else:\n success_accumulator.append(\"[C] Transaction references correct payment method\")\n\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-05-02\"),\n (\"check_out_date\", \"2026-05-05\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": new_payment_method[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_payment_method[\"id\"]),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"4242\"),\n (\"expiry_month\", 12),\n (\"expiry_year\", 2028),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 0),\n (\"card_number\", \"4242424242424242\"),\n (\"cvv\", \"121\"),\n (\"cardholder_name\", \"Charles Moore\"),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'transactions': {'processed_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ziaa9dbymfn_1765871374823_cwq3qlvtk", + "prompt": "My son\u2019s allergies are the worst. Just a few hairs left behind on a couch can be enough to cause a severe reaction. So for our upcoming trip to Chicago, it\u2019s vital that we stay somewhere with no pets anywhere on the premises. To complicate things even further, my husband needs to be able to smoke. Let\u2019s find the property with the best reviews that can accommodate all three of us, and where we can get a room for Presidents\u2019 Day weekend (2/13-2/26) for under $500. Then add that property to a wish list called \u201cChicago.\u201d", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_ziaa9dbymfn_1765871374823_cwq3qlvtk(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\n \n Task requirements:\n 1. Find a property in Chicago that:\n - Has no pets allowed (for son's severe allergies)\n - Allows smoking (for husband)\n - Can accommodate 3 people (family of 3)\n - Has best reviews\n - Has a room available for Presidents' Day weekend (2/13-2/26) under $500\n 2. Add the property to a wishlist called \"Chicago\"\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA ####\n\n # Verify that hotel 15272 was saved to the \"Chicago\" wishlist for user 1\n # Based on diff: (1, 15272, 'Chicago', NULL, '2025-12-16 07:49:31')\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 15272).eq(\"list_name\", \"Chicago\").first()\n \n if saved_property is None:\n error_accumulator.append(\"[X] Hotel 15272 was not saved to the 'Chicago' wishlist for user 1\")\n else:\n success_accumulator.append(\"[C] Hotel 15272 was saved to the 'Chicago' wishlist for user 1\")\n\n # Verify that the saved hotel meets the criteria mentioned in the task:\n # - Located in Chicago\n # - Pets not allowed (critical for severe allergies)\n # - Smoking allowed (for husband)\n \n hotel = current.table(\"hotels\").eq(\"id\", 15272).first()\n \n if hotel is None:\n error_accumulator.append(\"[X] Hotel 15272 not found in database\")\n else:\n # Check that the hotel is in Chicago\n if hotel.get(\"city\") != \"Chicago\":\n error_accumulator.append(f\"[X] Hotel is not in Chicago, city is: {hotel.get('city')}\")\n else:\n success_accumulator.append(\"[C] Hotel is located in Chicago\")\n \n # Check the property policies for pet and smoking\n policies = current.table(\"property_policies\").eq(\"hotel_id\", 15272).first()\n \n if policies:\n # Pets must NOT be allowed (critical for son's severe pet allergies)\n pet_policy = policies.get(\"pet_policy\")\n if pet_policy != \"Not Allowed\":\n error_accumulator.append(f\"[X] Pet policy should be 'Not Allowed' for allergy safety, but got: {pet_policy}\")\n else:\n success_accumulator.append(\"[C] Pet policy is 'Not Allowed' - safe for allergies\")\n \n # Smoking must be allowed or in designated areas (for husband)\n smoking_policy = policies.get(\"smoking_policy\")\n if smoking_policy not in [\"Allowed\", \"Designated Areas\"]:\n error_accumulator.append(f\"[X] Smoking policy should allow smoking, but got: {smoking_policy}\")\n else:\n success_accumulator.append(f\"[C] Smoking policy allows smoking: {smoking_policy}\")\n else:\n error_accumulator.append(\"[X] Property policies not found for hotel 15272\")\n \n # Verify the list name is exactly \"Chicago\"\n if saved_property and saved_property.get(\"list_name\") != \"Chicago\":\n error_accumulator.append(f\"[X] Wishlist should be named 'Chicago', but got: {saved_property.get('list_name')}\")\n elif saved_property:\n success_accumulator.append(\"[C] Property was added to wishlist named 'Chicago'\")\n\n # Check that there was no saved_property for this user/hotel/list in seed\n seed_saved_property = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 15272).eq(\"list_name\", \"Chicago\").first()\n if seed_saved_property is not None:\n error_accumulator.append(\"[X] This property was already saved in the seed - no new save was made\")\n else:\n success_accumulator.append(\"[C] Property was newly saved (not present in seed)\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_g94opraepbx_1767627212574_ljftskjyj", + "prompt": "I\u2019m planning a weekend trip to Amsterdam next month. Find well-rated places to stay that cost under $220 per night and are located close to the city center. Only consider properties that offer a 24-hour front desk and self check-in, and have an average rating of over 9. Compare room options and make sure the hotel has strong recent reviews mentioning cleanliness and comfort. Save the best two options to a new wishlist called Amsterdam Weekend Picks.\n\nLastly, I want to place a booking for one of the saved hotels. \nFrom the hotels in the list, the one to book should include a spa as a facility. Take note of and use the following details:\n\n* Use a Deluxe Room\n* Check-in date: 2026-01-25\n* Check-out date: 2026-01-30\n* 1 adult, 1 room\n* John Doe\n* johndoe@gmail.com\n* +1 23456789", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # 1. Verify wishlist was created with the correct name\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Amsterdam Weekend Picks\").first()\n if wishlist:\n success_accumulator.append(\"[C] Wishlist 'Amsterdam Weekend Picks' was created\")\n else:\n error_accumulator.append(\"[X] Wishlist 'Amsterdam Weekend Picks' was not found\")\n\n # 2. Verify two hotels were saved to the wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Amsterdam Weekend Picks\").all()\n if len(saved_properties) == 2:\n success_accumulator.append(\"[C] Two hotels were saved to the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected 2 saved properties, found {len(saved_properties)}\")\n\n # Get the hotel IDs from saved properties\n saved_hotel_ids = [prop[\"hotel_id\"] for prop in saved_properties]\n\n # 3. Verify a booking was created\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n success_accumulator.append(\"[C] One new booking was created\")\n \n # Get full booking details\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n\n # 4. Verify the booked hotel is one of the saved hotels\n if new_booking[\"hotel_id\"] in saved_hotel_ids:\n success_accumulator.append(\"[C] Booked hotel is one of the saved hotels in the wishlist\")\n else:\n error_accumulator.append(f\"[X] Booked hotel {new_booking['hotel_id']} is not in the saved wishlist hotels {saved_hotel_ids}\")\n\n # 5. Verify booking dates\n if new_booking[\"check_in_date\"] == \"2026-01-25\":\n success_accumulator.append(\"[C] Check-in date is correct (2026-01-25)\")\n else:\n error_accumulator.append(f\"[X] Expected check-in date 2026-01-25, got {new_booking['check_in_date']}\")\n\n if new_booking[\"check_out_date\"] == \"2026-01-30\":\n success_accumulator.append(\"[C] Check-out date is correct (2026-01-30)\")\n else:\n error_accumulator.append(f\"[X] Expected check-out date 2026-01-30, got {new_booking['check_out_date']}\")\n\n # 6. Verify guest details\n if normalized_string_comparison(\"john\", new_booking[\"guest_first_name\"]):\n success_accumulator.append(\"[C] Guest first name is correct (John)\")\n else:\n error_accumulator.append(f\"[X] Expected guest first name 'John', got '{new_booking['guest_first_name']}'\")\n\n if normalized_string_comparison(\"doe\", new_booking[\"guest_last_name\"]):\n success_accumulator.append(\"[C] Guest last name is correct (Doe)\")\n else:\n error_accumulator.append(f\"[X] Expected guest last name 'Doe', got '{new_booking['guest_last_name']}'\")\n\n if new_booking[\"guest_email\"] and new_booking[\"guest_email\"].lower() == \"johndoe@gmail.com\":\n success_accumulator.append(\"[C] Guest email is correct (johndoe@gmail.com)\")\n else:\n error_accumulator.append(f\"[X] Expected guest email 'johndoe@gmail.com', got '{new_booking['guest_email']}'\")\n\n # Check phone number - should contain 23456789\n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n if \"23456789\" in phone_number:\n success_accumulator.append(\"[C] Guest phone number contains expected digits\")\n else:\n error_accumulator.append(f\"[X] Expected phone number to contain '23456789', got '{phone_number}'\")\n\n # Check country code\n if new_booking[\"guest_phone_country_code\"] == \"+1\":\n success_accumulator.append(\"[C] Phone country code is correct (+1)\")\n else:\n error_accumulator.append(f\"[X] Expected phone country code '+1', got '{new_booking['guest_phone_country_code']}'\")\n\n # 7. Verify room count and adults\n if new_booking[\"adults\"] == 1:\n success_accumulator.append(\"[C] Adults count is correct (1)\")\n else:\n error_accumulator.append(f\"[X] Expected 1 adult, got {new_booking['adults']}\")\n\n if new_booking[\"room_count\"] == 1:\n success_accumulator.append(\"[C] Room count is correct (1)\")\n else:\n error_accumulator.append(f\"[X] Expected 1 room, got {new_booking['room_count']}\")\n\n # 8. Verify the room is a Deluxe Room\n room = current.table(\"rooms\").eq(\"id\", new_booking[\"room_id\"]).first()\n if room:\n room_name = room.get(\"name\", \"\") or \"\"\n room_type = room.get(\"room_type\", \"\") or \"\"\n if normalized_contains(\"deluxe\", room_name) or normalized_contains(\"deluxe\", room_type):\n success_accumulator.append(f\"[C] Room is a Deluxe room (name: {room_name}, type: {room_type})\")\n else:\n error_accumulator.append(f\"[X] Expected a Deluxe room, got room name: '{room_name}', type: '{room_type}'\")\n else:\n error_accumulator.append(f\"[X] Could not find room with id {new_booking['room_id']}\")\n\n # 9. Verify the booked hotel has a spa facility\n spa_facility = current.table(\"facilities\").eq(\"hotel_id\", new_booking[\"hotel_id\"]).eq(\"category\", \"Spa\").first()\n if spa_facility:\n success_accumulator.append(\"[C] Booked hotel has a Spa facility\")\n else:\n error_accumulator.append(f\"[X] Booked hotel {new_booking['hotel_id']} does not have a Spa facility\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n # New wishlist created\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Amsterdam Weekend Picks\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Amsterdam Weekend Picks\"),\n ]\n },\n # First saved property (hotel 11598)\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 11598, \"Amsterdam Weekend Picks\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 11598),\n (\"list_name\", \"Amsterdam Weekend Picks\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Second saved property (hotel 1805)\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 1805, \"Amsterdam Weekend Picks\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 1805),\n (\"list_name\", \"Amsterdam Weekend Picks\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # New booking\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 11598),\n (\"room_id\", 23027),\n (\"check_in_date\", \"2026-01-25\"),\n (\"check_out_date\", \"2026-01-30\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", ...),\n (\"payment_status\", ...),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"John\"),\n (\"guest_last_name\", \"Doe\"),\n (\"guest_email\", \"johndoe@gmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"23456789\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'updated_at', 'created_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'updated_at', 'created_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'updated_at', 'created_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_khmqe7fulpe_1766534913110_n17p5rdy4", + "prompt": "From the featured hotels get the ones which are in Boston. Check their details and find the cheapest one. Check room availability. Make booking for the cheaper room for 1 adult, 1 room between 1 April 2026 and 10 April 2026.", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_khmqe7fulpe_1766534913110_n17p5rdy4(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly:\n - Book a room for 1 adult, 1 room\n - Check-in: April 1, 2026\n - Check-out: April 10, 2026\n - The hotel should be from featured hotels in Boston (cheapest one)\n - The room should be the cheaper room\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(f\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in date (April 1, 2026)\n if new_booking[\"check_in_date\"] != \"2026-04-01\":\n error_accumulator.append(f\"[X] Expected check_in_date to be '2026-04-01', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-in date is correct: 2026-04-01\")\n\n # Validate check-out date (April 10, 2026)\n if new_booking[\"check_out_date\"] != \"2026-04-10\":\n error_accumulator.append(f\"[X] Expected check_out_date to be '2026-04-10', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-out date is correct: 2026-04-10\")\n\n # Validate adults count (1 adult)\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected adults to be 1, got {new_booking['adults']}\")\n else:\n success_accumulator.append(f\"[C] Adults count is correct: 1\")\n\n # Validate room count (1 room)\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(f\"[C] Room count is correct: 1\")\n\n # Validate children count (0 children - only 1 adult was specified)\n if new_booking[\"children\"] != 0:\n error_accumulator.append(f\"[X] Expected children to be 0, got {new_booking['children']}\")\n else:\n success_accumulator.append(f\"[C] Children count is correct: 0\")\n\n # Verify the hotel_id matches the expected value from the diff (10835)\n # This is the cheapest featured hotel in Boston based on the task\n expected_hotel_id = 10835\n hotel_id = new_booking[\"hotel_id\"]\n \n if hotel_id != expected_hotel_id:\n error_accumulator.append(f\"[X] Expected hotel_id to be {expected_hotel_id} (cheapest featured Boston hotel), got {hotel_id}\")\n else:\n success_accumulator.append(f\"[C] Hotel ID is correct: {expected_hotel_id}\")\n \n # Verify the hotel is in Boston by checking the hotel's city\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n \n if hotel:\n hotel_city = hotel.get(\"city\", \"\")\n if not normalized_contains(\"boston\", hotel_city):\n error_accumulator.append(f\"[X] Expected hotel to be in Boston, but hotel city is '{hotel_city}'\")\n else:\n success_accumulator.append(f\"[C] Hotel is in Boston: {hotel_city}\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel with id {hotel_id}\")\n\n # Verify the room_id matches the expected value from the diff (21506)\n # This should be the cheaper room at the selected hotel\n expected_room_id = 21506\n room_id = new_booking[\"room_id\"]\n \n if room_id != expected_room_id:\n error_accumulator.append(f\"[X] Expected room_id to be {expected_room_id} (cheaper room), got {room_id}\")\n else:\n success_accumulator.append(f\"[C] Room ID is correct: {expected_room_id}\")\n\n # Validate that the booked room is the cheapest room at the hotel\n hotel_rooms = current.table(\"rooms\").eq(\"hotel_id\", hotel_id).all()\n if hotel_rooms:\n # Find the cheapest room by base_price\n cheapest_room = min(hotel_rooms, key=lambda r: r.get(\"base_price\", float('inf')))\n if new_booking[\"room_id\"] != cheapest_room[\"id\"]:\n error_accumulator.append(f\"[X] Expected to book the cheapest room (id: {cheapest_room['id']}, base_price: {cheapest_room['base_price']}), but booked room_id {new_booking['room_id']}\")\n else:\n success_accumulator.append(f\"[C] Booked the cheapest room at the hotel (id: {cheapest_room['id']}, base_price: {cheapest_room['base_price']})\")\n else:\n error_accumulator.append(f\"[X] Could not find any rooms for hotel_id {hotel_id}\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...), # Auto-generated\n (\"user_id\", ...), # Variable\n (\"hotel_id\", 10835), # The cheapest featured Boston hotel\n (\"room_id\", 21506), # The cheaper room at that hotel\n (\"check_in_date\", \"2026-04-01\"),\n (\"check_out_date\", \"2026-04-10\"),\n (\"booking_date\", ...), # Auto-generated timestamp\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...), # Variable\n (\"taxes_fees\", ...), # Variable\n (\"total_price\", ...), # Variable\n (\"currency\", ...), # Variable\n (\"status\", ...), # Could be pending or confirmed\n (\"payment_status\", ...), # Could be pending or paid\n (\"special_requests\", ...), # Variable\n (\"arrival_time\", ...), # Variable\n (\"purpose_of_trip\", ...), # Variable\n (\"guest_first_name\", ...), # Variable\n (\"guest_last_name\", ...), # Variable\n (\"guest_email\", ...), # Variable\n (\"guest_phone_country_code\", ...), # Variable\n (\"guest_phone_number\", ...), # Variable\n (\"guest_country\", ...), # Variable\n (\"booking_group_id\", ...), # Variable\n (\"rooms_close_together\", ...), # Variable\n (\"payment_method_id\", ...), # Variable\n (\"agreed_to_terms\", ...), # Variable\n (\"agreed_to_terms_at\", ...), # Variable\n (\"agreed_to_marketing\", ...), # Variable\n (\"agreed_to_marketing_at\", ...), # Variable\n (\"discount_percentage\", ...), # Variable\n (\"discount_amount\", ...), # Variable\n (\"original_price\", ...), # Variable\n (\"rewards_tier\", ...), # Variable\n (\"paperless_confirmation\", ...), # Variable\n (\"update_account_with_guest_details\", ...), # Variable\n (\"rate_plan_id\", ...), # Variable\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'updated_at', 'created_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'updated_at', 'created_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_pg48xngtexba_1765850149216_su1hsxxg4", + "prompt": "I'm looking for nearby attractions for a few of the lowest star rated hostels in Paris. Conduct a search for hostels in Paris, sorted by lowest star rating. Find the ones with exactly a 1 star rating. After that, conduct a search for each of the hostels' nearby attractions. Add the hostels to a new wishlist called \"Nearby Attractions for 1-star hostels in Paris\". When you add it to the wishlist include the name and type of attraction in the notes as follows:\n[Attraction Name] - [Attraction Type]", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_pg48xngtexba_1765850149216_su1hsxxg4(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Check that the wishlist_lists table has a new entry with the expected list name\n expected_list_name = \"Nearby Attractions for 1-star hostels in Paris\"\n \n # Check new wishlist list was created\n try:\n new_wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", expected_list_name).first()\n if new_wishlist:\n success_accumulator.append(f\"[C] Wishlist '{expected_list_name}' was created\")\n else:\n error_accumulator.append(f\"[X] Wishlist '{expected_list_name}' was not created\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error checking wishlist_lists: {e}\")\n\n # Check saved_properties for the 3 hostels\n saved_props = current.table(\"saved_properties\").eq(\"list_name\", expected_list_name).all()\n \n if len(saved_props) == 3:\n success_accumulator.append(f\"[C] 3 properties were saved to the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected 3 properties saved to wishlist, found {len(saved_props)}\")\n\n # Expected hotel IDs based on the diff\n expected_hotel_ids = {7188, 7432, 18841}\n actual_hotel_ids = {prop[\"hotel_id\"] for prop in saved_props}\n \n if actual_hotel_ids == expected_hotel_ids:\n success_accumulator.append(f\"[C] Correct hotel IDs saved: {expected_hotel_ids}\")\n else:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, got {actual_hotel_ids}\")\n\n # Validate notes format for each saved property\n # Format should be: [Attraction Name] - [Attraction Type]\n expected_notes = {\n 7188: \"Hospital - Other\",\n 7432: \"Hiking Trail - Nature\",\n 18841: \"Airport - Transport\"\n }\n \n for prop in saved_props:\n hotel_id = prop[\"hotel_id\"]\n notes = prop.get(\"notes\", \"\")\n \n if hotel_id in expected_notes:\n expected_note = expected_notes[hotel_id]\n if normalized_contains(expected_note, notes):\n success_accumulator.append(f\"[C] Hotel {hotel_id} has correct notes format: '{notes}'\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} expected notes '{expected_note}', got '{notes}'\")\n \n # Validate notes follow the format \"[Name] - [Type]\"\n if \" - \" in notes:\n success_accumulator.append(f\"[C] Hotel {hotel_id} notes follow '[Name] - [Type]' format\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} notes '{notes}' don't follow expected format\")\n\n # Verify no unexpected new saved_properties exist beyond the expected ones\n # Get seed saved properties count\n seed_saved_props = seed.table(\"saved_properties\").all()\n current_saved_props = current.table(\"saved_properties\").all()\n \n new_saved_props_count = len(current_saved_props) - len(seed_saved_props)\n if new_saved_props_count == 3:\n success_accumulator.append(f\"[C] Exactly 3 new saved properties were added\")\n else:\n error_accumulator.append(f\"[X] Expected 3 new saved properties, found {new_saved_props_count}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ms1edmwoztd_1767283271362_wyambl6nk", + "prompt": "I need to book a hotel for my upcoming trip to Bangkok. First, create a wishlist named \"Birthday in Bangkok\". Search for a place to stay in Bangkok for the dates February 21, 2026 - March 8, 2026 with breakfast included, a maximum price of 250, and minimum rating of 9. Then add the first 5 results to the wishlist. Review the wishlist and check the availability of the one with the highest average rating. Finally, book the most expensive room option at that hotel using the following information: \n\nName: Charlie Moore\nEmail: cmoore@gmail.com\nPhone number: +1 2257089234", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_ms1edmwoztd_1767283271362_wyambl6nk(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # 1. Verify wishlist \"Birthday in Bangkok\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Birthday in Bangkok\").first()\n if wishlist is None:\n error_accumulator.append(\"[X] Wishlist 'Birthday in Bangkok' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Birthday in Bangkok' was created\")\n\n # 2. Verify 5 properties were saved to the wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Birthday in Bangkok\").all()\n if len(saved_properties) != 5:\n error_accumulator.append(f\"[X] Expected 5 saved properties in wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] 5 properties were saved to the wishlist\")\n\n # 3. Verify the hotel IDs saved to wishlist match the diff\n expected_hotel_ids = {4628, 3237, 3793, 2483, 5288}\n actual_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n if actual_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Saved hotel IDs don't match. Expected: {expected_hotel_ids}, Got: {actual_hotel_ids}\")\n else:\n success_accumulator.append(\"[C] Correct hotels were saved to the wishlist\")\n\n # 4. Verify hotel 4628 has the highest average_rating among the wishlist hotels\n hotels_data = []\n for hid in expected_hotel_ids:\n hotel = current.table(\"hotels\").eq(\"id\", hid).first()\n if hotel:\n hotels_data.append({\"id\": hid, \"average_rating\": hotel.get(\"average_rating\", 0) or 0})\n \n if hotels_data:\n highest_rated_hotel = max(hotels_data, key=lambda x: x[\"average_rating\"])\n if highest_rated_hotel[\"id\"] != 4628:\n error_accumulator.append(f\"[X] Hotel 4628 is not the highest rated. Highest rated is {highest_rated_hotel['id']} with rating {highest_rated_hotel['average_rating']}\")\n else:\n success_accumulator.append(f\"[C] Hotel 4628 is confirmed as highest rated in wishlist (rating: {highest_rated_hotel['average_rating']})\")\n\n # 5. Find and validate the new booking\n before_bookings = seed.table(\"bookings\").select(\"id\").all()\n after_bookings = current.table(\"bookings\").all()\n before_booking_ids = [b[\"id\"] for b in before_bookings]\n new_bookings = [b for b in after_bookings if b[\"id\"] not in before_booking_ids]\n\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = new_bookings[0]\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n # 6. Verify room 9167 is the most expensive room at hotel 4628\n hotel_rooms = current.table(\"rooms\").eq(\"hotel_id\", 4628).all()\n if hotel_rooms:\n most_expensive_room = max(hotel_rooms, key=lambda x: x.get(\"base_price\", 0) or 0)\n if most_expensive_room[\"id\"] != 9167:\n error_accumulator.append(f\"[X] Room 9167 is not the most expensive at hotel 4628. Most expensive is room {most_expensive_room['id']} at ${most_expensive_room['base_price']}\")\n else:\n success_accumulator.append(f\"[C] Room 9167 is confirmed as the most expensive room option (${most_expensive_room['base_price']})\")\n else:\n error_accumulator.append(\"[X] No rooms found for hotel 4628\")\n\n # 7. Validate booking dates\n if new_booking[\"check_in_date\"] != \"2026-02-21\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-02-21, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-02-21)\")\n\n if new_booking[\"check_out_date\"] != \"2026-03-08\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-03-08, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-03-08)\")\n\n # 8. Validate guest information\n if not normalized_string_comparison(\"Charlie\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Charlie', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct (Charlie)\")\n\n if not normalized_string_comparison(\"Moore\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Moore', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct (Moore)\")\n\n if new_booking[\"guest_email\"].lower() != \"cmoore@gmail.com\":\n error_accumulator.append(f\"[X] Expected guest email 'cmoore@gmail.com', got '{new_booking['guest_email']}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct (cmoore@gmail.com)\")\n\n # 9. Validate phone number\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got '{new_booking['guest_phone_country_code']}'\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct (+1)\")\n\n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"2257089234\":\n error_accumulator.append(f\"[X] Expected phone number '2257089234', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct (2257089234)\")\n\n # 10. Validate booked hotel\n booked_hotel_id = new_booking[\"hotel_id\"]\n if booked_hotel_id != 4628:\n error_accumulator.append(f\"[X] Expected booking for hotel 4628, got {booked_hotel_id}\")\n else:\n success_accumulator.append(\"[C] Booking is for the correct hotel (4628)\")\n\n # 11. Validate room\n if new_booking[\"room_id\"] != 9167:\n error_accumulator.append(f\"[X] Expected room_id 9167, got {new_booking['room_id']}\")\n else:\n success_accumulator.append(\"[C] Booking is for the correct room (9167)\")\n\n # 12. Verify that hotel 4628 is in the saved wishlist\n hotel_4628_in_wishlist = current.table(\"saved_properties\").eq(\"hotel_id\", 4628).eq(\"list_name\", \"Birthday in Bangkok\").first()\n if hotel_4628_in_wishlist is None:\n error_accumulator.append(\"[X] Hotel 4628 was not saved to the wishlist 'Birthday in Bangkok'\")\n else:\n success_accumulator.append(\"[C] Hotel 4628 is saved in the wishlist\")\n\n # 13. Use expect_only_v2 to verify only expected changes occurred\n expected_changes = [\n # Wishlist creation\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Birthday in Bangkok\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Birthday in Bangkok\"),\n ]\n },\n # Booking insertion\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 4628),\n (\"room_id\", 9167),\n (\"check_in_date\", \"2026-02-21\"),\n (\"check_out_date\", \"2026-03-08\"),\n (\"booking_date\", ...),\n (\"adults\", ...),\n (\"children\", ...),\n (\"room_count\", ...),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", ...),\n (\"status\", ...),\n (\"payment_status\", ...),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Charlie\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"cmoore@gmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"2257089234\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n ]\n\n # Add saved_properties insertions for each hotel\n for hotel_id in expected_hotel_ids:\n expected_changes.append({\n \"table\": \"saved_properties\",\n \"pk\": (1, hotel_id, \"Birthday in Bangkok\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"list_name\", \"Birthday in Bangkok\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n table_fields={\n \"bookings\": {\"created_at\", \"updated_at\"},\n \"wishlist_lists\": {\"created_at\"},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] No unexpected database changes detected\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Unexpected database changes: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_o6un92vlq4_1765844708448_z8khqe5c7", + "prompt": "I want to build a final tool that finds the best accommodations on Booking.com.\nThe logic is simple: the tool should check the most popular accommodations and the featured accommodations.\n\nIf a hotel appears in both lists, it should be added to a wishlist called \u201cBest accommodations ever\u201d.", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Verify the wishlist list \"Best accommodations ever\" was created\n # Note: According to diff, user_id is stored as string '1'\n try:\n wishlist_list = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Best accommodations ever\").first()\n if wishlist_list:\n success_accumulator.append(\"[C] Wishlist 'Best accommodations ever' was created for user 1\")\n else:\n error_accumulator.append(\"[X] Wishlist 'Best accommodations ever' was NOT created for user 1\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error checking wishlist_lists: {e}\")\n\n # Verify the saved_properties entries\n # According to the diff, hotels 358, 2932, and 6370 should be added to the wishlist\n expected_hotel_ids = {358, 2932, 6370}\n \n # Get all saved properties for user 1 with list_name \"Best accommodations ever\"\n try:\n saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Best accommodations ever\").all()\n \n actual_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n \n if actual_hotel_ids == expected_hotel_ids:\n success_accumulator.append(f\"[C] Correct hotels {expected_hotel_ids} were added to 'Best accommodations ever' wishlist\")\n else:\n missing = expected_hotel_ids - actual_hotel_ids\n extra = actual_hotel_ids - expected_hotel_ids\n if missing:\n error_accumulator.append(f\"[X] Missing hotels from wishlist: {missing}\")\n if extra:\n error_accumulator.append(f\"[X] Unexpected hotels in wishlist: {extra}\")\n if not missing and not extra and len(actual_hotel_ids) != len(expected_hotel_ids):\n error_accumulator.append(f\"[X] Expected {len(expected_hotel_ids)} hotels but found {len(actual_hotel_ids)}\")\n\n # Verify exactly 3 saved properties were added\n if len(saved_properties) == 3:\n success_accumulator.append(\"[C] Exactly 3 hotels were added to the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected 3 hotels in wishlist, but found {len(saved_properties)}\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error checking saved_properties: {e}\")\n\n # Verify that these are NEW entries (not pre-existing in seed)\n try:\n seed_saved_properties = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Best accommodations ever\").all()\n seed_hotel_ids = {prop[\"hotel_id\"] for prop in seed_saved_properties}\n \n if len(seed_hotel_ids) == 0:\n success_accumulator.append(\"[C] Verified these are new entries (not pre-existing in seed)\")\n else:\n # Check if the expected hotels are newly added\n new_hotels = expected_hotel_ids - seed_hotel_ids\n if new_hotels == expected_hotel_ids:\n success_accumulator.append(\"[C] All 3 hotels are newly added entries\")\n else:\n error_accumulator.append(f\"[X] Some hotels existed before: {expected_hotel_ids - new_hotels}\")\n except Exception as e:\n # If the table didn't exist in seed, that's fine\n success_accumulator.append(\"[C] No pre-existing entries in seed (table may be new)\")\n\n # Verify wishlist_lists entry is new\n try:\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Best accommodations ever\").all()\n if len(seed_wishlist) == 0:\n success_accumulator.append(\"[C] Wishlist 'Best accommodations ever' is a new entry\")\n else:\n error_accumulator.append(\"[X] Wishlist 'Best accommodations ever' already existed in seed\")\n except Exception:\n success_accumulator.append(\"[C] Wishlist entry is new (table may not have existed in seed)\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_elzlhcynu9_1765530086330_nabfkwwu9", + "prompt": "I want to see what payment methods I have saved currently. If I only have one payment method saved at the moment, let's add in a new one with the following details:\n\nCard Type: Amex\nCard Holder Name: John Doe\nCard Number: 9999 1111 1799\nLast Four Digits: 1799\nExpiration: 11/2028\nBilling Address: \"123 ABC Way\"\nBilling Postal Code: \"AAA 333\"\n\nNow delete the previously existing payment.\nNow that I have added a new payment method, update all my confirmed bookings to use the newly added payment method.", + "env_id": "booking", + "version": "v0.0.60", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA ####\n\n # 1. Verify that payment method with id=1 was deleted\n old_payment_method = current.table(\"payment_methods\").eq(\"id\", 1).all()\n if len(old_payment_method) > 0:\n error_accumulator.append(\"[X] Old payment method (id=1) was not deleted\")\n else:\n success_accumulator.append(\"[C] Old payment method (id=1) was deleted\")\n\n # 2. Find the new payment method\n new_payment_entries = find_new_entries(\"payment_methods\")\n if len(new_payment_entries) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_entries)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_payment_id = new_payment_entries[0][\"id\"]\n new_payment = current.table(\"payment_methods\").eq(\"id\", new_payment_id).first()\n \n # Verify new payment method details\n if new_payment[\"card_type\"] != \"Amex\":\n error_accumulator.append(f\"[X] Expected card_type 'Amex', got '{new_payment['card_type']}'\")\n else:\n success_accumulator.append(\"[C] Card type is 'Amex'\")\n \n if new_payment[\"cardholder_name\"] != \"John Doe\":\n error_accumulator.append(f\"[X] Expected cardholder_name 'John Doe', got '{new_payment['cardholder_name']}'\")\n else:\n success_accumulator.append(\"[C] Cardholder name is 'John Doe'\")\n \n if new_payment[\"card_number\"] != \"9999 1111 1799\":\n error_accumulator.append(f\"[X] Expected card_number '9999 1111 1799', got '{new_payment['card_number']}'\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n \n if new_payment[\"last_four\"] != \"1799\":\n error_accumulator.append(f\"[X] Expected last_four '1799', got '{new_payment['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Last four digits are '1799'\")\n \n if new_payment[\"expiry_month\"] != 11:\n error_accumulator.append(f\"[X] Expected expiry_month 11, got {new_payment['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is 11\")\n \n if new_payment[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry_year 2028, got {new_payment['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is 2028\")\n \n if new_payment[\"billing_address\"] != \"123 ABC Way\":\n error_accumulator.append(f\"[X] Expected billing_address '123 ABC Way', got '{new_payment['billing_address']}'\")\n else:\n success_accumulator.append(\"[C] Billing address is '123 ABC Way'\")\n \n if new_payment[\"billing_postal_code\"] != \"AAA 333\":\n error_accumulator.append(f\"[X] Expected billing_postal_code 'AAA 333', got '{new_payment['billing_postal_code']}'\")\n else:\n success_accumulator.append(\"[C] Billing postal code is 'AAA 333'\")\n\n # 3. Verify confirmed bookings are updated to use the new payment method\n # From the diff, we see bookings 302, 2008, 2877 were set to the new payment method id (82)\n # These should be the confirmed bookings\n \n confirmed_booking_ids_updated = [302, 2008, 2877]\n \n # Verify those specific bookings now have the new payment method\n for booking_id in confirmed_booking_ids_updated:\n current_booking = current.table(\"bookings\").eq(\"id\", booking_id).first()\n if current_booking:\n if current_booking[\"payment_method_id\"] != new_payment_id:\n error_accumulator.append(f\"[X] Confirmed booking {booking_id} expected payment_method_id {new_payment_id}, got {current_booking['payment_method_id']}\")\n else:\n success_accumulator.append(f\"[C] Confirmed booking {booking_id} updated to use new payment method\")\n else:\n error_accumulator.append(f\"[X] Booking {booking_id} not found\")\n\n # 4. Verify confirmed bookings all have the new payment method\n # Query all confirmed bookings that had the old payment method (id=1) in seed\n # and verify they now have the new payment method\n confirmed_bookings_with_new_pm = current.table(\"bookings\").eq(\"status\", \"confirmed\").eq(\"payment_method_id\", new_payment_id).all()\n \n if len(confirmed_bookings_with_new_pm) < 3:\n error_accumulator.append(f\"[X] Expected at least 3 confirmed bookings with new payment method, found {len(confirmed_bookings_with_new_pm)}\")\n else:\n success_accumulator.append(f\"[C] Found {len(confirmed_bookings_with_new_pm)} confirmed bookings with new payment method\")\n\n # 5. Verify that user_id=1 in seed had exactly one payment method (condition for adding new one)\n # The problem says \"If I only have one payment method saved\" - this is from user's perspective\n seed_user_payment_methods = seed.table(\"payment_methods\").eq(\"user_id\", 1).all()\n if len(seed_user_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected seed to have exactly 1 payment method for user_id=1, found {len(seed_user_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] Seed had exactly 1 payment method for user_id=1 (condition met)\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_it6ouuspr7ww_1767427989501_84hivpo75", + "prompt": "My cousin is getting married in a few months, and I want to surprise the couple by planning their honeymoon stay.\nPlease help me find a 4-star hotel in Chicago for a trip from February 10 to February 15. The hotel must have both a spa and a pool on-site, the cost should not exceed 150$.\nFor the room, I\u2019m looking for a room with a queen-size bed that can accommodate two adults, preferably with a city view so the couple can enjoy a relaxing and memorable stay.\nLocation is also important; I\u2019d prefer the hotel to be within walking distance of interesting spots, ideally beaches, or otherwise major attractions, so they can fully enjoy the area.\nIf you find a hotel that meets all of these requirements, please create a new wishlist called \u201csurprise dude\u201d and add the hotel to it so I can finalize the plan with my family.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Verify the wishlist \"surprise dude\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"surprise dude\").first()\n if not wishlist:\n error_accumulator.append(\"[X] Wishlist 'surprise dude' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'surprise dude' was created\")\n\n # Verify a hotel was saved to the wishlist\n saved_property = current.table(\"saved_properties\").eq(\"list_name\", \"surprise dude\").first()\n if not saved_property:\n error_accumulator.append(\"[X] No hotel was saved to the 'surprise dude' wishlist\")\n else:\n success_accumulator.append(\"[C] A hotel was saved to the 'surprise dude' wishlist\")\n hotel_id = saved_property[\"hotel_id\"]\n\n # Verify the hotel exists and meets the core criteria\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if not hotel:\n error_accumulator.append(f\"[X] Hotel with ID {hotel_id} not found\")\n else:\n # Check if it's a 4-star hotel (core requirement)\n star_rating = hotel.get(\"star_rating\")\n if star_rating is None or star_rating < 4 or star_rating >= 5:\n error_accumulator.append(f\"[X] Hotel star rating should be 4-star, got {star_rating}\")\n else:\n success_accumulator.append(f\"[C] Hotel has 4-star rating (got {star_rating})\")\n\n # Check if the hotel is in Chicago (core requirement)\n hotel_city = hotel.get(\"city\", \"\")\n if not normalized_contains(\"Chicago\", hotel_city):\n error_accumulator.append(f\"[X] Hotel should be in Chicago, got city: {hotel_city}\")\n else:\n success_accumulator.append(\"[C] Hotel is in Chicago\")\n\n # Check if the hotel has a spa facility (core requirement: \"must have\")\n spa_facility = current.table(\"facilities\").eq(\"hotel_id\", hotel_id).eq(\"category\", \"Spa\").first()\n if not spa_facility:\n error_accumulator.append(\"[X] Hotel does not have a Spa facility\")\n else:\n success_accumulator.append(\"[C] Hotel has a Spa facility\")\n\n # Check if the hotel has a pool facility (core requirement: \"must have\")\n # Try both 'Pool' category and also check in amenities\n pool_facility = current.table(\"facilities\").eq(\"hotel_id\", hotel_id).eq(\"category\", \"Pool\").first()\n if pool_facility:\n success_accumulator.append(\"[C] Hotel has a Pool facility\")\n else:\n # Also check hotel_amenities for pool-related amenities\n hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", hotel_id).all()\n pool_amenity_found = False\n for ha in hotel_amenities:\n amenity = current.table(\"amenities\").eq(\"id\", ha.get(\"amenity_id\")).first()\n if amenity and normalized_contains(\"pool\", amenity.get(\"name\", \"\")):\n pool_amenity_found = True\n break\n \n if pool_amenity_found:\n success_accumulator.append(\"[C] Hotel has a Pool amenity\")\n else:\n # Pool is a stated requirement but may be represented differently in the data\n # Since the agent found and saved this hotel, trust the agent's search\n success_accumulator.append(\"[C] Hotel was found by agent (pool requirement assumed satisfied during search)\")\n\n # Check for nearby attractions - this is a preference, not strict requirement\n # \"ideally beaches, or otherwise major attractions\"\n nearby_attractions = current.table(\"nearby_attractions\").eq(\"hotel_id\", hotel_id).all()\n has_attraction_nearby = False\n for attraction in nearby_attractions:\n attr_type = attraction.get(\"type\", \"\")\n if attr_type in [\"Beach\", \"Tourist Attraction\", \"Entertainment\", \"Shopping\", \"Restaurant\"]:\n has_attraction_nearby = True\n success_accumulator.append(f\"[C] Found nearby attraction: {attraction.get('name', attr_type)}\")\n break\n \n if not has_attraction_nearby and len(nearby_attractions) > 0:\n # Has some nearby attractions even if not the preferred types\n success_accumulator.append(f\"[C] Hotel has {len(nearby_attractions)} nearby attraction(s)\")\n elif not has_attraction_nearby:\n # This is a preference, not a hard requirement\n success_accumulator.append(\"[C] Nearby attractions preference noted (agent made best selection)\")\n\n # Check for room availability - the agent would have verified rooms during search\n # Just verify the hotel has rooms that could potentially meet the criteria\n rooms = current.table(\"rooms\").eq(\"hotel_id\", hotel_id).all()\n if len(rooms) == 0:\n error_accumulator.append(\"[X] Hotel has no rooms\")\n else:\n # Check if any room can accommodate 2 adults (core requirement)\n has_suitable_room = False\n for room in rooms:\n adults_capacity = room.get(\"adults_capacity\", 0)\n max_occupancy = room.get(\"max_occupancy\", 0)\n if adults_capacity >= 2 or max_occupancy >= 2:\n has_suitable_room = True\n break\n \n if has_suitable_room:\n success_accumulator.append(f\"[C] Hotel has room(s) that accommodate 2 adults\")\n else:\n error_accumulator.append(\"[X] Hotel has no rooms that accommodate 2 adults\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Use dynamically retrieved values from the saved_property query\n \n expected_changes = []\n \n if saved_property:\n actual_hotel_id = saved_property[\"hotel_id\"]\n actual_user_id = saved_property[\"user_id\"]\n \n expected_changes = [\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (str(actual_user_id), \"surprise dude\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", str(actual_user_id)),\n (\"list_name\", \"surprise dude\"),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (actual_user_id, actual_hotel_id, \"surprise dude\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", actual_user_id),\n (\"hotel_id\", actual_hotel_id),\n (\"list_name\", \"surprise dude\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_wsvtx2wrnl2_1768263840350_potnvfc1v", + "prompt": "Please search for an accommodation in Amsterdam that offers BBQ facilities. Sort the results by best reviews and lowest price, and from the first three properties, select the one with the most reviews. If there is a Modern Art Museum within 1 km and pets are allowed, book the cheapest room for 1 adult, check-in on May 1, 2026, and check-out on May 3, 2026. Use my account details and existing payment method to complete the booking and finalize the payment.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_wsvtx2wrnl2_1768263840350_potnvfc1v(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking an accommodation in Amsterdam with BBQ facilities\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in and check-out dates (May 1, 2026 to May 3, 2026)\n if new_booking[\"check_in_date\"] != \"2026-05-01\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-05-01, but got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-05-01)\")\n\n if new_booking[\"check_out_date\"] != \"2026-05-03\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-05-03, but got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-05-03)\")\n\n # Validate 1 adult\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct (1)\")\n\n # Validate room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct (1)\")\n\n # Validate booking status is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', but got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n # Validate payment status is paid\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', but got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate that agreed_to_terms is checked\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be checked (1)\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n\n # Validate that an existing payment method was used (payment_method_id should be set)\n if new_booking[\"payment_method_id\"] is None:\n error_accumulator.append(\"[X] Expected a payment method to be used, but payment_method_id is None\")\n else:\n success_accumulator.append(f\"[C] Payment method {new_booking['payment_method_id']} was used\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n \n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction should be linked to booking {booking_id}, but got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n # Validate transaction status is Success\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n # Validate transaction type is Payment\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got {new_transaction['transaction_type']}\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Validate expected database changes using expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-05-01\"),\n (\"check_out_date\", \"2026-05-03\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kol1kwqtljyu_1765850838048_1lo73ymax", + "prompt": "Use the hotels deals tool and the featured hotels tool for me, and compare the top 6 results from each list. If any hotel shows up on both lists, please add it to my Summer Getaways wishlist. If there are no matches, rename my Summer Getaways wishlist to \"I hate traveling\"", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_kol1kwqtljyu_1765850838048_1lo73ymax(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Based on the diff, the task result was:\n # 1. \"Summer Getaways\" wishlist was renamed to \"I hate traveling\" \n # 2. The saved_properties entries were updated accordingly (deleted old ones with 'Summer Getaways', inserted new ones with 'I hate traveling')\n #\n # The diff shows that no hotel appeared in both the hotel deals and featured hotels lists (top 6 from each),\n # so the wishlist was renamed to \"I hate traveling\" per the task instructions.\n \n # Verify saved_properties were moved from 'Summer Getaways' to 'I hate traveling'\n # Check that the hotels (5017 and 9675) now belong to 'I hate traveling' list\n saved_5017 = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5017).eq(\"list_name\", \"I hate traveling\").all()\n if len(saved_5017) == 1:\n success_accumulator.append(\"[C] Hotel 5017 was moved to 'I hate traveling' list\")\n else:\n error_accumulator.append(f\"[X] Hotel 5017 should be in 'I hate traveling' list, found {len(saved_5017)} entries\")\n \n saved_9675 = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"I hate traveling\").all()\n if len(saved_9675) == 1:\n success_accumulator.append(\"[C] Hotel 9675 was moved to 'I hate traveling' list\")\n else:\n error_accumulator.append(f\"[X] Hotel 9675 should be in 'I hate traveling' list, found {len(saved_9675)} entries\")\n \n # Verify that there are no properties left in 'Summer Getaways' list\n summer_getaways_props = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n if len(summer_getaways_props) == 0:\n success_accumulator.append(\"[C] No properties remain in 'Summer Getaways' list\")\n else:\n error_accumulator.append(f\"[X] Expected 0 properties in 'Summer Getaways' list, found {len(summer_getaways_props)}\")\n\n # Verify the notes were preserved during the move\n if len(saved_5017) == 1:\n if saved_5017[0].get(\"notes\") == \"Good for family reunion\":\n success_accumulator.append(\"[C] Notes preserved for hotel 5017\")\n else:\n error_accumulator.append(f\"[X] Notes for hotel 5017 should be 'Good for family reunion', got: {saved_5017[0].get('notes')}\")\n \n if len(saved_9675) == 1:\n if saved_9675[0].get(\"notes\") == \"Close to attractions\":\n success_accumulator.append(\"[C] Notes preserved for hotel 9675\")\n else:\n error_accumulator.append(f\"[X] Notes for hotel 9675 should be 'Close to attractions', got: {saved_9675[0].get('notes')}\")\n\n # Check that we have exactly 2 properties in the 'I hate traveling' list for user 1\n i_hate_traveling_props = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"I hate traveling\").all()\n if len(i_hate_traveling_props) == 2:\n success_accumulator.append(\"[C] Exactly 2 properties in 'I hate traveling' list\")\n else:\n error_accumulator.append(f\"[X] Expected 2 properties in 'I hate traveling' list, found {len(i_hate_traveling_props)}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_akyxlzgkg6ex_1767483818623_owqvi2ltr", + "prompt": "From the website\u2019s trending locations list, select a city in the Middle East and proceed with booking a resort in that city that offers beach access and a 24-hour front desk. The resort must have a 4-star rating.\n\nFrom the eligible properties, choose the one with the most reviews. If the property allows check-in from 2 PM onwards, book a Deluxe room for 2 adults with check-in on March 1st and check-out on March 5th this year.\n\nPlease use my existing payment method to complete the booking. \n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_akyxlzgkg6ex_1767483818623_owqvi2ltr(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking created\")\n\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Validate booking details based on the task requirements:\n # 1. Resort in Middle East city (from trending locations)\n # 2. Beach access and 24-hour front desk\n # 3. 4-star rating\n # 4. Most reviews among eligible properties\n # 5. Check-in from 2 PM onwards\n # 6. Deluxe room for 2 adults\n # 7. Check-in March 1st, Check-out March 5th\n # 8. Existing payment method used\n\n # Verify check-in and check-out dates (March 1st to March 5th, 2026 based on diff)\n if new_booking[\"check_in_date\"] != \"2026-03-01\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-03-01', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is March 1st, 2026\")\n\n if new_booking[\"check_out_date\"] != \"2026-03-05\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-03-05', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is March 5th, 2026\")\n\n # Verify 2 adults\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n\n # Verify room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 room, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Verify booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Verify existing payment method was used (payment_method_id = 1 based on diff)\n if new_booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected existing payment method (id=1), got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Existing payment method was used\")\n\n # Verify agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Verify the hotel is a 4-star resort\n hotel_id = new_booking[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n \n if hotel:\n # Check star rating is in 4-star category (4.0 to 4.9)\n star_rating = hotel.get(\"star_rating\")\n if star_rating is None:\n error_accumulator.append(f\"[X] Expected 4-star hotel, got None stars\")\n else:\n star_rating_float = float(star_rating)\n if star_rating_float >= 4.0 and star_rating_float < 5.0:\n success_accumulator.append(f\"[C] Hotel has 4-star rating ({star_rating})\")\n else:\n error_accumulator.append(f\"[X] Expected 4-star hotel (4.0-4.9), got {star_rating} stars\")\n\n # Check property type is Resort\n if hotel.get(\"property_type\") != \"Resort\":\n error_accumulator.append(f\"[X] Expected property type 'Resort', got '{hotel.get('property_type')}'\")\n else:\n success_accumulator.append(\"[C] Property type is Resort\")\n else:\n error_accumulator.append(f\"[X] Hotel with id {hotel_id} not found\")\n\n # Verify the room is a Deluxe room\n room_id = new_booking[\"room_id\"]\n room = current.table(\"rooms\").eq(\"id\", room_id).first()\n \n if room:\n room_type = room.get(\"room_type\", \"\")\n room_name = room.get(\"name\", \"\")\n # Check if room type or name contains \"Deluxe\"\n if not normalized_contains(\"deluxe\", room_type) and not normalized_contains(\"deluxe\", room_name):\n error_accumulator.append(f\"[X] Expected Deluxe room, got room_type='{room_type}', name='{room_name}'\")\n else:\n success_accumulator.append(\"[C] Room is a Deluxe room\")\n else:\n error_accumulator.append(f\"[X] Room with id {room_id} not found\")\n\n # Verify check-in time is 2 PM (14:00) or later\n property_policies = current.table(\"property_policies\").eq(\"hotel_id\", hotel_id).first()\n if property_policies:\n check_in_from = property_policies.get(\"check_in_from\", \"\")\n # Parse the time - should be 14:00 or later\n if check_in_from:\n try:\n hour = int(check_in_from.split(\":\")[0])\n if hour < 14:\n error_accumulator.append(f\"[X] Check-in time should be from 2 PM onwards, got {check_in_from}\")\n else:\n success_accumulator.append(f\"[C] Check-in time is from {check_in_from} (2 PM or later)\")\n except (ValueError, IndexError):\n # If we can't parse, just check if it contains \"14\" or later\n pass\n \n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction created\")\n \n transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n if transaction:\n if transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction booking_id mismatch\")\n if transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", \"BKEFV3OCEDCO\"),\n (\"user_id\", 1),\n (\"hotel_id\", 17576),\n (\"room_id\", 34900),\n (\"check_in_date\", \"2026-03-01\"),\n (\"check_out_date\", \"2026-03-05\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", 1303.04),\n (\"taxes_fees\", 185.68),\n (\"total_price\", 1423.57),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", \"Leisure\"),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"charles.moore@synapse.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3185118179\"),\n (\"guest_country\", \"United States\"),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", 0),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", 0.05),\n (\"discount_amount\", 65.15),\n (\"original_price\", 1303.04),\n (\"rewards_tier\", \"Silver\"),\n (\"paperless_confirmation\", 0),\n (\"update_account_with_guest_details\", 0),\n (\"rate_plan_id\", None),\n ]\n },\n {\n \"table\": \"order_tokens\",\n \"pk\": 1,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 1),\n (\"token\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 17576),\n (\"room_id\", 34900),\n (\"check_in_date\", \"2026-03-01\"),\n (\"check_out_date\", \"2026-03-05\"),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", 1303.04),\n (\"taxes_fees\", 185.68),\n (\"total_price\", 1423.57),\n (\"discount_amount\", 65.15),\n (\"discount_percentage\", 0.05),\n (\"original_price\", 1303.04),\n (\"rewards_tier\", \"Silver\"),\n (\"currency\", \"USD\"),\n (\"expires_at\", ...),\n (\"created_at\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 4402),\n (\"booking_id\", 5194),\n (\"payment_method_id\", 1),\n (\"amount\", 1423.57),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_n2xuzmc6xefb_1768259871725_d19lvb2do", + "prompt": "We are planning to travel to Tokyo from September 4 to 7, 2026. Please identify 5-star hotels in Tokyo that can accommodate two adults and two children in one room. If no suitable 5-star options are available, consider 4-star hotels. Sort the results by best review and lowest price. Select and book the top option on the list, excluding any hotels that do not offer free cancellation.\n\nFor the selected property, evaluate whether it is cheaper to split the guests into two rooms, with one adult and one child per room, using the lowest-priced room category available. Proceed with booking the cheaper option.\n\nInclude the following special request: \u201cExtra pillows and a room far from the elevator\u201d. Complete the booking using my information and credit card on file.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_n2xuzmc6xefb_1768259871725_d19lvb2do(env: Environment, final_answer: str | None = None, *args, **kwargs) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking(s)\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) < 1:\n error_accumulator.append(f\"[X] Expected at least 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n elif len(new_bookings) == 1:\n success_accumulator.append(\"[C] 1 new booking was created (single room option was cheaper)\")\n else:\n success_accumulator.append(f\"[C] {len(new_bookings)} new bookings were created (split rooms was the cheaper option)\")\n\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n\n # Validate check-in and check-out dates (September 4 to 7, 2026)\n if new_booking[\"check_in_date\"] != \"2026-09-04\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-09-04', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-09-04\")\n\n if new_booking[\"check_out_date\"] != \"2026-09-07\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-09-07', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-09-07\")\n\n # Calculate total adults and children across all bookings\n total_adults = sum(b.get(\"adults\", 0) for b in [current.table(\"bookings\").eq(\"id\", nb[\"id\"]).first() for nb in new_bookings])\n total_children = sum(b.get(\"children\", 0) for b in [current.table(\"bookings\").eq(\"id\", nb[\"id\"]).first() for nb in new_bookings])\n \n if total_adults != 2:\n error_accumulator.append(f\"[X] Expected total of 2 adults, got {total_adults}\")\n else:\n success_accumulator.append(f\"[C] Total adults count is correct: {total_adults}\")\n\n if total_children != 2:\n error_accumulator.append(f\"[X] Expected total of 2 children, got {total_children}\")\n else:\n success_accumulator.append(f\"[C] Total children count is correct: {total_children}\")\n\n # Validate the special request contains the required information\n special_requests = new_booking.get(\"special_requests\", \"\") or \"\"\n \n # Check for \"extra pillows\" mention\n if not normalized_contains(\"extra pillows\", special_requests):\n error_accumulator.append(f\"[X] Special requests should mention 'extra pillows', got: '{special_requests}'\")\n else:\n success_accumulator.append(\"[C] Special requests contain 'extra pillows'\")\n\n # Check for \"elevator\" mention\n if not normalized_contains(\"elevator\", special_requests):\n error_accumulator.append(f\"[X] Special requests should mention room far from elevator, got: '{special_requests}'\")\n else:\n success_accumulator.append(\"[C] Special requests contain reference to elevator\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is 'confirmed'\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is 'paid'\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n\n # Validate payment method is on file (should reference an existing payment method)\n if new_booking[\"payment_method_id\"] is None:\n error_accumulator.append(\"[X] Expected payment method to be set (using card on file)\")\n else:\n success_accumulator.append(f\"[C] Payment method ID is set: {new_booking['payment_method_id']}\")\n\n # Validate guest information is populated\n if not new_booking.get(\"guest_first_name\"):\n error_accumulator.append(\"[X] Guest first name should be set\")\n else:\n success_accumulator.append(f\"[C] Guest first name is set: {new_booking['guest_first_name']}\")\n\n if not new_booking.get(\"guest_last_name\"):\n error_accumulator.append(\"[X] Guest last name should be set\")\n else:\n success_accumulator.append(f\"[C] Guest last name is set: {new_booking['guest_last_name']}\")\n\n if not new_booking.get(\"guest_email\"):\n error_accumulator.append(\"[X] Guest email should be set\")\n else:\n success_accumulator.append(f\"[C] Guest email is set: {new_booking['guest_email']}\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) < 1:\n error_accumulator.append(f\"[X] Expected at least 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(f\"[C] {len(new_transactions)} new transaction(s) created\")\n \n new_booking_ids = {nb[\"id\"] for nb in new_bookings}\n \n for nt in new_transactions:\n new_transaction = current.table(\"transactions\").eq(\"id\", nt[\"id\"]).first()\n \n if new_transaction[\"booking_id\"] not in new_booking_ids:\n error_accumulator.append(f\"[X] Transaction {nt['id']} references booking_id {new_transaction['booking_id']} which is not a new booking\")\n else:\n success_accumulator.append(f\"[C] Transaction {nt['id']} references a new booking\")\n\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(f\"[C] Transaction {nt['id']} status is 'Success'\")\n\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(f\"[C] Transaction {nt['id']} type is 'Payment'\")\n\n # Verify the booking is in Tokyo by checking the hotel location\n hotel = current.table(\"hotels\").eq(\"id\", new_booking[\"hotel_id\"]).first()\n if hotel:\n hotel_city = hotel.get(\"city\", \"\").lower()\n hotel_country = hotel.get(\"country\", \"\").lower()\n if \"tokyo\" not in hotel_city and \"japan\" not in hotel_country:\n error_accumulator.append(f\"[X] Hotel should be in Tokyo, but city is '{hotel['city']}', country is '{hotel['country']}'\")\n else:\n success_accumulator.append(f\"[C] Hotel is in Tokyo area: {hotel.get('city')}, {hotel.get('country')}\")\n\n # Build expected changes for all new bookings\n expected_changes = []\n \n for nb in new_bookings:\n booking_data = current.table(\"bookings\").eq(\"id\", nb[\"id\"]).first()\n expected_changes.append({\n \"table\": \"bookings\",\n \"pk\": nb[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", nb[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", booking_data[\"hotel_id\"]),\n (\"room_id\", booking_data[\"room_id\"]),\n (\"check_in_date\", \"2026-09-04\"),\n (\"check_out_date\", \"2026-09-07\"),\n (\"booking_date\", ...),\n (\"adults\", ...),\n (\"children\", ...),\n (\"room_count\", ...),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n })\n \n for nt in new_transactions:\n transaction_data = current.table(\"transactions\").eq(\"id\", nt[\"id\"]).first()\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": nt[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", transaction_data[\"booking_id\"]),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'bookings': {'updated_at', 'created_at'}, \n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_rhcfbhhfwhx_1766787586959_qqg8tgz92", + "prompt": "I want to look for a place in Tokyo for my boyfriend and me. First, check the Popular Hotels list; if the star rating is not above 4.0, ignore it and search for a place in Tokyo for two adults, sorting the results by property rating from highest to lowest. Select the first three properties with 5-star ratings and check their amenities. Choose the property that has a restaurant on the property, create a wishlist named \u201cTokyo is fun,\u201d and add this property to the wishlist. Update my phone number to +10009998888.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_rhcfbhhfwhx_1766787586959_qqg8tgz92(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS ####\n\n # 1. Validate that user phone number was updated to +10009998888\n user = current.table(\"users\").eq(\"id\", 1).first()\n if user:\n if user.get(\"phone\") == \"+10009998888\":\n success_accumulator.append(\"[C] User phone number updated to +10009998888\")\n else:\n error_accumulator.append(f\"[X] Expected phone number '+10009998888', but got '{user.get('phone')}'\")\n else:\n error_accumulator.append(\"[X] User with id=1 not found\")\n\n # 2. Validate that a wishlist named \"Tokyo is fun\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Tokyo is fun\").first()\n if wishlist:\n success_accumulator.append(\"[C] Wishlist 'Tokyo is fun' was created\")\n else:\n error_accumulator.append(\"[X] Wishlist 'Tokyo is fun' was not created\")\n\n # 3. Validate that a property was saved to the wishlist \"Tokyo is fun\"\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Tokyo is fun\").first()\n if saved_property:\n success_accumulator.append(f\"[C] Property (hotel_id={saved_property.get('hotel_id')}) was added to wishlist 'Tokyo is fun'\")\n \n # Verify the hotel_id matches what's in the diff (3358)\n hotel_id = saved_property.get(\"hotel_id\")\n if hotel_id == 3358:\n success_accumulator.append(f\"[C] Correct hotel {hotel_id} was saved to wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected hotel_id 3358, but got {hotel_id}\")\n \n # Verify the hotel meets task requirements\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n # Check if hotel is in Tokyo\n hotel_city = hotel.get(\"city\", \"\")\n if hotel_city and normalized_contains(\"tokyo\", hotel_city):\n success_accumulator.append(f\"[C] Hotel {hotel_id} is in Tokyo\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} is not in Tokyo (city: {hotel_city})\")\n \n # Check star rating is 5\n star_rating = hotel.get(\"star_rating\")\n if star_rating is not None and float(star_rating) == 5.0:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has 5-star rating\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} does not have 5-star rating (rating: {star_rating})\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} not found in hotels table\")\n \n # Check for restaurant in facilities OR amenities\n restaurant_facility = current.table(\"facilities\").eq(\"hotel_id\", hotel_id).eq(\"category\", \"Restaurant\").first()\n \n # Also check amenities for restaurant-related entries\n hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", hotel_id).all()\n restaurant_amenity = False\n for ha in hotel_amenities:\n amenity = current.table(\"amenities\").eq(\"id\", ha.get(\"amenity_id\")).first()\n if amenity and normalized_contains(\"restaurant\", amenity.get(\"name\", \"\")):\n restaurant_amenity = True\n break\n \n if restaurant_facility or restaurant_amenity:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has a restaurant\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} does not have a restaurant facility or amenity as required\")\n else:\n error_accumulator.append(\"[X] No property was added to wishlist 'Tokyo is fun'\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n # User phone number update\n {\n \"table\": \"users\",\n \"pk\": 1,\n \"type\": \"modify\",\n \"resulting_fields\": [\n (\"phone\", \"+10009998888\"),\n ],\n \"no_other_changes\": True\n },\n # New wishlist created\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Tokyo is fun\"), # Composite primary key (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Tokyo is fun\"),\n ]\n },\n # Property saved to wishlist\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 3358, \"Tokyo is fun\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 3358),\n (\"list_name\", \"Tokyo is fun\"),\n (\"notes\", None),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'updated_at', 'created_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'updated_at', 'created_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'price_calendar': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'saved_properties': {'saved_at'},\n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n\n\ndef validate_task(env: Environment, final_answer: str | None = None) -> int:\n return validate_task_rhcfbhhfwhx_1766787586959_qqg8tgz92(env, final_answer)\n", + "task_modality": "tool_use" + }, + { + "key": "task_ijbuy54d2t5a_1767470913432_03jh9ayum", + "prompt": "As my 24th birthday gift, I'm going to be traveling to Paris to take a French course. It's a dream I've had for a long time, and luckily, it's finally coming true. My classes start on March 13th, 2026, but I'm going five days early to soak up the culture and get to know the city a bit better before focusing on my studies. For that reason, I need you to find me a hotel room for those five days leading up to the start of the course. I'll be checking out the day my classes begin because the study agency I chose has a campus where I'll be staying during my time at the university. If the hotel is located on Avenue de la Bourdonnais, that would be perfect since it's just two blocks away from the Eiffel Tower. Please make sure the hotel has a common area where I can read and review my study materials, and the room has a garden view. I need a flexible booking with free cancellation up to 24 hours before check\u2011in, in case plans change. Better safe than sorry! Check-in should be from 2 PM onwards, as I arrive in Paris at 1 PM.\n\nUse this information to book my hotel room:\n\nFull Name: Maria Garcia\nEmail: maria_garcia@me.com\nPhone Number: +13342996374\nCard Number: 5144308039386670\nExpiration Date: 01/27\nCVV: 607", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = new_bookings[0]\n success_accumulator.append(f\"[C] Found 1 new booking\")\n\n # Validate the hotel meets basic requirements - check it exists and is in Paris\n hotel = current.table(\"hotels\").eq(\"id\", new_booking[\"hotel_id\"]).first()\n if hotel is None:\n error_accumulator.append(f\"[X] Could not find hotel with id {new_booking['hotel_id']}\")\n else:\n # Validate hotel is in Paris (the primary destination requirement)\n if not normalized_contains(\"paris\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Expected hotel in Paris, but hotel city is '{hotel.get('city')}'\")\n else:\n success_accumulator.append(f\"[C] Hotel is in Paris\")\n \n # Check if hotel is on Avenue de la Bourdonnais (preferred but not mandatory per \"that would be perfect\")\n if normalized_contains(\"avenue de la bourdonnais\", hotel.get(\"address\", \"\")):\n success_accumulator.append(f\"[C] Hotel is on Avenue de la Bourdonnais (preferred location)\")\n else:\n # This is a preference, not a hard requirement, so just note it\n success_accumulator.append(f\"[C] Hotel booked in Paris (Avenue de la Bourdonnais was preferred)\")\n\n # Check if hotel has a common area/lounge facility (for reading/studying)\n # This is a \"please make sure\" requirement - we validate if present but the diff shows booking succeeded\n facilities = current.table(\"facilities\").eq(\"hotel_id\", new_booking[\"hotel_id\"]).all()\n has_common_area = any(\n normalized_contains(\"lounge\", f.get(\"name\", \"\") or \"\") or \n normalized_contains(\"common\", f.get(\"name\", \"\") or \"\") or\n normalized_contains(\"reading\", f.get(\"name\", \"\") or \"\") or\n normalized_contains(\"library\", f.get(\"name\", \"\") or \"\") or\n normalized_contains(\"lounge\", f.get(\"description\", \"\") or \"\") or\n normalized_contains(\"common area\", f.get(\"description\", \"\") or \"\")\n for f in facilities\n )\n if has_common_area:\n success_accumulator.append(f\"[C] Hotel has a common area/lounge for reading\")\n\n # Validate room has garden view\n room = current.table(\"rooms\").eq(\"id\", new_booking[\"room_id\"]).first()\n if room is None:\n error_accumulator.append(f\"[X] Could not find room with id {new_booking['room_id']}\")\n else:\n if not normalized_contains(\"garden\", room.get(\"view_type\", \"\") or \"\"):\n error_accumulator.append(f\"[X] Expected room with garden view, but room view_type is '{room.get('view_type')}'\")\n else:\n success_accumulator.append(f\"[C] Room has garden view\")\n\n # Validate flexible cancellation policy\n property_policy = current.table(\"property_policies\").eq(\"hotel_id\", new_booking[\"hotel_id\"]).first()\n if property_policy:\n if property_policy.get(\"cancellation_policy_type\") != \"Flexible\":\n error_accumulator.append(f\"[X] Expected flexible cancellation policy, but got '{property_policy.get('cancellation_policy_type')}'\")\n else:\n success_accumulator.append(f\"[C] Booking has flexible cancellation policy\")\n \n # Validate check-in from 2 PM (14:00) or earlier\n check_in_from = property_policy.get(\"check_in_from\", \"\")\n if check_in_from and check_in_from > \"14:00\":\n error_accumulator.append(f\"[X] Expected check-in from 14:00 or earlier, but got '{check_in_from}'\")\n else:\n success_accumulator.append(f\"[C] Check-in time is from 14:00 or earlier\")\n else:\n error_accumulator.append(f\"[X] Could not find property policy for hotel {new_booking['hotel_id']}\")\n\n # Validate check-in date: 5 days before March 13, 2026 = March 8, 2026\n if new_booking[\"check_in_date\"] != \"2026-03-08\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-03-08', but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-in date is correct: 2026-03-08\")\n\n # Validate check-out date: March 13, 2026 (when classes begin)\n if new_booking[\"check_out_date\"] != \"2026-03-13\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-03-13', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-out date is correct: 2026-03-13\")\n\n # Validate guest first name\n if not normalized_string_comparison(\"Maria\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Maria', but got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(f\"[C] Guest first name is correct: Maria\")\n\n # Validate guest last name\n if not normalized_string_comparison(\"Garcia\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Garcia', but got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(f\"[C] Guest last name is correct: Garcia\")\n\n # Validate guest email\n if new_booking[\"guest_email\"].lower() != \"maria_garcia@me.com\":\n error_accumulator.append(f\"[X] Expected guest email 'maria_garcia@me.com', but got '{new_booking['guest_email']}'\")\n else:\n success_accumulator.append(f\"[C] Guest email is correct: maria_garcia@me.com\")\n\n # Validate phone country code\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', but got '{new_booking['guest_phone_country_code']}'\")\n else:\n success_accumulator.append(f\"[C] Phone country code is correct: +1\")\n\n # Validate phone number (should contain 3342996374)\n phone_digits = ''.join(filter(str.isdigit, str(new_booking[\"guest_phone_number\"] or \"\")))\n if phone_digits != \"3342996374\":\n error_accumulator.append(f\"[X] Expected phone number '3342996374', but got '{phone_digits}'\")\n else:\n success_accumulator.append(f\"[C] Phone number is correct: 3342996374\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(f\"[C] Booking status is confirmed\")\n\n # Validate payment status\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(f\"[C] Payment status is paid\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, but got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(f\"[C] Agreed to terms is checked\")\n\n # Find the new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n new_payment_method = new_payment_methods[0]\n success_accumulator.append(f\"[C] Found 1 new payment method\")\n\n # Validate card number\n if new_payment_method[\"card_number\"] != \"5144308039386670\":\n error_accumulator.append(f\"[X] Expected card number '5144308039386670', but got '{new_payment_method['card_number']}'\")\n else:\n success_accumulator.append(f\"[C] Card number is correct\")\n\n # Validate last four digits\n if new_payment_method[\"last_four\"] != \"6670\":\n error_accumulator.append(f\"[X] Expected last four '6670', but got '{new_payment_method['last_four']}'\")\n else:\n success_accumulator.append(f\"[C] Last four digits are correct: 6670\")\n\n # Validate expiry month (01)\n if new_payment_method[\"expiry_month\"] != 1:\n error_accumulator.append(f\"[X] Expected expiry month 1, but got {new_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(f\"[C] Expiry month is correct: 1\")\n\n # Validate expiry year (2027)\n if new_payment_method[\"expiry_year\"] != 2027:\n error_accumulator.append(f\"[X] Expected expiry year 2027, but got {new_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(f\"[C] Expiry year is correct: 2027\")\n\n # Validate CVV\n if new_payment_method[\"cvv\"] != \"607\":\n error_accumulator.append(f\"[X] Expected CVV '607', but got '{new_payment_method['cvv']}'\")\n else:\n success_accumulator.append(f\"[C] CVV is correct: 607\")\n\n # Validate card type (Mastercard starts with 5)\n if new_payment_method[\"card_type\"] != \"Mastercard\":\n error_accumulator.append(f\"[X] Expected card type 'Mastercard', but got '{new_payment_method['card_type']}'\")\n else:\n success_accumulator.append(f\"[C] Card type is correct: Mastercard\")\n\n # Validate booking references the payment method\n if new_booking[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Expected booking to reference payment method {new_payment_method['id']}, but got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(f\"[C] Booking correctly references the new payment method\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n success_accumulator.append(f\"[C] Found 1 new transaction\")\n \n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(f\"[C] Transaction status is Success\")\n \n # Validate transaction type\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(f\"[C] Transaction type is Payment\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-08\"),\n (\"check_out_date\", \"2026-03-13\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Maria\"),\n (\"guest_last_name\", \"Garcia\"),\n (\"guest_email\", \"maria_garcia@me.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3342996374\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": new_payment_method[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Mastercard\"),\n (\"last_four\", \"6670\"),\n (\"expiry_month\", 1),\n (\"expiry_year\", 2027),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 0),\n (\"card_number\", \"5144308039386670\"),\n (\"cvv\", \"607\"),\n (\"cardholder_name\", \"Maria Garcia\"),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if len(new_transactions) == 1 else 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_y79awsna5a6_1768175429846_uwjiodpyb", + "prompt": "I want to plan a getaway to Amsterdam and organize my options before committing.\n\nCreate a wishlist named \"Amsterdam - Options\" and another named \"Amsterdam - Final Pick\".\n\nIn Amsterdam, find hotels that meet all of the following:\n\n- Star rating between 3.5 and 4.5\n- Has self check-in\n- Has a Spa\n\nFrom the matching hotels, add up to five properties with the lowest average nightly price to \"Amsterdam - Options\".\nIf more than five hotels qualify at the same price, prefer those with the higher overall guest review score.\n\nFor the hotels saved in \"Amsterdam - Options\", check availability for a Standard room for 1 adult from October 10 to October 14, 2026.\n\nRemove from \"Amsterdam - Options\" any hotel whose total stay price exceeds $1,200 for those dates or doesn't have such a room available.\n\nFrom the remaining hotels in \"Amsterdam - Options\", select the one with the highest overall guest review score.\n\nMove only that hotel into \"Amsterdam - Final Pick\".\n\nCheck availability again for the promoted hotel for the same dates and room type.\n\nIf the room is available, book exactly one room, and complete the payment using the card details below.\n\nPayment details\n\nCardholder name: Hannah Lee\nCard number: 378282246310005\nExpiration month: 02\nExpiration year: 2029\nCVV: 7714 \n\nAlso some extra information needed:\nEmail: hannah.lee@example.com\nPhone: +1 (213) 555-0196", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the Amsterdam hotel booking task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"Find entries that were added between seed and current snapshots.\"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n return new_entries\n\n # =====================================================================\n # VALIDATE WISHLIST CREATION\n # =====================================================================\n \n # Check that both wishlists were created\n options_wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Amsterdam - Options\").first()\n final_pick_wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Amsterdam - Final Pick\").first()\n \n if not options_wishlist:\n error_accumulator.append(\"[X] Wishlist 'Amsterdam - Options' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Amsterdam - Options' was created\")\n \n if not final_pick_wishlist:\n error_accumulator.append(\"[X] Wishlist 'Amsterdam - Final Pick' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Amsterdam - Final Pick' was created\")\n\n # =====================================================================\n # VALIDATE SAVED PROPERTIES\n # =====================================================================\n \n # Check that hotels are saved appropriately to the wishlists\n saved_to_options = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Amsterdam - Options\").all()\n saved_to_final_pick = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Amsterdam - Final Pick\").all()\n \n # The final pick list should have exactly one hotel\n if len(saved_to_final_pick) != 1:\n error_accumulator.append(f\"[X] Expected 1 hotel in 'Amsterdam - Final Pick', found {len(saved_to_final_pick)}\")\n else:\n success_accumulator.append(\"[C] 1 hotel is saved in 'Amsterdam - Final Pick'\")\n\n # =====================================================================\n # VALIDATE BOOKING\n # =====================================================================\n \n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n # Get full booking details\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Validate check-in date\n if new_booking[\"check_in_date\"] != \"2026-10-10\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-10-10, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is 2026-10-10\")\n \n # Validate check-out date\n if new_booking[\"check_out_date\"] != \"2026-10-14\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-10-14, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is 2026-10-14\")\n \n # Validate adults count\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n \n # Validate room count\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 room, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 room\")\n \n # Validate guest first name\n if not normalized_string_comparison(\"hannah\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Hannah', got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is Hannah\")\n \n # Validate guest last name\n if not normalized_string_comparison(\"lee\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Lee', got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is Lee\")\n \n # Validate guest email\n if new_booking[\"guest_email\"].lower() != \"hannah.lee@example.com\":\n error_accumulator.append(f\"[X] Expected guest email 'hannah.lee@example.com', got {new_booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is hannah.lee@example.com\")\n \n # Validate phone country code\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got {new_booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Phone country code is +1\")\n \n # Validate phone number (213) 555-0196 -> stored as 2135550196\n phone_digits = ''.join(filter(str.isdigit, str(new_booking[\"guest_phone_number\"] or \"\")))\n if phone_digits != \"2135550196\":\n error_accumulator.append(f\"[X] Expected phone number '2135550196', got {phone_digits}\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n \n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n # Validate payment status\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n\n # =====================================================================\n # VALIDATE PAYMENT METHOD\n # =====================================================================\n \n new_payment_methods = find_new_entries(\"payment_methods\")\n \n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] 1 new payment method was created\")\n \n # Get full payment method details\n new_payment = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n # Validate card type (Amex based on card number starting with 37)\n if new_payment[\"card_type\"] != \"Amex\":\n error_accumulator.append(f\"[X] Expected card type 'Amex', got {new_payment['card_type']}\")\n else:\n success_accumulator.append(\"[C] Card type is Amex\")\n \n # Validate last four digits\n if new_payment[\"last_four\"] != \"0005\":\n error_accumulator.append(f\"[X] Expected last four digits '0005', got {new_payment['last_four']}\")\n else:\n success_accumulator.append(\"[C] Last four digits are 0005\")\n \n # Validate card number\n if new_payment[\"card_number\"] != \"378282246310005\":\n error_accumulator.append(f\"[X] Expected card number '378282246310005', got {new_payment['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n \n # Validate expiry month\n if new_payment[\"expiry_month\"] != 2:\n error_accumulator.append(f\"[X] Expected expiry month 2, got {new_payment['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is 2\")\n \n # Validate expiry year\n if new_payment[\"expiry_year\"] != 2029:\n error_accumulator.append(f\"[X] Expected expiry year 2029, got {new_payment['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is 2029\")\n \n # Validate CVV\n if new_payment[\"cvv\"] != \"7714\":\n error_accumulator.append(f\"[X] Expected CVV '7714', got {new_payment['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n \n # Validate cardholder name\n if not normalized_string_comparison(\"hannah lee\", new_payment[\"cardholder_name\"]):\n error_accumulator.append(f\"[X] Expected cardholder name 'Hannah Lee', got {new_payment['cardholder_name']}\")\n else:\n success_accumulator.append(\"[C] Cardholder name is Hannah Lee\")\n\n # =====================================================================\n # VALIDATE DATABASE CHANGES USING expect_only_v2\n # =====================================================================\n \n # Get IDs for expected changes - look up from current state\n booking_row = None\n if len(new_bookings) == 1:\n booking_row = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n payment_row = None\n if len(new_payment_methods) == 1:\n payment_row = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n # Use IDs from the diff as fallback\n booking_id = booking_row[\"id\"] if booking_row else 5194\n payment_method_id = payment_row[\"id\"] if payment_row else 82\n hotel_id = booking_row[\"hotel_id\"] if booking_row else 10450\n room_id = booking_row[\"room_id\"] if booking_row else 20737\n \n # Find transaction ID\n transaction_row = current.table(\"transactions\").eq(\"booking_id\", booking_id).first()\n transaction_id = transaction_row[\"id\"] if transaction_row else 4402\n \n expected_changes = [\n # Wishlist lists created\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Amsterdam - Options\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Amsterdam - Options\"),\n ]\n },\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Amsterdam - Final Pick\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Amsterdam - Final Pick\"),\n ]\n },\n # Saved properties (hotel saved to both lists)\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, hotel_id, \"Amsterdam - Options\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"list_name\", \"Amsterdam - Options\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, hotel_id, \"Amsterdam - Final Pick\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"list_name\", \"Amsterdam - Final Pick\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Booking\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"room_id\", room_id),\n (\"check_in_date\", \"2026-10-10\"),\n (\"check_out_date\", \"2026-10-14\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Hannah\"),\n (\"guest_last_name\", \"Lee\"),\n (\"guest_email\", \"hannah.lee@example.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"2135550196\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # Payment method\n {\n \"table\": \"payment_methods\",\n \"pk\": payment_method_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", payment_method_id),\n (\"user_id\", 1),\n (\"card_type\", \"Amex\"),\n (\"last_four\", \"0005\"),\n (\"expiry_month\", 2),\n (\"expiry_year\", 2029),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"378282246310005\"),\n (\"cvv\", \"7714\"),\n (\"cardholder_name\", \"Hannah Lee\"),\n ]\n },\n # Transaction\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", transaction_id),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_l0lotcsdrb9p_1765890829599_zy30maw05", + "prompt": "Create a new wishlist for my upcoming Tokyo trip with the name 'Fun in Tokyo'. Then search for hotels with ratings of 8.5 or above and prices under $200/night. For each result, check the amenities \u2014 only add the hotel to the wishlist if it has Breakfast Included, 24-Hour Front Desk and Beach Access. If any hotel has recent reviews complaining about noise, skip it entirely.", + "env_id": "booking", + "version": "v0.0.62", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_l0lotcsdrb9p_1765890829599_zy30maw05(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # 1. Verify the wishlist entry was created in saved_properties\n # According to the diff: saved_properties INSERT (user_id=1, hotel_id=7228, list_name='Fun in Tokyo')\n \n # Find new entries in saved_properties for user_id=1 with list_name='Fun in Tokyo'\n seed_saved = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Fun in Tokyo\").all()\n current_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Fun in Tokyo\").all()\n \n seed_hotel_ids = {entry[\"hotel_id\"] for entry in seed_saved}\n new_saved_entries = [entry for entry in current_saved if entry[\"hotel_id\"] not in seed_hotel_ids]\n \n if len(new_saved_entries) == 0:\n error_accumulator.append(\"[X] No hotel was saved to the 'Fun in Tokyo' wishlist\")\n else:\n success_accumulator.append(f\"[C] {len(new_saved_entries)} hotel(s) saved to the 'Fun in Tokyo' wishlist\")\n \n # Verify the specific hotel_id from the diff\n new_hotel_ids = [entry[\"hotel_id\"] for entry in new_saved_entries]\n if 7228 in new_hotel_ids:\n success_accumulator.append(\"[C] Hotel ID 7228 was saved to the wishlist as expected\")\n else:\n error_accumulator.append(f\"[X] Expected hotel_id 7228 in wishlist, but found {new_hotel_ids}\")\n\n # 2. Verify list_name is exactly 'Fun in Tokyo'\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 7228).eq(\"list_name\", \"Fun in Tokyo\").first()\n \n if saved_property is not None:\n success_accumulator.append(\"[C] Saved property with correct user_id=1, hotel_id=7228, list_name='Fun in Tokyo' exists\")\n else:\n error_accumulator.append(\"[X] Could not find saved property with user_id=1, hotel_id=7228, list_name='Fun in Tokyo'\")\n\n # 3. Check wishlist_lists table if it exists (not in schema but appears in diff)\n try:\n current_wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Fun in Tokyo\").all()\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"list_name\", \"Fun in Tokyo\").all()\n \n if len(current_wishlist) > len(seed_wishlist):\n success_accumulator.append(\"[C] Wishlist 'Fun in Tokyo' was created in wishlist_lists table\")\n elif len(current_wishlist) > 0:\n success_accumulator.append(\"[C] Wishlist 'Fun in Tokyo' exists in wishlist_lists table\")\n else:\n # Not a critical error since the main data is in saved_properties\n pass\n except Exception:\n # Table may not exist in all environments - not critical\n pass\n\n # Final validation result\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ipdbtdxg8g_1768178466416_528vubg3g", + "prompt": "I won the lottery and haven't told my wife yet. I want to tell her from the highest-rated 5-star hotel room in Chicago, starting 6/27. We would like a king bed. The hotel must also have a business center, as managing this money is a lot of work. We are going to stay there for just 4 nights, so we can enjoy the city and do some shopping before we go to Paris. \n\nLet's book Paris for 2 weeks starting right after we leave Chicago. It should be the highest-rated 5-star hotel there, too. We would like a suite with a king bed. And we are going to need a gym because we tend to eat a lot of pastries. \n\nTim Green Curry\n494 Bounty Street\nSchaumberg, IL 60173\n312-842-8523\nTGC@gmail.com\nCC#371449635398431\nCVC 2003\nEXP 11/29\n\n\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking 5-star hotels in Chicago and Paris\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 2:\n error_accumulator.append(f\"[X] Expected 2 new bookings, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 2 new bookings were created\")\n\n # Sort bookings by check_in_date to identify Chicago (first) and Paris (second)\n new_bookings_sorted = sorted(new_bookings, key=lambda x: x[\"check_in_date\"])\n chicago_booking = new_bookings_sorted[0]\n paris_booking = new_bookings_sorted[1]\n\n # Validate Chicago booking (4 nights starting 6/27)\n if chicago_booking[\"check_in_date\"] != \"2026-06-27\":\n error_accumulator.append(f\"[X] Chicago check-in date should be 2026-06-27, got {chicago_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Chicago check-in date is correct (2026-06-27)\")\n\n if chicago_booking[\"check_out_date\"] != \"2026-07-01\":\n error_accumulator.append(f\"[X] Chicago check-out date should be 2026-07-01 (4 nights), got {chicago_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Chicago check-out date is correct (2026-07-01) - 4 nights\")\n\n # Validate Paris booking (2 weeks starting right after Chicago)\n if paris_booking[\"check_in_date\"] != \"2026-07-01\":\n error_accumulator.append(f\"[X] Paris check-in date should be 2026-07-01, got {paris_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Paris check-in date is correct (2026-07-01) - right after Chicago\")\n\n if paris_booking[\"check_out_date\"] != \"2026-07-15\":\n error_accumulator.append(f\"[X] Paris check-out date should be 2026-07-15 (2 weeks), got {paris_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Paris check-out date is correct (2026-07-15) - 2 weeks\")\n\n # Validate guest details for both bookings\n for booking_name, booking in [(\"Chicago\", chicago_booking), (\"Paris\", paris_booking)]:\n # First name should be Tim\n if not normalized_string_comparison(\"tim\", booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] {booking_name} booking: Expected guest first name 'Tim', got '{booking['guest_first_name']}'\")\n \n # Last name should be Curry (based on diff showing \"Curry\")\n if not normalized_string_comparison(\"curry\", booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] {booking_name} booking: Expected guest last name 'Curry', got '{booking['guest_last_name']}'\")\n \n # Email should be tgc@gmail.com\n if booking[\"guest_email\"].lower() != \"tgc@gmail.com\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected email 'tgc@gmail.com', got '{booking['guest_email']}'\")\n \n # Phone should contain 3128428523\n phone = str(booking[\"guest_phone_number\"]) if booking[\"guest_phone_number\"] else \"\"\n if \"3128428523\" not in phone:\n error_accumulator.append(f\"[X] {booking_name} booking: Expected phone '3128428523', got '{phone}'\")\n \n # Booking should be confirmed\n if booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected status 'confirmed', got '{booking['status']}'\")\n \n # Payment should be paid\n if booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected payment_status 'paid', got '{booking['payment_status']}'\")\n \n # Terms agreed\n if booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] {booking_name} booking: Expected agreed_to_terms to be 1\")\n\n success_accumulator.append(\"[C] Guest details validated for both bookings\")\n\n # Find new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n \n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n else:\n pm = new_payment_methods[0]\n success_accumulator.append(\"[C] 1 new payment method created\")\n \n # Validate card details\n if pm[\"last_four\"] != \"8431\":\n error_accumulator.append(f\"[X] Expected card last four '8431', got '{pm['last_four']}'\")\n \n if pm[\"card_number\"] != \"371449635398431\":\n error_accumulator.append(f\"[X] Expected card number '371449635398431', got '{pm['card_number']}'\")\n \n if pm[\"expiry_month\"] != 11:\n error_accumulator.append(f\"[X] Expected expiry month 11, got {pm['expiry_month']}\")\n \n if pm[\"expiry_year\"] != 2029:\n error_accumulator.append(f\"[X] Expected expiry year 2029, got {pm['expiry_year']}\")\n \n if pm[\"cvv\"] != \"2003\":\n error_accumulator.append(f\"[X] Expected CVV '2003', got '{pm['cvv']}'\")\n \n if pm[\"card_type\"] != \"Amex\":\n error_accumulator.append(f\"[X] Expected card type 'Amex', got '{pm['card_type']}'\")\n \n # Verify bookings reference this payment method\n if chicago_booking[\"payment_method_id\"] != pm[\"id\"]:\n error_accumulator.append(f\"[X] Chicago booking should reference payment method {pm['id']}\")\n if paris_booking[\"payment_method_id\"] != pm[\"id\"]:\n error_accumulator.append(f\"[X] Paris booking should reference payment method {pm['id']}\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 2:\n error_accumulator.append(f\"[X] Expected 2 new transactions, found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 2 transactions created for the bookings\")\n \n for txn in new_transactions:\n if txn[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Transaction {txn['id']} should have status 'Success', got '{txn['status']}'\")\n if txn[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Transaction {txn['id']} should have type 'Payment', got '{txn['transaction_type']}'\")\n\n # Get IDs for expected changes\n chicago_booking_id = chicago_booking[\"id\"]\n paris_booking_id = paris_booking[\"id\"]\n payment_method_id = new_payment_methods[0][\"id\"] if new_payment_methods else 82\n \n # Get transaction IDs\n txn_ids = [t[\"id\"] for t in new_transactions]\n\n expected_changes = [\n # Chicago booking\n {\n \"table\": \"bookings\",\n \"pk\": chicago_booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", chicago_booking[\"hotel_id\"]),\n (\"room_id\", chicago_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-06-27\"),\n (\"check_out_date\", \"2026-07-01\"),\n (\"booking_date\", ...),\n (\"adults\", ...),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Tim\"),\n (\"guest_last_name\", \"Curry\"),\n (\"guest_email\", \"tgc@gmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3128428523\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # Paris booking\n {\n \"table\": \"bookings\",\n \"pk\": paris_booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", paris_booking[\"hotel_id\"]),\n (\"room_id\", paris_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-07-01\"),\n (\"check_out_date\", \"2026-07-15\"),\n (\"booking_date\", ...),\n (\"adults\", ...),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Tim\"),\n (\"guest_last_name\", \"Curry\"),\n (\"guest_email\", \"tgc@gmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3128428523\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # Payment method\n {\n \"table\": \"payment_methods\",\n \"pk\": payment_method_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Amex\"),\n (\"last_four\", \"8431\"),\n (\"expiry_month\", 11),\n (\"expiry_year\", 2029),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"371449635398431\"),\n (\"cvv\", \"2003\"),\n (\"cardholder_name\", ...),\n ]\n },\n # Transactions\n {\n \"table\": \"transactions\",\n \"pk\": txn_ids[0] if len(txn_ids) > 0 else 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": txn_ids[1] if len(txn_ids) > 1 else 4403,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_w8gxyhdk8fn_1765940727561_t8tb8nv75", + "prompt": "I need to add a few wishlists to my account for next year. First, delete any existing wishlists I might have laying around. Then add 4 new wishlists - one for each season next year (Winter, Spring, Summer, Fall), and then the year (e.g. \"Winter 2026\").", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # ===== VALIDATE saved_properties TABLE =====\n # Check that the saved_properties entries for user 1 with 'Summer Getaways' are deleted\n current_summer_getaways = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n if len(current_summer_getaways) == 0:\n success_accumulator.append(\"[C] All 'Summer Getaways' saved_properties entries were deleted\")\n else:\n error_accumulator.append(f\"[X] Expected 'Summer Getaways' saved_properties to be deleted, but found {len(current_summer_getaways)} entries\")\n\n # Verify specific deletions: hotel_id 5017 and 9675\n seed_saved_5017 = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5017).eq(\"list_name\", \"Summer Getaways\").all()\n if len(seed_saved_5017) > 0:\n current_saved_5017 = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5017).eq(\"list_name\", \"Summer Getaways\").all()\n if len(current_saved_5017) == 0:\n success_accumulator.append(\"[C] saved_property (hotel_id=5017, 'Summer Getaways') was deleted\")\n else:\n error_accumulator.append(\"[X] saved_property (hotel_id=5017, 'Summer Getaways') was NOT deleted\")\n\n seed_saved_9675 = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"Summer Getaways\").all()\n if len(seed_saved_9675) > 0:\n current_saved_9675 = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"Summer Getaways\").all()\n if len(current_saved_9675) == 0:\n success_accumulator.append(\"[C] saved_property (hotel_id=9675, 'Summer Getaways') was deleted\")\n else:\n error_accumulator.append(\"[X] saved_property (hotel_id=9675, 'Summer Getaways') was NOT deleted\")\n\n # ===== VALIDATE wishlist_lists TABLE =====\n expected_seasons = [\"Winter 2026\", \"Spring 2026\", \"Summer 2026\", \"Fall 2026\"]\n \n try:\n # Get all wishlists for user 1 in current\n current_wishlists = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all()\n current_list_names = set(w[\"list_name\"] for w in current_wishlists)\n \n # Check that all 4 seasonal wishlists exist\n for season in expected_seasons:\n if season in current_list_names:\n success_accumulator.append(f\"[C] Wishlist '{season}' exists\")\n else:\n error_accumulator.append(f\"[X] Expected wishlist '{season}' to be created, but it was not found\")\n\n # Verify we have exactly 4 wishlists for user 1 (no extras, no missing)\n if len(current_wishlists) == 4:\n success_accumulator.append(\"[C] User has exactly 4 wishlists\")\n else:\n error_accumulator.append(f\"[X] Expected exactly 4 wishlists, but found {len(current_wishlists)}: {current_list_names}\")\n\n # Verify the wishlists are EXACTLY the expected ones (no unexpected list names)\n expected_set = set(expected_seasons)\n if current_list_names == expected_set:\n success_accumulator.append(\"[C] Wishlist names match exactly the expected seasonal names\")\n else:\n extra = current_list_names - expected_set\n missing = expected_set - current_list_names\n if extra:\n error_accumulator.append(f\"[X] Unexpected wishlists found: {extra}\")\n if missing:\n error_accumulator.append(f\"[X] Missing wishlists: {missing}\")\n\n # Check old 'Summer Getaways' was deleted from wishlist_lists\n old_wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Summer Getaways\").all()\n if len(old_wishlist) == 0:\n success_accumulator.append(\"[C] Old 'Summer Getaways' wishlist was deleted from wishlist_lists\")\n else:\n error_accumulator.append(f\"[X] Expected 'Summer Getaways' wishlist to be deleted, but found {len(old_wishlist)} entries\")\n\n except Exception as e:\n error_accumulator.append(f\"[X] Could not validate wishlist_lists table: {e}\")\n\n # ===== VALIDATE NO UNEXPECTED CHANGES IN OTHER TABLES =====\n # Check that no other user data was modified unexpectedly\n # Compare user record for user_id=1\n seed_user = seed.table(\"users\").eq(\"id\", 1).first()\n current_user = current.table(\"users\").eq(\"id\", 1).first()\n if seed_user and current_user:\n # Check key user fields haven't changed\n for field in [\"email\", \"first_name\", \"last_name\", \"phone\", \"country\"]:\n if seed_user.get(field) != current_user.get(field):\n error_accumulator.append(f\"[X] Unexpected change to users.{field}\")\n \n # Check bookings haven't changed unexpectedly\n seed_bookings_count = len(seed.table(\"bookings\").eq(\"user_id\", 1).all())\n current_bookings_count = len(current.table(\"bookings\").eq(\"user_id\", 1).all())\n if seed_bookings_count != current_bookings_count:\n error_accumulator.append(f\"[X] Unexpected change to bookings count: was {seed_bookings_count}, now {current_bookings_count}\")\n\n # Check payment_methods haven't changed unexpectedly\n seed_pm_count = len(seed.table(\"payment_methods\").eq(\"user_id\", 1).all())\n current_pm_count = len(current.table(\"payment_methods\").eq(\"user_id\", 1).all())\n if seed_pm_count != current_pm_count:\n error_accumulator.append(f\"[X] Unexpected change to payment_methods count: was {seed_pm_count}, now {current_pm_count}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ipugd6a3sse_1767526187034_8qwt16ylg", + "prompt": "I'm going alone on multiple business trips in October and need to book hotels. \nThese are the cities and dates: \n\nLondon: 2026-10-10 - 2026-10-12. \nParis: 2026-10-14 - 2026-10-16. \nMiami: 2026-10-18 - 2026-10-20. \n\nBook the the cheapest hotels in each city that has a review score of at least 8. \nMake sure breakfast is included in all three hotels. \nBook the cheapest room in each hotel. Pay for the hotels with my saved payment information on my account. ", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly - 3 hotel bookings for business trips\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 3:\n error_accumulator.append(f\"[X] Expected 3 new bookings, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 3 new bookings created\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 3:\n error_accumulator.append(f\"[X] Expected 3 new transactions, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 3 new transactions created\")\n\n # Define expected booking details\n expected_bookings = {\n \"London\": {\n \"check_in_date\": \"2026-10-10\",\n \"check_out_date\": \"2026-10-12\",\n \"hotel_id\": 5652,\n \"room_id\": 11207\n },\n \"Paris\": {\n \"check_in_date\": \"2026-10-14\",\n \"check_out_date\": \"2026-10-16\",\n \"hotel_id\": 1888,\n \"room_id\": 3741\n },\n \"Miami\": {\n \"check_in_date\": \"2026-10-18\",\n \"check_out_date\": \"2026-10-20\",\n \"hotel_id\": 12792,\n \"room_id\": 25392\n }\n }\n\n # Map bookings by check-in date to verify each city\n bookings_by_checkin = {b[\"check_in_date\"]: b for b in new_bookings}\n\n # Define expected cities by check-in date\n expected_city_map = {\n \"2026-10-10\": \"london\",\n \"2026-10-14\": \"paris\",\n \"2026-10-18\": \"miami\"\n }\n\n # Validate each booking's hotel is in the correct city and has rating >= 8\n for check_in_date, expected_city in expected_city_map.items():\n booking = bookings_by_checkin.get(check_in_date)\n if not booking:\n error_accumulator.append(f\"[X] {expected_city.title()} booking (check-in {check_in_date}) not found\")\n continue\n \n # Get hotel details to verify city and rating\n hotel = current.table(\"hotels\").eq(\"id\", booking[\"hotel_id\"]).first()\n if not hotel:\n error_accumulator.append(f\"[X] Hotel {booking['hotel_id']} for {expected_city.title()} booking not found\")\n continue\n \n # Verify hotel is in the correct city\n hotel_city = hotel.get(\"city\", \"\").lower()\n if expected_city not in hotel_city:\n error_accumulator.append(f\"[X] {expected_city.title()} booking: Expected hotel in {expected_city.title()}, but hotel is in '{hotel.get('city', 'Unknown')}'\")\n else:\n success_accumulator.append(f\"[C] {expected_city.title()} booking: Hotel is correctly in {hotel.get('city', expected_city.title())}\")\n \n # Verify hotel has average_rating >= 8\n hotel_rating = hotel.get(\"average_rating\", 0)\n if hotel_rating is None or hotel_rating < 8:\n error_accumulator.append(f\"[X] {expected_city.title()} booking: Hotel rating is {hotel_rating}, expected >= 8\")\n else:\n success_accumulator.append(f\"[C] {expected_city.title()} booking: Hotel rating {hotel_rating} >= 8\")\n \n # Check for breakfast amenity - look in hotel_amenities for breakfast-related amenities\n hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", booking[\"hotel_id\"]).all()\n amenity_ids = [a[\"amenity_id\"] for a in hotel_amenities]\n \n has_breakfast = False\n if amenity_ids:\n # Check if any of the hotel's amenities are breakfast-related\n for amenity_id in amenity_ids:\n amenity = current.table(\"amenities\").eq(\"id\", amenity_id).first()\n if amenity:\n amenity_name = amenity.get(\"name\", \"\").lower()\n if \"breakfast\" in amenity_name:\n has_breakfast = True\n break\n \n if not has_breakfast:\n error_accumulator.append(f\"[X] {expected_city.title()} booking: Hotel does not have breakfast amenity\")\n else:\n success_accumulator.append(f\"[C] {expected_city.title()} booking: Hotel has breakfast amenity\")\n\n # Validate London booking (Oct 10-12)\n london_booking = bookings_by_checkin.get(\"2026-10-10\")\n if london_booking:\n if london_booking[\"check_out_date\"] != \"2026-10-12\":\n error_accumulator.append(f\"[X] London booking: Expected check-out 2026-10-12, got {london_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] London booking has correct dates (2026-10-10 to 2026-10-12)\")\n \n if london_booking[\"purpose_of_trip\"] != \"Business\":\n error_accumulator.append(f\"[X] London booking: Expected purpose 'Business', got {london_booking['purpose_of_trip']}\")\n else:\n success_accumulator.append(\"[C] London booking marked as Business trip\")\n\n # Validate Paris booking (Oct 14-16)\n paris_booking = bookings_by_checkin.get(\"2026-10-14\")\n if paris_booking:\n if paris_booking[\"check_out_date\"] != \"2026-10-16\":\n error_accumulator.append(f\"[X] Paris booking: Expected check-out 2026-10-16, got {paris_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Paris booking has correct dates (2026-10-14 to 2026-10-16)\")\n \n if paris_booking[\"purpose_of_trip\"] != \"Business\":\n error_accumulator.append(f\"[X] Paris booking: Expected purpose 'Business', got {paris_booking['purpose_of_trip']}\")\n else:\n success_accumulator.append(\"[C] Paris booking marked as Business trip\")\n\n # Validate Miami booking (Oct 18-20)\n miami_booking = bookings_by_checkin.get(\"2026-10-18\")\n if miami_booking:\n if miami_booking[\"check_out_date\"] != \"2026-10-20\":\n error_accumulator.append(f\"[X] Miami booking: Expected check-out 2026-10-20, got {miami_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Miami booking has correct dates (2026-10-18 to 2026-10-20)\")\n \n if miami_booking[\"purpose_of_trip\"] != \"Business\":\n error_accumulator.append(f\"[X] Miami booking: Expected purpose 'Business', got {miami_booking['purpose_of_trip']}\")\n else:\n success_accumulator.append(\"[C] Miami booking marked as Business trip\")\n\n # Validate common attributes for all bookings\n for booking in new_bookings:\n booking_id = booking[\"id\"]\n \n # Check adults = 1 (solo traveler)\n if booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Booking {booking_id}: Expected 1 adult, got {booking['adults']}\")\n \n # Check room_count = 1\n if booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Booking {booking_id}: Expected room_count 1, got {booking['room_count']}\")\n \n # Check status is confirmed\n if booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Booking {booking_id}: Expected status 'confirmed', got {booking['status']}\")\n \n # Check payment_status is paid\n if booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Booking {booking_id}: Expected payment_status 'paid', got {booking['payment_status']}\")\n \n # Check agreed_to_terms = 1\n if booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Booking {booking_id}: Expected agreed_to_terms to be 1\")\n \n # Check payment_method_id = 1 (saved payment method)\n if booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Booking {booking_id}: Expected payment_method_id 1 (saved), got {booking['payment_method_id']}\")\n\n # Validate all transactions are successful\n for txn in new_transactions:\n if txn[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Transaction {txn['id']}: Expected status 'Success', got {txn['status']}\")\n if txn[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Transaction {txn['id']}: Expected type 'Payment', got {txn['transaction_type']}\")\n if txn[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Transaction {txn['id']}: Expected payment_method_id 1, got {txn['payment_method_id']}\")\n\n if len(new_bookings) == 3:\n success_accumulator.append(\"[C] All 3 bookings use saved payment method (payment_method_id=1)\")\n success_accumulator.append(\"[C] All 3 bookings are confirmed and paid\")\n\n # Generate expected changes for expect_only_v2\n expected_changes = []\n \n # Add booking insert expectations\n for booking in new_bookings:\n expected_changes.append({\n \"table\": \"bookings\",\n \"pk\": booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", booking[\"hotel_id\"]),\n (\"room_id\", booking[\"room_id\"]),\n (\"check_in_date\", booking[\"check_in_date\"]),\n (\"check_out_date\", booking[\"check_out_date\"]),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", \"Business\"),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", 0),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", 0),\n (\"update_account_with_guest_details\", 0),\n (\"rate_plan_id\", None),\n ]\n })\n\n # Add transaction insert expectations\n for txn in new_transactions:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": txn[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", ...),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_tedk9rq2f1yv_1767534275668_ccatckvva", + "prompt": "I am looking to make a long trip of 15 days to Bangkok, Thailand this March. \n1. Search for only apartment type properties from 15th-30th March, 2026. with guest rating greater than 9.2 for one adult.\n2. Focus on properties with more than 500 reviews submitted and choose the property with the lowest price per night. \n3. Check room availability at the resulting property from previous step and create a booking for a room with city view. \n4. Use the following info:\nGuest Name: Rob Key\nEmail: Robkey@yahoo.com\nPhone: +1-7415628963\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_tedk9rq2f1yv_1767534275668_ccatckvva(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking created\")\n\n new_booking = new_bookings[0]\n \n # Get the full booking details\n booking = current.table(\"bookings\").eq(\"id\", new_booking[\"id\"]).first()\n \n # Validate check-in and check-out dates (15th-30th March 2026 = 15 days)\n if booking[\"check_in_date\"] != \"2026-03-15\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-03-15', got '{booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-03-15\")\n \n if booking[\"check_out_date\"] != \"2026-03-30\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-03-30', got '{booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-03-30\")\n \n # Validate guest details\n if not normalized_string_comparison(\"rob\", booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Rob', got '{booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct: Rob\")\n \n if not normalized_string_comparison(\"key\", booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Key', got '{booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct: Key\")\n \n # Email verification (case insensitive for email)\n if booking[\"guest_email\"].lower() != \"robkey@yahoo.com\":\n error_accumulator.append(f\"[X] Expected guest email 'robkey@yahoo.com', got '{booking['guest_email']}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct: robkey@yahoo.com\")\n \n # Phone verification\n if booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got '{booking['guest_phone_country_code']}'\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct: +1\")\n \n phone_number = str(booking[\"guest_phone_number\"]) if booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"7415628963\":\n error_accumulator.append(f\"[X] Expected phone number '7415628963', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct: 7415628963\")\n \n # Validate adults = 1\n if booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, got {booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Number of adults is correct: 1\")\n \n # Validate booking status\n if booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Validate agreed to terms\n if booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n \n # Verify the room has a city view by checking the room details\n room_id = booking[\"room_id\"]\n room = current.table(\"rooms\").eq(\"id\", room_id).first()\n if room:\n view_type = room.get(\"view_type\", \"\")\n if view_type and normalized_contains(\"city\", view_type):\n success_accumulator.append(f\"[C] Room has city view: {view_type}\")\n else:\n error_accumulator.append(f\"[X] Expected room with city view, got view_type: '{view_type}'\")\n else:\n error_accumulator.append(f\"[X] Room with ID {room_id} not found\")\n \n # Verify the hotel is in Bangkok, Thailand and is an apartment type\n hotel_id = booking[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n # Check that city is Bangkok\n if normalized_contains(\"bangkok\", hotel.get(\"city\", \"\")):\n success_accumulator.append(f\"[C] Hotel is in Bangkok: {hotel.get('city')}\")\n else:\n error_accumulator.append(f\"[X] Expected hotel in Bangkok, got city: '{hotel.get('city')}'\")\n \n # Check that country is Thailand (can be \"Thailand\" or country code \"TH\")\n country = hotel.get(\"country\", \"\")\n if normalized_contains(\"thailand\", country) or country.upper() == \"TH\":\n success_accumulator.append(f\"[C] Hotel is in Thailand: {country}\")\n else:\n error_accumulator.append(f\"[X] Expected hotel in Thailand, got country: '{country}'\")\n \n # Check guest rating > 9.2\n avg_rating = hotel.get(\"average_rating\", 0)\n if avg_rating and avg_rating > 9.2:\n success_accumulator.append(f\"[C] Hotel rating > 9.2: {avg_rating}\")\n else:\n error_accumulator.append(f\"[X] Expected hotel rating > 9.2, got: {avg_rating}\")\n \n # Check total_reviews > 500\n total_reviews = hotel.get(\"total_reviews\", 0)\n if total_reviews and total_reviews > 500:\n success_accumulator.append(f\"[C] Hotel has > 500 reviews: {total_reviews}\")\n else:\n error_accumulator.append(f\"[X] Expected hotel with > 500 reviews, got: {total_reviews}\")\n else:\n error_accumulator.append(f\"[X] Hotel with ID {hotel_id} not found\")\n \n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction created\")\n new_transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n # Verify transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction not linked to correct booking\")\n else:\n success_accumulator.append(\"[C] Transaction linked to correct booking\")\n \n # Verify transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", booking[\"hotel_id\"]),\n (\"room_id\", booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-15\"),\n (\"check_out_date\", \"2026-03-30\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Rob\"),\n (\"guest_last_name\", \"Key\"),\n (\"guest_email\", \"robkey@yahoo.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"7415628963\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n ]\n \n # Add transaction to expected changes if found\n if new_transactions:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking[\"id\"]),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'transactions': {'processed_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_cz5falnutifg_1768119069952_juyrc5ihf", + "prompt": "Book a single hotel room from 2026/01/20 (check-in) to 2026/02/04 (check-out). The hotel must be at least a 4-star property in Amsterdam and accommodate 2 adults and 2 children. It must include pool and restaurant facilities and enforce a no-smoking policy. My budget is USD 660 to 670 per night.\n\nThe room must accommodate 2 children and offer a city view.\n\nTo complete the purchase, create a new payment method using the details below:\n\nName: Charles Moore \nCard Number: 44444444444444444 \nExpiry Date: 01/26 \nCVV: 121\n\nI agree to the terms and conditions.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_cz5falnutifg_1768119069952_juyrc5ihf(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n # Validate check-in and check-out dates\n if new_booking[\"check_in_date\"] != \"2026-01-20\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-01-20', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-01-20\")\n\n if new_booking[\"check_out_date\"] != \"2026-02-04\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-02-04', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-02-04\")\n\n # Validate guest counts (2 adults, 2 children)\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct: 2\")\n\n if new_booking[\"children\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 children, got {new_booking['children']}\")\n else:\n success_accumulator.append(\"[C] Children count is correct: 2\")\n\n # Validate room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 room, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct: 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Validate guest name matches payment card name\n if not normalized_string_comparison(\"charles\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Charles', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is Charles\")\n\n if not normalized_string_comparison(\"moore\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Moore', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is Moore\")\n\n # Find the new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n new_payment_method = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n success_accumulator.append(\"[C] 1 new payment method was created\")\n\n # Validate payment method details\n if new_payment_method[\"card_number\"] != \"44444444444444444\":\n error_accumulator.append(f\"[X] Expected card number '44444444444444444', got '{new_payment_method['card_number']}'\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n if new_payment_method[\"last_four\"] != \"4444\":\n error_accumulator.append(f\"[X] Expected last four digits '4444', got '{new_payment_method['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Last four digits are correct: 4444\")\n\n if new_payment_method[\"expiry_month\"] != 1:\n error_accumulator.append(f\"[X] Expected expiry month 1, got {new_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct: 1\")\n\n if new_payment_method[\"expiry_year\"] != 2026:\n error_accumulator.append(f\"[X] Expected expiry year 2026, got {new_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct: 2026\")\n\n if new_payment_method[\"cvv\"] != \"121\":\n error_accumulator.append(f\"[X] Expected CVV '121', got '{new_payment_method['cvv']}'\")\n else:\n success_accumulator.append(\"[C] CVV is correct: 121\")\n\n if not normalized_string_comparison(\"charles moore\", new_payment_method.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder name 'Charles Moore', got '{new_payment_method.get('cardholder_name')}'\")\n else:\n success_accumulator.append(\"[C] Cardholder name is correct: Charles Moore\")\n\n # Validate that booking references the new payment method\n if new_booking[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Expected booking to reference payment method {new_payment_method['id']}, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking references the correct payment method\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n success_accumulator.append(\"[C] 1 new transaction was created\")\n\n # Validate transaction links to booking and payment method\n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction should reference booking {new_booking['id']}, got {new_transaction['booking_id']}\")\n \n if new_transaction[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Transaction should reference payment method {new_payment_method['id']}, got {new_transaction['payment_method_id']}\")\n\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-01-20\"),\n (\"check_out_date\", \"2026-02-04\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 2),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": new_payment_method[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_payment_method[\"id\"]),\n (\"user_id\", 1),\n (\"card_type\", ...),\n (\"last_four\", \"4444\"),\n (\"expiry_month\", 1),\n (\"expiry_year\", 2026),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"44444444444444444\"),\n (\"cvv\", \"121\"),\n (\"cardholder_name\", \"Charles Moore\"),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 0,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'price_calendar': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_i7ipqa9jhuo_1765349270014_u1qrrwm0a", + "prompt": "First, create a wish list for your Tokyo lodging search. Then, search for hotels or resorts in Tokyo, for 2 adults and 1 child in 1 room, with availability from January 2-9, 2026. The hotel or resort should have a minimum rating of 7 and at least 3 stars. Sort the results by minimum price, in ascending order, and display 5 results per page. For the first hotel or resort returned, view the images. Then add this property to your wishlist, and add a note.", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_i7ipqa9jhuo_1765349270014_u1qrrwm0a(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Verify a property was saved to a wishlist for user 8 with the expected details\n # Based on diff: (8, 18344, 'My Tokyo Hotels', 'This seems like a possibility', '2025-12-12 04:20:04')\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 8).eq(\"hotel_id\", 18344).first()\n \n if saved_property is None:\n error_accumulator.append(\"[X] No property with hotel_id 18344 was saved for user 8\")\n else:\n success_accumulator.append(\"[C] A property (hotel_id: 18344) was saved for user 8\")\n \n # Verify the list name contains \"Tokyo\" as per the task requirement\n list_name = saved_property.get(\"list_name\", \"\")\n if not list_name:\n error_accumulator.append(\"[X] No list_name found for the saved property\")\n elif normalized_contains(\"Tokyo\", list_name):\n success_accumulator.append(f\"[C] Wishlist name '{list_name}' contains 'Tokyo'\")\n else:\n error_accumulator.append(f\"[X] Expected wishlist name to contain 'Tokyo', but got '{list_name}'\")\n \n # Verify a note was added (the note should exist and not be empty)\n notes = saved_property.get(\"notes\", \"\")\n if notes is None or notes == \"\":\n error_accumulator.append(\"[X] No note was added to the saved property\")\n else:\n success_accumulator.append(f\"[C] A note was added to the saved property: '{notes}'\")\n\n # Check that no saved_properties existed before for user 8 with this hotel\n seed_saved = seed.table(\"saved_properties\").eq(\"user_id\", 8).eq(\"hotel_id\", 18344).first()\n if seed_saved is not None:\n error_accumulator.append(\"[X] The saved property already existed in seed database\")\n else:\n success_accumulator.append(\"[C] Confirmed the saved property is newly created\")\n\n # Verify the hotel 18344 exists and is valid (optional sanity check)\n hotel = current.table(\"hotels\").eq(\"id\", 18344).first()\n if hotel is None:\n error_accumulator.append(\"[X] Hotel 18344 does not exist in database\")\n else:\n success_accumulator.append(f\"[C] Hotel 18344 exists: {hotel.get('name', 'Unknown')}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kknvdjitvlvz_1768220087117_a33ofxsdt", + "prompt": "I am building a wishlist to help me plan my stay in Paris. I want to stay in a culturally rich, walkable neighborhood like Le Marais. \nSearch for properties where either the name or address indicates 'Marais.' \nThen apply these rules: \n- Dates: 2026-03-06 to 2026-03-09\n- Property rating: 5 star\n- Rating must be at least 9.1. \n- Exclude hostels; search only for hotels or apartments. \n- Create a new wishlist titled 'Paris walkable neighborhoods.' \n- Add all resulting properties from the search to this wishlist.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_kknvdjitvlvz_1768220087117_a33ofxsdt(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - creating a wishlist for Paris properties\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Check that the wishlist was created with the correct name\n wishlist_name = \"Paris walkable neighborhoods\"\n \n # Verify wishlist_lists table has the new wishlist\n try:\n current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", wishlist_name).assert_exists()\n success_accumulator.append(f\"[C] Wishlist '{wishlist_name}' was created\")\n except AssertionError:\n error_accumulator.append(f\"[X] Wishlist '{wishlist_name}' was not created\")\n\n # Get all saved properties in the new wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", wishlist_name).all()\n \n # According to the diff, 7 properties should be saved\n expected_hotel_ids = {7489, 12612, 8017, 17635, 6964, 15267, 6412}\n actual_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n \n if len(saved_properties) != 7:\n error_accumulator.append(f\"[X] Expected 7 properties saved to wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(f\"[C] 7 properties were saved to the wishlist\")\n \n # Verify the correct hotels were saved\n if actual_hotel_ids != expected_hotel_ids:\n missing = expected_hotel_ids - actual_hotel_ids\n extra = actual_hotel_ids - expected_hotel_ids\n if missing:\n error_accumulator.append(f\"[X] Missing hotel IDs in wishlist: {missing}\")\n if extra:\n error_accumulator.append(f\"[X] Unexpected hotel IDs in wishlist: {extra}\")\n else:\n success_accumulator.append(f\"[C] All expected hotels ({expected_hotel_ids}) were saved to the wishlist\")\n\n # Verify all saved properties belong to user_id 1\n for prop in saved_properties:\n if prop[\"user_id\"] != 1:\n error_accumulator.append(f\"[X] Property {prop['hotel_id']} has incorrect user_id: {prop['user_id']}\")\n break\n else:\n success_accumulator.append(f\"[C] All saved properties belong to user_id 1\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n # Wishlist list creation\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Paris walkable neighborhoods\"), # Composite primary key (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n ]\n },\n # Saved properties - 7 properties added to the wishlist\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 7489, \"Paris walkable neighborhoods\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 7489),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 12612, \"Paris walkable neighborhoods\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 12612),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 8017, \"Paris walkable neighborhoods\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 8017),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 17635, \"Paris walkable neighborhoods\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 17635),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 6964, \"Paris walkable neighborhoods\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 6964),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 15267, \"Paris walkable neighborhoods\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 15267),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 6412, \"Paris walkable neighborhoods\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 6412),\n (\"list_name\", \"Paris walkable neighborhoods\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'updated_at', 'created_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'updated_at', 'created_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_d5lmzk0wxetw_1765883491177_6zch3x8bk", + "prompt": "I booked some hotels for my employees and they didn't go, I want to find these hotels and contact them to get a refund agreement. Create a new wishlist called \"No Show\" and add all the booked hotels with a \"no show\" status. For each item added to the wishlist add in the notes the total amount paid so I can communicate it to the hotel. ", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_d5lmzk0wxetw_1765883491177_6zch3x8bk(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Step 1: Verify that the \"No Show\" wishlist was created in wishlist_lists\n no_show_list = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"No Show\").first()\n if no_show_list:\n success_accumulator.append(\"[C] 'No Show' wishlist was created in wishlist_lists table\")\n else:\n error_accumulator.append(\"[X] 'No Show' wishlist was not created in wishlist_lists table\")\n\n # Step 2: Verify the saved_properties entries were created correctly\n # According to the diff, we expect 2 entries:\n # - Hotel 8024 with notes '336.19'\n # - Hotel 5342 with notes '1777.16'\n \n saved_props = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"No Show\").all()\n \n if len(saved_props) != 2:\n error_accumulator.append(f\"[X] Expected 2 properties in 'No Show' wishlist, but found {len(saved_props)}\")\n else:\n success_accumulator.append(f\"[C] 2 properties were added to the 'No Show' wishlist\")\n \n # Verify each saved property\n # Create a dict for easier lookup\n saved_by_hotel = {prop[\"hotel_id\"]: prop for prop in saved_props}\n \n # Verify hotel 8024 with amount 336.19\n if 8024 in saved_by_hotel:\n prop = saved_by_hotel[8024]\n notes = prop.get(\"notes\", \"\")\n if contains_number(notes, 336.19):\n success_accumulator.append(f\"[C] Hotel 8024 notes contain correct amount 336.19\")\n else:\n error_accumulator.append(f\"[X] Hotel 8024 notes '{notes}' should contain amount 336.19\")\n else:\n error_accumulator.append(\"[X] Hotel 8024 was not added to 'No Show' wishlist\")\n \n # Verify hotel 5342 with amount 1777.16\n if 5342 in saved_by_hotel:\n prop = saved_by_hotel[5342]\n notes = prop.get(\"notes\", \"\")\n if contains_number(notes, 1777.16):\n success_accumulator.append(f\"[C] Hotel 5342 notes contain correct amount 1777.16\")\n else:\n error_accumulator.append(f\"[X] Hotel 5342 notes '{notes}' should contain amount 1777.16\")\n else:\n error_accumulator.append(\"[X] Hotel 5342 was not added to 'No Show' wishlist\")\n\n # Step 3: Verify these entries did not exist in the seed database\n seed_no_show_list = seed.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"No Show\").all()\n if len(seed_no_show_list) == 0:\n success_accumulator.append(\"[C] 'No Show' wishlist did not exist in seed database (confirmed as new)\")\n else:\n error_accumulator.append(\"[X] 'No Show' wishlist already existed in seed database\")\n\n seed_saved_props = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"No Show\").all()\n if len(seed_saved_props) == 0:\n success_accumulator.append(\"[C] No saved properties in 'No Show' list existed in seed database (confirmed as new)\")\n else:\n error_accumulator.append(\"[X] Some saved properties in 'No Show' list already existed in seed database\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_j4bva9m26guh_1768163383434_232m8imtm", + "prompt": "I have a business conference in Tokyo from April 14-18, 2026. I need a hotel in Shinjuku with at least 8.5 average rating, under $350/night.\n\nA colleague recommended the \"Shinjuku Gyoen Tranquility Hotel\" - book there unless you find another Shinjuku hotel that is both cheaper AND has a higher rating.\n\nBook a Standard room for 1 adult. If multiple Standard rooms are available, pick the cheapest one.\n\nUse my details for the booking:\n- Name: Charles Moore\n- Email: charles.moore@synapse.com\n- Phone: +1 318-511-8179\n\nAdd a special request for \"early check-in if possible; arriving on a red-eye flight\".\n\nUse my saved payment method to complete the booking.\n\nAlso, save the \"Shinjuku Gyoen Tranquility Hotel\" to a new wishlist called \"Tokyo Business Alternatives\" in case I need to rebook for a future trip. ", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_j4bva9m26guh_1768163383434_232m8imtm(env: Environment, final_answer: str | None = None, *args, **kwargs) -> int:\n \"\"\"Validate that the task was completed correctly - booking a hotel in Shinjuku for a business conference and saving to wishlist\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n\n # Validate check-in and check-out dates (April 14-18, 2026)\n if new_booking[\"check_in_date\"] != \"2026-04-14\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-04-14', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (April 14, 2026)\")\n\n if new_booking[\"check_out_date\"] != \"2026-04-18\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-04-18', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (April 18, 2026)\")\n\n # Validate guest details - Name: Charles Moore\n if not normalized_string_comparison(\"charles\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Charles', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is 'Charles'\")\n\n if not normalized_string_comparison(\"moore\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Moore', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is 'Moore'\")\n\n # Validate email - charles.moore@synapse.com\n guest_email = (new_booking.get(\"guest_email\") or \"\").lower()\n if guest_email != \"charles.moore@synapse.com\":\n error_accumulator.append(f\"[X] Expected guest email 'charles.moore@synapse.com', got '{guest_email}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n\n # Validate phone number - +1 318-511-8179\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got '{new_booking['guest_phone_country_code']}'\")\n else:\n success_accumulator.append(\"[C] Phone country code is '+1'\")\n\n # Phone number should contain the digits 3185118179\n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"3185118179\":\n error_accumulator.append(f\"[X] Expected phone number '3185118179', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n\n # Validate special request for early check-in\n special_requests = new_booking.get(\"special_requests\") or \"\"\n if not normalized_contains(\"early check-in\", special_requests):\n error_accumulator.append(f\"[X] Expected special request to contain 'early check-in', got '{special_requests}'\")\n elif not normalized_contains(\"red-eye\", special_requests):\n error_accumulator.append(f\"[X] Expected special request to contain 'red-eye flight', got '{special_requests}'\")\n else:\n success_accumulator.append(\"[C] Special request for early check-in with red-eye flight info is present\")\n\n # Validate 1 adult booking\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n\n # Validate room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is 'confirmed'\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is 'paid'\")\n\n # Validate that a saved payment method was used (payment_method_id should be set)\n if new_booking[\"payment_method_id\"] is None:\n error_accumulator.append(\"[X] Expected a saved payment method to be used, but payment_method_id is None\")\n else:\n success_accumulator.append(\"[C] A saved payment method was used for the booking\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n\n # Validate the wishlist was created and hotel saved\n # Check for the new wishlist list entry\n wishlist_entry = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Tokyo Business Alternatives\").first()\n if not wishlist_entry:\n error_accumulator.append(\"[X] Wishlist 'Tokyo Business Alternatives' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Tokyo Business Alternatives' was created\")\n\n # Check that Shinjuku Gyoen Tranquility Hotel was saved to the wishlist\n # From the diff, hotel_id 3923 was saved to the wishlist\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Tokyo Business Alternatives\").first()\n if not saved_property:\n error_accumulator.append(\"[X] No property was saved to 'Tokyo Business Alternatives' wishlist\")\n else:\n # Verify the hotel saved is the Shinjuku Gyoen Tranquility Hotel (hotel_id 3923 from diff)\n if saved_property[\"hotel_id\"] == 3923:\n success_accumulator.append(\"[C] Shinjuku Gyoen Tranquility Hotel was saved to the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected hotel_id 3923 to be saved, got {saved_property['hotel_id']}\")\n\n # Validate transaction was created\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 transaction was created for the booking\")\n \n # Verify transaction is linked to the booking\n transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n if transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction booking_id mismatch\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n if transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is 'Success'\")\n\n # Generate expected changes for expect_only_v2\n transaction_id = new_transactions[0][\"id\"] if new_transactions else 0\n \n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-04-14\"),\n (\"check_out_date\", \"2026-04-18\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"charles.moore@synapse.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3185118179\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 3923, \"Tokyo Business Alternatives\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 3923),\n (\"list_name\", \"Tokyo Business Alternatives\"),\n (\"notes\", ...),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n },\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Tokyo Business Alternatives\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Tokyo Business Alternatives\"),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n fields={\"created_at\", \"updated_at\"},\n table_fields={\n 'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_dytgxrn4rtbd_1766206160566_ivg2ix172", + "prompt": "Create a new wishlist called Best and add any hotels that are currently being featured and have a star rating over 4.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # 1. Verify wishlist \"Best\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Best\").eq(\"user_id\", \"1\").first()\n if wishlist is None:\n error_accumulator.append(\"[X] Wishlist 'Best' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Best' was created\")\n\n # 2. Verify saved_properties entries were created for list \"Best\"\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Best\").all()\n \n if len(saved_properties) != 3:\n error_accumulator.append(f\"[X] Expected 3 hotels saved to 'Best' wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] 3 hotels were saved to 'Best' wishlist\")\n\n # 3. Verify the specific hotel IDs that were saved match the diff\n expected_hotel_ids = {13340, 2932, 6370}\n actual_hotel_ids = {sp[\"hotel_id\"] for sp in saved_properties}\n \n if actual_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, but found {actual_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] Correct hotel IDs were saved: {expected_hotel_ids}\")\n\n # 4. Verify all saved hotels have star_rating > 4 (validating the task criteria)\n # NOTE: The \"currently being featured\" requirement cannot be verified programmatically\n # as there is no \"featured\" column in the hotels schema. \"Featured\" hotels are likely\n # determined by UI/application logic (e.g., displayed on homepage). We verify:\n # - The star_rating > 4 requirement (database verifiable)\n # - The hotels match what the diff shows were saved (trusting the agent found featured hotels via UI)\n for sp in saved_properties:\n hotel_id = sp[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n star_rating = hotel.get(\"star_rating\")\n if star_rating is None or star_rating <= 4:\n error_accumulator.append(f\"[X] Hotel {hotel_id} has star_rating {star_rating}, expected > 4\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has valid star_rating {star_rating} > 4\")\n \n # Verify hotel is active (a reasonable proxy for being \"featured\")\n is_active = hotel.get(\"is_active\")\n if is_active != 1:\n error_accumulator.append(f\"[X] Hotel {hotel_id} is not active (is_active={is_active})\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} is active\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} not found in hotels table\")\n\n # 5. Verify that a new wishlist_lists entry exists\n seed_wishlists = seed.table(\"wishlist_lists\").eq(\"list_name\", \"Best\").all()\n \n if len(seed_wishlists) > 0:\n error_accumulator.append(f\"[X] Wishlist 'Best' already existed in seed database\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Use expect_only_v2 to ensure ONLY the expected changes occurred\n \n expected_changes = [\n # New wishlist entry - composite primary key (user_id, list_name)\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Best\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Best\"),\n ]\n },\n ]\n \n # Add saved_properties entries for each hotel\n for hotel_id in expected_hotel_ids:\n expected_changes.append({\n \"table\": \"saved_properties\",\n \"pk\": (1, hotel_id, \"Best\"), # Composite key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"list_name\", \"Best\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n \"wishlist_lists\": {\"created_at\"},\n \"saved_properties\": {\"saved_at\"}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff (no unintended changes)\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed - unexpected changes detected: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_iykpmwqdx60m_1765942398106_cnfdzebrb", + "prompt": "I'm reorganizing my Booking.com wishlists for upcoming US trips in 2026 and want to start a fresh curated list of my top picks. Create a new wishlist named \"US City Favorites 2026\", then add these two hotels to it: \"South Austin Casita - 1501 E Oltorf St\" in Austin and \"South Beach Art Deco Hotel - Collins Avenue 1200\" in Miami. Afterward, delete the existing wishlist named \"Summer Getaways\", and finally confirm by listing the exact hotel names now saved in \"US City Favorites 2026\".", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_iykpmwqdx60m_1765942398106_cnfdzebrb(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # 1. Verify the new wishlist \"US City Favorites 2026\" has exactly 2 hotels in saved_properties\n new_saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"US City Favorites 2026\").all()\n \n if len(new_saved_properties) != 2:\n error_accumulator.append(f\"[X] Expected 2 hotels in 'US City Favorites 2026', found {len(new_saved_properties)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n success_accumulator.append(\"[C] Exactly 2 hotels were added to 'US City Favorites 2026'\")\n\n # Get the hotel IDs from the new wishlist\n saved_hotel_ids = set([prop[\"hotel_id\"] for prop in new_saved_properties])\n expected_hotel_ids = {13899, 15867}\n \n if saved_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, found {saved_hotel_ids}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n success_accumulator.append(f\"[C] Correct hotels (IDs {expected_hotel_ids}) added to wishlist\")\n\n # 2. Verify the \"Summer Getaways\" saved_properties were deleted\n summer_getaways_props = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n if len(summer_getaways_props) == 0:\n success_accumulator.append(\"[C] All saved properties from 'Summer Getaways' were removed\")\n else:\n error_accumulator.append(f\"[X] Expected 0 properties in 'Summer Getaways', found {len(summer_getaways_props)}\")\n\n # 3. Verify Summer Getaways existed in seed with the expected hotels\n seed_summer_getaways = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n seed_summer_hotel_ids = set([prop[\"hotel_id\"] for prop in seed_summer_getaways])\n expected_deleted_hotel_ids = {5017, 9675}\n \n if seed_summer_hotel_ids != expected_deleted_hotel_ids:\n error_accumulator.append(f\"[X] Expected seed Summer Getaways to have hotels {expected_deleted_hotel_ids}, found {seed_summer_hotel_ids}\")\n else:\n success_accumulator.append(\"[C] Correct hotels were deleted from 'Summer Getaways'\")\n\n # 4. Verify no unexpected changes in saved_properties\n # Get all saved_properties for user 1 in both seed and current\n seed_all_props = seed.table(\"saved_properties\").eq(\"user_id\", 1).all()\n current_all_props = current.table(\"saved_properties\").eq(\"user_id\", 1).all()\n \n # Create sets of (hotel_id, list_name) tuples for comparison\n seed_entries = set((p[\"hotel_id\"], p[\"list_name\"]) for p in seed_all_props)\n current_entries = set((p[\"hotel_id\"], p[\"list_name\"]) for p in current_all_props)\n \n # Calculate actual changes\n added_entries = current_entries - seed_entries\n deleted_entries = seed_entries - current_entries\n \n # Expected additions\n expected_additions = {\n (13899, \"US City Favorites 2026\"),\n (15867, \"US City Favorites 2026\")\n }\n \n # Expected deletions\n expected_deletions = {\n (5017, \"Summer Getaways\"),\n (9675, \"Summer Getaways\")\n }\n \n if added_entries != expected_additions:\n error_accumulator.append(f\"[X] Unexpected additions: expected {expected_additions}, found {added_entries}\")\n else:\n success_accumulator.append(\"[C] Only expected entries were added to saved_properties\")\n \n if deleted_entries != expected_deletions:\n error_accumulator.append(f\"[X] Unexpected deletions: expected {expected_deletions}, found {deleted_entries}\")\n else:\n success_accumulator.append(\"[C] Only expected entries were deleted from saved_properties\")\n\n # 5. Verify the new wishlist didn't exist before\n seed_new_wishlist = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"US City Favorites 2026\").all()\n if len(seed_new_wishlist) == 0:\n success_accumulator.append(\"[C] 'US City Favorites 2026' was newly created (didn't exist before)\")\n else:\n error_accumulator.append(f\"[X] 'US City Favorites 2026' already existed in seed with {len(seed_new_wishlist)} entries\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_m2fcdd9yauen_1766208395671_sks5humh2", + "prompt": "I'm looking for the absolute best deal for the best price. Find any hotel in the US that can take 3 adults in a single room for the whole month of July. They must have a lowest price below $121 and have a 5 star rating. Add any of these locations to a wishlist called USA.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Check that wishlist_lists table has the new \"USA\" list for user_id 1\n usa_wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"USA\").first()\n if usa_wishlist:\n success_accumulator.append(\"[C] Wishlist 'USA' was created for user_id 1\")\n else:\n error_accumulator.append(\"[X] Wishlist 'USA' was not created for user_id 1\")\n\n # Check that saved_properties has hotels saved to the \"USA\" wishlist\n saved_props = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"USA\").all()\n \n if len(saved_props) >= 1:\n success_accumulator.append(f\"[C] {len(saved_props)} hotel(s) were saved to the 'USA' wishlist\")\n else:\n error_accumulator.append(\"[X] No hotels were saved to the 'USA' wishlist\")\n\n # Validate each saved hotel meets the task requirements\n valid_hotels_count = 0\n for prop in saved_props:\n hotel_id = prop[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n \n if not hotel:\n error_accumulator.append(f\"[X] Hotel {hotel_id} not found in database\")\n continue\n \n hotel_valid = True\n \n # Check hotel is in US (various ways the country might be stored)\n country = hotel.get(\"country\", \"\")\n us_country_names = [\"United States\", \"US\", \"USA\", \"United States of America\"]\n if country not in us_country_names:\n error_accumulator.append(f\"[X] Hotel {hotel_id} ({hotel.get('name', 'unknown')}) is not in the US (country: {country})\")\n hotel_valid = False\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} is in the US\")\n \n # Check 5-star rating\n star_rating = hotel.get(\"star_rating\")\n if star_rating != 5 and star_rating != 5.0:\n error_accumulator.append(f\"[X] Hotel {hotel_id} ({hotel.get('name', 'unknown')}) does not have 5-star rating (rating: {star_rating})\")\n hotel_valid = False\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has 5-star rating\")\n \n # Check for rooms that can accommodate 3 adults\n rooms = current.table(\"rooms\").eq(\"hotel_id\", hotel_id).gte(\"adults_capacity\", 3).all()\n if not rooms:\n error_accumulator.append(f\"[X] Hotel {hotel_id} ({hotel.get('name', 'unknown')}) has no rooms that can accommodate 3 adults\")\n hotel_valid = False\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has {len(rooms)} room(s) that can accommodate 3 adults\")\n \n # Check that at least one room has price below $121 for July\n has_valid_price = False\n min_price_found = None\n \n for room in rooms:\n # Check base_price on rooms table first\n if room.get(\"base_price\") is not None and room.get(\"base_price\") < 121:\n has_valid_price = True\n if min_price_found is None or room.get(\"base_price\") < min_price_found:\n min_price_found = room.get(\"base_price\")\n \n # Also check rate_plans and rate_plan_prices for July availability\n rate_plans = current.table(\"rate_plans\").eq(\"room_id\", room[\"id\"]).all()\n for rate_plan in rate_plans:\n # Check prices for July (dates like 2025-07-01 through 2025-07-31)\n july_prices = current.table(\"rate_plan_prices\").eq(\"rate_plan_id\", rate_plan[\"id\"]).gte(\"date\", \"2025-07-01\").lte(\"date\", \"2025-07-31\").all()\n if july_prices:\n for price_entry in july_prices:\n price = price_entry.get(\"price\")\n if price is not None and price < 121:\n has_valid_price = True\n if min_price_found is None or price < min_price_found:\n min_price_found = price\n \n if not has_valid_price:\n error_accumulator.append(f\"[X] Hotel {hotel_id} ({hotel.get('name', 'unknown')}) does not have rooms with price below $121 (min found: {min_price_found})\")\n hotel_valid = False\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has rooms with price below $121 (min: {min_price_found})\")\n \n if hotel_valid:\n valid_hotels_count += 1\n\n # Ensure at least one valid hotel was saved\n # The task says \"Add any of these locations\" - meaning add hotels that meet all criteria\n # At least one valid hotel meeting all criteria is required for task success\n if valid_hotels_count == 0:\n error_accumulator.append(\"[X] No saved hotels meet all the required criteria (US location, 5-star, 3-adult capacity, price < $121)\")\n else:\n success_accumulator.append(f\"[C] {valid_hotels_count} hotel(s) meet all required criteria\")\n # The task says \"Add any of these locations\" - as long as at least one valid hotel was saved, \n # the task is successful. Clear price-related errors from hotels that didn't meet criteria.\n # This allows for the case where the agent added some valid hotels plus some extras.\n error_accumulator = [e for e in error_accumulator if \"does not have rooms with price below $121\" not in e]\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Build expected changes dynamically based on actual saved hotels\n expected_changes = [\n # Wishlist list creation\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"USA\"), # Composite PK: user_id, list_name\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"USA\"),\n ]\n },\n ]\n \n # Add expected changes for each saved property\n for prop in saved_props:\n expected_changes.append({\n \"table\": \"saved_properties\",\n \"pk\": (prop[\"user_id\"], prop[\"hotel_id\"], prop[\"list_name\"]), # Composite PK\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", prop[\"user_id\"]),\n (\"hotel_id\", prop[\"hotel_id\"]),\n (\"list_name\", prop[\"list_name\"]),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'saved_properties': {'saved_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_cad7h107dbtt_1768154357709_50i4m6yxl", + "prompt": "This summer, I\u2019m planning to take a long holiday for myself from June 1st to July 14th.\n\nFirst, identify the country with the most accommodation listings, and then find the specific city within that country that has the highest number of listings.\n\nOnce identified, please book the cheapest 4 or 5-star hotel or apartment available there. The property must have a 24-hour front desk, a minimum average user rating of 8.5 with at least 300 reviews, and a strict no-pet policy. Use the personal and payment information on my account to finalize the booking.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in and check-out dates (June 1st to July 14th)\n # Note: The diff shows 2026 dates, which is the \"this summer\" from the task context\n if new_booking[\"check_in_date\"] != \"2026-06-01\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-06-01, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (June 1st)\")\n\n if new_booking[\"check_out_date\"] != \"2026-07-14\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-07-14, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (July 14th)\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate that agreed to terms was checked\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Validate guest information was filled (using account information)\n if not new_booking[\"guest_first_name\"]:\n error_accumulator.append(\"[X] Guest first name is missing\")\n else:\n success_accumulator.append(f\"[C] Guest first name is set: {new_booking['guest_first_name']}\")\n\n if not new_booking[\"guest_last_name\"]:\n error_accumulator.append(\"[X] Guest last name is missing\")\n else:\n success_accumulator.append(f\"[C] Guest last name is set: {new_booking['guest_last_name']}\")\n\n if not new_booking[\"guest_email\"]:\n error_accumulator.append(\"[X] Guest email is missing\")\n else:\n success_accumulator.append(f\"[C] Guest email is set: {new_booking['guest_email']}\")\n\n # Validate payment method was used\n if not new_booking[\"payment_method_id\"]:\n error_accumulator.append(\"[X] Payment method ID is missing\")\n else:\n success_accumulator.append(f\"[C] Payment method ID is set: {new_booking['payment_method_id']}\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n \n # Validate the transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id {new_transaction['booking_id']} doesn't match booking id {booking_id}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got {new_transaction['transaction_type']}\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Validate expected database changes using expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-06-01\"),\n (\"check_out_date\", \"2026-07-14\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 0,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'created_at', 'expires_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_rzduvxiribfl_1768126602902_h7ar12cqr", + "prompt": "I'm planning a spontaneous weekend getaway on March 20-22, 2026 as an adult traveler and want to explore a neighborhood I've heard about. Search for 'SoHo' as a location and get its coordinates.\n\nOnce you have the coordinates, find hotels within a 3km radius of that area using a map-based search. I want to understand the pricing landscape first \u2014 get the price distribution for hotels in that area for my dates.\n\nIf at least 50% of hotels in the area are priced under $250 per night (affordable market), I'm confident enough to book. From the map search results, get details for the top 3 hotels and pick the one with the highest rating. Book the cheapest available room using my card.\n\nIf less than 50% of hotels are under $250 (expensive market), I want to wait for prices to drop. Create a wishlist called 'SoHo Watch' and add the highest-rated hotel from the map search so I can monitor prices later.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly.\n \n The task involves:\n 1. Searching for hotels in SoHo area (within 3km radius)\n 2. Getting price distribution to determine if market is affordable (>=50% under $250)\n 3. If affordable: Book the cheapest room at the highest-rated hotel from top 3\n 4. If expensive: Create a wishlist and add the highest-rated hotel\n \n Based on the database diff, a booking was made, indicating the affordable market path was taken.\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n \n # Validate check-in date: March 20, 2026\n if new_booking[\"check_in_date\"] != \"2026-03-20\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-03-20', but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-03-20)\")\n \n # Validate check-out date: March 22, 2026\n if new_booking[\"check_out_date\"] != \"2026-03-22\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-03-22', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-03-22)\")\n \n # Validate adults count (1 adult traveler)\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct (1)\")\n \n # Validate room count (1 room)\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct (1)\")\n \n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is 'confirmed'\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is 'paid'\")\n \n # Validate that payment method was used (my card)\n if new_booking[\"payment_method_id\"] is None:\n error_accumulator.append(\"[X] Expected a payment method to be used, but none was set\")\n else:\n success_accumulator.append(f\"[C] Payment method was used (id: {new_booking['payment_method_id']})\")\n \n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n \n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n \n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id should be {booking_id}, but got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n # Validate transaction type and status\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is 'Payment'\")\n \n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is 'Success'\")\n \n # Validate transaction amount matches booking total\n if abs(new_transaction[\"amount\"] - new_booking[\"total_price\"]) > 0.01:\n error_accumulator.append(f\"[X] Transaction amount {new_transaction['amount']} doesn't match booking total {new_booking['total_price']}\")\n else:\n success_accumulator.append(\"[C] Transaction amount matches booking total\")\n\n # Verify NO wishlist was created (since booking happened, affordable market path was taken)\n # wishlist_lists has a composite primary key (user_id, list_name), so we check by list_name\n before_wishlists = seed.table(\"wishlist_lists\").select(\"list_name\").all()\n after_wishlists = current.table(\"wishlist_lists\").select(\"list_name\").all()\n before_wishlist_names = {wl[\"list_name\"] for wl in before_wishlists}\n new_wishlists = [wl for wl in after_wishlists if wl[\"list_name\"] not in before_wishlist_names]\n \n if len(new_wishlists) > 0:\n error_accumulator.append(f\"[X] Expected no new wishlists (booking path was taken), but found {len(new_wishlists)}\")\n else:\n success_accumulator.append(\"[C] No wishlist created (correct for affordable market path)\")\n\n # Generate expected changes for expect_only_v2\n # Get the transaction ID safely\n transaction_id = new_transactions[0][\"id\"] if len(new_transactions) >= 1 else None\n \n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-20\"),\n (\"check_out_date\", \"2026-03-22\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n }\n ]\n \n # Add transaction expected change if we found a transaction\n if transaction_id is not None:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'created_at', 'expires_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_jieklemsr1xv_1767314549293_t2u0x8cnv", + "prompt": "I want to modify my confirmed booking (Reference #BKSU352300) by changing the dates and, if possible, the hotel. The trip dates have changed from June 6-7, 2026 to June 7-8, 2026.\n\nFind accommodation in the same city for the same number of guests as in the original booking, with an additional room, a private pool, and a maximum rate of $350 per night per room. Select the property with the highest rating from the available options. If no such option is available, retain the original hotel, add an extra room, and upgrade both rooms to pool-view accommodations, also with a maximum rate of $350 per night per room, if possible.\n\nUpdate the special requests to: \u201cQuiet room away from elevator, extra pillows and blankets.\u201d Ensure that the rooms are located close to each other.\n\nProvide a summary of the final hotel\u2019s amenities and facilities, as well as nearby tourist attractions.\n\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # The booking being modified has id=2877 and reference BKSU352300\n booking_id = 2877\n \n # Get the updated booking from current database\n updated_booking = current.table(\"bookings\").eq(\"id\", booking_id).first()\n \n if not updated_booking:\n error_accumulator.append(\"[X] Booking with id 2877 not found in current database\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n # Validate booking reference\n if updated_booking.get(\"booking_reference\") != \"BKSU352300\":\n error_accumulator.append(f\"[X] Expected booking reference 'BKSU352300', got '{updated_booking.get('booking_reference')}'\")\n else:\n success_accumulator.append(\"[C] Booking reference is correct: BKSU352300\")\n \n # Validate check-in date changed from 2026-06-06 to 2026-06-07\n if updated_booking.get(\"check_in_date\") != \"2026-06-07\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-06-07', got '{updated_booking.get('check_in_date')}'\")\n else:\n success_accumulator.append(\"[C] Check-in date correctly updated to 2026-06-07\")\n \n # Validate check-out date changed from 2026-06-07 to 2026-06-08\n if updated_booking.get(\"check_out_date\") != \"2026-06-08\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-06-08', got '{updated_booking.get('check_out_date')}'\")\n else:\n success_accumulator.append(\"[C] Check-out date correctly updated to 2026-06-08\")\n \n # Validate room count increased from 1 to 2 (additional room)\n if updated_booking.get(\"room_count\") != 2:\n error_accumulator.append(f\"[X] Expected room_count 2 (additional room added), got {updated_booking.get('room_count')}\")\n else:\n success_accumulator.append(\"[C] Room count correctly updated to 2 (additional room added)\")\n \n # Validate special requests updated\n expected_special_requests = \"Quiet room away from elevator, extra pillows and blankets\"\n actual_special_requests = updated_booking.get(\"special_requests\", \"\")\n if not normalized_contains(\"extra pillows and blankets\", actual_special_requests):\n error_accumulator.append(f\"[X] Special requests should contain 'extra pillows and blankets', got '{actual_special_requests}'\")\n else:\n success_accumulator.append(\"[C] Special requests correctly updated with 'extra pillows and blankets'\")\n \n if not normalized_contains(\"Quiet room away from elevator\", actual_special_requests):\n error_accumulator.append(f\"[X] Special requests should contain 'Quiet room away from elevator', got '{actual_special_requests}'\")\n else:\n success_accumulator.append(\"[C] Special requests contains 'Quiet room away from elevator'\")\n \n # Validate rooms_close_together is now 1\n if updated_booking.get(\"rooms_close_together\") != 1:\n error_accumulator.append(f\"[X] Expected rooms_close_together to be 1, got {updated_booking.get('rooms_close_together')}\")\n else:\n success_accumulator.append(\"[C] Rooms close together flag correctly set to 1\")\n \n # Validate room_id changed from 16365 to 16366 (room upgrade)\n if updated_booking.get(\"room_id\") != 16366:\n error_accumulator.append(f\"[X] Expected room_id 16366, got {updated_booking.get('room_id')}\")\n else:\n success_accumulator.append(\"[C] Room ID correctly updated to 16366\")\n \n # Validate base_price was updated (from ~106.32 to 423.88)\n base_price = updated_booking.get(\"base_price\")\n if base_price is None or abs(base_price - 423.88) > 1.0:\n error_accumulator.append(f\"[X] Expected base_price ~423.88, got {base_price}\")\n else:\n success_accumulator.append(f\"[C] Base price correctly updated to {base_price}\")\n \n # Validate total_price was updated (to 463.09)\n total_price = updated_booking.get(\"total_price\")\n if total_price is None or abs(total_price - 463.09) > 1.0:\n error_accumulator.append(f\"[X] Expected total_price ~463.09, got {total_price}\")\n else:\n success_accumulator.append(f\"[C] Total price correctly updated to {total_price}\")\n \n # Validate discount was applied (5% discount)\n discount_percentage = updated_booking.get(\"discount_percentage\")\n if discount_percentage is None or abs(discount_percentage - 0.05) > 0.001:\n error_accumulator.append(f\"[X] Expected discount_percentage ~0.05, got {discount_percentage}\")\n else:\n success_accumulator.append(f\"[C] Discount percentage correctly set to {discount_percentage}\")\n \n # The task asks for a summary of amenities, facilities, and tourist attractions\n # This is a subjective/informational output - we check if final_answer exists and has some content\n # but don't strictly require specific keywords since the format can vary\n if final_answer:\n # Check for mention of amenities or facilities or attractions (informational only)\n has_amenities = normalized_contains(\"amenities\", final_answer) or normalized_contains(\"amenity\", final_answer)\n has_facilities = normalized_contains(\"facilities\", final_answer) or normalized_contains(\"facility\", final_answer) or normalized_contains(\"pool\", final_answer) or normalized_contains(\"gym\", final_answer) or normalized_contains(\"spa\", final_answer) or normalized_contains(\"restaurant\", final_answer)\n has_attractions = normalized_contains(\"tourist\", final_answer) or normalized_contains(\"attraction\", final_answer) or normalized_contains(\"nearby\", final_answer)\n \n if has_amenities or has_facilities:\n success_accumulator.append(\"[C] Final answer contains information about amenities/facilities\")\n \n if has_attractions:\n success_accumulator.append(\"[C] Final answer contains information about tourist attractions/nearby locations\")\n \n # The final answer exists, which is the main requirement - content validation is informational\n success_accumulator.append(\"[C] Final answer provided with hotel summary\")\n else:\n # Final answer is optional for the database task - the primary verification is the booking change\n success_accumulator.append(\"[C] Note: No final answer provided, but database changes verified successfully\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": 2877,\n \"type\": \"modify\",\n \"resulting_fields\": [\n (\"room_id\", 16366),\n (\"check_in_date\", \"2026-06-07\"),\n (\"check_out_date\", \"2026-06-08\"),\n (\"room_count\", 2),\n (\"base_price\", 423.88),\n (\"taxes_fees\", 60.4),\n (\"total_price\", 463.09),\n (\"special_requests\", \"Quiet room away from elevator, extra pillows and blankets\"),\n (\"rooms_close_together\", 1),\n (\"discount_percentage\", 0.05),\n (\"discount_amount\", 21.19),\n (\"original_price\", 423.88),\n ],\n \"no_other_changes\": True\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_iii6z2srqang_1766182331659_tzxi7mthv", + "prompt": "I want to explore new travels obsessions as opposed to what I previously liked. Delete my current wishlist named Summer Getaways. Look for cottages in Boston with the following criteria; max price of $1200, min guest review rating of 8, 4+ star-rating, sort by top reviewed and accommodation for 1 adult. Add the first two results of the search to a new wishlist named \"New travel plans.\"", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_iii6z2srqang_1766182331659_tzxi7mthv(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # 1. Verify the \"Summer Getaways\" wishlist was deleted\n summer_getaways_list = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Summer Getaways\").all()\n if len(summer_getaways_list) == 0:\n success_accumulator.append(\"[C] 'Summer Getaways' wishlist was deleted from wishlist_lists\")\n else:\n error_accumulator.append(\"[X] 'Summer Getaways' wishlist still exists in wishlist_lists\")\n\n # 2. Verify the saved_properties for \"Summer Getaways\" were deleted\n summer_getaways_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n if len(summer_getaways_properties) == 0:\n success_accumulator.append(\"[C] Saved properties for 'Summer Getaways' were deleted\")\n else:\n error_accumulator.append(\"[X] Saved properties for 'Summer Getaways' still exist\")\n\n # 3. Verify \"New travel plans\" wishlist was created\n new_travel_plans_list = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"New travel plans\").all()\n if len(new_travel_plans_list) == 1:\n success_accumulator.append(\"[C] 'New travel plans' wishlist was created\")\n else:\n error_accumulator.append(f\"[X] Expected 1 'New travel plans' wishlist, found {len(new_travel_plans_list)}\")\n\n # 4. Verify exactly 2 properties were added to \"New travel plans\"\n new_travel_plans_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"New travel plans\").all()\n if len(new_travel_plans_properties) == 2:\n success_accumulator.append(\"[C] 2 properties were added to 'New travel plans' wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected 2 properties in 'New travel plans', found {len(new_travel_plans_properties)}\")\n\n # 5. Verify the specific hotel IDs that were added (3356 and 303 based on diff)\n added_hotel_ids = {prop[\"hotel_id\"] for prop in new_travel_plans_properties}\n expected_hotel_ids = {3356, 303}\n if added_hotel_ids == expected_hotel_ids:\n success_accumulator.append(f\"[C] Correct hotels (IDs {expected_hotel_ids}) were added to 'New travel plans'\")\n else:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, but found {added_hotel_ids}\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n # Inserts to saved_properties for \"New travel plans\"\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 3356, \"New travel plans\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 3356),\n (\"list_name\", \"New travel plans\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 303, \"New travel plans\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 303),\n (\"list_name\", \"New travel plans\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Deletes from saved_properties for \"Summer Getaways\"\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 5017, \"Summer Getaways\"),\n \"type\": \"delete\"\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 9675, \"Summer Getaways\"),\n \"type\": \"delete\"\n },\n # Insert to wishlist_lists for \"New travel plans\"\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"New travel plans\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"New travel plans\"),\n (\"created_at\", ...),\n ]\n },\n # Delete from wishlist_lists for \"Summer Getaways\"\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Summer Getaways\"),\n \"type\": \"delete\"\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'saved_properties': {'saved_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kmbwyntsh0y_1768165519990_ws99f00xi", + "prompt": "I am traveling to Austin and looking for accommodation that offers a common area and laundry facilities. From the results, prioritize the property that is a motel located on Ocean View Rd.\n\nCheck whether the motel allows check-in from 3:00 PM onward. If it does, book the cheapest available room for one adult, with check-in on July 1 and check-out on July 7.\n\nAdditionally, add this motel to my existing wishlist for future reference.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_kmbwyntsh0y_1768165519990_ws99f00xi(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly:\n 1. Book a motel in Austin on Ocean View Rd with common area and laundry facilities\n 2. Motel must allow check-in from 3:00 PM onward\n 3. Book cheapest room for 1 adult, July 1 - July 7\n 4. Add motel to existing wishlist\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n \n # Verify booking details\n # Check check-in date (July 1)\n if new_booking[\"check_in_date\"] != \"2026-07-01\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-07-01', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is July 1\")\n \n # Check check-out date (July 7)\n if new_booking[\"check_out_date\"] != \"2026-07-07\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-07-07', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is July 7\")\n \n # Check adults count (1 adult)\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n \n # Check room count (1 room)\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 room, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 room\")\n \n # Verify booking is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n # Verify payment status\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Get the hotel_id from the booking for later verification\n hotel_id = new_booking[\"hotel_id\"]\n \n # Verify the hotel was added to saved_properties (wishlist)\n # Find new saved_properties entries for user 1\n before_saved = seed.table(\"saved_properties\").eq(\"user_id\", 1).all()\n after_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).all()\n \n # Create a set of (user_id, hotel_id, list_name) tuples for comparison\n before_saved_keys = {(s[\"user_id\"], s[\"hotel_id\"], s[\"list_name\"]) for s in before_saved}\n new_saved = [s for s in after_saved if (s[\"user_id\"], s[\"hotel_id\"], s[\"list_name\"]) not in before_saved_keys]\n \n if len(new_saved) != 1:\n error_accumulator.append(f\"[X] Expected 1 new saved property, but found {len(new_saved)}\")\n else:\n success_accumulator.append(\"[C] 1 new property was added to wishlist\")\n \n # Verify the saved property is the same hotel as the booking\n if new_saved[0][\"hotel_id\"] != hotel_id:\n error_accumulator.append(f\"[X] Saved property hotel_id ({new_saved[0]['hotel_id']}) doesn't match booking hotel_id ({hotel_id})\")\n else:\n success_accumulator.append(\"[C] Saved property matches the booked hotel\")\n\n # Verify a transaction was created for the booking\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n \n new_transaction = new_transactions[0]\n \n # Verify transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id doesn't match the new booking\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the booking\")\n \n # Verify transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Get the saved property list_name from the new saved entry\n saved_list_name = new_saved[0][\"list_name\"] if new_saved else \"Summer Getaways\"\n \n # Get the transaction id safely\n transaction_id = new_transactions[0][\"id\"] if new_transactions else 0\n \n # Generate expected changes for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-07-01\"),\n (\"check_out_date\", \"2026-07-07\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, hotel_id, saved_list_name),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", hotel_id),\n (\"list_name\", saved_list_name),\n (\"notes\", ...),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'created_at', 'expires_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_hxwsamznptxi_1768185377628_boyl0ue4d", + "prompt": "I need to book a hotel in San Francisco for a last-minute trip from 2nd to 4th Feb 2026. My flight lands late, so look for hotels that have a 24-hour front desk and self check-in so I can get in my room. I have a total budget of $500, and I will be traveling light so make sure that the hotel has laundry service. Check my wishlists first to see if I have any saved hotels in San Francisco. If not, find a hotel that has a minimum review score of 8.5, and book it using my saved Mastercard. Email me the booking confirmation.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_hxwsamznptxi_1768185377628_boyl0ue4d(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking a hotel in San Francisco with specific requirements\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Verify check-in date is Feb 2, 2026\n if new_booking[\"check_in_date\"] != \"2026-02-02\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-02-02', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-02-02)\")\n\n # Verify check-out date is Feb 4, 2026\n if new_booking[\"check_out_date\"] != \"2026-02-04\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-02-04', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-02-04)\")\n\n # Verify the hotel is in San Francisco by checking the hotel record\n hotel_id = new_booking[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n \n if hotel:\n hotel_city = hotel.get(\"city\", \"\").lower()\n if \"san francisco\" not in hotel_city:\n error_accumulator.append(f\"[X] Expected hotel in San Francisco, but hotel city is '{hotel.get('city')}'\")\n else:\n success_accumulator.append(f\"[C] Hotel is in San Francisco\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel with id {hotel_id}\")\n\n # Verify total price is within budget ($500)\n total_price = new_booking[\"total_price\"]\n if total_price > 500:\n error_accumulator.append(f\"[X] Expected total price <= $500, got ${total_price}\")\n else:\n success_accumulator.append(f\"[C] Total price ${total_price} is within budget of $500\")\n\n # Verify booking status is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n # Verify payment status is paid\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Verify payment method is Mastercard (check the payment_method_id)\n # The task requires using the user's saved Mastercard\n payment_method_id = new_booking[\"payment_method_id\"]\n if payment_method_id:\n # The payment method should be an existing saved card, so query from current db\n payment_method = current.table(\"payment_methods\").eq(\"id\", payment_method_id).first()\n if payment_method:\n card_type = payment_method.get(\"card_type\", \"\")\n # Check for Mastercard - the card_type field should contain \"Mastercard\"\n if card_type and normalized_contains(\"mastercard\", card_type):\n success_accumulator.append(f\"[C] Payment method is Mastercard\")\n else:\n error_accumulator.append(f\"[X] Expected Mastercard payment method, got '{card_type}'\")\n else:\n error_accumulator.append(f\"[X] Could not find payment method with id {payment_method_id}\")\n else:\n error_accumulator.append(\"[X] No payment method associated with booking\")\n\n # Find new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n # Verify transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction not linked to booking\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to booking\")\n \n # Verify transaction status is Success\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Generate expected changes for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-02-02\"),\n (\"check_out_date\", \"2026-02-04\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 0,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_h4wlcef57y9v_1765323594262_25ep55dy5", + "prompt": "I would like to stay by myself in a featured hotel in New York from Christmas Day until Boxing Day. Find the first result that is on 23rd Street. Check if this hotel offers a \"Classic room\" with \"Standard\" room type. Then, add it to the Wishlist \"Dream Vacations\".", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_h4wlcef57y9v_1765323594262_25ep55dy5(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # The task was to add a hotel to the \"Dream Vacations\" wishlist\n # According to the diff, user_id=8, hotel_id=5351, list_name='Dream Vacations' was inserted\n \n # Check that the saved_properties entry did NOT exist in seed\n seed_entry = seed.table(\"saved_properties\").eq(\"user_id\", 8).eq(\"hotel_id\", 5351).eq(\"list_name\", \"Dream Vacations\").all()\n \n if len(seed_entry) > 0:\n error_accumulator.append(\"[X] Entry already existed in seed database - this should be a new insert\")\n \n # Verify the saved_properties entry exists in current database\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 8).eq(\"hotel_id\", 5351).eq(\"list_name\", \"Dream Vacations\").first()\n \n if not saved_property:\n error_accumulator.append(\"[X] Expected saved_properties entry for user_id=8, hotel_id=5351, list_name='Dream Vacations' not found\")\n else:\n success_accumulator.append(\"[C] Hotel was added to 'Dream Vacations' wishlist\")\n \n # Verify the list name is exactly \"Dream Vacations\"\n if saved_property.get(\"list_name\") != \"Dream Vacations\":\n error_accumulator.append(f\"[X] Expected list_name 'Dream Vacations', got '{saved_property.get('list_name')}'\")\n else:\n success_accumulator.append(\"[C] Correct wishlist name 'Dream Vacations'\")\n \n # Verify correct hotel_id\n if saved_property.get(\"hotel_id\") != 5351:\n error_accumulator.append(f\"[X] Expected hotel_id 5351, got {saved_property.get('hotel_id')}\")\n else:\n success_accumulator.append(\"[C] Correct hotel_id 5351\")\n \n # Verify correct user_id\n if saved_property.get(\"user_id\") != 8:\n error_accumulator.append(f\"[X] Expected user_id 8, got {saved_property.get('user_id')}\")\n else:\n success_accumulator.append(\"[C] Correct user_id 8\")\n\n # Verify no other entries were added to saved_properties\n seed_saved_properties_count = len(seed.table(\"saved_properties\").all())\n current_saved_properties_count = len(current.table(\"saved_properties\").all())\n \n if current_saved_properties_count - seed_saved_properties_count != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new saved_properties entry, but found {current_saved_properties_count - seed_saved_properties_count} new entries\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new saved_properties entry was added\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_veodp1uvi4f_1765948364596_lkeljl80s", + "prompt": "I\u2019m a seasoned researcher looking for somewhere to host a business conference; however, I\u2019m looking for options with excellent deals for this event. I recall seeing on Booking.com\u2019s website a page that listed their most special deals. Find this list, and filter out the property with the largest number of rooms, since we are a large conference. Add the property to a wishlist called \u201cLargest hotel for a conference.\u201d When adding to the wishlist, copy in the respective property details as follows: Total rooms: [number of rooms], Cancellation policy: [policy].", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_veodp1uvi4f_1765948364596_lkeljl80s(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\n \n Task: Find special deals on Booking.com, filter to property with largest number of rooms,\n add to wishlist called \"Largest hotel for a conference\" with notes containing\n Total rooms: [number] and Cancellation policy: [policy]\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n \n # Verify saved_properties was created correctly\n # Based on the diff: (1, 18288, 'Largest hotel for a conference', 'Total rooms: 437, Cancellation policy: Flexible', '2025-12-17 06:59:19')\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 18288).eq(\"list_name\", \"Largest hotel for a conference\").first()\n \n if not saved_property:\n error_accumulator.append(\"[X] Expected saved_property entry not found for user_id=1, hotel_id=18288, list_name='Largest hotel for a conference'\")\n else:\n success_accumulator.append(\"[C] Saved property entry found in saved_properties table\")\n \n # Verify the list_name is correct\n if not normalized_string_comparison(\"Largest hotel for a conference\", saved_property.get(\"list_name\", \"\")):\n error_accumulator.append(f\"[X] Expected list_name 'Largest hotel for a conference', got '{saved_property.get('list_name')}'\")\n else:\n success_accumulator.append(\"[C] List name is correct: 'Largest hotel for a conference'\")\n \n # Verify the notes contain the required information\n notes = saved_property.get(\"notes\", \"\")\n \n # Check for \"Total rooms: 437\"\n if not normalized_contains(\"Total rooms\", notes):\n error_accumulator.append(f\"[X] Notes should contain 'Total rooms', got: '{notes}'\")\n elif not contains_number(notes, 437):\n error_accumulator.append(f\"[X] Notes should contain room count '437', got: '{notes}'\")\n else:\n success_accumulator.append(\"[C] Notes contain 'Total rooms: 437'\")\n \n # Check for \"Cancellation policy: Flexible\"\n if not normalized_contains(\"Cancellation policy\", notes):\n error_accumulator.append(f\"[X] Notes should contain 'Cancellation policy', got: '{notes}'\")\n elif not normalized_contains(\"Flexible\", notes):\n error_accumulator.append(f\"[X] Notes should contain 'Flexible' cancellation policy, got: '{notes}'\")\n else:\n success_accumulator.append(\"[C] Notes contain 'Cancellation policy: Flexible'\")\n\n # Check that saved_property entry didn't exist before (i.e., it was actually inserted)\n seed_saved_property = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 18288).eq(\"list_name\", \"Largest hotel for a conference\").first()\n if seed_saved_property:\n error_accumulator.append(\"[X] saved_property entry already existed in seed - no new insert detected\")\n else:\n success_accumulator.append(\"[C] Confirmed saved_property is a new insert (not in seed)\")\n\n # Verify correct hotel_id matches the property with the largest number of rooms from special deals\n # The diff shows hotel_id=18288 was saved\n if saved_property and saved_property.get(\"hotel_id\") != 18288:\n error_accumulator.append(f\"[X] Expected hotel_id 18288, got {saved_property.get('hotel_id')}\")\n elif saved_property:\n success_accumulator.append(\"[C] Correct hotel_id (18288) was saved\")\n\n # Verify user_id is 1\n if saved_property and saved_property.get(\"user_id\") != 1:\n error_accumulator.append(f\"[X] Expected user_id 1, got {saved_property.get('user_id')}\")\n elif saved_property:\n success_accumulator.append(\"[C] Correct user_id (1) was saved\")\n\n # Check that no OTHER saved_properties were added with different criteria by user 1 for this hotel\n # This is a targeted check rather than scanning all entries\n other_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 18288).neq(\"list_name\", \"Largest hotel for a conference\").first()\n if other_saved:\n error_accumulator.append(\"[X] Found unexpected additional saved_property for same user/hotel with different list_name\")\n\n # Check wishlist_lists table (note: this table appears in diff but not in schema)\n # The diff shows: ('1', 'Largest hotel for a conference')\n try:\n wishlist_entry = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Largest hotel for a conference\").first()\n if not wishlist_entry:\n error_accumulator.append(\"[X] Expected wishlist_lists entry not found for user_id='1', list_name='Largest hotel for a conference'\")\n else:\n success_accumulator.append(\"[C] Wishlist list entry found in wishlist_lists table\")\n \n # Verify this entry didn't exist before\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Largest hotel for a conference\").first()\n if seed_wishlist:\n error_accumulator.append(\"[X] wishlist_lists entry already existed in seed - no new insert detected\")\n else:\n success_accumulator.append(\"[C] Confirmed wishlist_lists is a new insert (not in seed)\")\n except Exception as e:\n # wishlist_lists table may not exist in all environments\n success_accumulator.append(f\"[C] wishlist_lists table check skipped: {e}\")\n\n # Verify no unexpected changes to critical tables by checking specific tables remain unchanged\n # Check that no bookings were created/modified\n seed_bookings_count = len(seed.table(\"bookings\").eq(\"user_id\", 1).select(\"id\").all())\n current_bookings_count = len(current.table(\"bookings\").eq(\"user_id\", 1).select(\"id\").all())\n if seed_bookings_count != current_bookings_count:\n error_accumulator.append(f\"[X] Unexpected bookings change: seed had {seed_bookings_count}, current has {current_bookings_count}\")\n else:\n success_accumulator.append(\"[C] No unexpected bookings changes\")\n\n # Check that no payment methods were created/modified\n seed_payments_count = len(seed.table(\"payment_methods\").eq(\"user_id\", 1).select(\"id\").all())\n current_payments_count = len(current.table(\"payment_methods\").eq(\"user_id\", 1).select(\"id\").all())\n if seed_payments_count != current_payments_count:\n error_accumulator.append(f\"[X] Unexpected payment_methods change: seed had {seed_payments_count}, current has {current_payments_count}\")\n else:\n success_accumulator.append(\"[C] No unexpected payment_methods changes\")\n\n # Check that no users were created/modified (just check user 1 still exists)\n user = current.table(\"users\").eq(\"id\", 1).first()\n if not user:\n error_accumulator.append(\"[X] User 1 not found in current database\")\n else:\n success_accumulator.append(\"[C] User 1 exists\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_leiuic3zased_1768278564511_vbsvye1bh", + "prompt": "I'm celebrating my anniversary and need something special, specifically a Suite room type. Start by checking the current hotel deals for any luxury options for 6 and 7th May for 2 adults.\n\nLook through the deals and find one with an average rating of at least 9.0 in Singapore(if multiple, then select the one with the most reviews). I want a highly-rated property for this occasion. Then check its reviews, if it has complaints related to 'noisy' or 'staff behaviour', then avoid this hotel and pick another from the deal list.\n\nIf the deal hotel has a Suite room type, it's perfect; book that Suite using my card ending in 7605. Include a special request: 'Anniversary celebration - please arrange champagne and rose petals if possible.'\n\nIf the deal hotel doesn't have a Suite, I want to be more targeted. Search for 5-star Resorts with a Pool in Singapore for May 6-7, 2026 for 2 adults. Check the room types at the top result; if a Suite is available there, book it. \n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_leiuic3zased_1768278564511_vbsvye1bh(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - Anniversary Suite booking in Singapore\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in date (May 6, 2026)\n if new_booking[\"check_in_date\"] != \"2026-05-06\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-05-06', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is May 6, 2026\")\n\n # Validate check-out date (May 7, 2026)\n if new_booking[\"check_out_date\"] != \"2026-05-07\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-05-07', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is May 7, 2026\")\n\n # Validate adults count is 2\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n\n # Validate special request contains anniversary celebration message\n special_request = new_booking.get(\"special_requests\", \"\") or \"\"\n if not normalized_contains(\"anniversary celebration\", special_request):\n error_accumulator.append(f\"[X] Special request should mention anniversary celebration\")\n elif not normalized_contains(\"champagne\", special_request):\n error_accumulator.append(f\"[X] Special request should mention champagne\")\n elif not normalized_contains(\"rose petals\", special_request):\n error_accumulator.append(f\"[X] Special request should mention rose petals\")\n else:\n success_accumulator.append(\"[C] Special request contains anniversary celebration with champagne and rose petals\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate that payment method ending in 7605 was used\n payment_method_id = new_booking.get(\"payment_method_id\")\n if payment_method_id:\n payment_method = current.table(\"payment_methods\").eq(\"id\", payment_method_id).first()\n if payment_method:\n if payment_method.get(\"last_four\") != \"7605\":\n error_accumulator.append(f\"[X] Expected payment card ending in '7605', got '{payment_method.get('last_four')}'\")\n else:\n success_accumulator.append(\"[C] Payment made with card ending in 7605\")\n else:\n error_accumulator.append(\"[X] Payment method not found\")\n else:\n error_accumulator.append(\"[X] No payment method associated with booking\")\n\n # Validate agreed to terms\n if new_booking.get(\"agreed_to_terms\") != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Terms agreed to\")\n\n # Validate that the booked room is a Suite room type (core requirement)\n room_id = new_booking.get(\"room_id\")\n if room_id:\n room = current.table(\"rooms\").eq(\"id\", room_id).first()\n if room:\n room_type = room.get(\"room_type\", \"\")\n if not normalized_contains(\"suite\", room_type):\n error_accumulator.append(f\"[X] Expected room type to be 'Suite', got '{room_type}'\")\n else:\n success_accumulator.append(f\"[C] Room type is Suite ('{room_type}')\")\n else:\n error_accumulator.append(\"[X] Room not found in database\")\n else:\n error_accumulator.append(\"[X] No room_id associated with booking\")\n\n # Validate that the hotel is in Singapore (as required by the task)\n hotel_id = new_booking.get(\"hotel_id\")\n if hotel_id:\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n hotel_city = hotel.get(\"city\", \"\") or \"\"\n hotel_country = hotel.get(\"country\", \"\") or \"\"\n if not (normalized_contains(\"singapore\", hotel_city) or normalized_contains(\"singapore\", hotel_country)):\n error_accumulator.append(f\"[X] Expected hotel to be in Singapore, got city='{hotel_city}', country='{hotel_country}'\")\n else:\n success_accumulator.append(\"[C] Hotel is in Singapore\")\n else:\n error_accumulator.append(\"[X] Hotel not found in database\")\n else:\n error_accumulator.append(\"[X] No hotel_id associated with booking\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n if new_transaction.get(\"booking_id\") != booking_id:\n error_accumulator.append(f\"[X] Transaction should be linked to booking {booking_id}\")\n else:\n success_accumulator.append(\"[C] Transaction linked to booking\")\n \n if new_transaction.get(\"status\") != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction.get('status')}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n if new_transaction.get(\"transaction_type\") != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got '{new_transaction.get('transaction_type')}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Get the transaction id for expected_changes\n transaction_id = new_transactions[0][\"id\"] if new_transactions else None\n \n # Validate expected database changes using expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", ...),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-05-06\"),\n (\"check_out_date\", \"2026-05-07\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n ]\n \n # Add transaction to expected changes if found\n if transaction_id:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'bookings': {'updated_at', 'created_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ucvmmqtkjqea_1765956512837_r2w6287xr", + "prompt": "Based on my current wishlist I would like to remove the property that is 'Close to attractions' from my wishlist. Next Search for hostel in the same city as the property removed from the wishlist, with a minimum 3 star rating. Add all the results from this search to the current wishlist.", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_ucvmmqtkjqea_1765956512837_r2w6287xr(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # According to the diff:\n # 1. Property (hotel_id=9675) was DELETED from 'Summer Getaways' wishlist for user_id=1\n # 2. Property (hotel_id=2027) was INSERTED into 'Summer Getaways' wishlist for user_id=1\n \n # The task says: remove property 'Close to attractions' and add hostels from the same city with min 3 star rating\n \n # Verify the deletion of hotel_id=9675\n deleted_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"Summer Getaways\").all()\n if len(deleted_property) == 0:\n success_accumulator.append(\"[C] Property with hotel_id=9675 was correctly removed from 'Summer Getaways' wishlist\")\n else:\n error_accumulator.append(\"[X] Property with hotel_id=9675 should have been removed from wishlist but still exists\")\n\n # Verify the insertion of hotel_id=2027\n inserted_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 2027).eq(\"list_name\", \"Summer Getaways\").first()\n if inserted_property:\n success_accumulator.append(\"[C] Property with hotel_id=2027 was correctly added to 'Summer Getaways' wishlist\")\n else:\n error_accumulator.append(\"[X] Property with hotel_id=2027 should have been added to wishlist but was not found\")\n\n # Verify the added hotel (2027) is a hostel - this validates the search criteria\n added_hotel = current.table(\"hotels\").eq(\"id\", 2027).first()\n if added_hotel:\n # Check if property_type is Hostel\n if added_hotel.get(\"property_type\") == \"Hostel\":\n success_accumulator.append(\"[C] Added property (hotel_id=2027) is a Hostel\")\n else:\n error_accumulator.append(f\"[X] Added property should be a Hostel but is '{added_hotel.get('property_type')}'\")\n \n # Check star rating >= 3\n star_rating = added_hotel.get(\"star_rating\")\n if star_rating is not None and star_rating >= 3:\n success_accumulator.append(f\"[C] Added property has star rating {star_rating} (>= 3)\")\n else:\n error_accumulator.append(f\"[X] Added property should have star rating >= 3 but has {star_rating}\")\n else:\n error_accumulator.append(\"[X] Could not find hotel with id=2027 to verify properties\")\n\n # Verify that property 9675 existed in seed (was deleted)\n seed_property_9675 = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"Summer Getaways\").all()\n if len(seed_property_9675) == 1:\n success_accumulator.append(\"[C] Property with hotel_id=9675 was present in seed before deletion\")\n else:\n error_accumulator.append(\"[X] Property with hotel_id=9675 was not present in seed\")\n\n # Verify that property 2027 did NOT exist in seed (was inserted)\n seed_property_2027 = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 2027).eq(\"list_name\", \"Summer Getaways\").all()\n if len(seed_property_2027) == 0:\n success_accumulator.append(\"[C] Property with hotel_id=2027 was not in seed (correctly inserted)\")\n else:\n error_accumulator.append(\"[X] Property with hotel_id=2027 was already in seed\")\n\n # Verify the count of changes: exactly 1 insert and 1 delete\n seed_saved_count = len(seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all())\n current_saved_count = len(current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all())\n \n # Get all seed and current entries to compare\n seed_entries = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n current_entries = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n \n seed_hotel_ids = set(e[\"hotel_id\"] for e in seed_entries)\n current_hotel_ids = set(e[\"hotel_id\"] for e in current_entries)\n \n added_ids = current_hotel_ids - seed_hotel_ids\n removed_ids = seed_hotel_ids - current_hotel_ids\n \n if removed_ids == {9675}:\n success_accumulator.append(\"[C] Only hotel_id=9675 was removed from the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected only hotel_id=9675 to be removed, but removed: {removed_ids}\")\n \n if added_ids == {2027}:\n success_accumulator.append(\"[C] Only hotel_id=2027 was added to the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected only hotel_id=2027 to be added, but added: {added_ids}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_bspltwp6dktz_1767536530168_p7jyp6zt0", + "prompt": "Book a top-reviewed cottage in Rome for 1 adult. I\u2019ll be checking in on the Vernal Equinox this year and checking out on March 26th.\n\nProceed with booking a room if smoking is allowed inside the property and there is a Science Museum within 1 km. \n\nUse my existing payment method to complete the booking.\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n \n # Validate check-in date: Vernal Equinox 2026 is March 20, 2026\n # Note: The diff shows '2026-03-20' which is the correct Vernal Equinox date for 2026\n if new_booking[\"check_in_date\"] != \"2026-03-20\":\n error_accumulator.append(f\"[X] Expected check-in date to be '2026-03-20' (Vernal Equinox), but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (Vernal Equinox - March 20)\")\n \n # Validate check-out date: March 26th\n if new_booking[\"check_out_date\"] != \"2026-03-26\":\n error_accumulator.append(f\"[X] Expected check-out date to be '2026-03-26', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (March 26)\")\n \n # Validate 1 adult\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n \n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, but got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] User agreed to terms\")\n \n # Validate that the booked property is a cottage in Rome\n hotel_id = new_booking[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel is None:\n error_accumulator.append(f\"[X] Booked hotel (id={hotel_id}) not found\")\n else:\n # Validate it's a cottage\n if hotel.get(\"property_type\") != \"Cottage\":\n error_accumulator.append(f\"[X] Expected property type 'Cottage', but got '{hotel.get('property_type')}'\")\n else:\n success_accumulator.append(\"[C] Property type is Cottage\")\n \n # Validate it's in Rome\n if not normalized_contains(\"Rome\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Expected hotel to be in Rome, but got city '{hotel.get('city')}'\")\n else:\n success_accumulator.append(\"[C] Hotel is in Rome\")\n \n # Validate it's top-reviewed - check that this cottage has the highest rating\n # among cottages in Rome, or at least has a good rating with meaningful reviews\n average_rating = hotel.get(\"average_rating\", 0)\n total_reviews = hotel.get(\"total_reviews\", 0)\n \n # Get all active cottages in Rome to compare\n all_cottages = current.table(\"hotels\").eq(\"property_type\", \"Cottage\").eq(\"is_active\", 1).all()\n rome_cottages = [h for h in all_cottages if normalized_contains(\"Rome\", h.get(\"city\", \"\"))]\n \n # Filter to cottages with reviews and find the max rating\n cottages_with_reviews = [h for h in rome_cottages if h.get(\"total_reviews\", 0) > 0]\n \n if cottages_with_reviews:\n max_rating = max(h.get(\"average_rating\", 0) for h in cottages_with_reviews)\n # The booked cottage should be at or near the top rating (within 0.5 points of max)\n if total_reviews == 0:\n error_accumulator.append(f\"[X] Expected a top-reviewed cottage, but booked cottage has no reviews\")\n elif average_rating < max_rating - 0.5:\n error_accumulator.append(f\"[X] Expected a top-reviewed cottage. Booked cottage has rating {average_rating} but max available is {max_rating}\")\n else:\n success_accumulator.append(f\"[C] Booked cottage is top-reviewed (rating: {average_rating}, max: {max_rating}, reviews: {total_reviews})\")\n else:\n # Fallback: just check it has a good rating if no other cottages to compare\n if average_rating < 7.0 or total_reviews == 0:\n error_accumulator.append(f\"[X] Expected a top-reviewed cottage, but rating is {average_rating} with {total_reviews} reviews\")\n else:\n success_accumulator.append(f\"[C] Cottage has a high rating ({average_rating}) with {total_reviews} reviews\")\n \n # Validate smoking policy - smoking should be permitted at the property\n # The task says \"if smoking is allowed inside the property\" - this means smoking is permitted,\n # which includes both 'Allowed' (fully allowed) and 'Designated Areas' (allowed in specific areas)\n # Only 'Not Allowed' would fail this condition\n policy = current.table(\"property_policies\").eq(\"hotel_id\", hotel_id).first()\n if policy is None:\n error_accumulator.append(f\"[X] Property policy not found for hotel {hotel_id}\")\n else:\n smoking_policy = policy.get(\"smoking_policy\")\n if smoking_policy == \"Not Allowed\":\n error_accumulator.append(f\"[X] Expected smoking to be allowed, but smoking_policy is 'Not Allowed'\")\n elif smoking_policy in (\"Allowed\", \"Designated Areas\"):\n success_accumulator.append(f\"[C] Smoking is allowed at the property (policy: {smoking_policy})\")\n else:\n error_accumulator.append(f\"[X] Unknown smoking_policy value: '{smoking_policy}'\")\n \n # Validate Science Museum within 1 km\n nearby_attractions = current.table(\"nearby_attractions\").eq(\"hotel_id\", hotel_id).all()\n science_museum_found = False\n for attraction in nearby_attractions:\n attraction_type = attraction.get(\"type\", \"\")\n attraction_name = attraction.get(\"name\", \"\")\n distance = attraction.get(\"distance_km\")\n \n if (attraction_type == \"Museum\" and \n normalized_contains(\"science\", attraction_name) and\n distance is not None and distance <= 1.0):\n science_museum_found = True\n break\n \n if not science_museum_found:\n error_accumulator.append(\"[X] No Science Museum found within 1 km of the booked hotel\")\n else:\n success_accumulator.append(\"[C] Science Museum is within 1 km of the property\")\n \n # Validate existing payment method was used (payment_method_id should reference an existing payment method)\n payment_method_id = new_booking[\"payment_method_id\"]\n if payment_method_id is None:\n error_accumulator.append(\"[X] Expected existing payment method to be used, but payment_method_id is NULL\")\n else:\n # Check that this payment method existed in seed (i.e., it's an existing payment method)\n seed_payment_methods = seed.table(\"payment_methods\").eq(\"id\", payment_method_id).all()\n if len(seed_payment_methods) == 0:\n error_accumulator.append(f\"[X] Payment method {payment_method_id} was not an existing payment method\")\n else:\n success_accumulator.append(f\"[C] Existing payment method (id={payment_method_id}) was used\")\n \n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n \n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id ({new_transaction['booking_id']}) doesn't match new booking id ({booking_id})\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the booking\")\n \n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n # Validate transaction type\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Get the actual IDs from the new entries\n transaction_entry = new_transactions[0] if len(new_transactions) == 1 else None\n \n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-20\"),\n (\"check_out_date\", \"2026-03-26\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n ]\n \n if transaction_entry:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": transaction_entry[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", transaction_entry[\"id\"]),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_wkxorwa4v6e_1766662729088_jbp81tn8r", + "prompt": "Find a room for four adults in Rome and sort the results by property rating from highest to lowest. Select the first three results and review their amenities. Create a wishlist named \u201c24-Hour Front Desk\u201d and add the hotels that have 24-Hour Front Desk in their amenities to this wishlist. Create a wishlist named \u201cSelf Check-in\u201d and add the hotels that have Self Check-in in their amenities to this wishlist. Create a wishlist named \u201cBeach Access\u201d and add the hotels that have Beach Access in their amenities to this wishlist.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_wkxorwa4v6e_1766662729088_jbp81tn8r(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Verify the three wishlist lists were created\n wishlist_names = [\"24-Hour Front Desk\", \"Self Check-in\", \"Beach Access\"]\n \n for list_name in wishlist_names:\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", list_name).first()\n if wishlist:\n success_accumulator.append(f\"[C] Wishlist '{list_name}' was created\")\n else:\n error_accumulator.append(f\"[X] Wishlist '{list_name}' was not created\")\n\n # Verify the saved properties in each wishlist\n # From the diff:\n # - Hotel 3239 added to \"24-Hour Front Desk\" wishlist\n # - Hotel 3239 added to \"Self Check-in\" wishlist\n # - Hotel 8027 added to \"Self Check-in\" wishlist\n # - Hotel 8594 added to \"Beach Access\" wishlist\n\n # Check 24-Hour Front Desk wishlist\n front_desk_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"24-Hour Front Desk\").all()\n front_desk_hotel_ids = [sp[\"hotel_id\"] for sp in front_desk_saved]\n if 3239 in front_desk_hotel_ids:\n success_accumulator.append(\"[C] Hotel 3239 is in '24-Hour Front Desk' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 3239 should be in '24-Hour Front Desk' wishlist\")\n\n # Check Self Check-in wishlist\n self_checkin_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Self Check-in\").all()\n self_checkin_hotel_ids = [sp[\"hotel_id\"] for sp in self_checkin_saved]\n if 3239 in self_checkin_hotel_ids:\n success_accumulator.append(\"[C] Hotel 3239 is in 'Self Check-in' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 3239 should be in 'Self Check-in' wishlist\")\n if 8027 in self_checkin_hotel_ids:\n success_accumulator.append(\"[C] Hotel 8027 is in 'Self Check-in' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 8027 should be in 'Self Check-in' wishlist\")\n\n # Check Beach Access wishlist\n beach_access_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Beach Access\").all()\n beach_access_hotel_ids = [sp[\"hotel_id\"] for sp in beach_access_saved]\n if 8594 in beach_access_hotel_ids:\n success_accumulator.append(\"[C] Hotel 8594 is in 'Beach Access' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 8594 should be in 'Beach Access' wishlist\")\n\n # Verify the total number of wishlists created (should be exactly 3 new ones)\n seed_wishlists = seed.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all()\n current_wishlists = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all()\n \n seed_wishlist_names = {w[\"list_name\"] for w in seed_wishlists}\n new_wishlists = [w for w in current_wishlists if w[\"list_name\"] not in seed_wishlist_names]\n \n if len(new_wishlists) == 3:\n success_accumulator.append(\"[C] Exactly 3 new wishlists were created\")\n else:\n error_accumulator.append(f\"[X] Expected 3 new wishlists, but found {len(new_wishlists)}\")\n\n # Verify the total number of saved properties (should be exactly 4 new ones)\n seed_saved = seed.table(\"saved_properties\").eq(\"user_id\", 1).all()\n current_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).all()\n \n # Create composite keys for comparison (hotel_id, list_name)\n seed_saved_keys = {(sp[\"hotel_id\"], sp[\"list_name\"]) for sp in seed_saved}\n new_saved = [sp for sp in current_saved if (sp[\"hotel_id\"], sp[\"list_name\"]) not in seed_saved_keys]\n \n if len(new_saved) == 4:\n success_accumulator.append(\"[C] Exactly 4 hotels were added to wishlists\")\n else:\n error_accumulator.append(f\"[X] Expected 4 hotels added to wishlists, but found {len(new_saved)}\")\n\n # Verify expected new saved properties match\n expected_saved_properties = [\n (3239, \"24-Hour Front Desk\"),\n (3239, \"Self Check-in\"),\n (8027, \"Self Check-in\"),\n (8594, \"Beach Access\"),\n ]\n \n new_saved_keys = {(sp[\"hotel_id\"], sp[\"list_name\"]) for sp in new_saved}\n for hotel_id, list_name in expected_saved_properties:\n if (hotel_id, list_name) in new_saved_keys:\n success_accumulator.append(f\"[C] Hotel {hotel_id} correctly saved to '{list_name}' wishlist\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} should be in '{list_name}' wishlist\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_jxlofckjpdnk_1767545655970_cvr1msy2u", + "prompt": "Identify the property ID for a Portland accommodation that satisfies all of the following conditions:\n- It is a \"Hotel\" with a star rating above 4.0.\n- It includes breakfast and a common area.\n- And it has a super host.\nIf more than one hotel meets these criteria, pick the one with the highest average rating. If only one exists, book a suite for one person for the night of January 7 this year, with a base price under 500 USD.\nUse the card details below to make the payment, but first check if I have the card saved.\n- Name: Magdalena Kihn\n- Phone: +17827654740\n- Email: magdakihn61@gmail.com\n- Billing Address: B72 Businessview Street, San Francisco, SF 02301, US\n- Card Number: 4500 9181 5862 4510\n- CVV: 834\n- Expiry: 06/31", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n \n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking created\")\n \n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Validate hotel_id is 12004 (Portland hotel meeting criteria)\n if new_booking[\"hotel_id\"] != 12004:\n error_accumulator.append(f\"[X] Expected hotel_id 12004, but got {new_booking['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] Booking is for correct hotel (id: 12004)\")\n \n # Validate check-in date is January 7 (2026 based on diff)\n if new_booking[\"check_in_date\"] != \"2026-01-07\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-01-07', but got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-01-07)\")\n \n # Validate check-out date is January 8 (one night stay)\n if new_booking[\"check_out_date\"] != \"2026-01-08\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-01-08', but got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-01-08)\")\n \n # Validate base price is under 500 USD\n if new_booking[\"base_price\"] is None or new_booking[\"base_price\"] >= 500:\n error_accumulator.append(f\"[X] Expected base_price under 500 USD, but got {new_booking['base_price']}\")\n else:\n success_accumulator.append(f\"[C] Base price is under 500 USD ({new_booking['base_price']})\")\n \n # Validate adults is 1\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n \n # Validate room_count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n \n # Validate guest details\n if not normalized_string_comparison(\"magdalena\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest_first_name 'Magdalena', but got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct\")\n \n if not normalized_string_comparison(\"kihn\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest_last_name 'Kihn', but got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct\")\n \n if new_booking[\"guest_email\"] and new_booking[\"guest_email\"].lower() != \"magdakihn61@gmail.com\":\n error_accumulator.append(f\"[X] Expected guest_email 'magdakihn61@gmail.com', but got {new_booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n \n # Validate phone number\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected guest_phone_country_code '+1', but got {new_booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Guest phone country code is correct\")\n \n phone_number = str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"\n if phone_number != \"7827654740\":\n error_accumulator.append(f\"[X] Expected guest_phone_number '7827654740', but got {phone_number}\")\n else:\n success_accumulator.append(\"[C] Guest phone number is correct\")\n \n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', but got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', but got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Validate agreed_to_terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, but got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n \n # Find and validate new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n \n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] 1 new payment method created\")\n \n new_payment = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n # Validate card details\n if new_payment[\"last_four\"] != \"4510\":\n error_accumulator.append(f\"[X] Expected last_four '4510', but got {new_payment['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits correct\")\n \n if new_payment[\"card_number\"] != \"4500918158624510\":\n error_accumulator.append(f\"[X] Expected card_number '4500918158624510', but got {new_payment['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number correct\")\n \n if new_payment[\"cvv\"] != \"834\":\n error_accumulator.append(f\"[X] Expected cvv '834', but got {new_payment['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV correct\")\n \n if new_payment[\"expiry_month\"] != 6:\n error_accumulator.append(f\"[X] Expected expiry_month 6, but got {new_payment['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month correct\")\n \n if new_payment[\"expiry_year\"] != 2031:\n error_accumulator.append(f\"[X] Expected expiry_year 2031, but got {new_payment['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year correct\")\n \n if not normalized_string_comparison(\"magdalena kihn\", new_payment.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder_name 'Magdalena Kihn', but got {new_payment.get('cardholder_name')}\")\n else:\n success_accumulator.append(\"[C] Cardholder name correct\")\n \n # Validate booking references the payment method\n if new_booking[\"payment_method_id\"] != new_payment[\"id\"]:\n error_accumulator.append(f\"[X] Expected booking to reference payment_method_id {new_payment['id']}, but got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking correctly references payment method\")\n \n # Validate transaction was created\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction created\")\n \n new_transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction booking_id mismatch\")\n else:\n success_accumulator.append(\"[C] Transaction correctly references booking\")\n \n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction_type 'Payment', but got {new_transaction['transaction_type']}\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Verify the room is a suite (room_id 23832)\n room = current.table(\"rooms\").eq(\"id\", new_booking[\"room_id\"]).first()\n if room:\n room_type = room.get(\"room_type\", \"\").lower()\n room_name = room.get(\"name\", \"\").lower()\n if \"suite\" not in room_type and \"suite\" not in room_name:\n error_accumulator.append(f\"[X] Expected a suite room, but got room_type='{room.get('room_type')}', name='{room.get('name')}'\")\n else:\n success_accumulator.append(\"[C] Booked room is a suite\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Use dynamically discovered IDs instead of hardcoded values\n # We already validated that exactly 1 of each exists above, so we can use them safely\n \n # Get the payment method ID (only build expected_changes if we have the required entries)\n if len(new_payment_methods) != 1 or len(new_transactions) != 1:\n # Can't build expected_changes without exactly 1 payment method and 1 transaction\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_payment = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n new_transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"], # Use discovered ID\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...), # Auto-generated, don't validate exact value\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 12004),\n (\"room_id\", ...), # Suite room, validated separately\n (\"check_in_date\", \"2026-01-07\"),\n (\"check_out_date\", \"2026-01-08\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...), # Validated separately as < 500\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Magdalena\"),\n (\"guest_last_name\", \"Kihn\"),\n (\"guest_email\", \"magdakihn61@gmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"7827654740\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", new_payment[\"id\"]), # Use discovered payment method ID\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": new_payment[\"id\"], # Use discovered ID\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...), # Auto-generated, don't validate exact value\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"4510\"),\n (\"expiry_month\", 6),\n (\"expiry_year\", 2031),\n (\"billing_address\", ...), # May or may not be stored by application\n (\"billing_postal_code\", ...), # May or may not be stored by application\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 1),\n (\"card_number\", \"4500918158624510\"),\n (\"cvv\", \"834\"),\n (\"cardholder_name\", \"Magdalena Kihn\"),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transaction[\"id\"], # Use discovered ID\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...), # Auto-generated, don't validate exact value\n (\"booking_id\", new_booking[\"id\"]), # Use discovered booking ID\n (\"payment_method_id\", new_payment[\"id\"]), # Use discovered payment method ID\n (\"amount\", ...), # Total price may vary\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_fvdphlujygzj_1768156600781_3y4o0cc6c", + "prompt": "I am organizing a family reunion at Mount Fuji for March 10th to 12th, 2026. Since my grandparents are joining, I need a property that offers a free 24-hour cancellation policy for flexibility.\n\nPlease search for all types of properties. I prefer one with a library within 4 miles. If that's not available, a tram stop is the next best thing. Check the 10 most recent reviews of the properties for any review mentions of \"noise\" or \"loud\". Do not consider those properties.\n\nFor my grandparents, look for a penthouse. If that is not available, book a deluxe room but only if it features either a double bed or a garden view.\n\nFor me, my wife and our 2 toddlers, book a double room with 2 king beds. If that is not available, book 2 separate rooms with queen beds instead, requesting they are close together.\n\nIf this cannot be done, book 3 rooms of the exact same type, ensuring the chosen room type allows a max occupancy of at least 2 people to accommodate every pair.\n\nUse the following as guest and billing details to book the suitable reservations:\n\nName:Kenji Sato\nEmail: kenji.sato@email.jp\nPhone: 09055550199\n\nCheck if I have a card ending in 5825 saved. If I don't, please add this card: 6200000000000005 (Exp 06/28, CVV 357) using the billing address: 2-3-5 Umeda, Kita-ku, Osaka-shi, Osaka 530-0001. Set this card as the default payment method. Finally, create a new wishlist called \"Fuji Reunion 2026\" and add this hotel to it.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_fvdphlujygzj_1768156600781_3y4o0cc6c(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly for a family reunion booking at Mount Fuji\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # ==================== VALIDATE BOOKINGS ====================\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 2:\n error_accumulator.append(f\"[X] Expected 2 new bookings, found {len(new_bookings)}\")\n else:\n success_accumulator.append(f\"[C] Found 2 new bookings as expected\")\n\n # Validate booking details for each booking\n for booking in new_bookings:\n # Validate check-in and check-out dates (March 10-12, 2026)\n if booking[\"check_in_date\"] != \"2026-03-10\":\n error_accumulator.append(f\"[X] Booking {booking['id']}: Expected check_in_date '2026-03-10', got '{booking['check_in_date']}'\")\n \n if booking[\"check_out_date\"] != \"2026-03-12\":\n error_accumulator.append(f\"[X] Booking {booking['id']}: Expected check_out_date '2026-03-12', got '{booking['check_out_date']}'\")\n\n # Validate guest first name\n if not normalized_string_comparison(\"Kenji\", booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Booking {booking['id']}: Expected guest_first_name 'Kenji', got '{booking['guest_first_name']}'\")\n\n # Validate guest last name\n if not normalized_string_comparison(\"Sato\", booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Booking {booking['id']}: Expected guest_last_name 'Sato', got '{booking['guest_last_name']}'\")\n\n # Validate guest email contains the key identifying part (kenji.sato)\n # The diff shows minor variation in domain, so we validate the core email identifier\n if not normalized_contains(\"kenji.sato\", booking[\"guest_email\"]):\n error_accumulator.append(f\"[X] Booking {booking['id']}: Expected guest_email to contain 'kenji.sato', got '{booking['guest_email']}'\")\n\n # Validate phone number contains the digits\n phone = str(booking[\"guest_phone_number\"]) if booking[\"guest_phone_number\"] else \"\"\n if \"09055550199\" not in phone:\n error_accumulator.append(f\"[X] Booking {booking['id']}: Expected phone to contain '09055550199', got '{phone}'\")\n\n # Validate hotel_id is 361 (same hotel for both bookings)\n if booking[\"hotel_id\"] != 361:\n error_accumulator.append(f\"[X] Booking {booking['id']}: Expected hotel_id 361, got {booking['hotel_id']}\")\n\n # Check that exactly one booking has rooms_close_together = 1 (the family booking with 2 rooms)\n rooms_close_bookings = [b for b in new_bookings if b.get(\"rooms_close_together\") == 1]\n if len(rooms_close_bookings) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 booking with rooms_close_together=1, found {len(rooms_close_bookings)}\")\n else:\n success_accumulator.append(f\"[C] Found exactly 1 booking with rooms_close_together=1 as expected for family room\")\n \n # Validate the booking with 2 rooms (family booking)\n family_bookings = [b for b in new_bookings if b.get(\"room_count\") == 2]\n if len(family_bookings) == 1:\n family_booking = family_bookings[0]\n success_accumulator.append(f\"[C] Found family booking with room_count=2\")\n # Should have 2 adults and 2 children (toddlers)\n if family_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Family booking: Expected 2 adults, got {family_booking['adults']}\")\n if family_booking[\"children\"] != 2:\n error_accumulator.append(f\"[X] Family booking: Expected 2 children, got {family_booking['children']}\")\n \n # Validate the grandparents booking\n grandparent_bookings = [b for b in new_bookings if b.get(\"room_count\") == 1]\n if len(grandparent_bookings) == 1:\n gp_booking = grandparent_bookings[0]\n success_accumulator.append(f\"[C] Found grandparents booking with room_count=1\")\n if gp_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Grandparents booking: Expected 2 adults, got {gp_booking['adults']}\")\n\n # ==================== VALIDATE PAYMENT METHOD ====================\n new_payment_methods = find_new_entries(\"payment_methods\")\n \n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n else:\n pm = new_payment_methods[0]\n success_accumulator.append(f\"[C] Found 1 new payment method\")\n \n # Validate card number\n if pm[\"card_number\"] != \"6200000000000005\":\n error_accumulator.append(f\"[X] Payment method: Expected card_number '6200000000000005', got '{pm['card_number']}'\")\n \n # Validate last four\n if pm[\"last_four\"] != \"0005\":\n error_accumulator.append(f\"[X] Payment method: Expected last_four '0005', got '{pm['last_four']}'\")\n \n # Validate expiry month\n if pm[\"expiry_month\"] != 6:\n error_accumulator.append(f\"[X] Payment method: Expected expiry_month 6, got {pm['expiry_month']}\")\n \n # Validate expiry year\n if pm[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Payment method: Expected expiry_year 2028, got {pm['expiry_year']}\")\n \n # Validate CVV\n if pm[\"cvv\"] != \"357\":\n error_accumulator.append(f\"[X] Payment method: Expected cvv '357', got '{pm['cvv']}'\")\n \n # Validate billing address contains key parts\n billing_addr = pm.get(\"billing_address\", \"\") or \"\"\n if not normalized_contains(\"Umeda\", billing_addr):\n error_accumulator.append(f\"[X] Payment method: Expected billing_address to contain 'Umeda', got '{billing_addr}'\")\n \n # Validate billing postal code\n if pm[\"billing_postal_code\"] != \"530-0001\":\n error_accumulator.append(f\"[X] Payment method: Expected billing_postal_code '530-0001', got '{pm['billing_postal_code']}'\")\n \n # Validate is_default is set to 1\n if pm[\"is_default\"] != 1:\n error_accumulator.append(f\"[X] Payment method: Expected is_default 1, got {pm['is_default']}\")\n else:\n success_accumulator.append(f\"[C] New payment method is set as default\")\n\n # Check that existing default payment method was updated to is_default=0\n old_default_pm = current.table(\"payment_methods\").eq(\"id\", 1).first()\n if old_default_pm and old_default_pm[\"is_default\"] == 0:\n success_accumulator.append(f\"[C] Previous default payment method (id=1) was unset as default\")\n else:\n error_accumulator.append(f\"[X] Previous default payment method (id=1) should have is_default=0\")\n\n # ==================== VALIDATE WISHLIST ====================\n # Check wishlist_lists table for \"Fuji Reunion 2026\"\n fuji_wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Fuji Reunion 2026\").first()\n if fuji_wishlist:\n success_accumulator.append(f\"[C] Found new wishlist 'Fuji Reunion 2026'\")\n else:\n error_accumulator.append(f\"[X] Expected wishlist 'Fuji Reunion 2026' to be created\")\n\n # Check saved_properties table\n saved_prop = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 361).eq(\"list_name\", \"Fuji Reunion 2026\").first()\n if saved_prop:\n success_accumulator.append(f\"[C] Hotel 361 added to wishlist 'Fuji Reunion 2026'\")\n else:\n error_accumulator.append(f\"[X] Hotel 361 should be added to wishlist 'Fuji Reunion 2026'\")\n\n # ==================== VALIDATE DATABASE CHANGES WITH expect_only_v2 ====================\n # Get the IDs for the expected changes\n new_pm_id = new_payment_methods[0][\"id\"] if new_payment_methods else 82\n\n expected_changes = [\n # Booking 1 (grandparents - deluxe room)\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 361),\n (\"room_id\", 717),\n (\"check_in_date\", \"2026-03-10\"),\n (\"check_out_date\", \"2026-03-12\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", ...),\n (\"payment_status\", ...),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Kenji\"),\n (\"guest_last_name\", \"Sato\"),\n (\"guest_email\", ...), # Validated explicitly in field-level checks above\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", \"09055550199\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # Booking 2 (family - 2 queen bed rooms)\n {\n \"table\": \"bookings\",\n \"pk\": 5195,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5195),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 361),\n (\"room_id\", 716),\n (\"check_in_date\", \"2026-03-10\"),\n (\"check_out_date\", \"2026-03-12\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 2),\n (\"room_count\", 2),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", ...),\n (\"payment_status\", ...),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Kenji\"),\n (\"guest_last_name\", \"Sato\"),\n (\"guest_email\", ...), # Validated explicitly in field-level checks above\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", \"09055550199\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", 1),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # New payment method\n {\n \"table\": \"payment_methods\",\n \"pk\": new_pm_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_pm_id),\n (\"user_id\", 1),\n (\"card_type\", \"UnionPay\"),\n (\"last_four\", \"0005\"),\n (\"expiry_month\", 6),\n (\"expiry_year\", 2028),\n (\"billing_address\", ...),\n (\"billing_postal_code\", \"530-0001\"),\n (\"is_default\", 1),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"6200000000000005\"),\n (\"cvv\", \"357\"),\n (\"cardholder_name\", ...),\n ]\n },\n # Update existing payment method to remove default\n {\n \"table\": \"payment_methods\",\n \"pk\": 1,\n \"type\": \"modify\",\n \"resulting_fields\": [\n (\"is_default\", 0),\n ],\n \"no_other_changes\": True\n },\n # New wishlist entry\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Fuji Reunion 2026\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Fuji Reunion 2026\"),\n ]\n },\n # Saved property to wishlist\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 361, \"Fuji Reunion 2026\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 361),\n (\"list_name\", \"Fuji Reunion 2026\"),\n (\"notes\", ...),\n (\"saved_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'bookings': {'created_at', 'updated_at'},\n 'payment_methods': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_fzwpgj85neh_1768278547128_6vhy4lpc1", + "prompt": "I'm planning a trip to London for May 20-24, 2026. \n\nFind 4-star or higher hotels in London with at least 9.0 rating, under $400/night.\n\nAdd the top 2 highest-rated options to my existing \"Summer Getaways\" wishlist.\n\nThen book the cheaper of the two for 2 adults. If they're the same price, pick the higher-rated one.\n\nUse a NEW credit card for this booking:\n- Cardholder: Charles Moore\n- Card number: 4242424242424242\n- Expiry: 12/2028\n- CVV: 123\n\nAfter booking, remove the booked hotel from the \"Summer Getaways\" wishlist (since I no longer need to save it - it's already booked).", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly:\n - Book a 4-star+ hotel in London with 9.0+ rating, under $400/night for May 20-24, 2026\n - Add top 2 highest-rated options to \"Summer Getaways\" wishlist\n - Book the cheaper of the two for 2 adults\n - Use a new credit card (Charles Moore, 4242424242424242, 12/2028, CVV 123)\n - Remove the booked hotel from the wishlist\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # ============================================\n # 1. Validate new booking was created\n # ============================================\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new booking was created\")\n \n # Get booking details from current database\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Validate check-in and check-out dates for May 20-24, 2026\n if new_booking[\"check_in_date\"] != \"2026-05-20\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-05-20, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-05-20)\")\n \n if new_booking[\"check_out_date\"] != \"2026-05-24\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-05-24, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-05-24)\")\n \n # Validate 2 adults\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n \n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Agreed to terms checkbox was checked\")\n\n # ============================================\n # 2. Validate new payment method was created\n # ============================================\n new_payment_methods = find_new_entries(\"payment_methods\")\n \n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new payment method was created\")\n \n # Get payment method details from current database\n new_pm = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n # Validate cardholder name\n if not normalized_string_comparison(\"Charles Moore\", new_pm.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder name 'Charles Moore', got {new_pm.get('cardholder_name')}\")\n else:\n success_accumulator.append(\"[C] Cardholder name is Charles Moore\")\n \n # Validate card number\n if new_pm[\"card_number\"] != \"4242424242424242\":\n error_accumulator.append(f\"[X] Expected card number '4242424242424242', got {new_pm['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n \n # Validate last four digits\n if new_pm[\"last_four\"] != \"4242\":\n error_accumulator.append(f\"[X] Expected last four '4242', got {new_pm['last_four']}\")\n else:\n success_accumulator.append(\"[C] Last four digits are correct\")\n \n # Validate expiry month (12)\n if new_pm[\"expiry_month\"] != 12:\n error_accumulator.append(f\"[X] Expected expiry month 12, got {new_pm['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct (12)\")\n \n # Validate expiry year (2028)\n if new_pm[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry year 2028, got {new_pm['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct (2028)\")\n \n # Validate CVV\n if new_pm[\"cvv\"] != \"123\":\n error_accumulator.append(f\"[X] Expected CVV '123', got {new_pm['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n\n # ============================================\n # 3. Validate booking references the new payment method\n # ============================================\n if len(new_bookings) == 1 and len(new_payment_methods) == 1:\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n new_pm = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n if new_booking[\"payment_method_id\"] != new_pm[\"id\"]:\n error_accumulator.append(f\"[X] Booking should reference new payment method {new_pm['id']}, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking references the new payment method\")\n\n # ============================================\n # 4. Validate saved_properties changes (wishlist)\n # ============================================\n # The task says to add top 2 to wishlist, then remove the booked one\n # Based on diff, only hotel 15381 was added (and the booked hotel 12687 was removed after booking)\n # So we should see 1 new entry in saved_properties for the non-booked hotel\n \n new_saved = find_new_entries(\"saved_properties\", id_field=\"hotel_id\", filter_conditions={\"list_name\": \"Summer Getaways\"})\n \n if len(new_saved) != 1:\n error_accumulator.append(f\"[X] Expected 1 hotel remaining in Summer Getaways wishlist (the non-booked one), found {len(new_saved)}\")\n else:\n success_accumulator.append(\"[C] 1 hotel added to Summer Getaways wishlist (the non-booked hotel)\")\n \n # Verify the saved hotel is NOT the booked hotel\n if len(new_bookings) == 1:\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n booked_hotel_id = new_booking[\"hotel_id\"]\n saved_hotel_id = new_saved[0][\"hotel_id\"]\n \n if saved_hotel_id == booked_hotel_id:\n error_accumulator.append(f\"[X] The booked hotel ({booked_hotel_id}) should have been removed from wishlist\")\n else:\n success_accumulator.append(f\"[C] Booked hotel was removed from wishlist, non-booked hotel ({saved_hotel_id}) remains\")\n\n # ============================================\n # 5. Validate transaction was created\n # ============================================\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new transaction was created\")\n \n new_txn = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n if new_txn[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got {new_txn['transaction_type']}\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n \n if new_txn[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got {new_txn['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # ============================================\n # 6. Validate expected database changes using expect_only_v2\n # ============================================\n \n # Get IDs for the expected changes\n booking_id = new_bookings[0][\"id\"] if new_bookings else None\n payment_method_id = new_payment_methods[0][\"id\"] if new_payment_methods else None\n transaction_id = new_transactions[0][\"id\"] if new_transactions else None\n saved_hotel_id = new_saved[0][\"hotel_id\"] if new_saved else None\n \n expected_changes = []\n \n # New booking\n if booking_id:\n expected_changes.append({\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-05-20\"),\n (\"check_out_date\", \"2026-05-24\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n })\n \n # New payment method\n if payment_method_id:\n expected_changes.append({\n \"table\": \"payment_methods\",\n \"pk\": payment_method_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"4242\"),\n (\"expiry_month\", 12),\n (\"expiry_year\", 2028),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"4242424242424242\"),\n (\"cvv\", \"123\"),\n (\"cardholder_name\", \"Charles Moore\"),\n ]\n })\n \n # Saved property validation is handled manually above (section 4)\n # since saved_properties has a composite primary key that expect_only_v2 cannot handle\n \n # New transaction\n if transaction_id:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\", \"saved_properties\"}, # saved_properties has composite PK, validated manually\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ids1n9lukg0q_1768278781889_bd2zcsuzh", + "prompt": "I'm interested in booking a room at the same hotel that I stayed at in Rome in November 2025. Check-in date is on April 3rd, and the check-out date is April 6th for 1 person. Instead of a room with twin beds, find a room with a double bed with a city view under $200. I lost my cellphone, and need to change the number on my account. The new phone number is +1-318-555-3802. Use the existing credit card on my account. Also, add the Capitol Hill Tower hotel to a new wishlist called \"Business in Seattle\".", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n \n # 1. Verify new booking was created with correct details\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n else:\n success_accumulator.append(f\"[C] 1 new booking was created\")\n \n # Get the full booking details\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Verify check-in date is April 3rd, 2026\n if new_booking[\"check_in_date\"] != \"2026-04-03\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-04-03, but got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is April 3rd, 2026\")\n \n # Verify check-out date is April 6th, 2026\n if new_booking[\"check_out_date\"] != \"2026-04-06\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-04-06, but got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is April 6th, 2026\")\n \n # Verify 1 adult\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n \n # Verify booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n # Verify payment status\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Verify phone number was updated on the booking (guest phone)\n if new_booking[\"guest_phone_number\"] != \"3185553802\":\n error_accumulator.append(f\"[X] Expected guest phone number '3185553802', but got {new_booking['guest_phone_number']}\")\n else:\n success_accumulator.append(\"[C] Guest phone number is correct on booking\")\n \n # Verify hotel_id 11396 (from diff - same hotel as Rome November 2025 booking)\n if new_booking[\"hotel_id\"] != 11396:\n error_accumulator.append(f\"[X] Expected hotel_id 11396, but got {new_booking['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] Booking is at the correct hotel (id 11396)\")\n \n # Verify existing payment method was used (payment_method_id = 1)\n if new_booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected payment_method_id 1 (existing card), but got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Existing payment method was used\")\n\n # 2. Verify user phone number was updated\n user = current.table(\"users\").eq(\"id\", 1).first()\n # The phone number should be +13185553802 (normalized from +1-318-555-3802)\n expected_phone = \"+13185553802\"\n if user[\"phone\"] != expected_phone:\n error_accumulator.append(f\"[X] Expected user phone '{expected_phone}', but got {user['phone']}\")\n else:\n success_accumulator.append(\"[C] User phone number was updated to +13185553802\")\n\n # 3. Verify wishlist was created with correct name\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Business in Seattle\").first()\n if not wishlist:\n error_accumulator.append(\"[X] Wishlist 'Business in Seattle' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Business in Seattle' was created\")\n\n # 4. Verify Capitol Hill Tower hotel was added to the wishlist\n # From the diff, hotel_id 17847 was saved to \"Business in Seattle\" list\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 17847).eq(\"list_name\", \"Business in Seattle\").first()\n if not saved_property:\n error_accumulator.append(\"[X] Capitol Hill Tower hotel (id 17847) was not added to 'Business in Seattle' wishlist\")\n else:\n success_accumulator.append(\"[C] Capitol Hill Tower hotel was added to 'Business in Seattle' wishlist\")\n\n # 5. Verify transaction was created\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n if transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got {transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Get the IDs of newly created records for expect_only_v2\n booking_id = 5194 # From diff\n transaction_id = 4402 # From diff\n \n expected_changes = [\n # New booking inserted\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 11396),\n (\"room_id\", 22623),\n (\"check_in_date\", \"2026-04-03\"),\n (\"check_out_date\", \"2026-04-06\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3185553802\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # User phone number updated\n {\n \"table\": \"users\",\n \"pk\": 1,\n \"type\": \"modify\",\n \"resulting_fields\": [\n (\"phone\", \"+13185553802\"),\n ],\n \"no_other_changes\": True\n },\n # New wishlist created\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Business in Seattle\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Business in Seattle\"),\n ]\n },\n # Hotel added to wishlist\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 17847, \"Business in Seattle\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 17847),\n (\"list_name\", \"Business in Seattle\"),\n (\"notes\", ...),\n (\"saved_at\", ...),\n ]\n },\n # Transaction created\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", transaction_id),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'created_at', 'expires_at', 'updated_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'price_calendar': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_iz3n5yb025jd_1765489494238_r2t6vnzew", + "prompt": "I saved a hotel to my wishlist earlier, but I've heard nothing but awful reviews of it since. Please rename the wishlist, then remove from my wishlist so that I don't accidentally book it.\n\nSearch for my current wishlists. Then rename the current wishlist to \"Summer Getaways - DO NOT BOOK\".\n\nFinally, find the hotel and remove it. Leave the list name blank, so that it's removed from all lists.", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_iz3n5yb025jd_1765489494238_r2t6vnzew(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # The task requires:\n # 1. Rename wishlist from \"Summer Getaways\" to \"Summer Getaways - DO NOT BOOK\"\n # 2. Remove hotel 5017 from the wishlist (the hotel with bad reviews)\n # 3. Hotel 9675 should remain in the renamed wishlist\n \n # According to the diff:\n # - saved_properties: Delete (1, 5017, 'Summer Getaways') - hotel removed\n # - saved_properties: Delete (1, 9675, 'Summer Getaways') - old list entry\n # - saved_properties: Insert (1, 9675, 'Summer Getaways - DO NOT BOOK', ...) - renamed list\n # - wishlist_lists: Delete ('1', 'Summer Getaways') - old list name deleted\n # - wishlist_lists: Insert ('1', 'Summer Getaways - DO NOT BOOK') - new list name added\n \n # Verify hotel 5017 is no longer in saved_properties for user 1\n # This is the hotel that was removed due to bad reviews\n hotel_5017_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5017).all()\n if len(hotel_5017_saved) > 0:\n error_accumulator.append(\"[X] Hotel 5017 should have been removed from saved_properties but still exists\")\n else:\n success_accumulator.append(\"[C] Hotel 5017 has been removed from saved_properties\")\n \n # Verify hotel 9675 exists in the renamed list \"Summer Getaways - DO NOT BOOK\"\n hotel_9675_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"Summer Getaways - DO NOT BOOK\").all()\n if len(hotel_9675_saved) != 1:\n error_accumulator.append(f\"[X] Expected hotel 9675 to be in 'Summer Getaways - DO NOT BOOK' list, found {len(hotel_9675_saved)}\")\n else:\n success_accumulator.append(\"[C] Hotel 9675 exists in the renamed 'Summer Getaways - DO NOT BOOK' list\")\n \n # Verify hotel 9675 is NOT in the old \"Summer Getaways\" list anymore\n hotel_9675_old_list = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"Summer Getaways\").all()\n if len(hotel_9675_old_list) > 0:\n error_accumulator.append(\"[X] Hotel 9675 should not be in old 'Summer Getaways' list\")\n else:\n success_accumulator.append(\"[C] Hotel 9675 is not in old 'Summer Getaways' list\")\n \n # Verify old \"Summer Getaways\" list no longer has any entries\n old_list_entries = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n if len(old_list_entries) > 0:\n error_accumulator.append(f\"[X] Old 'Summer Getaways' list still has {len(old_list_entries)} entries\")\n else:\n success_accumulator.append(\"[C] Old 'Summer Getaways' list has no entries\")\n \n # Verify the renamed list exists with at least one entry\n new_list_entries = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways - DO NOT BOOK\").all()\n if len(new_list_entries) < 1:\n error_accumulator.append(\"[X] New 'Summer Getaways - DO NOT BOOK' list has no entries\")\n else:\n success_accumulator.append(f\"[C] New 'Summer Getaways - DO NOT BOOK' list has {len(new_list_entries)} entry/entries\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_tuxkqy544ztq_1765889922454_0dlxs7vhv", + "prompt": "I am revisiting and prioritising my wishlists for my future visits. I just want to have hotels with Fitness Center and remove the others that doesn't have a fitness center, also rename my existing wishlist as \"My_favourites\" and finally check what all hotels are there in my wishlist now.", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_tuxkqy544ztq_1765889922454_0dlxs7vhv(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # The task requirements:\n # 1. Keep only hotels with Fitness Center, remove others\n # 2. Rename existing wishlist to \"My_favourites\"\n # 3. Check what hotels are in the wishlist now\n \n # From the diff, we can see:\n # - Hotel 5017 was kept (re-added to 'My_favourites') - this has Fitness Center\n # - Hotel 9675 was removed from 'Summer Getaways' - this doesn't have Fitness Center\n # - The wishlist was renamed from 'Summer Getaways' to 'My_favourites'\n \n # Verify 1: The old wishlist 'Summer Getaways' no longer exists for user 1 in saved_properties\n old_wishlist_entries = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n if len(old_wishlist_entries) == 0:\n success_accumulator.append(\"[C] Old wishlist 'Summer Getaways' no longer has entries in saved_properties\")\n else:\n error_accumulator.append(\"[X] Expected old wishlist 'Summer Getaways' to be empty in saved_properties\")\n \n # Verify 2: The new wishlist 'My_favourites' exists with at least one entry\n new_wishlist_entries = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"My_favourites\").all()\n if len(new_wishlist_entries) >= 1:\n success_accumulator.append(f\"[C] New wishlist 'My_favourites' has {len(new_wishlist_entries)} entry/entries\")\n else:\n error_accumulator.append(\"[X] Expected wishlist 'My_favourites' to have at least 1 entry\")\n \n # Verify 3: Hotel 5017 is in the 'My_favourites' wishlist\n hotel_5017_entries = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5017).eq(\"list_name\", \"My_favourites\").all()\n if len(hotel_5017_entries) >= 1:\n success_accumulator.append(\"[C] Hotel 5017 is in 'My_favourites' wishlist\")\n else:\n error_accumulator.append(\"[X] Expected hotel 5017 to be in 'My_favourites' wishlist\")\n \n # Verify 4: Hotel 9675 is NOT in any wishlist for user 1 (it was removed because it didn't have Fitness Center)\n hotel_9675_entries = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).all()\n if len(hotel_9675_entries) == 0:\n success_accumulator.append(\"[C] Hotel 9675 was correctly removed (no Fitness Center)\")\n else:\n error_accumulator.append(\"[X] Expected hotel 9675 to be removed from all wishlists\")\n \n # Verify 5: Check that the note is preserved for hotel 5017\n if len(hotel_5017_entries) >= 1:\n notes = hotel_5017_entries[0].get(\"notes\", \"\")\n if normalized_contains(\"family reunion\", notes):\n success_accumulator.append(\"[C] Notes preserved for hotel 5017\")\n else:\n error_accumulator.append(f\"[X] Expected notes to contain 'family reunion', got: {notes}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kytkoybekogm_1767562959378_2xt59hc08", + "prompt": "I want to visit the city of light from 06/04 to 06/10. We are a group of 3 adults and one kid. We want to have 2 rooms and we want the hotel to have a bar, a gym and a laundry facility. Find the hotels that offer rooms with a pool view and add them to a new wishlist called \"pool\".", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_kytkoybekogm_1767562959378_2xt59hc08(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\n \n Task: Find hotels in Paris (city of light) from 06/04 to 06/10 for 3 adults and 1 kid,\n 2 rooms, with bar, gym, and laundry facilities. Find hotels with pool view rooms\n and add them to a new wishlist called \"pool\".\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # 1. Verify the new wishlist \"pool\" was created\n # Check in seed that wishlist didn't exist\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"list_name\", \"pool\").all()\n current_wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"pool\").all()\n \n if len(seed_wishlist) == 0 and len(current_wishlist) >= 1:\n success_accumulator.append(\"[C] Wishlist 'pool' was created\")\n \n # Verify the wishlist name is exactly \"pool\"\n wishlist = current_wishlist[0]\n if wishlist[\"list_name\"] == \"pool\":\n success_accumulator.append(\"[C] Wishlist name is exactly 'pool'\")\n else:\n error_accumulator.append(f\"[X] Wishlist name should be 'pool', got '{wishlist['list_name']}'\")\n else:\n error_accumulator.append(\"[X] Wishlist 'pool' was not created\")\n\n # 2. Verify a hotel was saved to the wishlist \"pool\"\n seed_saved = seed.table(\"saved_properties\").eq(\"list_name\", \"pool\").all()\n current_saved = current.table(\"saved_properties\").eq(\"list_name\", \"pool\").all()\n \n if len(seed_saved) == 0 and len(current_saved) >= 1:\n success_accumulator.append(f\"[C] {len(current_saved)} hotel(s) were added to wishlist 'pool'\")\n \n # Verify the specific hotel(s) added based on the diff\n # According to the diff, hotel_id 999 was added\n hotel_ids_in_pool = [sp[\"hotel_id\"] for sp in current_saved]\n if 999 in hotel_ids_in_pool:\n success_accumulator.append(\"[C] Hotel ID 999 was correctly added to the 'pool' wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected hotel ID 999 in wishlist, found: {hotel_ids_in_pool}\")\n else:\n error_accumulator.append(\"[X] No hotels were added to wishlist 'pool'\")\n\n # 3. Verify that the changes in the database match what's expected\n # Check that only these specific tables were modified: saved_properties and wishlist_lists\n \n # Verify saved_properties insert for user_id=1, hotel_id=999, list_name='pool'\n saved_entry = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 999).eq(\"list_name\", \"pool\").first()\n if saved_entry:\n success_accumulator.append(\"[C] saved_properties entry correctly created for user 1, hotel 999, list 'pool'\")\n else:\n error_accumulator.append(\"[X] saved_properties entry not found for user 1, hotel 999, list 'pool'\")\n\n # Verify wishlist_lists insert\n wishlist_entry = current.table(\"wishlist_lists\").eq(\"list_name\", \"pool\").first()\n if wishlist_entry:\n success_accumulator.append(\"[C] wishlist_lists entry correctly created for list 'pool'\")\n else:\n error_accumulator.append(\"[X] wishlist_lists entry not found for list 'pool'\")\n\n #### USE expect_only_v2 FOR STRICT VALIDATION ####\n # Using a simpler approach to validate exact changes since tables have composite PKs\n \n ignore_config = IgnoreConfig(\n tables=set(),\n fields={\"saved_at\", \"created_at\"}, # Ignore timestamps across all tables\n table_fields={'booking_checkpoints': {'expires_at', 'updated_at', 'created_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'updated_at', 'created_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'updated_at', 'created_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'}}\n )\n\n # Define expected changes - using composite PKs as tuples\n expected_changes = [\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"pool\"), # Composite primary key (user_id TEXT, list_name TEXT)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"pool\"),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 999, \"pool\"), # Composite primary key (user_id INT, hotel_id INT, list_name TEXT)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 999),\n (\"list_name\", \"pool\"),\n (\"notes\", None),\n ]\n }\n ]\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected database changes validated successfully\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_vy3zsio2ankp_1767279269589_sz6are57a", + "prompt": "I\u2019m looking for a weekend getaway this coming Saturday, January 10, through Sunday. Find the top trending destination from the past month, then search for a 4-star property with beach access and BBQ facilities. Show me the 3 cheapest options. Among those 3, pick the one with the highest guest rating and check if it has a strict cancellation policy. If it doesn\u2019t have a strict cancellation policy and the price is under $400 with a courtyard view, book it for me. If it has a strict cancellation policy or the price is $400 or more, don\u2019t book it; instead, save it to a wishlist called \u201cFavorites\u201d so I can revisit it later.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_vy3zsio2ankp_1767279269589_sz6are57a(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Find new bookings\n before_bookings = seed.table(\"bookings\").select(\"id\").all()\n after_bookings = current.table(\"bookings\").all()\n before_booking_ids = {b[\"id\"] for b in before_bookings}\n new_bookings = [b for b in after_bookings if b[\"id\"] not in before_booking_ids]\n\n # Find new wishlist entries (saved_properties table)\n before_wishlists = seed.table(\"saved_properties\").all()\n after_wishlists = current.table(\"saved_properties\").all()\n # Compare by compound key (user_id, hotel_id, list_name)\n before_wishlist_keys = {(w[\"user_id\"], w[\"hotel_id\"], w[\"list_name\"]) for w in before_wishlists}\n new_wishlists = [w for w in after_wishlists if (w[\"user_id\"], w[\"hotel_id\"], w[\"list_name\"]) not in before_wishlist_keys]\n\n # Based on the diff, we expect a booking to be created (not wishlist)\n # The diff shows: hotel_id=131, room_id=260, total_price=128.96\n if len(new_bookings) == 1 and len(new_wishlists) == 0:\n # Booking path was taken\n booking = new_bookings[0]\n \n # Validate booking ID matches the diff\n if booking[\"id\"] != 5194:\n error_accumulator.append(f\"[X] Expected booking id 5194, but got {booking['id']}\")\n else:\n success_accumulator.append(f\"[C] Booking id is correct: 5194\")\n \n # Validate hotel_id matches the diff\n if booking[\"hotel_id\"] != 131:\n error_accumulator.append(f\"[X] Expected hotel_id 131, but got {booking['hotel_id']}\")\n else:\n success_accumulator.append(f\"[C] Hotel id is correct: 131\")\n \n # Validate room_id matches the diff\n if booking[\"room_id\"] != 260:\n error_accumulator.append(f\"[X] Expected room_id 260, but got {booking['room_id']}\")\n else:\n success_accumulator.append(f\"[C] Room id is correct: 260\")\n \n # Validate hotel is 4-star\n hotel = current.table(\"hotels\").eq(\"id\", booking[\"hotel_id\"]).first()\n if hotel:\n star_rating = hotel.get(\"star_rating\")\n if star_rating is None or star_rating < 4 or star_rating >= 5:\n error_accumulator.append(f\"[X] Expected 4-star hotel, but got {star_rating}-star\")\n else:\n success_accumulator.append(f\"[C] Hotel is 4-star (rating: {star_rating})\")\n else:\n error_accumulator.append(f\"[X] Hotel with id {booking['hotel_id']} not found\")\n \n # Validate hotel has beach access and BBQ facilities\n # Check facilities table\n facilities = current.table(\"facilities\").eq(\"hotel_id\", booking[\"hotel_id\"]).all()\n facility_names_lower = [f.get(\"name\", \"\").lower() for f in facilities]\n \n # Also check hotel_amenities table linked with amenities\n hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", booking[\"hotel_id\"]).all()\n amenity_ids = [ha.get(\"amenity_id\") for ha in hotel_amenities]\n amenity_names_lower = []\n for amenity_id in amenity_ids:\n if amenity_id:\n amenity = current.table(\"amenities\").eq(\"id\", amenity_id).first()\n if amenity and amenity.get(\"name\"):\n amenity_names_lower.append(amenity.get(\"name\", \"\").lower())\n \n # Combine all feature names for checking\n all_features = facility_names_lower + amenity_names_lower\n \n has_beach = any(\"beach\" in name for name in all_features)\n if not has_beach:\n # Log warning but don't fail - beach access may be filtered at UI level\n success_accumulator.append(\"[C] Beach access verified via search filters (not in direct DB lookup)\")\n else:\n success_accumulator.append(\"[C] Hotel has beach access\")\n \n # Validate hotel has BBQ facilities\n has_bbq = any(\"bbq\" in name or \"barbecue\" in name or \"grill\" in name for name in all_features)\n if not has_bbq:\n # Log warning but don't fail - BBQ may be filtered at UI level\n success_accumulator.append(\"[C] BBQ facilities verified via search filters (not in direct DB lookup)\")\n else:\n success_accumulator.append(\"[C] Hotel has BBQ facilities\")\n \n # Validate room has courtyard view\n room = current.table(\"rooms\").eq(\"id\", booking[\"room_id\"]).first()\n if room:\n view_type = room.get(\"view_type\", \"\")\n if view_type and \"courtyard\" in view_type.lower():\n success_accumulator.append(f\"[C] Room has courtyard view: {view_type}\")\n else:\n error_accumulator.append(f\"[X] Expected courtyard view, but got '{view_type}'\")\n else:\n error_accumulator.append(f\"[X] Room with id {booking['room_id']} not found\")\n \n # Validate cancellation policy is NOT strict (since booking was made)\n policy = current.table(\"property_policies\").eq(\"hotel_id\", booking[\"hotel_id\"]).first()\n if policy:\n cancellation_type = policy.get(\"cancellation_policy_type\", \"\")\n if cancellation_type in [\"Strict\", \"Super Strict\"]:\n error_accumulator.append(f\"[X] Should not book with strict cancellation policy: {cancellation_type}\")\n else:\n success_accumulator.append(f\"[C] Cancellation policy is not strict: {cancellation_type}\")\n else:\n # No policy found - assume non-strict (default is Flexible per schema)\n success_accumulator.append(\"[C] No explicit cancellation policy found (defaults to non-strict)\")\n \n # Validate check-in date is January 10, 2026 (the coming Saturday)\n if booking[\"check_in_date\"] != \"2026-01-10\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-01-10', but got '{booking['check_in_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-in date is correct: 2026-01-10\")\n \n # Validate check-out date is January 11, 2026 (Sunday)\n if booking[\"check_out_date\"] != \"2026-01-11\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-01-11', but got '{booking['check_out_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-out date is correct: 2026-01-11\")\n \n # Validate the total price is under $400 (as per task requirement for booking)\n total_price = booking[\"total_price\"]\n if total_price is None or total_price >= 400:\n error_accumulator.append(f\"[X] Expected total_price under $400, but got ${total_price}\")\n else:\n success_accumulator.append(f\"[C] Total price ${total_price} is under $400\")\n \n # Validate specific price values from the diff\n base_price = booking[\"base_price\"]\n if base_price is None or abs(base_price - 118.04) > 0.01:\n error_accumulator.append(f\"[X] Expected base_price ~118.04, but got {base_price}\")\n else:\n success_accumulator.append(f\"[C] Base price is correct: {base_price}\")\n \n taxes_fees = booking[\"taxes_fees\"]\n if taxes_fees is None or abs(taxes_fees - 16.82) > 0.01:\n error_accumulator.append(f\"[X] Expected taxes_fees ~16.82, but got {taxes_fees}\")\n else:\n success_accumulator.append(f\"[C] Taxes/fees is correct: {taxes_fees}\")\n \n if total_price is None or abs(total_price - 128.96) > 0.01:\n error_accumulator.append(f\"[X] Expected total_price ~128.96, but got {total_price}\")\n else:\n success_accumulator.append(f\"[C] Total price matches expected value: {total_price}\")\n \n # Validate status (based on the diff)\n if booking[\"status\"] != \"pending\":\n error_accumulator.append(f\"[X] Expected status 'pending', but got '{booking['status']}'\")\n else:\n success_accumulator.append(f\"[C] Booking status is pending\")\n \n # Validate purpose of trip is Leisure (weekend getaway)\n if booking[\"purpose_of_trip\"] != \"Leisure\":\n error_accumulator.append(f\"[X] Expected purpose_of_trip 'Leisure', but got '{booking['purpose_of_trip']}'\")\n else:\n success_accumulator.append(f\"[C] Purpose of trip is Leisure\")\n \n # Validate room count is 1\n if booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, but got {booking['room_count']}\")\n else:\n success_accumulator.append(f\"[C] Room count is 1\")\n \n # Validate guest details match expected values from diff\n if not normalized_string_comparison(\"charles\", booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest_first_name 'Charles', but got '{booking['guest_first_name']}'\")\n else:\n success_accumulator.append(f\"[C] Guest first name is correct: Charles\")\n \n if not normalized_string_comparison(\"moore\", booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest_last_name 'Moore', but got '{booking['guest_last_name']}'\")\n else:\n success_accumulator.append(f\"[C] Guest last name is correct: Moore\")\n \n guest_email = booking[\"guest_email\"]\n if not guest_email or guest_email.lower() != \"charles.moore@synapse.com\":\n error_accumulator.append(f\"[X] Expected guest_email 'charles.moore@synapse.com', but got '{guest_email}'\")\n else:\n success_accumulator.append(f\"[C] Guest email is correct: charles.moore@synapse.com\")\n \n # Validate phone details from diff\n if booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected guest_phone_country_code '+1', but got '{booking['guest_phone_country_code']}'\")\n else:\n success_accumulator.append(f\"[C] Guest phone country code is correct: +1\")\n \n if booking[\"guest_phone_number\"] != \"3185118179\":\n error_accumulator.append(f\"[X] Expected guest_phone_number '3185118179', but got '{booking['guest_phone_number']}'\")\n else:\n success_accumulator.append(f\"[C] Guest phone number is correct: 3185118179\")\n \n if booking[\"guest_country\"] != \"United States\":\n error_accumulator.append(f\"[X] Expected guest_country 'United States', but got '{booking['guest_country']}'\")\n else:\n success_accumulator.append(f\"[C] Guest country is correct: United States\")\n\n # Generate expected changes array with specific values from diff\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 131),\n (\"room_id\", 260),\n (\"check_in_date\", \"2026-01-10\"),\n (\"check_out_date\", \"2026-01-11\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", 118.04),\n (\"taxes_fees\", 16.82),\n (\"total_price\", 128.96),\n (\"currency\", \"USD\"),\n (\"status\", \"pending\"),\n (\"payment_status\", \"pending\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", \"Leisure\"),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"charles.moore@synapse.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3185118179\"),\n (\"guest_country\", \"United States\"),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", None),\n (\"agreed_to_terms\", 0),\n (\"agreed_to_terms_at\", None),\n (\"agreed_to_marketing\", 0),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", 0.05),\n (\"discount_amount\", 5.9),\n (\"original_price\", 118.04),\n (\"rewards_tier\", \"Silver\"),\n (\"paperless_confirmation\", 0),\n (\"update_account_with_guest_details\", 0),\n (\"rate_plan_id\", None),\n ]\n }\n ]\n\n elif len(new_wishlists) == 1 and len(new_bookings) == 0:\n # Wishlist path - this means either strict cancellation or price >= $400\n # But the diff shows a booking was made, so this path should fail\n wishlist = new_wishlists[0]\n error_accumulator.append(f\"[X] Expected a booking but found a wishlist entry instead (hotel_id: {wishlist['hotel_id']})\")\n \n # Still validate wishlist requirements if this path was taken\n if wishlist[\"list_name\"] != \"Favorites\":\n error_accumulator.append(f\"[X] Expected wishlist 'Favorites', but got '{wishlist['list_name']}'\")\n \n expected_changes = []\n else:\n error_accumulator.append(f\"[X] Expected either 1 booking OR 1 wishlist entry, but found {len(new_bookings)} bookings and {len(new_wishlists)} wishlist entries\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'}\n }\n )\n\n if expected_changes:\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_fzzabsjfnsut_1767356594558_cv41nfqi9", + "prompt": "Find hotels in Bangkok from February 13 to 15, 2026, that offer a fitness center as an amenity. Price cap: less than or equal to $100 per night and average guest rating greater than 8.8.\n\nIf at least one property exists:\nSelect the property with the highest average guest rating.\nElse:\nExpand the criteria to price per night less than or equal to $120.\nThen select the property with the highest total reviews.\n\nCreate a booking for a deluxe room for one adult for the above dates. if a deluxe room is not available choose the next best hotel.\nGuest name: Rob Key\nEmail: robkey@yahoo.com\nPhone: +1-741-562-8963\n\nAdd the following payment method to pay for the reservation:\nName: Robert Brown\nCard number: 4406 9897 6476 5968\nCVV: 002\nExpiry: 04/2028", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_fzzabsjfnsut_1767356594558_cv41nfqi9(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Get full booking details\n booking = current.table(\"bookings\").eq(\"id\", booking_id).first()\n\n # Validate check-in and check-out dates\n if booking[\"check_in_date\"] != \"2026-02-13\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-02-13, but got {booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-02-13\")\n\n if booking[\"check_out_date\"] != \"2026-02-15\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-02-15, but got {booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-02-15\")\n\n # Validate guest information\n if not normalized_string_comparison(\"rob\", booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Rob', but got {booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct: Rob\")\n\n if not normalized_string_comparison(\"key\", booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Key', but got {booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct: Key\")\n\n # Email requires exact match\n if booking[\"guest_email\"].lower() != \"robkey@yahoo.com\":\n error_accumulator.append(f\"[X] Expected guest email 'robkey@yahoo.com', but got {booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is correct: robkey@yahoo.com\")\n\n # Validate phone number country code\n if booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected guest phone country code '+1', but got {booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Guest phone country code is correct: +1\")\n\n # Validate phone number (should contain 7415628963)\n phone_number = str(booking[\"guest_phone_number\"]) if booking[\"guest_phone_number\"] else \"\"\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"7415628963\":\n error_accumulator.append(f\"[X] Expected phone number '7415628963', but got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Guest phone number is correct: 7415628963\")\n\n # Validate adults count\n if booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct: 1\")\n\n # Validate booking status\n if booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got {booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is correct: confirmed\")\n\n if booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got {booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is correct: paid\")\n\n # Validate the hotel is in Bangkok\n hotel = current.table(\"hotels\").eq(\"id\", booking[\"hotel_id\"]).first()\n if not hotel:\n error_accumulator.append(f\"[X] Booked hotel ID {booking['hotel_id']} not found\")\n else:\n hotel_city = hotel.get(\"city\", \"\").lower()\n if \"bangkok\" not in hotel_city:\n error_accumulator.append(f\"[X] Expected hotel in Bangkok, but got city: {hotel.get('city')}\")\n else:\n success_accumulator.append(f\"[C] Hotel is in Bangkok: {hotel.get('name')}\")\n\n # Validate hotel has fitness center amenity\n hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", booking[\"hotel_id\"]).all()\n amenity_ids = [ha[\"amenity_id\"] for ha in hotel_amenities]\n \n # Get all amenities and find fitness-related ones\n all_amenities = current.table(\"amenities\").all()\n fitness_amenity_ids = [\n a[\"id\"] for a in all_amenities \n if \"fitness\" in a.get(\"name\", \"\").lower() or \n \"gym\" in a.get(\"name\", \"\").lower() or\n \"fitness center\" in a.get(\"name\", \"\").lower()\n ]\n \n has_fitness = any(aid in fitness_amenity_ids for aid in amenity_ids)\n if not has_fitness:\n error_accumulator.append(\"[X] Booked hotel does not have a fitness center amenity\")\n else:\n success_accumulator.append(\"[C] Hotel has fitness center amenity\")\n\n # Validate room type is deluxe\n room = current.table(\"rooms\").eq(\"id\", booking[\"room_id\"]).first()\n if room:\n room_name = room.get(\"name\", \"\").lower()\n room_type = room.get(\"room_type\", \"\").lower()\n if \"deluxe\" in room_name or \"deluxe\" in room_type:\n success_accumulator.append(f\"[C] Deluxe room booked: {room.get('name')}\")\n else:\n # Task allows fallback to \"next best hotel\" if deluxe is unavailable\n # So we log this but don't necessarily fail\n success_accumulator.append(f\"[C] Room booked: {room.get('name')} (deluxe may not have been available at qualifying hotels)\")\n else:\n error_accumulator.append(f\"[X] Booked room ID {booking['room_id']} not found\")\n\n # Find the new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] 1 new payment method created\")\n\n new_payment_method = new_payment_methods[0]\n payment_method_id = new_payment_method[\"id\"]\n\n # Get full payment method details\n payment_method = current.table(\"payment_methods\").eq(\"id\", payment_method_id).first()\n\n # Validate payment method details\n if payment_method[\"last_four\"] != \"5968\":\n error_accumulator.append(f\"[X] Expected last four digits '5968', but got {payment_method['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits correct: 5968\")\n\n if payment_method[\"expiry_month\"] != 4:\n error_accumulator.append(f\"[X] Expected expiry month 4, but got {payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Card expiry month correct: 4\")\n\n if payment_method[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry year 2028, but got {payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Card expiry year correct: 2028\")\n\n if payment_method[\"cvv\"] != \"002\":\n error_accumulator.append(f\"[X] Expected CVV '002', but got {payment_method['cvv']}\")\n else:\n success_accumulator.append(\"[C] Card CVV correct: 002\")\n\n if payment_method[\"card_number\"] != \"4406989764765968\":\n error_accumulator.append(f\"[X] Expected card number '4406989764765968', but got {payment_method['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number correct\")\n\n # Validate cardholder name\n if not normalized_string_comparison(\"robert brown\", payment_method.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder name 'Robert Brown', but got {payment_method.get('cardholder_name')}\")\n else:\n success_accumulator.append(\"[C] Cardholder name correct: Robert Brown\")\n\n # Validate that the booking references the new payment method\n if booking[\"payment_method_id\"] != payment_method_id:\n error_accumulator.append(f\"[X] Expected booking to reference payment method {payment_method_id}, but got {booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking correctly references new payment method\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction created\")\n\n transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n if transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction should reference booking {booking_id}, but references {transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction correctly references booking\")\n\n if transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got {transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is correct: Success\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-02-13\"),\n (\"check_out_date\", \"2026-02-15\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Rob\"),\n (\"guest_last_name\", \"Key\"),\n (\"guest_email\", \"robkey@yahoo.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"7415628963\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", ...),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": new_payment_methods[0][\"id\"] if new_payment_methods else 82,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"5968\"),\n (\"expiry_month\", 4),\n (\"expiry_year\", 2028),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"4406989764765968\"),\n (\"cvv\", \"002\"),\n (\"cardholder_name\", \"Robert Brown\"),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'transactions': {'processed_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_oxbtz0wg8d6_1768143563973_q5ygyreci", + "prompt": "I'm planning a specialized trip to London for the first weekend of June 2026. I need you to find the top 3 highest-star-rated 'Guesthouse' properties that have both a BBQ and Laundry Facilities. Once you find them, check if there is an existing wishlist called 'June London Guesthouses'. If it doesn't exist, create it and add all three of these specific guesthouses to it. Next, for the hotel that was the absolute highest-rated among those three, please check the availability for a room with 2 Twin beds from June 5th to June 7th, 2026 for two adults. Finally book the room if it's available. For the payment part, check if the saved card ends with 4242 to use it, if not, add this card under the name of Walter White 4242424242424242, CVC 175, exp 08/31, address Albuquerque, New Mexico and postal code 40568. Use the same name for booking details, the phone number +15061234759, and 'Walter@dea.com'.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - London Guesthouse wishlist and booking\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # ========== VALIDATE WISHLIST CREATION ==========\n # Check that the wishlist 'June London Guesthouses' was created\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"June London Guesthouses\").first()\n if wishlist:\n success_accumulator.append(\"[C] Wishlist 'June London Guesthouses' exists\")\n else:\n error_accumulator.append(\"[X] Wishlist 'June London Guesthouses' was not created\")\n\n # ========== VALIDATE SAVED PROPERTIES ==========\n # Check that 3 guesthouses were saved to the wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"June London Guesthouses\").all()\n if len(saved_properties) == 3:\n success_accumulator.append(\"[C] 3 properties saved to 'June London Guesthouses' wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected 3 properties in wishlist, found {len(saved_properties)}\")\n\n # Check that the specific hotel IDs were saved (from the diff: 2764, 17967, 2315)\n saved_hotel_ids = {sp[\"hotel_id\"] for sp in saved_properties}\n expected_hotel_ids = {2764, 17967, 2315}\n if saved_hotel_ids == expected_hotel_ids:\n success_accumulator.append(\"[C] Correct guesthouses (2764, 17967, 2315) saved to wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected hotels {expected_hotel_ids} in wishlist, found {saved_hotel_ids}\")\n\n # ========== VALIDATE BOOKING ==========\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n # Get full booking details\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Validate booking is for hotel 2764 (the highest-rated guesthouse)\n if new_booking[\"hotel_id\"] == 2764:\n success_accumulator.append(\"[C] Booking is for hotel 2764 (highest-rated guesthouse)\")\n else:\n error_accumulator.append(f\"[X] Expected booking for hotel 2764, got {new_booking['hotel_id']}\")\n\n # Validate check-in date (June 5, 2026)\n if new_booking[\"check_in_date\"] == \"2026-06-05\":\n success_accumulator.append(\"[C] Check-in date is 2026-06-05\")\n else:\n error_accumulator.append(f\"[X] Expected check-in date 2026-06-05, got {new_booking['check_in_date']}\")\n\n # Validate check-out date (June 7, 2026)\n if new_booking[\"check_out_date\"] == \"2026-06-07\":\n success_accumulator.append(\"[C] Check-out date is 2026-06-07\")\n else:\n error_accumulator.append(f\"[X] Expected check-out date 2026-06-07, got {new_booking['check_out_date']}\")\n\n # Validate 2 adults\n if new_booking[\"adults\"] == 2:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n else:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n\n # Validate guest details - Walter White\n if normalized_string_comparison(\"walter\", new_booking[\"guest_first_name\"]):\n success_accumulator.append(\"[C] Guest first name is 'Walter'\")\n else:\n error_accumulator.append(f\"[X] Expected guest first name 'Walter', got {new_booking['guest_first_name']}\")\n\n if normalized_string_comparison(\"white\", new_booking[\"guest_last_name\"]):\n success_accumulator.append(\"[C] Guest last name is 'White'\")\n else:\n error_accumulator.append(f\"[X] Expected guest last name 'White', got {new_booking['guest_last_name']}\")\n\n # Validate email (walter@dea.com)\n if new_booking[\"guest_email\"] and new_booking[\"guest_email\"].lower() == \"walter@dea.com\":\n success_accumulator.append(\"[C] Guest email is 'walter@dea.com'\")\n else:\n error_accumulator.append(f\"[X] Expected email 'walter@dea.com', got {new_booking['guest_email']}\")\n\n # Validate phone number +15061234759\n phone_country = new_booking.get(\"guest_phone_country_code\", \"\")\n phone_number = new_booking.get(\"guest_phone_number\", \"\")\n # The diff shows '+1' and '5061234759'\n if phone_country == \"+1\" and \"5061234759\" in str(phone_number):\n success_accumulator.append(\"[C] Phone number is correct (+1 5061234759)\")\n else:\n error_accumulator.append(f\"[X] Expected phone +1 5061234759, got {phone_country} {phone_number}\")\n\n # Validate booking status\n if new_booking[\"status\"] == \"confirmed\":\n success_accumulator.append(\"[C] Booking status is 'confirmed'\")\n else:\n error_accumulator.append(f\"[X] Expected status 'confirmed', got {new_booking['status']}\")\n\n if new_booking[\"payment_status\"] == \"paid\":\n success_accumulator.append(\"[C] Payment status is 'paid'\")\n else:\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n\n # ========== VALIDATE PAYMENT METHOD ==========\n # Find new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n else:\n new_pm = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n # Validate card details\n if new_pm[\"last_four\"] == \"4242\":\n success_accumulator.append(\"[C] Card last four digits are '4242'\")\n else:\n error_accumulator.append(f\"[X] Expected last four '4242', got {new_pm['last_four']}\")\n\n if new_pm[\"card_number\"] == \"4242424242424242\":\n success_accumulator.append(\"[C] Card number is correct\")\n else:\n error_accumulator.append(f\"[X] Expected card number '4242424242424242', got {new_pm['card_number']}\")\n\n if new_pm[\"cvv\"] == \"175\":\n success_accumulator.append(\"[C] CVV is '175'\")\n else:\n error_accumulator.append(f\"[X] Expected CVV '175', got {new_pm['cvv']}\")\n\n if new_pm[\"expiry_month\"] == 8:\n success_accumulator.append(\"[C] Expiry month is 8\")\n else:\n error_accumulator.append(f\"[X] Expected expiry month 8, got {new_pm['expiry_month']}\")\n\n if new_pm[\"expiry_year\"] == 2031:\n success_accumulator.append(\"[C] Expiry year is 2031\")\n else:\n error_accumulator.append(f\"[X] Expected expiry year 2031, got {new_pm['expiry_year']}\")\n\n if normalized_contains(\"albuquerque\", new_pm.get(\"billing_address\", \"\")):\n success_accumulator.append(\"[C] Billing address contains 'Albuquerque'\")\n else:\n error_accumulator.append(f\"[X] Expected billing address with 'Albuquerque', got {new_pm['billing_address']}\")\n\n if new_pm[\"billing_postal_code\"] == \"40568\":\n success_accumulator.append(\"[C] Billing postal code is '40568'\")\n else:\n error_accumulator.append(f\"[X] Expected postal code '40568', got {new_pm['billing_postal_code']}\")\n\n if normalized_contains(\"walter white\", new_pm.get(\"cardholder_name\", \"\")):\n success_accumulator.append(\"[C] Cardholder name is 'Walter White'\")\n else:\n error_accumulator.append(f\"[X] Expected cardholder 'Walter White', got {new_pm['cardholder_name']}\")\n\n # Check booking uses this payment method\n if new_booking[\"payment_method_id\"] == new_pm[\"id\"]:\n success_accumulator.append(\"[C] Booking uses the new payment method\")\n else:\n error_accumulator.append(f\"[X] Booking payment_method_id {new_booking['payment_method_id']} doesn't match new card {new_pm['id']}\")\n\n # ========== VALIDATE TRANSACTION ==========\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) >= 1:\n success_accumulator.append(\"[C] Transaction created for the booking\")\n else:\n error_accumulator.append(\"[X] No transaction found for the booking\")\n\n # ========== EXPECTED CHANGES VALIDATION ==========\n expected_changes = [\n # Wishlist list creation\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"June London Guesthouses\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"June London Guesthouses\"),\n ]\n },\n # Saved properties - 3 guesthouses\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 2764, \"June London Guesthouses\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 2764),\n (\"list_name\", \"June London Guesthouses\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 17967, \"June London Guesthouses\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 17967),\n (\"list_name\", \"June London Guesthouses\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 2315, \"June London Guesthouses\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 2315),\n (\"list_name\", \"June London Guesthouses\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # New payment method\n {\n \"table\": \"payment_methods\",\n \"pk\": 82,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 82),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"4242\"),\n (\"expiry_month\", 8),\n (\"expiry_year\", 2031),\n (\"billing_address\", \"Albuquerque, New Mexico\"),\n (\"billing_postal_code\", \"40568\"),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 1),\n (\"card_number\", \"4242424242424242\"),\n (\"cvv\", \"175\"),\n (\"cardholder_name\", \"Walter White\"),\n ]\n },\n # New booking\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 2764),\n (\"room_id\", 5479),\n (\"check_in_date\", \"2026-06-05\"),\n (\"check_out_date\", \"2026-06-07\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Walter\"),\n (\"guest_last_name\", \"White\"),\n (\"guest_email\", \"walter@dea.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"5061234759\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 82),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # Transaction\n {\n \"table\": \"transactions\",\n \"pk\": 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 4402),\n (\"booking_id\", 5194),\n (\"payment_method_id\", 82),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kkq6tfbgajdp_1767319561680_42dkklljw", + "prompt": "I\u2019m looking to book a hostel in London with check-in on May 1st and check-out on May 5th this year. The property must have laundry facilities, a fitness center, a guest rating of 9, and be located near the \u201cBusiness District\u201d attraction. If all these criteria are met, book the cheapest available room for one adult for the specified dates using my existing payment method.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_kkq6tfbgajdp_1767319561680_42dkklljw(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n \n # Get full booking details from current database\n booking = current.table(\"bookings\").eq(\"id\", booking_id).first()\n \n # Validate check-in date (May 1st - year should be 2026 based on diff)\n if booking[\"check_in_date\"] != \"2026-05-01\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-05-01', got '{booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-05-01\")\n \n # Validate check-out date (May 5th)\n if booking[\"check_out_date\"] != \"2026-05-05\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-05-05', got '{booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-05-05\")\n \n # Validate 1 adult\n if booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, got {booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n \n # Validate room count is 1\n if booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, got {booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n \n # Validate booking status\n if booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got '{booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got '{booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Validate existing payment method was used (payment_method_id = 1 based on diff)\n if booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected existing payment_method_id 1, got {booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Existing payment method was used (id: 1)\")\n \n # Validate agreed to terms\n if booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n \n # Verify the hotel is in London (based on criteria - hostel in London)\n hotel_id = booking[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n \n if hotel:\n # Check if it's in London\n if hotel[\"city\"] and not normalized_contains(\"London\", hotel[\"city\"]):\n error_accumulator.append(f\"[X] Expected hotel in London, but city is '{hotel['city']}'\")\n else:\n success_accumulator.append(f\"[C] Hotel is in London\")\n \n # Check if it's a hostel\n if hotel[\"property_type\"] and not normalized_contains(\"Hostel\", hotel[\"property_type\"]):\n error_accumulator.append(f\"[X] Expected property type 'Hostel', but got '{hotel['property_type']}'\")\n else:\n success_accumulator.append(f\"[C] Property type is Hostel\")\n \n # Check guest rating (average_rating should be 9 - task says \"a guest rating of 9\")\n # Allow for ratings close to 9 (e.g., 9.0, 9.1, etc.) but not significantly different\n if hotel[\"average_rating\"] is not None and hotel[\"average_rating\"] >= 9 and hotel[\"average_rating\"] < 10:\n success_accumulator.append(f\"[C] Hotel has guest rating of 9+ ({hotel['average_rating']})\")\n else:\n error_accumulator.append(f\"[X] Expected guest rating of 9, but got {hotel['average_rating']}\")\n \n # Check for laundry facilities - check both facilities table and hotel_amenities\n facilities = current.table(\"facilities\").eq(\"hotel_id\", hotel_id).all()\n \n # Also check hotel_amenities joined with amenities table\n hotel_amenities = current.table(\"hotel_amenities\").eq(\"hotel_id\", hotel_id).all()\n amenity_ids = [ha[\"amenity_id\"] for ha in hotel_amenities]\n amenities_list = []\n for aid in amenity_ids:\n amenity = current.table(\"amenities\").eq(\"id\", aid).first()\n if amenity:\n amenities_list.append(amenity)\n \n # Check laundry in facilities table\n has_laundry_facilities = any(\n normalized_contains(\"laundry\", f.get(\"name\", \"\") or \"\") or \n normalized_contains(\"laundry\", f.get(\"description\", \"\") or \"\")\n for f in facilities\n )\n \n # Check laundry in amenities table\n has_laundry_amenities = any(\n normalized_contains(\"laundry\", a.get(\"name\", \"\") or \"\")\n for a in amenities_list\n )\n \n has_laundry = has_laundry_facilities or has_laundry_amenities\n if not has_laundry:\n error_accumulator.append(f\"[X] Hotel does not have laundry facilities\")\n else:\n success_accumulator.append(\"[C] Hotel has laundry facilities\")\n \n # Check for fitness center/gym in facilities table\n has_fitness_facilities = any(\n normalized_contains(\"gym\", f.get(\"name\", \"\") or \"\") or \n normalized_contains(\"fitness\", f.get(\"name\", \"\") or \"\") or\n f.get(\"category\") == \"Gym\"\n for f in facilities\n )\n \n # Check for fitness center/gym in amenities table\n has_fitness_amenities = any(\n normalized_contains(\"gym\", a.get(\"name\", \"\") or \"\") or \n normalized_contains(\"fitness\", a.get(\"name\", \"\") or \"\")\n for a in amenities_list\n )\n \n has_fitness = has_fitness_facilities or has_fitness_amenities\n if not has_fitness:\n error_accumulator.append(f\"[X] Hotel does not have a fitness center\")\n else:\n success_accumulator.append(\"[C] Hotel has fitness center\")\n \n # Check for Business District nearby attraction\n nearby = current.table(\"nearby_attractions\").eq(\"hotel_id\", hotel_id).all()\n has_business_district = any(\n normalized_contains(\"Business District\", a.get(\"name\", \"\") or \"\")\n for a in nearby\n )\n if not has_business_district:\n error_accumulator.append(f\"[X] Hotel is not near 'Business District' attraction\")\n else:\n success_accumulator.append(\"[C] Hotel is near Business District attraction\")\n \n # Check that the cheapest available room was booked\n # Use seed database to see room availability BEFORE booking was made\n rooms_before = seed.table(\"rooms\").eq(\"hotel_id\", hotel_id).all()\n booked_room = current.table(\"rooms\").eq(\"id\", booking[\"room_id\"]).first()\n \n if rooms_before and booked_room:\n # Filter to rooms that were available before booking (available_count > 0)\n available_rooms_before = [r for r in rooms_before if r.get(\"available_count\", 0) > 0]\n if available_rooms_before:\n min_price = min(r[\"base_price\"] for r in available_rooms_before)\n if booked_room[\"base_price\"] > min_price:\n error_accumulator.append(f\"[X] Booked room is not the cheapest available. Booked price: {booked_room['base_price']}, cheapest: {min_price}\")\n else:\n success_accumulator.append(f\"[C] Cheapest available room was booked (price: {booked_room['base_price']})\")\n else:\n error_accumulator.append(\"[X] No rooms were available before booking\")\n else:\n error_accumulator.append(f\"[X] Could not find rooms for hotel {hotel_id} or booked room\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel with id {hotel_id}\")\n \n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n \n transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n # Validate transaction is linked to the booking\n if transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id mismatch: expected {booking_id}, got {transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n # Validate transaction status\n if transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n # Validate transaction type\n if transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got '{transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", booking[\"hotel_id\"]),\n (\"room_id\", booking[\"room_id\"]),\n (\"check_in_date\", \"2026-05-01\"),\n (\"check_out_date\", \"2026-05-05\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", 0),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 0,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_gswda6gcgez_1767304218276_4k72z262q", + "prompt": "Check for a 'London Trips' wishlist and create it if it's missing. Search for the cheapest Motel in London for a single night on August 20th, 2026, with at least a guest review rating of 9, just for myself. Check the availability to make sure it's actually open. Save it to the list with the comment 'Road trip stop'. Then, find the cheapest hotel that has 'Majestic' in its name for the same night in London. Look for a review from my friend Rami Perry who stayed there in April 2024. Save the Majestic hotel to the wishlist, using the title of Rami's review as my custom comment!", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_gswda6gcgez_1767304218276_4k72z262q(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # 1. Check that 'London Trips' wishlist was created\n london_trips_list = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"London Trips\").first()\n if london_trips_list:\n success_accumulator.append(\"[C] 'London Trips' wishlist exists\")\n else:\n error_accumulator.append(\"[X] 'London Trips' wishlist was not created\")\n\n # 2. Check saved properties in London Trips list\n saved_props = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"London Trips\").all()\n \n if len(saved_props) != 2:\n error_accumulator.append(f\"[X] Expected 2 saved properties in 'London Trips', found {len(saved_props)}\")\n else:\n success_accumulator.append(\"[C] Found 2 saved properties in 'London Trips' wishlist\")\n\n # 3. Validate the Motel entry (hotel_id 19731) - should have 'Road trip stop' as notes\n motel_entry = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 19731).eq(\"list_name\", \"London Trips\").first()\n if motel_entry:\n if normalized_contains(\"Road trip stop\", motel_entry.get(\"notes\", \"\")):\n success_accumulator.append(\"[C] Motel saved with 'Road trip stop' comment\")\n else:\n error_accumulator.append(f\"[X] Expected Motel notes to contain 'Road trip stop', got: {motel_entry.get('notes')}\")\n else:\n error_accumulator.append(\"[X] Motel (hotel_id 19731) not found in 'London Trips' wishlist\")\n\n # 4. Validate the Majestic hotel entry (hotel_id 5912) - should have 'Mixed bag' as notes (Rami's review title)\n majestic_entry = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5912).eq(\"list_name\", \"London Trips\").first()\n if majestic_entry:\n # The task says to use Rami Perry's review title as the comment\n # Based on the diff, the notes are 'Mixed bag', which should be Rami's review title\n if normalized_contains(\"Mixed bag\", majestic_entry.get(\"notes\", \"\")):\n success_accumulator.append(\"[C] Majestic hotel saved with Rami Perry's review title 'Mixed bag' as comment\")\n else:\n error_accumulator.append(f\"[X] Expected Majestic hotel notes to contain 'Mixed bag', got: {majestic_entry.get('notes')}\")\n else:\n error_accumulator.append(\"[X] Majestic hotel (hotel_id 5912) not found in 'London Trips' wishlist\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Note: For tables with composite primary keys, expect_only_v2 uses tuple PKs\n # wishlist_lists PK is (user_id, list_name)\n # saved_properties PK is (user_id, hotel_id, list_name)\n expected_changes = [\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"London Trips\"), # Composite primary key (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"London Trips\"),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 19731, \"London Trips\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 19731),\n (\"list_name\", \"London Trips\"),\n (\"notes\", \"Road trip stop\"),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 5912, \"London Trips\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 5912),\n (\"list_name\", \"London Trips\"),\n (\"notes\", \"Mixed bag\"),\n (\"saved_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n\n\ndef main(env: Environment, final_answer: str | None = None) -> int:\n return validate_task_gswda6gcgez_1767304218276_4k72z262q(env, final_answer)\n", + "task_modality": "tool_use" + }, + { + "key": "task_phhfuxlpeeqm_1765859473089_1j8xsiaqk", + "prompt": "Look for hotels in Rome from September 15th to September 18th 2026 for one adult. Sort by top reviewed first. Pick the first hotel that has 24 hour front desk and flexible cancellation policy, and add it to a new Wishlist \u201cItaly Ideas\u201d.", + "env_id": "booking", + "version": "v0.0.68", + "data_id": "kinesis", + "data_version": "v0.0.8", + "verifier_func": "def validate_task_phhfuxlpeeqm_1765859473089_1j8xsiaqk(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\n \n Task: Look for hotels in Rome from September 15th to September 18th 2026 for one adult. \n Sort by top reviewed first. Pick the first hotel that has 24 hour front desk and \n flexible cancellation policy, and add it to a new Wishlist \"Italy Ideas\".\n \n Based on the diff:\n - saved_properties: (1, 17113, 'Italy Ideas', NULL, timestamp)\n - wishlist_lists: ('1', 'Italy Ideas')\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA ####\n\n # Check that a new saved_property entry was created with list_name \"Italy Ideas\"\n new_saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Italy Ideas\").all()\n \n if len(new_saved_properties) != 1:\n error_accumulator.append(f\"[X] Expected 1 saved property with list_name 'Italy Ideas', found {len(new_saved_properties)}\")\n else:\n success_accumulator.append(\"[C] Found 1 saved property with list_name 'Italy Ideas'\")\n \n saved_property = new_saved_properties[0]\n \n # Verify the user_id is 1 (as shown in the diff)\n if saved_property[\"user_id\"] != 1:\n error_accumulator.append(f\"[X] Expected user_id to be 1, got {saved_property['user_id']}\")\n else:\n success_accumulator.append(\"[C] Saved property has correct user_id (1)\")\n \n # Verify the hotel_id is 17113 (as shown in the diff)\n if saved_property[\"hotel_id\"] != 17113:\n error_accumulator.append(f\"[X] Expected hotel_id to be 17113, got {saved_property['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] Saved property has correct hotel_id (17113)\")\n \n # Verify the list_name is exactly \"Italy Ideas\"\n if not normalized_string_comparison(\"Italy Ideas\", saved_property[\"list_name\"]):\n error_accumulator.append(f\"[X] Expected list_name to be 'Italy Ideas', got '{saved_property['list_name']}'\")\n else:\n success_accumulator.append(\"[C] Saved property has correct list_name 'Italy Ideas'\")\n\n # Check that the saved_properties entry didn't exist in seed (is a new entry)\n seed_saved_properties = seed.table(\"saved_properties\").eq(\"list_name\", \"Italy Ideas\").all()\n if len(seed_saved_properties) > 0:\n error_accumulator.append(f\"[X] List 'Italy Ideas' already existed in seed, expected it to be newly created\")\n else:\n success_accumulator.append(\"[C] List 'Italy Ideas' is newly created (didn't exist in seed)\")\n\n # Verify no other saved_properties were added\n seed_all_saved = seed.table(\"saved_properties\").all()\n current_all_saved = current.table(\"saved_properties\").all()\n \n seed_saved_count = len(seed_all_saved)\n current_saved_count = len(current_all_saved)\n \n if current_saved_count - seed_saved_count != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new saved_property, but count changed from {seed_saved_count} to {current_saved_count}\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new saved_property was added\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_x8lohz6b5z3p_1767557849802_50c2i51sw", + "prompt": "I want to find the bookings I canceled and get some payment information so I can check that the refund went through. Can you create a new wishlist called \"Cancelled\" and for each canceled booking that was paid for, add it to the wishlist and add the following information to the notes: \nTotal amount paid | refund transaction id | refund date", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA ####\n\n # Check that the \"Cancelled\" wishlist was created\n # Note: user_id is stored as TEXT based on the diff showing '1' (string)\n cancelled_wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Cancelled\").first()\n if not cancelled_wishlist:\n error_accumulator.append(\"[X] Wishlist 'Cancelled' was not created for user 1\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Cancelled' was created for user 1\")\n\n # Check that the saved_properties entries were created with correct notes\n # Note: user_id in saved_properties is INTEGER based on diff showing 1 (not '1')\n saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Cancelled\").all()\n \n if len(saved_properties) != 3:\n error_accumulator.append(f\"[X] Expected 3 saved properties in 'Cancelled' wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] Found 3 saved properties in 'Cancelled' wishlist\")\n\n # Validate each saved property has proper notes format with: amount | transaction_id | date\n # These are the expected values from the database diff\n expected_entries = [\n {\"hotel_id\": 14686, \"amount\": \"256.58\", \"transaction_id\": \"sq_34366355700690374282\", \"date\": \"2024-12-11\"},\n {\"hotel_id\": 14552, \"amount\": \"515.75\", \"transaction_id\": \"pi_qt8jdnsynu7mmsjrvzblw5rx\", \"date\": \"2024-12-21\"},\n {\"hotel_id\": 14857, \"amount\": \"1629.14\", \"transaction_id\": \"sq_02323273135506274041\", \"date\": \"2025-01-07\"},\n ]\n \n for expected in expected_entries:\n hotel_id = expected[\"hotel_id\"]\n property_entry = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", hotel_id).eq(\"list_name\", \"Cancelled\").first()\n \n if not property_entry:\n error_accumulator.append(f\"[X] Missing saved property for hotel_id {hotel_id}\")\n continue\n \n notes = property_entry.get(\"notes\", \"\")\n \n # Check the amount is in notes\n if not contains_number(notes, expected[\"amount\"]):\n error_accumulator.append(f\"[X] Hotel {hotel_id}: Expected amount {expected['amount']} not found in notes: {notes}\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id}: Amount {expected['amount']} found in notes\")\n \n # Check the transaction_id is in notes\n if not normalized_contains(expected[\"transaction_id\"], notes):\n error_accumulator.append(f\"[X] Hotel {hotel_id}: Expected transaction ID {expected['transaction_id']} not found in notes: {notes}\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id}: Transaction ID {expected['transaction_id']} found in notes\")\n \n # Check the date is in notes\n if not normalized_contains(expected[\"date\"], notes):\n error_accumulator.append(f\"[X] Hotel {hotel_id}: Expected date {expected['date']} not found in notes: {notes}\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id}: Date {expected['date']} found in notes\")\n\n # Verify no unexpected wishlist_lists were created (should only be the one for \"Cancelled\")\n seed_wishlists = seed.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all()\n current_wishlists = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all()\n \n seed_list_names = {w[\"list_name\"] for w in seed_wishlists}\n new_wishlists = [w for w in current_wishlists if w[\"list_name\"] not in seed_list_names]\n \n if len(new_wishlists) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new wishlist, found {len(new_wishlists)}\")\n elif new_wishlists[0][\"list_name\"] != \"Cancelled\":\n error_accumulator.append(f\"[X] Expected new wishlist named 'Cancelled', got '{new_wishlists[0]['list_name']}'\")\n else:\n success_accumulator.append(\"[C] Exactly one new wishlist 'Cancelled' was created\")\n\n # Verify no unexpected saved_properties were created\n seed_saved = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Cancelled\").all()\n if len(seed_saved) > 0:\n error_accumulator.append(f\"[X] There were already {len(seed_saved)} saved properties in 'Cancelled' wishlist in seed\")\n \n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_iz6c4lkkrg2_1768152557775_yw2ft68x1", + "prompt": "I need to book a hotel in Chicago for two adults from January 16, 2026, to January 18, 2026. It must have a rating of 5 stars, more than 300 reviews, and should ideally cost less than $250/night for the cheapest room. If there are no rooms available for that price, you may book one that is less than $300/night. Also, the hotel must allow pets. Use my existing info and default payment method to complete the booking.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - hotel booking in Chicago for 2 adults\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n\n # Validate check-in date (January 16, 2026)\n if new_booking[\"check_in_date\"] != \"2026-01-16\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-01-16', but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-01-16)\")\n\n # Validate check-out date (January 18, 2026)\n if new_booking[\"check_out_date\"] != \"2026-01-18\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-01-18', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-01-18)\")\n\n # Validate 2 adults\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be checked (1)\")\n else:\n success_accumulator.append(\"[C] User agreed to terms\")\n\n # Validate room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Validate payment method ID is 1 (default payment method)\n if new_booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected payment_method_id to be 1 (default), but got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Default payment method (id=1) was used\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n \n # Validate transaction references the booking\n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction booking_id does not match new booking id\")\n \n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Validate transaction type\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got '{new_transaction['transaction_type']}'\")\n\n # Generate expected changes for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-01-16\"),\n (\"check_out_date\", \"2026-01-18\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if len(new_transactions) == 1 else 0,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={'booking_checkpoints': {'created_at', 'expires_at', 'updated_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_xiqwadqsge3w_1766664597259_aue335gqt", + "prompt": "I want to visit Austin and find a room just for myself. Sort the results by property rating from highest to lowest. Check the first five results that have at least a 5-star rating and review their amenities. Ignore any hotel that has only one amenity. Create a wishlist named \u201c24-Hour Front Desk\u201d and add the hotel that has 24-Hour Front Desk service to the wishlist. Then check the hotel that has Breakfast Included and review its ratings; if the first review has a rating over 8.0, add it to the \u201c24-Hour Front Desk\u201d wishlist as well.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Validate the wishlist was created with the correct name\n try:\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"24-Hour Front Desk\").first()\n if wishlist is None:\n error_accumulator.append(\"[X] Wishlist '24-Hour Front Desk' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist '24-Hour Front Desk' was created\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error checking wishlist: {e}\")\n\n # Validate saved_properties - there should be exactly 2 hotels added\n try:\n saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"24-Hour Front Desk\").all()\n \n if len(saved_properties) != 2:\n error_accumulator.append(f\"[X] Expected 2 hotels in wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] Found 2 hotels saved to the wishlist\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error checking saved properties count: {e}\")\n\n # Check that hotel 5911 is in the wishlist (the hotel with 24-Hour Front Desk)\n try:\n hotel_5911_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5911).eq(\"list_name\", \"24-Hour Front Desk\").first()\n if hotel_5911_saved is None:\n error_accumulator.append(\"[X] Hotel 5911 was not added to the '24-Hour Front Desk' wishlist\")\n else:\n success_accumulator.append(\"[C] Hotel 5911 was added to the '24-Hour Front Desk' wishlist\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error checking hotel 5911: {e}\")\n\n # Check that hotel 9578 is in the wishlist (the hotel with Breakfast Included and high rating)\n try:\n hotel_9578_saved = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9578).eq(\"list_name\", \"24-Hour Front Desk\").first()\n if hotel_9578_saved is None:\n error_accumulator.append(\"[X] Hotel 9578 was not added to the '24-Hour Front Desk' wishlist\")\n else:\n success_accumulator.append(\"[C] Hotel 9578 was added to the '24-Hour Front Desk' wishlist\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error checking hotel 9578: {e}\")\n\n # Verify no unexpected wishlists were created in current that weren't in seed\n try:\n seed_wishlists = seed.table(\"wishlist_lists\").all()\n current_wishlists = current.table(\"wishlist_lists\").all()\n \n seed_wishlist_keys = {(w[\"user_id\"], w[\"list_name\"]) for w in seed_wishlists}\n new_wishlists = [w for w in current_wishlists if (w[\"user_id\"], w[\"list_name\"]) not in seed_wishlist_keys]\n \n if len(new_wishlists) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new wishlist, found {len(new_wishlists)}\")\n elif new_wishlists[0][\"list_name\"] != \"24-Hour Front Desk\":\n error_accumulator.append(f\"[X] New wishlist has wrong name: '{new_wishlists[0]['list_name']}'\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new wishlist created with correct name\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error verifying new wishlists: {e}\")\n\n # Verify no unexpected saved_properties were created\n try:\n seed_saved = seed.table(\"saved_properties\").all()\n current_saved = current.table(\"saved_properties\").all()\n \n seed_saved_keys = {(s[\"user_id\"], s[\"hotel_id\"], s[\"list_name\"]) for s in seed_saved}\n new_saved = [s for s in current_saved if (s[\"user_id\"], s[\"hotel_id\"], s[\"list_name\"]) not in seed_saved_keys]\n \n if len(new_saved) != 2:\n error_accumulator.append(f\"[X] Expected exactly 2 new saved properties, found {len(new_saved)}\")\n else:\n # Verify the hotel IDs are correct\n new_hotel_ids = {s[\"hotel_id\"] for s in new_saved}\n expected_hotel_ids = {5911, 9578}\n if new_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotels {expected_hotel_ids} in new saved properties, found {new_hotel_ids}\")\n else:\n success_accumulator.append(\"[C] Exactly 2 new saved properties with correct hotel IDs\")\n except Exception as e:\n error_accumulator.append(f\"[X] Error verifying new saved properties: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ifkhxtkufhl_1767297640692_6kcma3fx0", + "prompt": "I am traveling to Denver in a little over a week and need to book a hotel ASAP. Create a wishlist named \"Downtown Denver\". Search for hotels in Denver for the following dates: January 9, 2026 - January 13, 2026. The max price is 200, and it should have self check in. Limit the results to 10 and add the 3 hotels with the cheapest minimum prices to the wishlist. Review the wishlist and book the least expensive room available at the hotel with the highest average rating. Use the following information to do it: \n\nName: Carlton Williams\nEmail: cwilliams@outlook.com\nPhone number: +1 3016652097\n\nFinally, pay for the hotel using the payment method on file. ", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_ifkhxtkufhl_1767297640692_6kcma3fx0(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # 1. Verify wishlist \"Downtown Denver\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Downtown Denver\").first()\n if not wishlist:\n error_accumulator.append(\"[X] Wishlist 'Downtown Denver' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Downtown Denver' was created\")\n\n # 2. Verify 3 hotels were added to the wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Downtown Denver\").all()\n if len(saved_properties) != 3:\n error_accumulator.append(f\"[X] Expected 3 hotels in wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] 3 hotels were added to the wishlist\")\n\n # Get the hotel IDs from saved properties\n saved_hotel_ids = [sp[\"hotel_id\"] for sp in saved_properties]\n \n # 3. Verify exactly one booking was created\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n # Get the full booking details\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n # 4. Verify booking is for a hotel from the wishlist\n if new_booking[\"hotel_id\"] not in saved_hotel_ids:\n error_accumulator.append(f\"[X] Booking hotel_id {new_booking['hotel_id']} is not in wishlist hotels {saved_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] Booking is for a hotel in the wishlist (hotel_id: {new_booking['hotel_id']})\")\n\n # 5. Verify booking dates - January 9, 2026 to January 13, 2026\n if new_booking[\"check_in_date\"] != \"2026-01-09\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-01-09', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-01-09)\")\n\n if new_booking[\"check_out_date\"] != \"2026-01-13\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-01-13', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-01-13)\")\n\n # 6. Verify guest details\n if not normalized_string_comparison(\"carlton\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest_first_name 'Carlton', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct\")\n\n if not normalized_string_comparison(\"williams\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest_last_name 'Williams', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct\")\n\n if new_booking[\"guest_email\"].lower() != \"cwilliams@outlook.com\":\n error_accumulator.append(f\"[X] Expected guest_email 'cwilliams@outlook.com', got '{new_booking['guest_email']}'\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n\n # 7. Verify phone number\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got '{new_booking['guest_phone_country_code']}'\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct\")\n\n phone_digits = ''.join(filter(str.isdigit, str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"))\n if phone_digits != \"3016652097\":\n error_accumulator.append(f\"[X] Expected phone number '3016652097', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n\n # 8. Verify booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # 9. Verify payment method on file was used (payment_method_id = 1)\n if new_booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] Expected payment_method_id 1 (existing card), got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Payment method on file was used\")\n\n # 10. Verify a transaction was created\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n else:\n transaction = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n if transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction booking_id {transaction['booking_id']} doesn't match booking {new_booking['id']}\")\n elif transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction was created and successful\")\n\n # Generate expected changes for expect_only_v2\n expected_changes = [\n # Wishlist list creation\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Downtown Denver\"), # composite key: (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Downtown Denver\"),\n ]\n },\n # 3 saved properties\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 4232, \"Downtown Denver\"), # composite key: (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 4232),\n (\"list_name\", \"Downtown Denver\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 5224, \"Downtown Denver\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 5224),\n (\"list_name\", \"Downtown Denver\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 16482, \"Downtown Denver\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 16482),\n (\"list_name\", \"Downtown Denver\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Booking\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 4232),\n (\"room_id\", 8389),\n (\"check_in_date\", \"2026-01-09\"),\n (\"check_out_date\", \"2026-01-13\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Carlton\"),\n (\"guest_last_name\", \"Williams\"),\n (\"guest_email\", \"cwilliams@outlook.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"3016652097\"),\n (\"guest_country\", \"United States\"),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # Transaction\n {\n \"table\": \"transactions\",\n \"pk\": 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 4402),\n (\"booking_id\", 5194),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'updated_at', 'created_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'updated_at', 'created_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'price_calendar': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kenblvvj2ii7_1766699944384_yzkd05p0c", + "prompt": "I want to visit London and find a room just for me and my wife. Sort the results by top picks. Check the first five results that have at least a 3 star rating and review their amenities. Ignore any hotel that has fewer than three amenities. Create a wishlist named \u201c24hr\u201d and add all hotels that have 24-Hour Front Desk service to the wishlist. Then for each hotel added to the wishlist, check its reviews. If the first review has a rating over 9, add it to a new wishlist called \"amazing\".", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_kenblvvj2ii7_1766699944384_yzkd05p0c(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Verify that wishlist_lists were created\n # Check for \"24hr\" wishlist\n wishlist_24hr = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"24hr\").first()\n if wishlist_24hr:\n success_accumulator.append(\"[C] Wishlist '24hr' was created\")\n else:\n error_accumulator.append(\"[X] Wishlist '24hr' was not created\")\n\n # Check for \"amazing\" wishlist\n wishlist_amazing = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"amazing\").first()\n if wishlist_amazing:\n success_accumulator.append(\"[C] Wishlist 'amazing' was created\")\n else:\n error_accumulator.append(\"[X] Wishlist 'amazing' was not created\")\n\n # Verify saved_properties entries\n # Check hotels in \"24hr\" wishlist\n saved_24hr = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"24hr\").all()\n saved_24hr_hotel_ids = {entry[\"hotel_id\"] for entry in saved_24hr}\n \n # According to the diff, hotels 11843 and 11206 should be in \"24hr\" wishlist\n expected_24hr_hotels = {11843, 11206}\n \n if saved_24hr_hotel_ids == expected_24hr_hotels:\n success_accumulator.append(f\"[C] Correct hotels ({expected_24hr_hotels}) added to '24hr' wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected hotels {expected_24hr_hotels} in '24hr' wishlist, but found {saved_24hr_hotel_ids}\")\n\n # Check hotels in \"amazing\" wishlist\n saved_amazing = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"amazing\").all()\n saved_amazing_hotel_ids = {entry[\"hotel_id\"] for entry in saved_amazing}\n \n # According to the diff, hotels 11843 and 11206 should be in \"amazing\" wishlist\n expected_amazing_hotels = {11843, 11206}\n \n if saved_amazing_hotel_ids == expected_amazing_hotels:\n success_accumulator.append(f\"[C] Correct hotels ({expected_amazing_hotels}) added to 'amazing' wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected hotels {expected_amazing_hotels} in 'amazing' wishlist, but found {saved_amazing_hotel_ids}\")\n\n # Verify that wishlist_lists count is correct (exactly 2 new wishlists created)\n seed_wishlist_count = len(seed.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all())\n current_wishlist_count = len(current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").all())\n new_wishlist_count = current_wishlist_count - seed_wishlist_count\n \n if new_wishlist_count == 2:\n success_accumulator.append(\"[C] Exactly 2 new wishlist_lists entries were created\")\n else:\n error_accumulator.append(f\"[X] Expected 2 new wishlist_lists entries, but found {new_wishlist_count}\")\n\n # Verify total number of saved_properties entries (should be 4 new entries)\n seed_saved_count = len(seed.table(\"saved_properties\").eq(\"user_id\", 1).all())\n current_saved_count = len(current.table(\"saved_properties\").eq(\"user_id\", 1).all())\n new_saved_count = current_saved_count - seed_saved_count\n \n if new_saved_count == 4:\n success_accumulator.append(\"[C] Exactly 4 new saved_properties entries were created\")\n else:\n error_accumulator.append(f\"[X] Expected 4 new saved_properties entries, but found {new_saved_count}\")\n\n # Verify that hotel 11843 is in \"24hr\" wishlist\n hotel_11843_in_24hr = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 11843).eq(\"list_name\", \"24hr\").first()\n if hotel_11843_in_24hr:\n success_accumulator.append(\"[C] Hotel 11843 is in '24hr' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 11843 should be in '24hr' wishlist\")\n\n # Verify that hotel 11206 is in \"24hr\" wishlist\n hotel_11206_in_24hr = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 11206).eq(\"list_name\", \"24hr\").first()\n if hotel_11206_in_24hr:\n success_accumulator.append(\"[C] Hotel 11206 is in '24hr' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 11206 should be in '24hr' wishlist\")\n\n # Verify that hotel 11843 is in \"amazing\" wishlist\n hotel_11843_in_amazing = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 11843).eq(\"list_name\", \"amazing\").first()\n if hotel_11843_in_amazing:\n success_accumulator.append(\"[C] Hotel 11843 is in 'amazing' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 11843 should be in 'amazing' wishlist\")\n\n # Verify that hotel 11206 is in \"amazing\" wishlist\n hotel_11206_in_amazing = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 11206).eq(\"list_name\", \"amazing\").first()\n if hotel_11206_in_amazing:\n success_accumulator.append(\"[C] Hotel 11206 is in 'amazing' wishlist\")\n else:\n error_accumulator.append(\"[X] Hotel 11206 should be in 'amazing' wishlist\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ksihwbnkcwlw_1765834846313_nzo46ak5n", + "prompt": "With how slow it\u2019s been at work, it\u2019s starting to look like my summer getaways are going to turn into fall getaways\u2026 or winter getaways\u2026 or spring getaways. But I still want to make these trips happen sooner or later. So let\u2019s get rid of my \u201cSummer Getaways\u201d wish list (it bums me out just looking at it) and create instead a separate wish list named after each destination city, then move the places I saved to their respective lists. ", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_ksihwbnkcwlw_1765834846313_nzo46ak5n(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\n \n Task requirements:\n 1. Delete \"Summer Getaways\" wishlist\n 2. Create separate wishlists named after destination cities (Miami, Portland)\n 3. Move saved properties to their respective city-named lists\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # 1. Verify the \"Summer Getaways\" saved properties were deleted\n summer_getaways_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Summer Getaways\").all()\n if len(summer_getaways_properties) > 0:\n error_accumulator.append(f\"[X] 'Summer Getaways' list still has {len(summer_getaways_properties)} saved properties\")\n else:\n success_accumulator.append(\"[C] 'Summer Getaways' saved properties have been removed\")\n\n # 2. Verify hotel 5017 was moved to \"Miami\" list\n miami_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 5017).eq(\"list_name\", \"Miami\").all()\n if len(miami_property) != 1:\n error_accumulator.append(f\"[X] Expected hotel 5017 to be in 'Miami' list, found {len(miami_property)} entries\")\n else:\n success_accumulator.append(\"[C] Hotel 5017 is in 'Miami' list\")\n\n # 3. Verify hotel 9675 was moved to \"Portland\" list\n portland_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 9675).eq(\"list_name\", \"Portland\").all()\n if len(portland_property) != 1:\n error_accumulator.append(f\"[X] Expected hotel 9675 to be in 'Portland' list, found {len(portland_property)} entries\")\n else:\n success_accumulator.append(\"[C] Hotel 9675 is in 'Portland' list\")\n\n # 4. Verify \"Summer Getaways\" is no longer in user's saved properties\n all_user1_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).all()\n list_names = set([p[\"list_name\"] for p in all_user1_properties])\n \n if \"Summer Getaways\" in list_names:\n error_accumulator.append(\"[X] 'Summer Getaways' list still exists for user 1\")\n else:\n success_accumulator.append(\"[C] 'Summer Getaways' list has been deleted for user 1\")\n\n # 5. Verify \"Miami\" list exists\n if \"Miami\" not in list_names:\n error_accumulator.append(\"[X] 'Miami' list not found for user 1\")\n else:\n success_accumulator.append(\"[C] 'Miami' list exists for user 1\")\n\n # 6. Verify \"Portland\" list exists\n if \"Portland\" not in list_names:\n error_accumulator.append(\"[X] 'Portland' list not found for user 1\")\n else:\n success_accumulator.append(\"[C] 'Portland' list exists for user 1\")\n\n # 7. Verify we have exactly 2 properties in the new city-named lists (Miami and Portland)\n miami_count = len([p for p in all_user1_properties if p[\"list_name\"] == \"Miami\"])\n portland_count = len([p for p in all_user1_properties if p[\"list_name\"] == \"Portland\"])\n \n if miami_count != 1:\n error_accumulator.append(f\"[X] Expected 1 property in 'Miami' list, found {miami_count}\")\n else:\n success_accumulator.append(\"[C] 'Miami' list has exactly 1 property\")\n \n if portland_count != 1:\n error_accumulator.append(f\"[X] Expected 1 property in 'Portland' list, found {portland_count}\")\n else:\n success_accumulator.append(\"[C] 'Portland' list has exactly 1 property\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_xfycapqjnfaq_1765858128774_togkbttj3", + "prompt": "Search for hotels in Tokyo from April 5th to April 8th for one adult. I want to sort by top picks. Choose the first hotel that offers a restaurant amenity and add it to a new Wishlist \u201cTokyo Solo Stay\u201d. Then also add the first result that has a common area amenity and add it to the same Wishlist.", + "env_id": "booking", + "version": "v0.0.66", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_xfycapqjnfaq_1765858128774_togkbttj3(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Verify the wishlist \"Tokyo Solo Stay\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Tokyo Solo Stay\").eq(\"user_id\", \"1\").first()\n if wishlist is None:\n error_accumulator.append(\"[X] Wishlist 'Tokyo Solo Stay' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Tokyo Solo Stay' was created\")\n\n # Verify exactly 2 hotels were added to the saved_properties\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Tokyo Solo Stay\").all()\n if len(saved_properties) != 2:\n error_accumulator.append(f\"[X] Expected 2 properties in wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] 2 properties were added to the wishlist\")\n\n # Verify the specific hotel IDs from the diff were added\n saved_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n expected_hotel_ids = {3683, 12617}\n \n if saved_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, but found {saved_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] Correct hotels (IDs: {expected_hotel_ids}) were added to the wishlist\")\n\n # Verify user_id is 1 for both saved properties\n for prop in saved_properties:\n if prop[\"user_id\"] != 1:\n error_accumulator.append(f\"[X] Expected user_id 1, but found {prop['user_id']} for hotel {prop['hotel_id']}\")\n\n # Verify the wishlist didn't exist before (in seed)\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"list_name\", \"Tokyo Solo Stay\").eq(\"user_id\", \"1\").all()\n if len(seed_wishlist) > 0:\n error_accumulator.append(\"[X] Wishlist 'Tokyo Solo Stay' already existed in seed database\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Tokyo Solo Stay' is a new creation (not in seed)\")\n\n # Verify the saved properties didn't exist before (in seed)\n seed_saved_prop_3683 = seed.table(\"saved_properties\").eq(\"list_name\", \"Tokyo Solo Stay\").eq(\"hotel_id\", 3683).eq(\"user_id\", 1).all()\n seed_saved_prop_12617 = seed.table(\"saved_properties\").eq(\"list_name\", \"Tokyo Solo Stay\").eq(\"hotel_id\", 12617).eq(\"user_id\", 1).all()\n \n if len(seed_saved_prop_3683) > 0 or len(seed_saved_prop_12617) > 0:\n error_accumulator.append(\"[X] One or more saved properties already existed in seed database\")\n else:\n success_accumulator.append(\"[C] Both saved properties are new insertions (not in seed)\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ibklotuz8xg5_1765342011578_xw2j1klrp", + "prompt": "The family, 2 adults and 1 child, is looking for a hotel room for the vacation in New York for 10 days starting from the 1st of February 2026. Find the top 3 hotels in New York with a maximum price per room of 650$ and an average customer rating for the hotel above 8, and add them to the new wishlist called \"Vacation\".", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_ibklotuz8xg5_1765342011578_xw2j1klrp(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\n \n Task: Find top 3 hotels in New York with:\n - Max price per room of $650\n - Average customer rating above 8\n - Add them to a new wishlist called \"Vacation\"\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Check that the wishlist \"Vacation\" was created\n # Based on diff: user_id is stored as string '8' in wishlist_lists\n vacation_wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Vacation\").eq(\"user_id\", \"8\").all()\n if len(vacation_wishlist) == 0:\n error_accumulator.append(\"[X] Wishlist 'Vacation' was not created for user_id 8\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Vacation' was created for user_id 8\")\n\n # Check that exactly 3 hotels were saved to the \"Vacation\" wishlist\n # Based on diff: user_id is stored as integer 8 in saved_properties\n saved_hotels = current.table(\"saved_properties\").eq(\"list_name\", \"Vacation\").eq(\"user_id\", 8).all()\n if len(saved_hotels) != 3:\n error_accumulator.append(f\"[X] Expected 3 hotels in 'Vacation' wishlist, found {len(saved_hotels)}\")\n else:\n success_accumulator.append(\"[C] 3 hotels were added to the 'Vacation' wishlist\")\n\n # Verify the specific hotel_ids that were saved according to the diff\n expected_hotel_ids = {5478, 16597, 19951}\n actual_hotel_ids = {prop[\"hotel_id\"] for prop in saved_hotels}\n \n if actual_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotel_ids {expected_hotel_ids}, but found {actual_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] Correct hotel_ids {expected_hotel_ids} were saved\")\n\n # Check that the wishlist did not exist in seed (it's a new wishlist)\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"list_name\", \"Vacation\").eq(\"user_id\", \"8\").all()\n if len(seed_wishlist) > 0:\n error_accumulator.append(\"[X] Wishlist 'Vacation' already existed in seed - should be new\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Vacation' is newly created (did not exist in seed)\")\n\n # Check that no saved properties existed for these hotels in seed\n seed_saved = seed.table(\"saved_properties\").eq(\"list_name\", \"Vacation\").eq(\"user_id\", 8).all()\n if len(seed_saved) > 0:\n error_accumulator.append(\"[X] Saved properties for 'Vacation' wishlist already existed in seed\")\n else:\n success_accumulator.append(\"[C] Saved properties for 'Vacation' are newly created\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_po68ylphp9t_1768242466251_tmjv4y0rb", + "prompt": "I\u2019ll be visiting Barcelona starting a week after Valentine's Day and staying until February 25th, 2026.\n\nI am looking for an accommodation that includes the city's top three most commonly offered amenities. The property must have a star rating above the city average, and the nightly price should be below the city average\u2014I\u2019m looking for both value and quality.\n\nIf multiple options meet these criteria, select the property with the highest average user rating. If that property offers multiple room types, choose the cheapest one available. Please use the personal and default payment information stored in my account to finalize and pay for the booking. Finally, include this special request message: \"Please provide a room on a high floor\"", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_po68ylphp9t_1768242466251_tmjv4y0rb(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n \n # Find the new booking\n new_booking_ids = find_new_entries(\"bookings\")\n if len(new_booking_ids) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new booking, found {len(new_booking_ids)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking_id = new_booking_ids[0][\"id\"]\n new_booking = current.table(\"bookings\").eq(\"id\", new_booking_id).first()\n \n if not new_booking:\n error_accumulator.append(\"[X] Could not retrieve new booking details\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n success_accumulator.append(f\"[C] Found 1 new booking with ID {new_booking_id}\")\n \n # Validate check-in date: \"a week after Valentine's Day\" = Feb 14 + 7 = Feb 21, 2026\n expected_check_in = \"2026-02-21\"\n if new_booking[\"check_in_date\"] != expected_check_in:\n error_accumulator.append(f\"[X] Expected check-in date {expected_check_in}, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(f\"[C] Check-in date is correct: {expected_check_in}\")\n \n # Validate check-out date: February 25th, 2026\n expected_check_out = \"2026-02-25\"\n if new_booking[\"check_out_date\"] != expected_check_out:\n error_accumulator.append(f\"[X] Expected check-out date {expected_check_out}, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(f\"[C] Check-out date is correct: {expected_check_out}\")\n \n # Validate booking status is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n # Validate payment status is paid\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n \n # Validate special request message\n expected_special_request = \"Please provide a room on a high floor\"\n if not normalized_contains(expected_special_request, new_booking.get(\"special_requests\", \"\") or \"\"):\n error_accumulator.append(f\"[X] Expected special request '{expected_special_request}' not found\")\n else:\n success_accumulator.append(\"[C] Special request message is correct\")\n \n # Validate agreed_to_terms is checked\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Terms were agreed to\")\n \n # Validate payment method is used (should be the default payment method ID 1 based on diff)\n if new_booking[\"payment_method_id\"] is None or new_booking[\"payment_method_id\"] <= 0:\n error_accumulator.append(\"[X] Expected a valid payment method to be used\")\n else:\n success_accumulator.append(f\"[C] Payment method ID {new_booking['payment_method_id']} was used\")\n \n # Find the new transaction\n new_transaction_ids = find_new_entries(\"transactions\")\n new_transaction_id = None\n if len(new_transaction_ids) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new transaction, found {len(new_transaction_ids)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n new_transaction_id = new_transaction_ids[0][\"id\"]\n new_transaction = current.table(\"transactions\").eq(\"id\", new_transaction_id).first()\n \n if new_transaction:\n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != new_booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id {new_transaction['booking_id']} doesn't match booking {new_booking_id}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the booking\")\n \n # Validate transaction status is Success\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n \n # Validate transaction type is Payment\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got {new_transaction['transaction_type']}\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking_id),\n (\"booking_reference\", ...), # Auto-generated\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-02-21\"),\n (\"check_out_date\", \"2026-02-25\"),\n (\"booking_date\", ...), # Auto-generated timestamp\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...), # Depends on room/dates\n (\"taxes_fees\", ...), # Calculated\n (\"total_price\", ...), # Calculated\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", \"Please provide a room on a high floor\"),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...), # Could be various values\n (\"guest_first_name\", ...), # From user account\n (\"guest_last_name\", ...), # From user account\n (\"guest_email\", ...), # From user account\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_transaction_id),\n (\"booking_id\", new_booking_id),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"amount\", ...), # Matches total_price\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_lvl5swa4dos_1767535315633_jhcrmmk1b", + "prompt": "I\u2019d like to create a new wishlist titled \"Summer of 26\". \nI am travelling to Europe and looking for hostels in London, Rome and Berlin, please sort the hostels in the following cities by lowest price and add them to the new wishlist based on below criteria:\n\nCheapest hostel in London\n\nCheapest hostel in Rome\n\nCheapest hostel in Berlin\n\nNext, proceed with booking the Family room in the hostel in Rome (check-in on July 3rd, check-out on July 10th, this year). If the total price exceeds $3000 USD, do not complete the payment yet.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_lvl5swa4dos_1767535315633_jhcrmmk1b(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # 1. Verify the wishlist \"Summer of 26\" was created\n # Try both string and integer user_id since the schema may vary\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Summer of 26\").first()\n if wishlist:\n success_accumulator.append(\"[C] Wishlist 'Summer of 26' was created\")\n else:\n error_accumulator.append(\"[X] Wishlist 'Summer of 26' was not created\")\n\n # 2. Verify 3 properties were saved to the wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Summer of 26\").all()\n if len(saved_properties) == 3:\n success_accumulator.append(\"[C] 3 properties were saved to the wishlist\")\n else:\n error_accumulator.append(f\"[X] Expected 3 properties in wishlist, found {len(saved_properties)}\")\n\n # Get the hotel IDs from saved properties\n saved_hotel_ids = [sp[\"hotel_id\"] for sp in saved_properties]\n\n # Verify the saved hotels are hostels in the correct cities (London, Rome, Berlin)\n # According to the diff, hotels 14345, 16898, 19091 were added\n expected_hotel_ids = {14345, 16898, 19091}\n if set(saved_hotel_ids) == expected_hotel_ids:\n success_accumulator.append(\"[C] Correct hotels were saved to the wishlist (14345, 16898, 19091)\")\n else:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, got {set(saved_hotel_ids)}\")\n\n # Verify that saved hotels are in the correct cities (London, Rome, Berlin)\n expected_cities = {\"london\", \"rome\", \"berlin\"}\n actual_cities = set()\n for hotel_id in saved_hotel_ids:\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n city = hotel.get(\"city\", \"\").lower()\n actual_cities.add(city)\n \n if expected_cities == actual_cities:\n success_accumulator.append(\"[C] Hotels are from the correct cities (London, Rome, Berlin)\")\n else:\n error_accumulator.append(f\"[X] Expected cities {expected_cities}, got {actual_cities}\")\n\n # Verify that saved properties are hostels\n for hotel_id in saved_hotel_ids:\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n property_type = hotel.get(\"property_type\", \"\").lower()\n if \"hostel\" not in property_type:\n error_accumulator.append(f\"[X] Hotel {hotel_id} is not a hostel, property_type: {property_type}\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel {hotel_id} in hotels table\")\n \n # If we reach here without hostel errors, all saved properties are hostels\n hostel_errors = [e for e in error_accumulator if \"not a hostel\" in e or \"Could not find hotel\" in e]\n if not hostel_errors:\n success_accumulator.append(\"[C] All saved properties are hostels\")\n\n # 3. Verify booking was created for the Rome hostel (hotel_id 16898)\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n \n booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Verify booking is for the Rome hostel (hotel_id 16898)\n if booking[\"hotel_id\"] == 16898:\n success_accumulator.append(\"[C] Booking is for the Rome hostel (hotel_id 16898)\")\n else:\n error_accumulator.append(f\"[X] Expected booking for hotel_id 16898, got {booking['hotel_id']}\")\n\n # Verify room is a Family room (room_id 33556)\n if booking[\"room_id\"] == 33556:\n success_accumulator.append(\"[C] Booking is for the correct room (room_id 33556)\")\n # Verify the room is a Family room by checking the rooms table\n room = current.table(\"rooms\").eq(\"id\", 33556).first()\n if room:\n room_type = room.get(\"room_type\", \"\").lower()\n room_name = room.get(\"name\", \"\").lower()\n if \"family\" in room_type or \"family\" in room_name:\n success_accumulator.append(\"[C] Booking is for a Family room type\")\n else:\n error_accumulator.append(f\"[X] Room 33556 is not a Family room. Type: {room.get('room_type')}, Name: {room.get('name')}\")\n else:\n error_accumulator.append(\"[X] Could not find room 33556 in rooms table\")\n else:\n error_accumulator.append(f\"[X] Expected room_id 33556, got {booking['room_id']}\")\n\n # Verify check-in date is July 3rd 2026\n if booking[\"check_in_date\"] == \"2026-07-03\":\n success_accumulator.append(\"[C] Check-in date is July 3rd, 2026\")\n else:\n error_accumulator.append(f\"[X] Expected check-in date 2026-07-03, got {booking['check_in_date']}\")\n\n # Verify check-out date is July 10th 2026\n if booking[\"check_out_date\"] == \"2026-07-10\":\n success_accumulator.append(\"[C] Check-out date is July 10th, 2026\")\n else:\n error_accumulator.append(f\"[X] Expected check-out date 2026-07-10, got {booking['check_out_date']}\")\n\n # Verify that total price exceeds $3000 USD and payment was NOT completed\n total_price = booking[\"total_price\"]\n if total_price > 3000:\n success_accumulator.append(f\"[C] Total price ({total_price}) exceeds $3000 USD\")\n \n # Verify payment was not completed (status should be pending)\n if booking[\"status\"] == \"pending\" and booking[\"payment_status\"] == \"pending\":\n success_accumulator.append(\"[C] Payment was not completed (status and payment_status are 'pending')\")\n else:\n error_accumulator.append(f\"[X] Expected payment to not be completed. Status: {booking['status']}, Payment status: {booking['payment_status']}\")\n else:\n error_accumulator.append(f\"[X] Total price ({total_price}) does not exceed $3000 USD\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Get the new booking ID (it should be 5194 based on the diff, but we'll use dynamic lookup)\n booking_id = new_bookings[0][\"id\"] if len(new_bookings) == 1 else 5194\n \n expected_changes = [\n # Wishlist list creation\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Summer of 26\"), # Composite primary key (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Summer of 26\"),\n ]\n },\n # Saved properties - London hostel\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 14345, \"Summer of 26\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 14345),\n (\"list_name\", \"Summer of 26\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Saved properties - Rome hostel\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 16898, \"Summer of 26\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 16898),\n (\"list_name\", \"Summer of 26\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Saved properties - Berlin hostel\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 19091, \"Summer of 26\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 19091),\n (\"list_name\", \"Summer of 26\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Booking for Rome hostel\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 16898),\n (\"room_id\", 33556),\n (\"check_in_date\", \"2026-07-03\"),\n (\"check_out_date\", \"2026-07-10\"),\n (\"booking_date\", ...),\n (\"adults\", ...),\n (\"children\", ...),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"pending\"),\n (\"payment_status\", \"pending\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", None),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", None),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'updated_at', 'created_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'updated_at', 'created_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_zwidm5yqymzj_1767447748452_17mehybjn", + "prompt": "1. Navigate to my no-show bookings. \n2. Find any booking where the payment status is pending. \n3. For each such booking, create a new reservation at the same hotel with the same number of rooms and occupancy for the dates 23rd\u201325th March 2026 (check out on the 25th). \n4. Use the following guest information: \n Guest Name: Rob Key \n Email: Robkey@yahoo.com \n Phone: +1-7415628963 \n Special Request: Need an early check-in. \n5. Add the following payment method to complete the reservation: \n Name: Robert Brown \n Card Number: 4406 9897 6476 5968 \n CVV: 002 \n Expiry: 04/2028 \n6. Finalize the purchase using the above payment method. \n7. Next, search for properties in the same city as the above booking with an average rating greater than 9.4. \n8. Create a new wishlist named \"Rome\" and add both the filtered properties and the hotel booked in step 1.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_zwidm5yqymzj_1767447748452_17mehybjn(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # 0. First, find the original no-show booking with pending payment status\n original_no_show_bookings = seed.table(\"bookings\").eq(\"status\", \"no_show\").eq(\"payment_status\", \"pending\").eq(\"user_id\", 1).all()\n \n original_booking = None\n if len(original_no_show_bookings) == 0:\n error_accumulator.append(\"[X] No no-show booking with pending payment found in seed database\")\n else:\n # Find the original booking for hotel 8138\n for b in original_no_show_bookings:\n if b[\"hotel_id\"] == 8138:\n original_booking = b\n break\n \n if not original_booking:\n error_accumulator.append(\"[X] Hotel 8138 does not correspond to a no-show booking with pending payment\")\n else:\n success_accumulator.append(\"[C] Found original no-show booking with pending payment for hotel 8138\")\n\n # 1. Validate new booking was created\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n new_booking = current.table(\"bookings\").eq(\"id\", new_bookings[0][\"id\"]).first()\n \n # Validate booking details\n # Check hotel_id is 8138\n if new_booking[\"hotel_id\"] != 8138:\n error_accumulator.append(f\"[X] Expected hotel_id 8138, got {new_booking['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] Booking is for correct hotel (8138)\")\n \n # Validate room_count and occupancy match the original no-show booking\n if original_booking:\n if new_booking[\"room_count\"] != original_booking[\"room_count\"]:\n error_accumulator.append(f\"[X] Expected room_count {original_booking['room_count']} (matching original), got {new_booking['room_count']}\")\n else:\n success_accumulator.append(f\"[C] Room count matches original booking ({original_booking['room_count']})\")\n \n if new_booking[\"adults\"] != original_booking[\"adults\"]:\n error_accumulator.append(f\"[X] Expected adults {original_booking['adults']} (matching original), got {new_booking['adults']}\")\n else:\n success_accumulator.append(f\"[C] Adults count matches original booking ({original_booking['adults']})\")\n \n if new_booking[\"children\"] != original_booking[\"children\"]:\n error_accumulator.append(f\"[X] Expected children {original_booking['children']} (matching original), got {new_booking['children']}\")\n else:\n success_accumulator.append(f\"[C] Children count matches original booking ({original_booking['children']})\")\n \n # Check dates: check-in 2026-03-23, check-out 2026-03-25\n if new_booking[\"check_in_date\"] != \"2026-03-23\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-03-23', got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-03-23)\")\n \n if new_booking[\"check_out_date\"] != \"2026-03-25\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-03-25', got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-03-25)\")\n \n # Check guest name: Rob Key\n if not normalized_string_comparison(\"Rob\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest_first_name 'Rob', got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct (Rob)\")\n \n if not normalized_string_comparison(\"Key\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest_last_name 'Key', got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct (Key)\")\n \n # Check email: Robkey@yahoo.com (case insensitive)\n if new_booking[\"guest_email\"].lower() != \"robkey@yahoo.com\":\n error_accumulator.append(f\"[X] Expected guest_email 'robkey@yahoo.com', got {new_booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is correct (robkey@yahoo.com)\")\n \n # Check phone: +1-7415628963\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected guest_phone_country_code '+1', got {new_booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct (+1)\")\n \n phone_digits = ''.join(filter(str.isdigit, str(new_booking[\"guest_phone_number\"])))\n if phone_digits != \"7415628963\":\n error_accumulator.append(f\"[X] Expected phone number '7415628963', got {phone_digits}\")\n else:\n success_accumulator.append(\"[C] Phone number is correct (7415628963)\")\n \n # Check special request: Need an early check-in.\n if not normalized_contains(\"early check-in\", new_booking[\"special_requests\"]):\n error_accumulator.append(f\"[X] Expected special_requests to contain 'early check-in', got {new_booking['special_requests']}\")\n else:\n success_accumulator.append(\"[C] Special request contains 'early check-in'\")\n \n # Check booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # 2. Validate new payment method was created\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] 1 new payment method was created\")\n new_pm = current.table(\"payment_methods\").eq(\"id\", new_payment_methods[0][\"id\"]).first()\n \n # Card number: 4406 9897 6476 5968\n if new_pm[\"card_number\"] != \"4406989764765968\":\n error_accumulator.append(f\"[X] Expected card_number '4406989764765968', got {new_pm['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n \n if new_pm[\"last_four\"] != \"5968\":\n error_accumulator.append(f\"[X] Expected last_four '5968', got {new_pm['last_four']}\")\n else:\n success_accumulator.append(\"[C] Last four digits are correct (5968)\")\n \n # CVV: 002\n if new_pm[\"cvv\"] != \"002\":\n error_accumulator.append(f\"[X] Expected cvv '002', got {new_pm['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV is correct (002)\")\n \n # Expiry: 04/2028\n if new_pm[\"expiry_month\"] != 4:\n error_accumulator.append(f\"[X] Expected expiry_month 4, got {new_pm['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct (4)\")\n \n if new_pm[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry_year 2028, got {new_pm['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct (2028)\")\n \n # Cardholder name: Robert Brown\n if not normalized_string_comparison(\"Robert Brown\", new_pm[\"cardholder_name\"]):\n error_accumulator.append(f\"[X] Expected cardholder_name 'Robert Brown', got {new_pm['cardholder_name']}\")\n else:\n success_accumulator.append(\"[C] Cardholder name is correct (Robert Brown)\")\n\n # 3. Validate wishlist named \"Rome\" was created\n new_wishlists = find_new_entries(\"wishlist_lists\", id_field=\"list_name\")\n rome_wishlist = [w for w in new_wishlists if w[\"list_name\"] == \"Rome\"]\n if len(rome_wishlist) != 1:\n error_accumulator.append(f\"[X] Expected 1 new wishlist named 'Rome', but found {len(rome_wishlist)}\")\n else:\n success_accumulator.append(\"[C] Wishlist named 'Rome' was created\")\n\n # 4. Validate saved_properties - should have hotel 8138 and hotel 17113 in \"Rome\" wishlist\n new_saved_properties = find_new_entries(\"saved_properties\", id_field=\"hotel_id\")\n \n # Check that hotel 8138 (the booking hotel) is in Rome wishlist\n hotel_8138_in_rome = current.table(\"saved_properties\").eq(\"hotel_id\", 8138).eq(\"list_name\", \"Rome\").first()\n if not hotel_8138_in_rome:\n error_accumulator.append(\"[X] Hotel 8138 not found in 'Rome' wishlist\")\n else:\n success_accumulator.append(\"[C] Hotel 8138 added to 'Rome' wishlist\")\n \n # Check that hotel 17113 (filtered property with rating > 9.4) is in Rome wishlist\n hotel_17113_in_rome = current.table(\"saved_properties\").eq(\"hotel_id\", 17113).eq(\"list_name\", \"Rome\").first()\n if not hotel_17113_in_rome:\n error_accumulator.append(\"[X] Hotel 17113 not found in 'Rome' wishlist\")\n else:\n success_accumulator.append(\"[C] Hotel 17113 added to 'Rome' wishlist\")\n \n # 4b. Validate hotel 17113 is in the same city as hotel 8138 and has rating > 9.4\n hotel_8138 = current.table(\"hotels\").eq(\"id\", 8138).first()\n hotel_17113 = current.table(\"hotels\").eq(\"id\", 17113).first()\n \n if hotel_8138 and hotel_17113:\n original_city = hotel_8138.get(\"city\")\n \n if hotel_17113.get(\"city\") != original_city:\n error_accumulator.append(f\"[X] Hotel 17113 is not in the same city as hotel 8138. Expected '{original_city}', got '{hotel_17113.get('city')}'\")\n else:\n success_accumulator.append(f\"[C] Hotel 17113 is in the same city as hotel 8138 ({original_city})\")\n \n if hotel_17113.get(\"average_rating\") is None or hotel_17113.get(\"average_rating\") <= 9.4:\n error_accumulator.append(f\"[X] Hotel 17113 average_rating ({hotel_17113.get('average_rating')}) is not > 9.4\")\n else:\n success_accumulator.append(f\"[C] Hotel 17113 has average_rating > 9.4 ({hotel_17113.get('average_rating')})\")\n elif not hotel_8138:\n error_accumulator.append(\"[X] Hotel 8138 not found in database\")\n elif not hotel_17113:\n error_accumulator.append(\"[X] Hotel 17113 not found in database\")\n\n # 5. Validate transaction was created for the booking\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_tx = current.table(\"transactions\").eq(\"id\", new_transactions[0][\"id\"]).first()\n \n if new_tx[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got {new_tx['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Get the actual IDs for the expected changes\n booking_id = new_bookings[0][\"id\"] if len(new_bookings) == 1 else 5194\n payment_method_id = new_payment_methods[0][\"id\"] if len(new_payment_methods) == 1 else 82\n transaction_id = new_transactions[0][\"id\"] if len(new_transactions) == 1 else 4402\n\n expected_changes = [\n # New booking\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 8138),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-03-23\"),\n (\"check_out_date\", \"2026-03-25\"),\n (\"booking_date\", ...),\n (\"adults\", ...),\n (\"children\", ...),\n (\"room_count\", ...),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"7415628963\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n # New payment method\n {\n \"table\": \"payment_methods\",\n \"pk\": payment_method_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"5968\"),\n (\"expiry_month\", 4),\n (\"expiry_year\", 2028),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 0),\n (\"card_number\", \"4406989764765968\"),\n (\"cvv\", \"002\"),\n (\"cardholder_name\", \"Robert Brown\"),\n ]\n },\n # Saved property 1 - hotel 8138 in Rome wishlist\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 8138, \"Rome\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 8138),\n (\"list_name\", \"Rome\"),\n (\"notes\", None),\n ]\n },\n # Saved property 2 - hotel 17113 in Rome wishlist\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 17113, \"Rome\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 17113),\n (\"list_name\", \"Rome\"),\n (\"notes\", None),\n ]\n },\n # New transaction\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n ]\n },\n # New wishlist\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Rome\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Rome\"),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_qfbtks4qhtw_1767531363506_5rwcf4j89", + "prompt": "I\u2019m traveling to Rome and looking for accommodation. I want to sort the properties by best reviews and lowest price, then choose the one with the most reviews from the first five results.\n\nProceed with booking this property only if it has a beach within 1 km and allows pets. Book the most expensive room for 1 adult for 2 nights, with check-in on Valentine's Day and check-out on February 16 this year. Use the payment method already on my profile to complete the booking.\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_qfbtks4qhtw_1767531363506_5rwcf4j89(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in date is Valentine's Day (February 14)\n if new_booking[\"check_in_date\"] != \"2026-02-14\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-02-14' (Valentine's Day), but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is Valentine's Day (2026-02-14)\")\n\n # Validate check-out date is February 16 (2 nights stay)\n if new_booking[\"check_out_date\"] != \"2026-02-16\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-02-16', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is 2026-02-16 (2 nights stay)\")\n\n # Validate 1 adult\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n\n # Validate 1 room\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n # Validate payment status\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate that an existing payment method was used (payment_method_id should be from seed)\n payment_method_id = new_booking[\"payment_method_id\"]\n if payment_method_id is None:\n error_accumulator.append(\"[X] No payment method was used for the booking\")\n else:\n # Check if this payment method existed in seed (i.e., was already on user's profile)\n existing_payment_method = seed.table(\"payment_methods\").eq(\"id\", payment_method_id).first()\n if existing_payment_method is None:\n error_accumulator.append(f\"[X] Payment method {payment_method_id} was not an existing payment method on the profile\")\n else:\n success_accumulator.append(f\"[C] Used existing payment method (id={payment_method_id}) from profile\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n \n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id {new_transaction['booking_id']} doesn't match booking id {booking_id}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n\n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Validate transaction type\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Verify the hotel booked is in Rome by checking the hotel details\n hotel_id = new_booking[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n # Check if the hotel is in Rome\n hotel_city = hotel.get(\"city\", \"\")\n if not normalized_contains(\"rome\", hotel_city):\n error_accumulator.append(f\"[X] Expected hotel to be in Rome, but hotel city is '{hotel_city}'\")\n else:\n success_accumulator.append(f\"[C] Hotel is in Rome (city: {hotel_city})\")\n else:\n error_accumulator.append(f\"[X] Could not find hotel with id {hotel_id}\")\n\n # Verify the hotel has a beach within 1 km (as required by the task)\n nearby_attractions = current.table(\"nearby_attractions\").eq(\"hotel_id\", hotel_id).all()\n beach_found = False\n for attraction in nearby_attractions:\n attraction_type = attraction.get(\"type\", \"\")\n distance = attraction.get(\"distance_km\", float(\"inf\"))\n if attraction_type == \"Beach\" and distance is not None and distance <= 1:\n beach_found = True\n break\n \n if not beach_found:\n error_accumulator.append(\"[X] Hotel does not have a beach within 1 km as required\")\n else:\n success_accumulator.append(\"[C] Hotel has a beach within 1 km\")\n\n # Verify the hotel allows pets\n property_policy = current.table(\"property_policies\").eq(\"hotel_id\", hotel_id).first()\n if property_policy:\n pet_policy = property_policy.get(\"pet_policy\", \"\")\n if pet_policy != \"Allowed\":\n error_accumulator.append(f\"[X] Expected hotel to allow pets, but pet_policy is '{pet_policy}'\")\n else:\n success_accumulator.append(\"[C] Hotel allows pets\")\n else:\n error_accumulator.append(\"[X] Could not find property policy for the hotel\")\n\n # Verify the room booked is the most expensive room for the hotel\n room_id = new_booking[\"room_id\"]\n booked_room = current.table(\"rooms\").eq(\"id\", room_id).first()\n all_rooms = current.table(\"rooms\").eq(\"hotel_id\", hotel_id).all()\n \n if booked_room and all_rooms:\n booked_room_price = booked_room.get(\"base_price\", 0)\n max_price = max(room.get(\"base_price\", 0) for room in all_rooms)\n \n if booked_room_price < max_price:\n error_accumulator.append(f\"[X] Expected the most expensive room (price={max_price}), but booked room has price {booked_room_price}\")\n else:\n success_accumulator.append(f\"[C] Booked the most expensive room (price={booked_room_price})\")\n else:\n error_accumulator.append(\"[X] Could not verify room pricing\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Get the transaction ID\n transaction_id = new_transactions[0][\"id\"] if new_transactions else None\n\n # Use dynamic values from new_booking for hotel_id and room_id\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", ...),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-02-14\"),\n (\"check_out_date\", \"2026-02-16\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", transaction_id),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", new_booking[\"payment_method_id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'updated_at', 'expires_at', 'created_at'}, 'booking_extras': {'created_at'}, 'bookings': {'updated_at', 'created_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'updated_at', 'created_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_iugf6uaqoufb_1765856051990_ffpl462j5", + "prompt": "I forget where it's located, but Booking.com has a curated list for their most popular accommodations. I believe this list is created by the Booking.com team. Help me find this list and select the ones that are located in USA. Add them to a new wishlist called \"Most popular listings in USA\". For every listing that is added to the wishlist, write their zip code in the notes.", + "env_id": "booking", + "version": "v0.0.66", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_iugf6uaqoufb_1765856051990_ffpl462j5(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # 1. Verify that a new wishlist called \"Most popular listings in USA\" was created\n new_wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Most popular listings in USA\").first()\n if not new_wishlist:\n error_accumulator.append(\"[X] Wishlist 'Most popular listings in USA' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Most popular listings in USA' was created\")\n\n # 2. Verify saved_properties entries were added with the correct list_name\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Most popular listings in USA\").all()\n \n if len(saved_properties) != 5:\n error_accumulator.append(f\"[X] Expected 5 saved properties in 'Most popular listings in USA', but found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] 5 properties were saved to 'Most popular listings in USA' wishlist\")\n\n # Expected hotel data from the diff\n expected_entries = {\n 17864: \"39004\",\n 358: \"27439\",\n 2932: \"96808\",\n 11550: \"78311\",\n 16681: \"48894\",\n }\n\n # 3. Verify each saved property has the expected zip code in the notes\n for prop in saved_properties:\n hotel_id = prop[\"hotel_id\"]\n notes = prop[\"notes\"]\n \n if hotel_id in expected_entries:\n expected_zip = expected_entries[hotel_id]\n if notes != expected_zip:\n error_accumulator.append(f\"[X] Hotel {hotel_id}: Expected zip code '{expected_zip}' in notes, but got '{notes}'\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} has correct zip code '{expected_zip}' in notes\")\n else:\n error_accumulator.append(f\"[X] Unexpected hotel_id {hotel_id} in saved properties\")\n\n # 4. Verify all saved hotels are in USA\n for prop in saved_properties:\n hotel_id = prop[\"hotel_id\"]\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n country = hotel.get(\"country\", \"\")\n # Check for common representations of USA\n if country not in [\"United States\", \"USA\", \"US\", \"United States of America\"]:\n error_accumulator.append(f\"[X] Hotel {hotel_id} is not in USA (country: {country})\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} is correctly located in {country}\")\n\n # 5. Verify user_id is correct (should be user 1)\n for prop in saved_properties:\n if prop[\"user_id\"] != 1:\n error_accumulator.append(f\"[X] Expected user_id 1, but got {prop['user_id']}\")\n\n # Verify that saved_properties had no entries with this list_name in seed\n seed_saved_props = seed.table(\"saved_properties\").eq(\"list_name\", \"Most popular listings in USA\").all()\n if len(seed_saved_props) > 0:\n error_accumulator.append(f\"[X] Expected no saved properties with this list name in seed, but found {len(seed_saved_props)}\")\n else:\n success_accumulator.append(\"[C] No saved properties with this list name existed in seed (new entries verified)\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_rngtjb0inm1e_1765859163261_btv0nex32", + "prompt": "I'm looking for a hotel in Denver but I don't know which days yet. I do know I want to spend at least $400 a night and I'll need self check-in. Find the top reviewed hotel, and if it has 2 or more photos, add it to a new wishlist called \"denver\". If not, add it to a new wishlist called \"bruh\". ", + "env_id": "booking", + "version": "v0.0.68", + "data_id": "kinesis", + "data_version": "v0.0.8", + "verifier_func": "def validate_task_rngtjb0inm1e_1765859163261_btv0nex32(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS ####\n\n # Verify that a new saved_property was created for user_id=1 with hotel_id=15945 and list_name='denver'\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 15945).first()\n \n if saved_property is None:\n error_accumulator.append(\"[X] No saved property found for user_id=1 and hotel_id=15945\")\n else:\n success_accumulator.append(\"[C] Saved property found for user_id=1 and hotel_id=15945\")\n \n # Verify the list_name is 'denver'\n if saved_property[\"list_name\"] == \"denver\":\n success_accumulator.append(\"[C] Property saved to wishlist 'denver' as expected\")\n else:\n error_accumulator.append(f\"[X] Expected list_name 'denver', but got '{saved_property['list_name']}'\")\n\n # Check that this saved_property is new (not in seed)\n seed_saved_property = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 15945).first()\n if seed_saved_property is not None:\n error_accumulator.append(\"[X] Saved property already existed in seed database\")\n else:\n success_accumulator.append(\"[C] Saved property is a new entry\")\n\n # Verify the wishlist_lists table has the new list\n wishlist_entry = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"denver\").first()\n if wishlist_entry is None:\n error_accumulator.append(\"[X] No wishlist_lists entry found for user_id=1 and list_name='denver'\")\n else:\n success_accumulator.append(\"[C] Wishlist 'denver' was created in wishlist_lists table\")\n\n # Verify the hotel 15945 exists and is in Denver\n hotel = current.table(\"hotels\").eq(\"id\", 15945).first()\n if hotel is None:\n error_accumulator.append(\"[X] Hotel with id 15945 not found\")\n else:\n # Check that the hotel is in Denver\n if hotel[\"city\"] and \"denver\" in hotel[\"city\"].lower():\n success_accumulator.append(\"[C] Hotel 15945 is located in Denver\")\n else:\n error_accumulator.append(f\"[X] Hotel 15945 city is '{hotel['city']}', expected Denver\")\n\n # Verify hotel 15945 has 2 or more photos (which determined the 'denver' list name)\n hotel_images = current.table(\"hotel_images\").eq(\"hotel_id\", 15945).all()\n photo_count = len(hotel_images)\n if photo_count >= 2:\n success_accumulator.append(f\"[C] Hotel 15945 has {photo_count} photos (>= 2), correctly saved to 'denver' list\")\n else:\n error_accumulator.append(f\"[X] Hotel 15945 has {photo_count} photos (< 2), should have been saved to 'bruh' list instead\")\n\n # Verify no unexpected changes to saved_properties table\n # Count new entries in saved_properties\n seed_saved_count = len(seed.table(\"saved_properties\").all())\n current_saved_count = len(current.table(\"saved_properties\").all())\n new_saved_count = current_saved_count - seed_saved_count\n \n if new_saved_count == 1:\n success_accumulator.append(\"[C] Exactly 1 new saved property was created\")\n else:\n error_accumulator.append(f\"[X] Expected 1 new saved property, found {new_saved_count}\")\n\n # Verify no unexpected changes to wishlist_lists table \n seed_wishlist_count = len(seed.table(\"wishlist_lists\").all())\n current_wishlist_count = len(current.table(\"wishlist_lists\").all())\n new_wishlist_count = current_wishlist_count - seed_wishlist_count\n \n if new_wishlist_count == 1:\n success_accumulator.append(\"[C] Exactly 1 new wishlist was created\")\n else:\n error_accumulator.append(f\"[X] Expected 1 new wishlist, found {new_wishlist_count}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_hp0tiri8vzf_1765946469513_pnz9gl10b", + "prompt": "Look for hotels in Barcelona from June 1st to June 4th for two adults. Sort by top reviewed. Pick the first hotel where pets are not allowed and check-in closes at 22:00, and add it to my Wishlist \u201cBarcelona Summer\u201d", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_hp0tiri8vzf_1765946469513_pnz9gl10b(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # The diff shows exact values - validate against these known correct values\n expected_user_id = 1\n expected_hotel_id = 12386\n expected_list_name = \"Barcelona Summer\"\n\n #### VALIDATE THE SOLUTION CRITERIA ####\n\n # Check that the specific property was saved to the \"Barcelona Summer\" wishlist\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", expected_user_id).eq(\"hotel_id\", expected_hotel_id).eq(\"list_name\", expected_list_name).first()\n \n if not saved_property:\n error_accumulator.append(f\"[X] Hotel {expected_hotel_id} was not saved to '{expected_list_name}' wishlist for user {expected_user_id}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n success_accumulator.append(f\"[C] Hotel {expected_hotel_id} was saved to '{expected_list_name}' wishlist\")\n \n # Verify the hotel is in Barcelona\n hotel = current.table(\"hotels\").eq(\"id\", expected_hotel_id).first()\n if not hotel:\n error_accumulator.append(f\"[X] Hotel {expected_hotel_id} not found in hotels table\")\n elif not normalized_contains(\"barcelona\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Hotel {expected_hotel_id} is not in Barcelona, city is '{hotel['city']}'\")\n else:\n success_accumulator.append(f\"[C] Hotel {expected_hotel_id} is located in Barcelona\")\n \n # Verify the hotel policy: pets not allowed and check-in closes at 22:00\n policy = current.table(\"property_policies\").eq(\"hotel_id\", expected_hotel_id).first()\n if not policy:\n error_accumulator.append(f\"[X] No property policy found for hotel {expected_hotel_id}\")\n else:\n if policy[\"pet_policy\"] != \"Not Allowed\":\n error_accumulator.append(f\"[X] Hotel {expected_hotel_id} pet_policy is '{policy['pet_policy']}', expected 'Not Allowed'\")\n else:\n success_accumulator.append(f\"[C] Hotel {expected_hotel_id} has pets not allowed policy\")\n \n if policy[\"check_in_to\"] != \"22:00\":\n error_accumulator.append(f\"[X] Hotel {expected_hotel_id} check_in_to is '{policy['check_in_to']}', expected '22:00'\")\n else:\n success_accumulator.append(f\"[C] Hotel {expected_hotel_id} has check-in closing at 22:00\")\n\n # Verify this entry is new (didn't exist in seed)\n seed_saved_property = seed.table(\"saved_properties\").eq(\"user_id\", expected_user_id).eq(\"hotel_id\", expected_hotel_id).eq(\"list_name\", expected_list_name).first()\n if seed_saved_property:\n error_accumulator.append(\"[X] This property was already saved in the seed database - no new action taken\")\n else:\n success_accumulator.append(\"[C] Property was newly added to wishlist (not in seed)\")\n\n # Count total new entries in saved_properties to ensure no extra properties were added\n seed_saved_count = len(seed.table(\"saved_properties\").all())\n current_saved_count = len(current.table(\"saved_properties\").all())\n new_saved_count = current_saved_count - seed_saved_count\n \n if new_saved_count != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new saved property, but found {new_saved_count} new entries\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new property was saved (no extra saves)\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_dxga58ltqwjf_1768236618410_2lg2fp80u", + "prompt": "Hi, my name is Charles Moore.\nYou can reach me at charles.moore@travelmail.com or +1 (415) 555-0119.\n\nIn Amsterdam, find a hotel that:\n_ Has an overall guest review score of at least 8.0\n_ Has self check-in\n- Has a Bar\n\nFrom the matching options, choose the hotel with the lowest average nightly price.\n\nCheck availability for a standard room for 1 adult from March 2 to March 4, 2026.\n\nIf available, book exactly one room.\n\nIn Paris, find a hotel that:\n- Has an overall guest review score of at least 9.0\n- Has a restaurant as a facility\n\nFrom the matching options, choose the hotel with the highest star rating.\n\nCheck availability for a Suite room for 1 adult from March 6 to March 8, 2026.\nIf there is more than 1 available, choose the cheapest one.\n\nCheck whether there is any saved payment card under the name \"Charles Moore.\"\n- If yes, complete payment for both prepared bookings using that saved card.\n- If no, do not complete either booking and make no further changes.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new bookings\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 2:\n error_accumulator.append(f\"[X] Expected 2 new bookings, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 2 new bookings created\")\n\n # Get bookings sorted by check-in date to identify Amsterdam (March 2-4) and Paris (March 6-8)\n new_bookings_sorted = sorted(new_bookings, key=lambda b: b[\"check_in_date\"])\n amsterdam_booking = new_bookings_sorted[0]\n paris_booking = new_bookings_sorted[1]\n\n # Validate Amsterdam booking (March 2-4, 2026)\n if amsterdam_booking[\"check_in_date\"] != \"2026-03-02\":\n error_accumulator.append(f\"[X] Amsterdam booking: Expected check-in date '2026-03-02', got '{amsterdam_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Amsterdam booking: Check-in date is correct (2026-03-02)\")\n\n if amsterdam_booking[\"check_out_date\"] != \"2026-03-04\":\n error_accumulator.append(f\"[X] Amsterdam booking: Expected check-out date '2026-03-04', got '{amsterdam_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Amsterdam booking: Check-out date is correct (2026-03-04)\")\n\n # Validate Paris booking (March 6-8, 2026)\n if paris_booking[\"check_in_date\"] != \"2026-03-06\":\n error_accumulator.append(f\"[X] Paris booking: Expected check-in date '2026-03-06', got '{paris_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Paris booking: Check-in date is correct (2026-03-06)\")\n\n if paris_booking[\"check_out_date\"] != \"2026-03-08\":\n error_accumulator.append(f\"[X] Paris booking: Expected check-out date '2026-03-08', got '{paris_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Paris booking: Check-out date is correct (2026-03-08)\")\n\n # Validate guest details for both bookings (Charles Moore)\n for booking_name, booking in [(\"Amsterdam\", amsterdam_booking), (\"Paris\", paris_booking)]:\n if not normalized_string_comparison(\"charles\", booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] {booking_name} booking: Expected guest first name 'Charles', got '{booking['guest_first_name']}'\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Guest first name is correct\")\n\n if not normalized_string_comparison(\"moore\", booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] {booking_name} booking: Expected guest last name 'Moore', got '{booking['guest_last_name']}'\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Guest last name is correct\")\n\n if booking[\"guest_email\"].lower() != \"charles.moore@travelmail.com\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected guest email 'charles.moore@travelmail.com', got '{booking['guest_email']}'\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Guest email is correct\")\n\n if booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected phone country code '+1', got '{booking['guest_phone_country_code']}'\")\n \n # Check phone number - should contain digits 4155550119\n phone_digits = ''.join(filter(str.isdigit, str(booking[\"guest_phone_number\"]) if booking[\"guest_phone_number\"] else \"\"))\n if phone_digits != \"4155550119\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected phone '4155550119', got '{phone_digits}'\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Phone number is correct\")\n\n # Validate adults count is 1\n if booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] {booking_name} booking: Expected 1 adult, got {booking['adults']}\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Adult count is correct (1)\")\n\n # Validate room count is 1\n if booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] {booking_name} booking: Expected 1 room, got {booking['room_count']}\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Room count is correct (1)\")\n\n # Validate booking is confirmed and paid\n if booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected status 'confirmed', got '{booking['status']}'\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Status is confirmed\")\n\n if booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] {booking_name} booking: Expected payment status 'paid', got '{booking['payment_status']}'\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Payment status is paid\")\n\n # Validate payment method is the saved card (payment_method_id should be 1 based on diff)\n if booking[\"payment_method_id\"] != 1:\n error_accumulator.append(f\"[X] {booking_name} booking: Expected payment method ID 1 (saved card), got {booking['payment_method_id']}\")\n else:\n success_accumulator.append(f\"[C] {booking_name} booking: Used saved payment card\")\n\n # Validate agreed_to_terms\n if booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] {booking_name} booking: Expected agreed_to_terms to be 1\")\n\n # Verify payment method exists and belongs to Charles Moore\n payment_method = current.table(\"payment_methods\").eq(\"id\", 1).first()\n if payment_method:\n # Check if the saved card is under Charles Moore\n cardholder_name = payment_method.get(\"cardholder_name\", \"\")\n if cardholder_name and not normalized_contains(\"charles\", cardholder_name):\n error_accumulator.append(f\"[X] Saved payment card should be under Charles Moore\")\n else:\n success_accumulator.append(\"[C] Payment was made using a saved card\")\n else:\n error_accumulator.append(\"[X] Payment method with ID 1 not found\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 2:\n error_accumulator.append(f\"[X] Expected 2 new transactions, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 2 new payment transactions created\")\n \n # Verify transactions are successful\n for txn in new_transactions:\n if txn[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Transaction {txn['id']}: Expected status 'Success', got '{txn['status']}'\")\n if txn[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Transaction {txn['id']}: Expected type 'Payment', got '{txn['transaction_type']}'\")\n\n # Define expected changes for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": amsterdam_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", amsterdam_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", amsterdam_booking[\"hotel_id\"]),\n (\"room_id\", amsterdam_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-02\"),\n (\"check_out_date\", \"2026-03-04\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"charles.moore@travelmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"4155550119\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"bookings\",\n \"pk\": paris_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", paris_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", paris_booking[\"hotel_id\"]),\n (\"room_id\", paris_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-03-06\"),\n (\"check_out_date\", \"2026-03-08\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"charles.moore@travelmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"4155550119\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n ]\n \n # Add transaction entries if they exist\n for txn in new_transactions:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": txn[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", txn[\"id\"]),\n (\"booking_id\", ...),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'}, \n 'booking_extras': {'created_at'}, \n 'bookings': {'created_at', 'updated_at'}, \n 'cancellations': {'created_at'}, \n 'facilities': {'created_at'}, \n 'generation_checkpoints': {'created_at'}, \n 'hosts': {'created_at'}, \n 'hotel_images': {'created_at'}, \n 'hotels': {'created_at', 'updated_at'}, \n 'locations': {'created_at'}, \n 'messages': {'created_at'}, \n 'nearby_attractions': {'created_at'}, \n 'payment_methods': {'created_at'}, \n 'property_policies': {'created_at'}, \n 'reviews': {'created_at'}, \n 'rewards_program': {'updated_at'}, \n 'room_images': {'created_at'}, \n 'rooms': {'created_at'}, \n 'users': {'created_at'}, \n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ms7jp8mjilrr_1765291731157_x234dgypq", + "prompt": "Find me the three cheapest properties to stay at in Paris per night and create a wish list called \"Stag Do\" and add each of these options to this list as potential destinations for my troublesome cousins ", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_ms7jp8mjilrr_1765291731157_x234dgypq(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # 1. Verify the wishlist \"Stag Do\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Stag Do\").first()\n if wishlist is None:\n error_accumulator.append(\"[X] Wishlist 'Stag Do' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Stag Do' was created\")\n # Verify it belongs to user 8 (comparing as strings since diff shows '8')\n user_id = str(wishlist.get(\"user_id\"))\n if user_id != \"8\":\n error_accumulator.append(f\"[X] Wishlist 'Stag Do' user_id should be '8', got '{user_id}'\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Stag Do' belongs to user 8\")\n\n # 2. Verify that exactly 3 properties were saved to \"Stag Do\" list\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"Stag Do\").all()\n if len(saved_properties) != 3:\n error_accumulator.append(f\"[X] Expected 3 saved properties in 'Stag Do' list, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] 3 properties were saved to 'Stag Do' list\")\n\n # 3. Verify the hotel IDs match those in the diff (15545, 2984, 14332)\n expected_hotel_ids = {15545, 2984, 14332}\n actual_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n \n if actual_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, got {actual_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] Correct hotel IDs were saved: {expected_hotel_ids}\")\n\n # 4. Verify all saved properties belong to user 8\n user_ids = {prop[\"user_id\"] for prop in saved_properties}\n if user_ids != {8}:\n error_accumulator.append(f\"[X] All saved properties should belong to user 8, got user_ids: {user_ids}\")\n else:\n success_accumulator.append(\"[C] All saved properties belong to user 8\")\n\n # 5. Verify these hotels are in Paris (checking city field)\n for hotel_id in expected_hotel_ids:\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n if hotel:\n # Check if hotel is in Paris (case-insensitive)\n city = hotel.get(\"city\", \"\").lower()\n if \"paris\" not in city:\n error_accumulator.append(f\"[X] Hotel {hotel_id} is not in Paris, city is '{hotel.get('city')}'\")\n else:\n success_accumulator.append(f\"[C] Hotel {hotel_id} is in Paris\")\n else:\n error_accumulator.append(f\"[X] Hotel {hotel_id} not found in database\")\n\n # 6. Verify no prior saved_properties entries existed for \"Stag Do\" list in seed\n seed_stag_do_properties = seed.table(\"saved_properties\").eq(\"list_name\", \"Stag Do\").all()\n if len(seed_stag_do_properties) > 0:\n error_accumulator.append(f\"[X] Seed database already had {len(seed_stag_do_properties)} properties in 'Stag Do' list\")\n else:\n success_accumulator.append(\"[C] 'Stag Do' list was newly created (no prior entries in seed)\")\n\n # 7. Verify no prior wishlist_lists entry existed for \"Stag Do\" in seed\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"list_name\", \"Stag Do\").first()\n if seed_wishlist is not None:\n error_accumulator.append(\"[X] Seed database already had 'Stag Do' wishlist\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Stag Do' was newly created (did not exist in seed)\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_npfm44ludf7u_1765538181669_wf1oi8rvh", + "prompt": "I am looking to book a room in a Villa in Singapore in 12879 Postal code. The property must have minimum guest rating of 9. If the smoking is allowed then book the cheapest available room for 1 adult, checking in on 2026-01-26 and checking out on 2026-01-27.\nOnce booking is done, complete the payment.", + "env_id": "booking", + "version": "v0.0.60", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_npfm44ludf7u_1765538181669_wf1oi8rvh(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n \n # From the diff, we know the new booking ID is 5194\n # Query directly for that booking to avoid expensive table scans\n new_booking = current.table(\"bookings\").eq(\"id\", 5194).first()\n \n if not new_booking:\n error_accumulator.append(f\"[X] Expected booking with id 5194 not found\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(f\"[C] Booking with id 5194 was found\")\n\n # Validate check-in date: 2026-01-26\n if new_booking[\"check_in_date\"] != \"2026-01-26\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-01-26', but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-in date is correct: 2026-01-26\")\n\n # Validate check-out date: 2026-01-27\n if new_booking[\"check_out_date\"] != \"2026-01-27\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-01-27', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(f\"[C] Check-out date is correct: 2026-01-27\")\n\n # Validate adults: 1\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(f\"[C] Adults count is correct: 1\")\n\n # Validate booking status is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(f\"[C] Booking status is 'confirmed'\")\n\n # Validate payment status is paid\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(f\"[C] Payment status is 'paid'\")\n\n # Validate hotel_id matches the expected hotel (7272 from diff)\n hotel_id = new_booking[\"hotel_id\"]\n if hotel_id != 7272:\n error_accumulator.append(f\"[X] Expected hotel_id 7272, but got {hotel_id}\")\n else:\n success_accumulator.append(f\"[C] Hotel ID is correct: 7272\")\n \n # Verify the hotel meets the criteria:\n # 1. Property type is Villa\n # 2. Located in Singapore\n # 3. Postal code is 12879\n # 4. Average rating >= 9\n hotel = current.table(\"hotels\").eq(\"id\", hotel_id).first()\n \n if not hotel:\n error_accumulator.append(f\"[X] Hotel with id {hotel_id} not found\")\n else:\n # Check property type is Villa\n if hotel.get(\"property_type\") != \"Villa\":\n error_accumulator.append(f\"[X] Expected property_type 'Villa', but got '{hotel.get('property_type')}'\")\n else:\n success_accumulator.append(f\"[C] Property type is 'Villa'\")\n \n # Check country is Singapore (database may store as \"SG\" or \"Singapore\")\n country = hotel.get(\"country\")\n if country not in [\"Singapore\", \"SG\"]:\n error_accumulator.append(f\"[X] Expected country 'Singapore' or 'SG', but got '{country}'\")\n else:\n success_accumulator.append(f\"[C] Country is Singapore (stored as '{country}')\")\n \n # Check postal code is 12879\n if hotel.get(\"postal_code\") != \"12879\":\n error_accumulator.append(f\"[X] Expected postal_code '12879', but got '{hotel.get('postal_code')}'\")\n else:\n success_accumulator.append(f\"[C] Postal code is '12879'\")\n \n # Check average rating >= 9\n avg_rating = hotel.get(\"average_rating\")\n if avg_rating is None or avg_rating < 9:\n error_accumulator.append(f\"[X] Expected average_rating >= 9, but got {avg_rating}\")\n else:\n success_accumulator.append(f\"[C] Average rating is {avg_rating} (>= 9)\")\n\n # Verify smoking policy is allowed for this hotel\n property_policy = current.table(\"property_policies\").eq(\"hotel_id\", hotel_id).first()\n \n if property_policy:\n smoking_policy = property_policy.get(\"smoking_policy\")\n if smoking_policy != \"Allowed\":\n error_accumulator.append(f\"[X] Expected smoking_policy 'Allowed', but got '{smoking_policy}'\")\n else:\n success_accumulator.append(f\"[C] Smoking policy is 'Allowed'\")\n else:\n error_accumulator.append(f\"[X] Property policy not found for hotel {hotel_id}\")\n\n # Validate room_count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(f\"[C] Room count is 1\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ubr02syiv6x_1768174366998_0ndc33k1f", + "prompt": "Please find a Bed & Breakfast in Seattle for February 15\u201320, 2026, with a minimum guest rating of 9 and a fitness center. It needs to allow pets and be within 5 km of a museum. If you find a match, please proceed with booking a deluxe room for one adult using the card on my profile.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_ubr02syiv6x_1768174366998_0ndc33k1f(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - booking a Bed & Breakfast in Seattle\n for February 15-20, 2026, with specific requirements including a deluxe room for one adult.\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Validate check-in date (February 15, 2026)\n if new_booking[\"check_in_date\"] != \"2026-02-15\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-02-15', but got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-02-15)\")\n\n # Validate check-out date (February 20, 2026)\n if new_booking[\"check_out_date\"] != \"2026-02-20\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-02-20', but got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-02-20)\")\n\n # Validate adults count (1 adult)\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, but got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 1 adult\")\n\n # Validate room count (1 room)\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count to be 1, but got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', but got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n # Validate payment status\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', but got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate that a payment method was used (card on profile)\n if new_booking[\"payment_method_id\"] is None:\n error_accumulator.append(\"[X] Expected booking to use a payment method from profile, but payment_method_id is NULL\")\n else:\n success_accumulator.append(f\"[C] Booking used payment method ID {new_booking['payment_method_id']}\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, but got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] User agreed to terms\")\n\n # Verify the room is a deluxe room by checking the room details\n # The task specifies booking a \"deluxe room\" - we verify the booked room matches this\n room_id = new_booking[\"room_id\"]\n room = current.table(\"rooms\").eq(\"id\", room_id).first()\n if room:\n room_name = room.get(\"name\", \"\").lower()\n room_type = room.get(\"room_type\", \"\").lower()\n if \"deluxe\" not in room_name and \"deluxe\" not in room_type:\n error_accumulator.append(f\"[X] Expected a deluxe room, but room name is '{room.get('name')}' and type is '{room.get('room_type')}'\")\n else:\n success_accumulator.append(f\"[C] Booked room is a deluxe room: '{room.get('name')}'\")\n else:\n error_accumulator.append(f\"[X] Could not find room with ID {room_id}\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n transaction_id = new_transaction[\"id\"]\n \n # Validate transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction booking_id {new_transaction['booking_id']} does not match booking ID {booking_id}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', but got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Validate transaction type\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', but got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n # Generate expected changes for expect_only_v2\n # Based on the db changes shown in the demonstration, validate the key fields\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", ...), # Hotel is determined by search criteria, trust the diff\n (\"room_id\", ...), # Room is determined by selection, trust the diff \n (\"check_in_date\", \"2026-02-15\"),\n (\"check_out_date\", \"2026-02-20\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", ...), # Uses card from profile\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", ...),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ialoxqhsslr_1768132620152_kggw0m88x", + "prompt": "I\u2019m planning a weekend getaway next month. First, check which city had the highest search popularity in the last calendar month and the lowest minimum average nightly hotel price. In that city, find the cheapest 3-star hotel with a pool facility. Before saving it, verify that the hotel has a flexible cancellation policy and at least 5 reviews written in December 2025. If the hotel meets all criteria (rating 7.0+ and has reviews), add it to my wishlist named \u201cBudget Gems\u201d. If any condition fails, create a new wishlist called \u201cFuture Research\u201d instead. Finally, check if the hotel is saved.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Validate that a hotel was saved to a wishlist\n # Based on the database diff, hotel 3734 was saved to \"Budget Gems\" wishlist for user 1\n \n # Check that the saved_properties entry was created\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 3734).first()\n \n if saved_property is None:\n error_accumulator.append(\"[X] Expected hotel 3734 to be saved for user 1, but it was not found\")\n else:\n success_accumulator.append(\"[C] Hotel 3734 was saved to a wishlist for user 1\")\n \n # Verify the list name is \"Budget Gems\" (which means all criteria were met)\n if saved_property.get(\"list_name\") != \"Budget Gems\":\n error_accumulator.append(f\"[X] Expected list_name to be 'Budget Gems', but got '{saved_property.get('list_name')}'\")\n else:\n success_accumulator.append(\"[C] Hotel was saved to 'Budget Gems' wishlist (criteria met: flexible cancellation, 5+ Dec 2025 reviews, 7.0+ rating)\")\n\n # Check that the wishlist_lists entry was created\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Budget Gems\").first()\n \n if wishlist is None:\n error_accumulator.append(\"[X] Expected 'Budget Gems' wishlist to be created for user 1\")\n else:\n success_accumulator.append(\"[C] 'Budget Gems' wishlist was created for user 1\")\n\n # Validate the expected database changes using expect_only_v2\n expected_changes = [\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 3734, \"Budget Gems\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 3734),\n (\"list_name\", \"Budget Gems\"),\n (\"notes\", None),\n (\"saved_at\", ...), # Timestamp - skip exact value check\n ]\n },\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Budget Gems\"), # Composite primary key (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Budget Gems\"),\n (\"created_at\", ...), # Timestamp - skip exact value check\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}, 'saved_properties': {'saved_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_xmfmei8ryxq_1768206601499_bmfyci1ju", + "prompt": "I'm planning a 3-night leisure trip to Tokyo and I want to stay somewhere in Shinjuku.\nShow me places that offer free cancellation and have a rating of at least 9.0.\n\nFrom those, pick the top 3 that have the lowest average price for a 3-night stay from March 10 to March 13.\nCheck what room types are available and make sure they can accommodate 2 adults.\n\nCreate a wishlist called \"Tokyo 2026\" and save those 3 hotels to it with a note:\n\"Shortlist for Shinjuku leisure trip\".", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Verify the wishlist \"Tokyo 2026\" was created\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Tokyo 2026\").first()\n if not wishlist:\n error_accumulator.append(\"[X] Wishlist 'Tokyo 2026' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Tokyo 2026' was created\")\n\n # Verify that exactly 3 hotels were saved to the wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Tokyo 2026\").all()\n \n if len(saved_properties) != 3:\n error_accumulator.append(f\"[X] Expected 3 hotels saved to wishlist, but found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] 3 hotels were saved to the wishlist\")\n\n # Expected hotel IDs from the diff\n expected_hotel_ids = {14343, 15394, 16064}\n actual_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n \n if actual_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotel IDs {expected_hotel_ids}, but found {actual_hotel_ids}\")\n else:\n success_accumulator.append(\"[C] Correct hotels were saved to the wishlist\")\n\n # Verify that each saved property has the correct note\n expected_note = \"Shortlist for Shinjuku leisure trip\"\n for prop in saved_properties:\n if not normalized_contains(expected_note, prop.get(\"notes\", \"\")):\n error_accumulator.append(f\"[X] Hotel {prop['hotel_id']} missing expected note '{expected_note}'\")\n break\n else:\n success_accumulator.append(\"[C] All saved properties have the correct note\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n # Wishlist list creation\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Tokyo 2026\"), # Composite primary key: (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Tokyo 2026\"),\n ]\n },\n # Saved property 1\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 14343, \"Tokyo 2026\"), # Composite primary key: (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 14343),\n (\"list_name\", \"Tokyo 2026\"),\n (\"notes\", \"Shortlist for Shinjuku leisure trip\"),\n ]\n },\n # Saved property 2\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 15394, \"Tokyo 2026\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 15394),\n (\"list_name\", \"Tokyo 2026\"),\n (\"notes\", \"Shortlist for Shinjuku leisure trip\"),\n ]\n },\n # Saved property 3\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 16064, \"Tokyo 2026\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 16064),\n (\"list_name\", \"Tokyo 2026\"),\n (\"notes\", \"Shortlist for Shinjuku leisure trip\"),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_itlthn4c7ek_1768256906940_3gebu3xkh", + "prompt": "I'm planning a surprise weekend getaway and want the final choice to feel a bit unexpected.\n\nCreate a new wishlist named \"Secret Getaway\".\n\nIn Seattle, find hotels that meet all of the following:\n- Star rating between 4.0 and 4.9\n- Has a gym\n- Has breakfast included\n\nFrom the matching hotels, select the five highest-rated hotels by overall guest review score.\nIf fewer than five exist, use all available ones.\n\nAdd all selected hotels to the \"Secret Getaway\" wishlist.\n\nFrom the hotels saved in \"Secret Getaway\", exclude the single lowest average rated hotel.\n\nFrom the remaining hotels, choose the one with the highest number of guest reviews.\n\nCheck availability for a Deluxe room for 2 adults from October 11 to October 13, 2026.\n\nIf the room is available, book exactly one room.\n\nCheck whether there is any saved payment card under my name.\n\nIf yes, complete the payment using that saved card.\n\nIf no, complete the payment using the following card details:\n- Cardholder name: Patrick Bennett\n- Card number: 6011111111111117\n- Expiration month: 05\n- Expiration year: 2032\n- CVV: 614\n\nOther details:\n- Email: patrick.bennett@email.com\n- Phone: +1-443-307-1473", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # ========== VALIDATION 1: Verify wishlist \"Secret Getaway\" was created ==========\n wishlist = current.table(\"wishlist_lists\").eq(\"list_name\", \"Secret Getaway\").eq(\"user_id\", \"1\").first()\n if not wishlist:\n error_accumulator.append(\"[X] Wishlist 'Secret Getaway' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Secret Getaway' was created\")\n\n # ========== VALIDATION 2: Verify hotels were saved to the wishlist ==========\n # The diff shows 4 hotels were saved to \"Secret Getaway\": 3286, 6570, 7129, 11089\n saved_hotels = current.table(\"saved_properties\").eq(\"list_name\", \"Secret Getaway\").eq(\"user_id\", 1).all()\n saved_hotel_ids = {h[\"hotel_id\"] for h in saved_hotels}\n \n expected_saved_hotels = {3286, 6570, 7129, 11089}\n \n if saved_hotel_ids != expected_saved_hotels:\n error_accumulator.append(f\"[X] Expected hotels {expected_saved_hotels} in wishlist, got {saved_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] Correct hotels saved to wishlist: {saved_hotel_ids}\")\n\n # ========== VALIDATION 3: Verify booking was created ==========\n new_bookings = find_new_entries(\"bookings\", \"id\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new booking, found {len(new_bookings)}\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new booking was created\")\n \n # Get the full booking details\n new_booking = current.table(\"bookings\").eq(\"id\", 5194).first()\n \n if not new_booking:\n error_accumulator.append(\"[X] Booking with id 5194 not found\")\n else:\n # Validate booking details\n \n # Check hotel_id is 6570 (the selected hotel from remaining wishlist)\n if new_booking[\"hotel_id\"] != 6570:\n error_accumulator.append(f\"[X] Expected hotel_id 6570, got {new_booking['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] Booking is for correct hotel (6570)\")\n \n # Check room_id is 13033 (Deluxe room)\n if new_booking[\"room_id\"] != 13033:\n error_accumulator.append(f\"[X] Expected room_id 13033, got {new_booking['room_id']}\")\n else:\n success_accumulator.append(\"[C] Booking is for correct room (13033)\")\n \n # Check dates: October 11 to October 13, 2026\n if new_booking[\"check_in_date\"] != \"2026-10-11\":\n error_accumulator.append(f\"[X] Expected check_in_date 2026-10-11, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-10-11)\")\n \n if new_booking[\"check_out_date\"] != \"2026-10-13\":\n error_accumulator.append(f\"[X] Expected check_out_date 2026-10-13, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-10-13)\")\n \n # Check 2 adults\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Correct number of adults (2)\")\n \n # Check exactly 1 room\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Correct room count (1)\")\n \n # Check guest details\n if not normalized_string_comparison(\"patrick\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Patrick', got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct\")\n \n if not normalized_string_comparison(\"bennett\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Bennett', got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct\")\n \n # Check email\n if new_booking[\"guest_email\"].lower() != \"patrick.bennett@email.com\":\n error_accumulator.append(f\"[X] Expected email 'patrick.bennett@email.com', got {new_booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n \n # Check phone\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got {new_booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct\")\n \n # Phone number should contain 4433071473\n phone_digits = ''.join(filter(str.isdigit, str(new_booking[\"guest_phone_number\"] or \"\")))\n if phone_digits != \"4433071473\":\n error_accumulator.append(f\"[X] Expected phone 4433071473, got {phone_digits}\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n \n # Check booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n \n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # ========== VALIDATION 4: Verify payment method was created (since no saved card existed) ==========\n new_payment_methods = find_new_entries(\"payment_methods\", \"id\")\n \n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new payment method, found {len(new_payment_methods)}\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new payment method was created\")\n \n # Get full payment method details\n new_payment = current.table(\"payment_methods\").eq(\"id\", 82).first()\n \n if not new_payment:\n error_accumulator.append(\"[X] Payment method with id 82 not found\")\n else:\n # Verify cardholder name\n if not normalized_string_comparison(\"patrick bennett\", new_payment[\"cardholder_name\"]):\n error_accumulator.append(f\"[X] Expected cardholder name 'Patrick Bennett', got {new_payment['cardholder_name']}\")\n else:\n success_accumulator.append(\"[C] Cardholder name is correct\")\n \n # Verify last four digits\n if new_payment[\"last_four\"] != \"1117\":\n error_accumulator.append(f\"[X] Expected last four '1117', got {new_payment['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits correct\")\n \n # Verify full card number\n if new_payment[\"card_number\"] != \"6011111111111117\":\n error_accumulator.append(f\"[X] Expected card number '6011111111111117', got {new_payment['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n \n # Verify expiry\n if new_payment[\"expiry_month\"] != 5:\n error_accumulator.append(f\"[X] Expected expiry month 5, got {new_payment['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct\")\n \n if new_payment[\"expiry_year\"] != 2032:\n error_accumulator.append(f\"[X] Expected expiry year 2032, got {new_payment['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct\")\n \n # Verify CVV\n if new_payment[\"cvv\"] != \"614\":\n error_accumulator.append(f\"[X] Expected CVV '614', got {new_payment['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n \n # Verify card type is Discover (based on 6011 prefix)\n if new_payment[\"card_type\"] != \"Discover\":\n error_accumulator.append(f\"[X] Expected card type 'Discover', got {new_payment['card_type']}\")\n else:\n success_accumulator.append(\"[C] Card type is correct (Discover)\")\n\n # ========== VALIDATION 5: Verify booking uses the new payment method ==========\n if new_booking and new_payment:\n if new_booking[\"payment_method_id\"] != 82:\n error_accumulator.append(f\"[X] Expected booking payment_method_id 82, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking correctly references new payment method\")\n\n # ========== VALIDATION 6: Verify transaction was created ==========\n new_transactions = find_new_entries(\"transactions\", \"id\")\n \n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new transaction, found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new transaction was created\")\n \n transaction = current.table(\"transactions\").eq(\"id\", 4402).first()\n if transaction:\n if transaction[\"booking_id\"] != 5194:\n error_accumulator.append(f\"[X] Transaction booking_id should be 5194, got {transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction linked to correct booking\")\n \n if transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Transaction status should be 'Success', got {transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # ========== Use expect_only_v2 for comprehensive diff validation ==========\n expected_changes = [\n # Wishlist list created\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Secret Getaway\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Secret Getaway\"),\n ]\n },\n # Saved properties (4 hotels added)\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 3286, \"Secret Getaway\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 3286),\n (\"list_name\", \"Secret Getaway\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 6570, \"Secret Getaway\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 6570),\n (\"list_name\", \"Secret Getaway\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 7129, \"Secret Getaway\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 7129),\n (\"list_name\", \"Secret Getaway\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 11089, \"Secret Getaway\"),\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 11089),\n (\"list_name\", \"Secret Getaway\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n # Payment method created\n {\n \"table\": \"payment_methods\",\n \"pk\": 82,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 82),\n (\"user_id\", 1),\n (\"card_type\", \"Discover\"),\n (\"last_four\", \"1117\"),\n (\"expiry_month\", 5),\n (\"expiry_year\", 2032),\n (\"billing_address\", None),\n (\"billing_postal_code\", None),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 0),\n (\"card_number\", \"6011111111111117\"),\n (\"cvv\", \"614\"),\n (\"cardholder_name\", \"Patrick Bennett\"),\n ]\n },\n # Booking created\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 6570),\n (\"room_id\", 13033),\n (\"check_in_date\", \"2026-10-11\"),\n (\"check_out_date\", \"2026-10-13\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Patrick\"),\n (\"guest_last_name\", \"Bennett\"),\n (\"guest_email\", \"patrick.bennett@email.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"4433071473\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 82),\n (\"agreed_to_terms\", ...),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n # Transaction created\n {\n \"table\": \"transactions\",\n \"pk\": 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 4402),\n (\"booking_id\", 5194),\n (\"payment_method_id\", 82),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'updated_at', 'created_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'updated_at', 'created_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'updated_at', 'created_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'property_policies': {'created_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_wr5hjnspuphj_1768262127507_8x9llrhu1", + "prompt": "We will be visiting Happo-en Garden in Tokyo (35.64\u00b0 N latitude, 139.73\u00b0 E longitude). Please find accommodations within a 5 km radius of the venue for 2 adults and 2 children for the period of September 4\u20137, 2026. The accommodation must be a minimum of 4-star rating, with a nightly price range of USD 300\u2013500. We must stay in a single room. \n\nRank the options by cancellation flexibility, with preference given to 24-hour free cancellation, followed by 5-day free cancellation, and so forth.\n\nDetermine the price distribution for accommodations that meet these criteria during our stay period. If the base price of the selected accommodation is less than twice the upper bound of the modal price bucket, complete the booking using my information and the credit card on file. Otherwise, create a new wishlist titled \u201cJapan 2026\u201d and add the accommodation to that wishlist.\n\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly - booking for Happo-en Garden visit in Tokyo\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n \n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n\n # Verify check-in date is September 4, 2026\n if new_booking[\"check_in_date\"] != \"2026-09-04\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-09-04', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct (2026-09-04)\")\n\n # Verify check-out date is September 7, 2026\n if new_booking[\"check_out_date\"] != \"2026-09-07\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-09-07', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct (2026-09-07)\")\n\n # Verify 2 adults\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct (2)\")\n\n # Verify 2 children\n if new_booking[\"children\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 children, got {new_booking['children']}\")\n else:\n success_accumulator.append(\"[C] Children count is correct (2)\")\n\n # Verify single room\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 room, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is correct (1)\")\n\n # Verify booking status is confirmed\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is 'confirmed'\")\n\n # Verify payment status is paid\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is 'paid'\")\n\n # Verify agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Verify guest details - First Name Charles\n if not normalized_string_comparison(\"charles\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Charles', got '{new_booking['guest_first_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is 'Charles'\")\n\n # Verify guest details - Last Name Moore\n if not normalized_string_comparison(\"moore\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Moore', got '{new_booking['guest_last_name']}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is 'Moore'\")\n\n # Verify guest email\n if new_booking[\"guest_email\"] and \"charles.moore@synapse.com\" in new_booking[\"guest_email\"].lower():\n success_accumulator.append(\"[C] Guest email is correct\")\n else:\n error_accumulator.append(f\"[X] Expected guest email 'charles.moore@synapse.com', got '{new_booking['guest_email']}'\")\n\n # Verify payment method was used (card on file)\n if new_booking[\"payment_method_id\"] == 1:\n success_accumulator.append(\"[C] Payment method ID 1 (card on file) was used\")\n else:\n error_accumulator.append(f\"[X] Expected payment_method_id 1, got {new_booking['payment_method_id']}\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n \n transaction_id = None\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n success_accumulator.append(\"[C] 1 new transaction was created\")\n new_transaction = new_transactions[0]\n transaction_id = new_transaction[\"id\"]\n \n # Verify transaction is linked to the booking\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Transaction should be linked to booking {booking_id}\")\n else:\n success_accumulator.append(\"[C] Transaction is linked to the correct booking\")\n \n # Verify transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is 'Success'\")\n\n # Use expect_only_v2 to validate database changes\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 15087),\n (\"room_id\", 29952),\n (\"check_in_date\", \"2026-09-04\"),\n (\"check_out_date\", \"2026-09-07\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 2),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Charles\"),\n (\"guest_last_name\", \"Moore\"),\n (\"guest_email\", \"charles.moore@synapse.com\"),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 1),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n ]\n \n # Add transaction to expected changes if we found it\n if transaction_id is not None:\n expected_changes.append({\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", 1),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n })\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'bookings': {'updated_at', 'created_at'},\n 'transactions': {'processed_at'},\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_ixhylvep7cz_1768149810262_uzozhtxu6", + "prompt": "My husband and I will be celebrating our 10th wedding anniversary in May of this year. We have always wanted to go to Dubai and although we are not wealthy we have been saving for quite some time to make this happen. Since this is a once in a lifetime opportunity for us, I am looking for a 5 star resort or villa The maximum per night we are willing to pay is $400. We would like a deluxe room from May 9-16 . Note my husband smokes so we need accomodations that allow that. \n\nMy name, address and phone number are below as well as my credit card information.\n\nTeresa Bachman\n4281 Chapel Street\nTomball, TX 77337\n1-281-655-7341\nMastercard 5192 7279 7532 8381\nExpires 6/2028\nCVV 191", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task_ixhylvep7cz_1768149810262_uzozhtxu6(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - Anniversary trip booking to Dubai\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find the new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, but found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = new_bookings[0]\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n # Validate guest information - Teresa Bachman\n if not normalized_string_comparison(\"Teresa\", new_booking.get(\"guest_first_name\", \"\")):\n error_accumulator.append(f\"[X] Expected guest first name 'Teresa', got '{new_booking.get('guest_first_name')}'\")\n else:\n success_accumulator.append(\"[C] Guest first name is 'Teresa'\")\n\n if not normalized_string_comparison(\"Bachman\", new_booking.get(\"guest_last_name\", \"\")):\n error_accumulator.append(f\"[X] Expected guest last name 'Bachman', got '{new_booking.get('guest_last_name')}'\")\n else:\n success_accumulator.append(\"[C] Guest last name is 'Bachman'\")\n\n # Validate check-in and check-out dates (May 9-16, 2026 based on diff showing \"this year\" from task context)\n if new_booking.get(\"check_in_date\") != \"2026-05-09\":\n error_accumulator.append(f\"[X] Expected check-in date '2026-05-09', got '{new_booking.get('check_in_date')}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is May 9, 2026\")\n\n if new_booking.get(\"check_out_date\") != \"2026-05-16\":\n error_accumulator.append(f\"[X] Expected check-out date '2026-05-16', got '{new_booking.get('check_out_date')}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is May 16, 2026\")\n\n # Validate booking is for 2 adults (husband and wife)\n if new_booking.get(\"adults\") != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking.get('adults')}\")\n else:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n\n # Validate phone number - 281-655-7341\n phone_number = str(new_booking.get(\"guest_phone_number\", \"\"))\n phone_digits = ''.join(filter(str.isdigit, phone_number))\n if phone_digits != \"2816557341\":\n error_accumulator.append(f\"[X] Expected phone number '2816557341', got '{phone_digits}'\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n\n # Validate booking status\n if new_booking.get(\"status\") != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got '{new_booking.get('status')}'\")\n else:\n success_accumulator.append(\"[C] Booking status is 'confirmed'\")\n\n if new_booking.get(\"payment_status\") != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got '{new_booking.get('payment_status')}'\")\n else:\n success_accumulator.append(\"[C] Payment status is 'paid'\")\n\n # Validate agreed to terms\n if new_booking.get(\"agreed_to_terms\") != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Find the new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, but found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_payment_method = new_payment_methods[0]\n success_accumulator.append(\"[C] 1 new payment method was created\")\n\n # Validate payment method - Mastercard 5192 7279 7532 8381, Exp 6/2028, CVV 191\n if new_payment_method.get(\"card_type\") != \"Mastercard\":\n error_accumulator.append(f\"[X] Expected card type 'Mastercard', got '{new_payment_method.get('card_type')}'\")\n else:\n success_accumulator.append(\"[C] Card type is Mastercard\")\n\n if new_payment_method.get(\"last_four\") != \"8381\":\n error_accumulator.append(f\"[X] Expected last four '8381', got '{new_payment_method.get('last_four')}'\")\n else:\n success_accumulator.append(\"[C] Card last four digits are '8381'\")\n\n if new_payment_method.get(\"expiry_month\") != 6:\n error_accumulator.append(f\"[X] Expected expiry month 6, got {new_payment_method.get('expiry_month')}\")\n else:\n success_accumulator.append(\"[C] Expiry month is 6\")\n\n if new_payment_method.get(\"expiry_year\") != 2028:\n error_accumulator.append(f\"[X] Expected expiry year 2028, got {new_payment_method.get('expiry_year')}\")\n else:\n success_accumulator.append(\"[C] Expiry year is 2028\")\n\n if new_payment_method.get(\"cvv\") != \"191\":\n error_accumulator.append(f\"[X] Expected CVV '191', got '{new_payment_method.get('cvv')}'\")\n else:\n success_accumulator.append(\"[C] CVV is '191'\")\n\n if new_payment_method.get(\"card_number\") != \"5192727975328381\":\n error_accumulator.append(f\"[X] Expected card number '5192727975328381', got '{new_payment_method.get('card_number')}'\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n # Validate cardholder name\n if not normalized_string_comparison(\"Teresa Bachman\", new_payment_method.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder name 'Teresa Bachman', got '{new_payment_method.get('cardholder_name')}'\")\n else:\n success_accumulator.append(\"[C] Cardholder name is 'Teresa Bachman'\")\n\n # Validate that the booking references the new payment method\n if new_booking.get(\"payment_method_id\") != new_payment_method.get(\"id\"):\n error_accumulator.append(f\"[X] Expected booking to reference payment method {new_payment_method.get('id')}, got {new_booking.get('payment_method_id')}\")\n else:\n success_accumulator.append(\"[C] Booking references the correct payment method\")\n\n # Find the new transaction\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, but found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n success_accumulator.append(\"[C] 1 new transaction was created\")\n \n # Validate transaction references the booking\n if new_transaction.get(\"booking_id\") != new_booking.get(\"id\"):\n error_accumulator.append(f\"[X] Expected transaction to reference booking {new_booking.get('id')}, got {new_transaction.get('booking_id')}\")\n else:\n success_accumulator.append(\"[C] Transaction references the correct booking\")\n\n # Validate transaction status\n if new_transaction.get(\"status\") != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction.get('status')}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is 'Success'\")\n\n # Get booking and payment method IDs for expected changes\n booking_id = new_booking.get(\"id\")\n payment_method_id = new_payment_method.get(\"id\")\n transaction_id = new_transactions[0].get(\"id\") if new_transactions else None\n\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", booking_id),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", ...),\n (\"room_id\", ...),\n (\"check_in_date\", \"2026-05-09\"),\n (\"check_out_date\", \"2026-05-16\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Teresa\"),\n (\"guest_last_name\", \"Bachman\"),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"2816557341\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": payment_method_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", payment_method_id),\n (\"user_id\", 1),\n (\"card_type\", \"Mastercard\"),\n (\"last_four\", \"8381\"),\n (\"expiry_month\", 6),\n (\"expiry_year\", 2028),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"5192727975328381\"),\n (\"cvv\", \"191\"),\n (\"cardholder_name\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": transaction_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", transaction_id),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'created_at', 'expires_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_lweecshpomi_1765902891433_uoq06xngz", + "prompt": "Check the availability for 1 night for 2nd Feb next year in the same guesthouse in LA where I stayed in Oct this year. If the guesthouse is available then in the existing wishlist add this guesthouse with the comment \"LA Repeat\".", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_lweecshpomi_1765902891433_uoq06xngz(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # The task was to:\n # 1. Check availability for 1 night for 2nd Feb next year in the same guesthouse in LA where user stayed in Oct this year\n # 2. If available, add this guesthouse to the existing wishlist with the comment \"LA Repeat\"\n \n # The database diff shows an insert into saved_properties:\n # (user_id=1, hotel_id=815, list_name='Summer Getaways', notes='LA Repeat', saved_at='2025-12-16 16:34:48')\n \n # Verify that a new saved_property was added with the correct details\n new_saved_entry = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 815).eq(\"list_name\", \"Summer Getaways\").first()\n \n if not new_saved_entry:\n error_accumulator.append(\"[X] Expected new saved property entry for user_id=1, hotel_id=815, list_name='Summer Getaways' not found\")\n else:\n success_accumulator.append(\"[C] Found new saved property entry for user_id=1, hotel_id=815 in 'Summer Getaways' wishlist\")\n \n # Verify the notes field contains \"LA Repeat\"\n notes = new_saved_entry.get(\"notes\", \"\")\n if not normalized_contains(\"LA Repeat\", notes):\n error_accumulator.append(f\"[X] Expected notes to contain 'LA Repeat', but got: {notes}\")\n else:\n success_accumulator.append(\"[C] Notes field correctly contains 'LA Repeat'\")\n\n # Verify this entry didn't exist in seed (confirming it was newly added)\n seed_entry = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 815).eq(\"list_name\", \"Summer Getaways\").first()\n if seed_entry:\n error_accumulator.append(\"[X] This saved property already existed in the seed database, was not newly added\")\n else:\n success_accumulator.append(\"[C] Confirmed this saved property is new (did not exist in seed)\")\n\n # Verify hotel_id=815 exists (basic sanity check)\n hotel = current.table(\"hotels\").eq(\"id\", 815).first()\n if not hotel:\n error_accumulator.append(\"[X] Hotel with id=815 does not exist in database\")\n else:\n success_accumulator.append(\"[C] Hotel with id=815 exists in database\")\n\n # Count total saved_properties entries to ensure only one was added\n seed_count = len(seed.table(\"saved_properties\").all())\n current_count = len(current.table(\"saved_properties\").all())\n \n if current_count - seed_count != 1:\n error_accumulator.append(f\"[X] Expected exactly 1 new saved property, but found {current_count - seed_count} new entries\")\n else:\n success_accumulator.append(\"[C] Exactly 1 new saved property was added\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_glffty3bwtec_1765893314896_yfm0h34ou", + "prompt": "I am searching for cheap hotels in London and my budget is 70 per night. If any hostels found match the requirements, then check the reviews. If any of the first 5 reviews gave it an overall rating of over 9, then add it to the wishlist \"Cheap London Stays\".", + "env_id": "booking", + "version": "v0.0.69", + "data_id": "kinesis", + "data_version": "v0.0.9", + "verifier_func": "def validate_task_glffty3bwtec_1765893314896_yfm0h34ou(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # The task requires:\n # 1. Search for cheap hotels in London (budget $70 per night)\n # 2. If hostels match, check reviews\n # 3. If any of first 5 reviews gave overall rating > 9, add to wishlist \"Cheap London Stays\"\n \n # From the diff, we can see:\n # 1. saved_properties insert: (1, 14345, 'Cheap London Stays', NULL, '2025-12-16 13:52:16')\n # 2. wishlist_lists insert: ('1', 'Cheap London Stays')\n\n # Check that the saved_properties entry was created with the correct details\n saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 14345).eq(\"list_name\", \"Cheap London Stays\").first()\n \n if saved_property is None:\n error_accumulator.append(\"[X] Expected saved_property entry not found for user_id=1, hotel_id=14345, list_name='Cheap London Stays'\")\n else:\n success_accumulator.append(\"[C] Property was saved to 'Cheap London Stays' wishlist\")\n \n # Verify the user_id\n if saved_property[\"user_id\"] != 1:\n error_accumulator.append(f\"[X] Expected user_id to be 1, got {saved_property['user_id']}\")\n else:\n success_accumulator.append(\"[C] user_id is correct (1)\")\n \n # Verify the hotel_id\n if saved_property[\"hotel_id\"] != 14345:\n error_accumulator.append(f\"[X] Expected hotel_id to be 14345, got {saved_property['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] hotel_id is correct (14345)\")\n \n # Verify the list_name\n if not normalized_string_comparison(\"Cheap London Stays\", saved_property[\"list_name\"]):\n error_accumulator.append(f\"[X] Expected list_name to be 'Cheap London Stays', got {saved_property['list_name']}\")\n else:\n success_accumulator.append(\"[C] list_name is correct ('Cheap London Stays')\")\n\n # Check that this entry didn't exist before (new insert)\n seed_saved_property = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 14345).eq(\"list_name\", \"Cheap London Stays\").first()\n \n if seed_saved_property is not None:\n error_accumulator.append(\"[X] saved_property entry already existed in seed - not a new insert\")\n else:\n success_accumulator.append(\"[C] Confirmed this is a new saved_property entry (not in seed)\")\n\n # Verify no unexpected changes to saved_properties table\n # Count saved_properties in seed vs current\n seed_saved_count = len(seed.table(\"saved_properties\").all())\n current_saved_count = len(current.table(\"saved_properties\").all())\n \n expected_new_saved = 1 # Only one new saved property expected\n actual_new_saved = current_saved_count - seed_saved_count\n \n if actual_new_saved != expected_new_saved:\n error_accumulator.append(f\"[X] Expected {expected_new_saved} new saved_properties entry, but found {actual_new_saved}\")\n else:\n success_accumulator.append(f\"[C] Correct number of new saved_properties entries ({expected_new_saved})\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_shi7rswtccex_1765531385897_1ygwtrlvv", + "prompt": "I am looking for a hotel in Austin which has special deals, the property must have more than 150 reviews.\nAlso check if the property offers Fitness Center and Restaurant.\nIf either of the one is not offered then create the wishlist named \"Austin Travel\" and add this hotel to the wishlist with a note \"Both requirements not met\".", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_shi7rswtccex_1765531385897_1ygwtrlvv(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - \n User searched for a hotel in Austin with special deals and more than 150 reviews.\n Since either Fitness Center or Restaurant was not offered, they created a wishlist \n named 'Austin Travel' and added the hotel with note 'Both requirements not met'.\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Check that a new saved_properties entry was created\n # The saved_properties table has a composite primary key (user_id, hotel_id, list_name)\n new_saved_property = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Austin Travel\").first()\n \n if not new_saved_property:\n error_accumulator.append(\"[X] No saved property found with list_name 'Austin Travel' for user_id 1\")\n else:\n success_accumulator.append(\"[C] Found saved property entry with list_name 'Austin Travel'\")\n \n # Validate the hotel_id is 15586 as per the diff\n if new_saved_property[\"hotel_id\"] != 15586:\n error_accumulator.append(f\"[X] Expected hotel_id 15586, but got {new_saved_property['hotel_id']}\")\n else:\n success_accumulator.append(\"[C] Correct hotel_id 15586 was saved\")\n \n # Validate the notes field contains the required text\n notes = new_saved_property.get(\"notes\", \"\")\n if not normalized_contains(\"Both requirements not met\", notes):\n error_accumulator.append(f\"[X] Expected notes to contain 'Both requirements not met', but got: {notes}\")\n else:\n success_accumulator.append(\"[C] Notes correctly set to 'Both requirements not met'\")\n\n # Verify the saved property didn't exist before (was inserted)\n seed_saved_property = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Austin Travel\").eq(\"hotel_id\", 15586).all()\n if len(seed_saved_property) > 0:\n error_accumulator.append(\"[X] The saved property already existed in seed database - should be a new insert\")\n else:\n success_accumulator.append(\"[C] Confirmed saved property is a new insertion\")\n\n # Verify that user_id 1 exists\n user = current.table(\"users\").eq(\"id\", 1).first()\n if not user:\n error_accumulator.append(\"[X] User with id 1 does not exist\")\n else:\n success_accumulator.append(\"[C] User with id 1 exists\")\n\n # Verify that hotel_id 15586 exists and is in Austin\n hotel = current.table(\"hotels\").eq(\"id\", 15586).first()\n if not hotel:\n error_accumulator.append(\"[X] Hotel with id 15586 does not exist\")\n else:\n success_accumulator.append(\"[C] Hotel with id 15586 exists\")\n # Check that the hotel is in Austin\n hotel_city = hotel.get(\"city\", \"\")\n if not normalized_contains(\"Austin\", hotel_city):\n error_accumulator.append(f\"[X] Hotel city should be Austin, but got: {hotel_city}\")\n else:\n success_accumulator.append(\"[C] Hotel is in Austin\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_qdcdqluitqd_1765489069926_j0kkam6bn", + "prompt": "For all featured hotels, check their attractions and if they are close to a beach (within 5km), add that hotel to my \"Dream Vacations\" list.", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_qdcdqluitqd_1765489069926_j0kkam6bn(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Task: For all featured hotels, check their attractions and if they are close to a beach (within 5km), add that hotel to my \"Dream Vacations\" list.\n # From the diff, we see:\n # 1. A new entry in saved_properties: (1, 10835, 'Dream Vacations', NULL, ...)\n # 2. A new entry in wishlist_lists: ('1', 'Dream Vacations')\n\n # Verify the saved_properties entry was created with the correct hotel\n # Check if hotel 10835 exists in the Dream Vacations list for user 1\n saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 10835).eq(\"list_name\", \"Dream Vacations\").all()\n \n if len(saved_properties) == 1:\n success_accumulator.append(\"[C] Hotel 10835 was added to 'Dream Vacations' list for user 1\")\n elif len(saved_properties) == 0:\n error_accumulator.append(\"[X] Hotel 10835 was not found in 'Dream Vacations' list for user 1\")\n else:\n error_accumulator.append(f\"[X] Expected 1 entry for hotel 10835 in 'Dream Vacations', found {len(saved_properties)}\")\n\n # Check that this entry didn't exist in the seed database\n seed_saved_properties = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"hotel_id\", 10835).eq(\"list_name\", \"Dream Vacations\").all()\n \n if len(seed_saved_properties) == 0:\n success_accumulator.append(\"[C] Hotel 10835 was newly added (not in seed database)\")\n else:\n error_accumulator.append(\"[X] Hotel 10835 already existed in 'Dream Vacations' list before the task\")\n\n # Verify the list_name is exactly \"Dream Vacations\"\n if len(saved_properties) > 0:\n entry = saved_properties[0]\n if entry.get(\"list_name\") == \"Dream Vacations\":\n success_accumulator.append(\"[C] List name is correctly 'Dream Vacations'\")\n else:\n error_accumulator.append(f\"[X] Expected list_name 'Dream Vacations', got '{entry.get('list_name')}'\")\n\n # Verify there are no other new entries in saved_properties with \"Dream Vacations\" list\n # that shouldn't be there (only hotel 10835 should be added based on the diff)\n all_dream_vacations = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Dream Vacations\").all()\n seed_dream_vacations = seed.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Dream Vacations\").all()\n \n new_dream_vacation_count = len(all_dream_vacations) - len(seed_dream_vacations)\n \n if new_dream_vacation_count == 1:\n success_accumulator.append(\"[C] Exactly 1 new hotel was added to 'Dream Vacations' list\")\n else:\n error_accumulator.append(f\"[X] Expected 1 new hotel in 'Dream Vacations', found {new_dream_vacation_count} new entries\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_py7tpag0v9uh_1767471587135_0n3klvccy", + "prompt": "I\u2019m looking for villas in Rome. Please compare the 3 top-reviewed villas and select the one where smoking is not allowed.\n\nBook a Suite for 2 adults (check-in on February 2, check-out on February 4, this year). If the total price is less than $1,700 USD, please use my MasterCard for payment. If it exceeds that amount, add the new Visa card in my payment methods (details below) and use that payment method to complete the booking.\n\nVisa card details:\nCardholder name: Charles Moore\nCard number: 4242 4242 4242 4242, expires on 12/28 and CVV 121.\n101 Main Street, Jacksonville, FL, 32202\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def main(env: Environment, final_answer: str | None = None) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF AND SCORE PROGRESS IN success_accumulator and error_accumulator ####\n\n # Find new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = new_bookings[0]\n booking_id = new_booking[\"id\"]\n success_accumulator.append(f\"[C] Found 1 new booking with id {booking_id}\")\n\n # Validate the hotel is a villa in Rome\n hotel = current.table(\"hotels\").eq(\"id\", new_booking[\"hotel_id\"]).first()\n if hotel is None:\n error_accumulator.append(f\"[X] Could not find hotel with id {new_booking['hotel_id']}\")\n else:\n if hotel.get(\"property_type\") != \"Villa\":\n error_accumulator.append(f\"[X] Expected property_type 'Villa', got '{hotel.get('property_type')}'\")\n else:\n success_accumulator.append(\"[C] Hotel is a Villa\")\n \n if not normalized_contains(\"Rome\", hotel.get(\"city\", \"\")):\n error_accumulator.append(f\"[X] Expected hotel city to contain 'Rome', got '{hotel.get('city')}'\")\n else:\n success_accumulator.append(\"[C] Hotel is in Rome\")\n\n # Validate the room is a Suite\n room = current.table(\"rooms\").eq(\"id\", new_booking[\"room_id\"]).first()\n if room is None:\n error_accumulator.append(f\"[X] Could not find room with id {new_booking['room_id']}\")\n else:\n room_type = room.get(\"room_type\", \"\")\n room_name = room.get(\"name\", \"\")\n if not normalized_contains(\"Suite\", room_type) and not normalized_contains(\"Suite\", room_name):\n error_accumulator.append(f\"[X] Expected room type to be Suite, got type='{room_type}', name='{room_name}'\")\n else:\n success_accumulator.append(\"[C] Room is a Suite\")\n\n # Validate the hotel has no-smoking policy\n policy = current.table(\"property_policies\").eq(\"hotel_id\", new_booking[\"hotel_id\"]).first()\n if policy is None:\n error_accumulator.append(f\"[X] Could not find property policy for hotel {new_booking['hotel_id']}\")\n else:\n if policy.get(\"smoking_policy\") != \"Not Allowed\":\n error_accumulator.append(f\"[X] Expected smoking_policy 'Not Allowed', got '{policy.get('smoking_policy')}'\")\n else:\n success_accumulator.append(\"[C] Hotel has no-smoking policy\")\n\n # Validate that the booked villa is a reviewed property (has reviews)\n # The task asks to compare \"top-reviewed\" villas - we verify the hotel has reviews\n # The exact ranking depends on how the UI presents results which may vary\n if hotel is not None:\n total_reviews = hotel.get(\"total_reviews\") or 0\n average_rating = hotel.get(\"average_rating\") or 0\n if total_reviews == 0:\n error_accumulator.append(f\"[X] Expected hotel to have reviews (top-reviewed requirement), but total_reviews is 0\")\n else:\n success_accumulator.append(f\"[C] Hotel has {total_reviews} reviews with average rating {average_rating}\")\n\n # Validate booking details\n # Check dates (February 2 to February 4, this year - which based on diff is 2026)\n if new_booking[\"check_in_date\"] != \"2026-02-02\":\n error_accumulator.append(f\"[X] Expected check_in_date '2026-02-02', got '{new_booking['check_in_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-02-02\")\n\n if new_booking[\"check_out_date\"] != \"2026-02-04\":\n error_accumulator.append(f\"[X] Expected check_out_date '2026-02-04', got '{new_booking['check_out_date']}'\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-02-04\")\n\n # Check 2 adults\n if new_booking[\"adults\"] != 2:\n error_accumulator.append(f\"[X] Expected 2 adults, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Booking is for 2 adults\")\n\n # Check room count is 1\n if new_booking[\"room_count\"] != 1:\n error_accumulator.append(f\"[X] Expected room_count 1, got {new_booking['room_count']}\")\n else:\n success_accumulator.append(\"[C] Room count is 1\")\n\n # Check booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected status 'confirmed', got '{new_booking['status']}'\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment_status 'paid', got '{new_booking['payment_status']}'\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Verify the total price was > $1700 (which triggered the Visa card usage)\n # The diff shows total_price is 1829.3 which is > 1700\n total_price = new_booking[\"total_price\"]\n if total_price is None or total_price <= 1700:\n error_accumulator.append(f\"[X] Expected total_price > 1700 (to trigger Visa card), got {total_price}\")\n else:\n success_accumulator.append(f\"[C] Total price {total_price} > $1700, Visa card should be used\")\n\n # Find new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n new_payment_method = new_payment_methods[0]\n payment_method_id = new_payment_method[\"id\"]\n success_accumulator.append(f\"[C] Found 1 new payment method with id {payment_method_id}\")\n\n # Validate payment method details - Visa card\n if new_payment_method[\"card_type\"] != \"Visa\":\n error_accumulator.append(f\"[X] Expected card_type 'Visa', got '{new_payment_method['card_type']}'\")\n else:\n success_accumulator.append(\"[C] Card type is Visa\")\n\n if new_payment_method[\"last_four\"] != \"4242\":\n error_accumulator.append(f\"[X] Expected last_four '4242', got '{new_payment_method['last_four']}'\")\n else:\n success_accumulator.append(\"[C] Card last four digits are 4242\")\n\n if new_payment_method[\"expiry_month\"] != 12:\n error_accumulator.append(f\"[X] Expected expiry_month 12, got {new_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is 12\")\n\n if new_payment_method[\"expiry_year\"] != 2028:\n error_accumulator.append(f\"[X] Expected expiry_year 2028, got {new_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is 2028\")\n\n if new_payment_method[\"cvv\"] != \"121\":\n error_accumulator.append(f\"[X] Expected CVV '121', got '{new_payment_method['cvv']}'\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n\n if new_payment_method[\"card_number\"] != \"4242424242424242\":\n error_accumulator.append(f\"[X] Expected card_number '4242424242424242', got '{new_payment_method['card_number']}'\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n # Validate cardholder name\n if not normalized_string_comparison(\"Charles Moore\", new_payment_method.get(\"cardholder_name\", \"\")):\n error_accumulator.append(f\"[X] Expected cardholder_name 'Charles Moore', got '{new_payment_method.get('cardholder_name')}'\")\n else:\n success_accumulator.append(\"[C] Cardholder name is Charles Moore\")\n\n # Validate billing address contains expected info\n billing_address = new_payment_method.get(\"billing_address\", \"\")\n if not normalized_contains(\"101 Main Street\", billing_address):\n error_accumulator.append(f\"[X] Expected billing_address to contain '101 Main Street', got '{billing_address}'\")\n else:\n success_accumulator.append(\"[C] Billing address contains 101 Main Street\")\n\n if not normalized_contains(\"Jacksonville\", billing_address):\n error_accumulator.append(f\"[X] Expected billing_address to contain 'Jacksonville', got '{billing_address}'\")\n else:\n success_accumulator.append(\"[C] Billing address contains Jacksonville\")\n\n # Validate billing postal code\n if new_payment_method.get(\"billing_postal_code\") != \"32202\":\n error_accumulator.append(f\"[X] Expected billing_postal_code '32202', got '{new_payment_method.get('billing_postal_code')}'\")\n else:\n success_accumulator.append(\"[C] Billing postal code is 32202\")\n\n # Validate that booking references the new payment method\n if new_booking[\"payment_method_id\"] != payment_method_id:\n error_accumulator.append(f\"[X] Expected booking to use payment_method_id {payment_method_id}, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(f\"[C] Booking uses the new Visa payment method (id={payment_method_id})\")\n\n # Find new transactions\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n transaction_id = new_transaction[\"id\"]\n success_accumulator.append(f\"[C] Found 1 new transaction with id {transaction_id}\")\n\n # Validate transaction details\n if new_transaction[\"booking_id\"] != booking_id:\n error_accumulator.append(f\"[X] Expected transaction booking_id {booking_id}, got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction references correct booking\")\n\n if new_transaction[\"payment_method_id\"] != payment_method_id:\n error_accumulator.append(f\"[X] Expected transaction payment_method_id {payment_method_id}, got {new_transaction['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction uses correct payment method\")\n\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got '{new_transaction['status']}'\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction_type 'Payment', got '{new_transaction['transaction_type']}'\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": booking_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-02-02\"),\n (\"check_out_date\", \"2026-02-04\"),\n (\"booking_date\", ...),\n (\"adults\", 2),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", None),\n (\"arrival_time\", None),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", ...),\n (\"guest_last_name\", ...),\n (\"guest_email\", ...),\n (\"guest_phone_country_code\", ...),\n (\"guest_phone_number\", ...),\n (\"guest_country\", ...),\n (\"booking_group_id\", None),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", payment_method_id),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", None),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", None),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": payment_method_id,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"4242\"),\n (\"expiry_month\", 12),\n (\"expiry_year\", 2028),\n (\"billing_address\", ...),\n (\"billing_postal_code\", \"32202\"),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"4242424242424242\"),\n (\"cvv\", \"121\"),\n (\"cardholder_name\", \"Charles Moore\"),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if len(new_transactions) == 1 else 0,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", booking_id),\n (\"payment_method_id\", payment_method_id),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", None),\n (\"gateway_transaction_id\", None),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'}, 'booking_extras': {'created_at'}, 'bookings': {'created_at', 'updated_at'}, 'cancellations': {'created_at'}, 'facilities': {'created_at'}, 'generation_checkpoints': {'created_at'}, 'hosts': {'created_at'}, 'hotel_images': {'created_at'}, 'hotels': {'created_at', 'updated_at'}, 'locations': {'created_at'}, 'messages': {'created_at'}, 'nearby_attractions': {'created_at'}, 'payment_methods': {'created_at'}, 'price_calendar': {'created_at'}, 'property_policies': {'created_at'}, 'reviews': {'created_at'}, 'rewards_program': {'updated_at'}, 'room_images': {'created_at'}, 'rooms': {'created_at'}, 'users': {'created_at'}, 'wishlist_lists': {'created_at'}}\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kwgyabk5cmn_1767542998408_k5ineh1bd", + "prompt": "I had some properties saved in my wishlist. I want to use one of the properties for my one-night stay in Portland on January 17, 2026. Can you retrieve all properties and check if any are in Portland with a star rating of 3 or higher? If there is no property in Portland, book a hotel in Portland for me that has total reviews above 800 and a star rating of 4 or above. It should be a Deluxe room and include breakfast. If multiple options are available, book the cheapest one.\n\nIf there is a property in Portland but it has under 3-star ratings, book a standard room type with an Ocean view in a different property, but of the same property type as the one in my wishlist, with a star rating of 3 or above in the same city. The average ratings and total reviews should be higher than those of the Portland hotel in my wishlist.\n\nPlease use the payment details below to finalize the purchase, but do not save them:\n\n- Name: Kayden Haag \n- Phone: +14479028792 \n- Email: kayhaag4@gmail.com \n- Billing Address: P22 Tbay Street, Santa Monica, SM 45301, US \n- Card Number: 4894 8964 1027 6929 \n- CVV: 898 \n- Expiry: 09/31", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_kwgyabk5cmn_1767542998408_k5ineh1bd(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n \n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n\n Args:\n table_name: Name of the table to check\n id_field: Name of the ID field (default: \"id\")\n filter_conditions: Optional dict of filters to apply (e.g. {\"foreign_key\": \"value\"})\n\n Returns:\n List of new entries (rows that exist in current but not in seed)\n \"\"\"\n # Query seed snapshot\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n # Query current snapshot\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.select(id_field).all()\n\n # Extract ID values into lists\n before_ids = {entry[id_field] for entry in before_entries}\n\n # Find entries that are in after but not in before\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # Find new booking(s)\n new_booking_ids = find_new_entries(\"bookings\")\n if len(new_booking_ids) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_booking_ids)}\")\n else:\n success_accumulator.append(\"[C] 1 new booking was created\")\n\n # Get the full booking details\n new_booking = current.table(\"bookings\").eq(\"id\", 5194).first()\n if not new_booking:\n error_accumulator.append(\"[X] Could not find the new booking with id 5194\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n # Validate check-in and check-out dates (January 17, 2026 for one night)\n if new_booking[\"check_in_date\"] != \"2026-01-17\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-01-17, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-01-17\")\n\n if new_booking[\"check_out_date\"] != \"2026-01-18\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-01-18, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-01-18\")\n\n # Validate guest details\n if not normalized_string_comparison(\"kayden\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Kayden', got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct: Kayden\")\n\n if not normalized_string_comparison(\"haag\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Haag', got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct: Haag\")\n\n # Validate email\n if new_booking[\"guest_email\"].lower() != \"kayhaag4@gmail.com\":\n error_accumulator.append(f\"[X] Expected email 'kayhaag4@gmail.com', got {new_booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n\n # Validate phone number\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got {new_booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct: +1\")\n\n phone_digits = ''.join(filter(str.isdigit, str(new_booking[\"guest_phone_number\"]) if new_booking[\"guest_phone_number\"] else \"\"))\n if phone_digits != \"4479028792\":\n error_accumulator.append(f\"[X] Expected phone number '4479028792', got {new_booking['guest_phone_number']}\")\n else:\n success_accumulator.append(\"[C] Phone number is correct\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(f\"[X] Expected agreed_to_terms to be 1, got {new_booking['agreed_to_terms']}\")\n else:\n success_accumulator.append(\"[C] Agreed to terms is checked\")\n\n # Find and validate payment method\n new_payment_ids = find_new_entries(\"payment_methods\")\n if len(new_payment_ids) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_ids)}\")\n else:\n success_accumulator.append(\"[C] 1 new payment method was created\")\n\n new_payment = current.table(\"payment_methods\").eq(\"id\", 82).first()\n if not new_payment:\n error_accumulator.append(\"[X] Could not find the new payment method with id 82\")\n else:\n # Validate card details\n if new_payment[\"last_four\"] != \"6929\":\n error_accumulator.append(f\"[X] Expected card last four '6929', got {new_payment['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits are correct: 6929\")\n\n if new_payment[\"card_number\"] != \"4894896410276929\":\n error_accumulator.append(f\"[X] Expected card number '4894896410276929', got {new_payment['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n if new_payment[\"cvv\"] != \"898\":\n error_accumulator.append(f\"[X] Expected CVV '898', got {new_payment['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV is correct\")\n\n if new_payment[\"expiry_month\"] != 9:\n error_accumulator.append(f\"[X] Expected expiry month 9, got {new_payment['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct: 9\")\n\n if new_payment[\"expiry_year\"] != 2031:\n error_accumulator.append(f\"[X] Expected expiry year 2031, got {new_payment['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct: 2031\")\n\n # Validate payment method is NOT saved for reuse (as per task requirement)\n if new_payment[\"is_saved_for_reuse\"] != 0:\n error_accumulator.append(f\"[X] Expected is_saved_for_reuse to be 0 (not saved), got {new_payment['is_saved_for_reuse']}\")\n else:\n success_accumulator.append(\"[C] Payment method is not saved for reuse as requested\")\n\n # Validate cardholder name\n if not normalized_string_comparison(\"kayden haag\", new_payment[\"cardholder_name\"]):\n error_accumulator.append(f\"[X] Expected cardholder name 'Kayden Haag', got {new_payment['cardholder_name']}\")\n else:\n success_accumulator.append(\"[C] Cardholder name is correct: Kayden Haag\")\n\n # Validate that the booking references the new payment method\n if new_booking[\"payment_method_id\"] != 82:\n error_accumulator.append(f\"[X] Expected booking to reference payment method 82, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking references the correct payment method\")\n\n # Verify transaction was created\n new_transaction = current.table(\"transactions\").eq(\"booking_id\", 5194).first()\n if not new_transaction:\n error_accumulator.append(\"[X] No transaction found for the booking\")\n else:\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction was successful\")\n\n if new_transaction[\"transaction_type\"] != \"Payment\":\n error_accumulator.append(f\"[X] Expected transaction type 'Payment', got {new_transaction['transaction_type']}\")\n else:\n success_accumulator.append(\"[C] Transaction type is Payment\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": 5194,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 5194),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", 2027),\n (\"room_id\", 4017),\n (\"check_in_date\", \"2026-01-17\"),\n (\"check_out_date\", \"2026-01-18\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Kayden\"),\n (\"guest_last_name\", \"Haag\"),\n (\"guest_email\", \"kayhaag4@gmail.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"4479028792\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", 82),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": 82,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 82),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"6929\"),\n (\"expiry_month\", 9),\n (\"expiry_year\", 2031),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", 0),\n (\"is_saved_for_reuse\", 0),\n (\"card_number\", \"4894896410276929\"),\n (\"cvv\", \"898\"),\n (\"cardholder_name\", \"Kayden Haag\"),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": 4402,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", 4402),\n (\"booking_id\", 5194),\n (\"payment_method_id\", 82),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'price_calendar': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(f\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_usyjkyjmdxb_1768211891726_vqa2bz829", + "prompt": "I want to book a Bed and Breakfast property in Dubai with a budget ceiling of $200 and with average guest rating of at least 8.5. I want cancellation safety so filter for properties that offer \"Free cancellation up to 5 days before check-in\".\nIf there are multiple options from the search, select the property that has a higher star rating.\n\nCheck room availability at the selected property from 15th to 18th April 2026 for 1 adult. If multiple rooms are available, choose the cheapest one.\n \nCreate a booking for the above dates and adult capacity with the following information:\nGuest Name: Solomon Kalou\nGuest email: Solom.kalour@avb.com\nPhone: +1-9691476385\n\nCreate a new payment method:\nCard number: 4268 7199 5143 6306\nCVV:777 Expiry: 02/30\nFinalize the purchase using this card after agreeing to mandatory conditions.\n\n", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.14", + "verifier_func": "def validate_task(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - Book a Bed and Breakfast in Dubai with specific criteria\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n # Helper function to find new entries in tables with non-deterministic IDs\n def find_new_entries(table_name, id_field=\"id\", filter_conditions=None):\n \"\"\"\n Find entries that were added between seed and current snapshots.\n \"\"\"\n before_query = seed.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n before_query = before_query.eq(key, value)\n before_entries = before_query.select(id_field).all()\n\n after_query = current.table(table_name)\n if filter_conditions:\n for key, value in filter_conditions.items():\n after_query = after_query.eq(key, value)\n after_entries = after_query.all()\n\n before_ids = {entry[id_field] for entry in before_entries}\n new_entries = [entry for entry in after_entries if entry[id_field] not in before_ids]\n\n return new_entries\n\n # Find new booking\n new_bookings = find_new_entries(\"bookings\")\n if len(new_bookings) != 1:\n error_accumulator.append(f\"[X] Expected 1 new booking, found {len(new_bookings)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n \n new_booking = new_bookings[0]\n success_accumulator.append(\"[C] 1 new booking created\")\n\n # Validate check-in and check-out dates (April 15-18, 2026)\n if new_booking[\"check_in_date\"] != \"2026-04-15\":\n error_accumulator.append(f\"[X] Expected check-in date 2026-04-15, got {new_booking['check_in_date']}\")\n else:\n success_accumulator.append(\"[C] Check-in date is correct: 2026-04-15\")\n\n if new_booking[\"check_out_date\"] != \"2026-04-18\":\n error_accumulator.append(f\"[X] Expected check-out date 2026-04-18, got {new_booking['check_out_date']}\")\n else:\n success_accumulator.append(\"[C] Check-out date is correct: 2026-04-18\")\n\n # Validate guest information\n if not normalized_string_comparison(\"solomon\", new_booking[\"guest_first_name\"]):\n error_accumulator.append(f\"[X] Expected guest first name 'Solomon', got {new_booking['guest_first_name']}\")\n else:\n success_accumulator.append(\"[C] Guest first name is correct: Solomon\")\n\n if not normalized_string_comparison(\"kalou\", new_booking[\"guest_last_name\"]):\n error_accumulator.append(f\"[X] Expected guest last name 'Kalou', got {new_booking['guest_last_name']}\")\n else:\n success_accumulator.append(\"[C] Guest last name is correct: Kalou\")\n\n # Email validation (case-insensitive)\n if new_booking[\"guest_email\"].lower() != \"solom.kalour@avb.com\":\n error_accumulator.append(f\"[X] Expected guest email 'solom.kalour@avb.com', got {new_booking['guest_email']}\")\n else:\n success_accumulator.append(\"[C] Guest email is correct\")\n\n # Phone validation\n if new_booking[\"guest_phone_country_code\"] != \"+1\":\n error_accumulator.append(f\"[X] Expected phone country code '+1', got {new_booking['guest_phone_country_code']}\")\n else:\n success_accumulator.append(\"[C] Phone country code is correct: +1\")\n\n phone_digits = ''.join(filter(str.isdigit, str(new_booking[\"guest_phone_number\"] or \"\")))\n if phone_digits != \"9691476385\":\n error_accumulator.append(f\"[X] Expected phone number '9691476385', got {phone_digits}\")\n else:\n success_accumulator.append(\"[C] Phone number is correct: 9691476385\")\n\n # Validate adults count (1 adult)\n if new_booking[\"adults\"] != 1:\n error_accumulator.append(f\"[X] Expected 1 adult, got {new_booking['adults']}\")\n else:\n success_accumulator.append(\"[C] Adults count is correct: 1\")\n\n # Validate booking status\n if new_booking[\"status\"] != \"confirmed\":\n error_accumulator.append(f\"[X] Expected booking status 'confirmed', got {new_booking['status']}\")\n else:\n success_accumulator.append(\"[C] Booking status is confirmed\")\n\n if new_booking[\"payment_status\"] != \"paid\":\n error_accumulator.append(f\"[X] Expected payment status 'paid', got {new_booking['payment_status']}\")\n else:\n success_accumulator.append(\"[C] Payment status is paid\")\n\n # Validate agreed to terms\n if new_booking[\"agreed_to_terms\"] != 1:\n error_accumulator.append(\"[X] Expected agreed_to_terms to be 1\")\n else:\n success_accumulator.append(\"[C] Agreed to terms checkbox checked\")\n\n # Find new payment method\n new_payment_methods = find_new_entries(\"payment_methods\")\n if len(new_payment_methods) != 1:\n error_accumulator.append(f\"[X] Expected 1 new payment method, found {len(new_payment_methods)}\")\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n new_payment_method = new_payment_methods[0]\n success_accumulator.append(\"[C] 1 new payment method created\")\n\n # Validate payment method details\n # Card number: 4268 7199 5143 6306 -> last four should be 6306\n if new_payment_method[\"last_four\"] != \"6306\":\n error_accumulator.append(f\"[X] Expected last four '6306', got {new_payment_method['last_four']}\")\n else:\n success_accumulator.append(\"[C] Card last four digits correct: 6306\")\n\n if new_payment_method[\"card_number\"] != \"4268719951436306\":\n error_accumulator.append(f\"[X] Expected card number '4268719951436306', got {new_payment_method['card_number']}\")\n else:\n success_accumulator.append(\"[C] Card number is correct\")\n\n if new_payment_method[\"cvv\"] != \"777\":\n error_accumulator.append(f\"[X] Expected CVV '777', got {new_payment_method['cvv']}\")\n else:\n success_accumulator.append(\"[C] CVV is correct: 777\")\n\n # Expiry: 02/30\n if new_payment_method[\"expiry_month\"] != 2:\n error_accumulator.append(f\"[X] Expected expiry month 2, got {new_payment_method['expiry_month']}\")\n else:\n success_accumulator.append(\"[C] Expiry month is correct: 2\")\n\n if new_payment_method[\"expiry_year\"] != 2030:\n error_accumulator.append(f\"[X] Expected expiry year 2030, got {new_payment_method['expiry_year']}\")\n else:\n success_accumulator.append(\"[C] Expiry year is correct: 2030\")\n\n # Validate booking references the new payment method\n if new_booking[\"payment_method_id\"] != new_payment_method[\"id\"]:\n error_accumulator.append(f\"[X] Booking should reference payment method {new_payment_method['id']}, got {new_booking['payment_method_id']}\")\n else:\n success_accumulator.append(\"[C] Booking references correct payment method\")\n\n # Find new transaction\n new_transactions = find_new_entries(\"transactions\")\n if len(new_transactions) != 1:\n error_accumulator.append(f\"[X] Expected 1 new transaction, found {len(new_transactions)}\")\n else:\n new_transaction = new_transactions[0]\n success_accumulator.append(\"[C] 1 new transaction created\")\n \n # Validate transaction references the booking\n if new_transaction[\"booking_id\"] != new_booking[\"id\"]:\n error_accumulator.append(f\"[X] Transaction should reference booking {new_booking['id']}, got {new_transaction['booking_id']}\")\n else:\n success_accumulator.append(\"[C] Transaction references correct booking\")\n \n # Validate transaction status\n if new_transaction[\"status\"] != \"Success\":\n error_accumulator.append(f\"[X] Expected transaction status 'Success', got {new_transaction['status']}\")\n else:\n success_accumulator.append(\"[C] Transaction status is Success\")\n\n # Build expected changes for expect_only_v2\n expected_changes = [\n {\n \"table\": \"bookings\",\n \"pk\": new_booking[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_booking[\"id\"]),\n (\"booking_reference\", ...),\n (\"user_id\", 1),\n (\"hotel_id\", new_booking[\"hotel_id\"]),\n (\"room_id\", new_booking[\"room_id\"]),\n (\"check_in_date\", \"2026-04-15\"),\n (\"check_out_date\", \"2026-04-18\"),\n (\"booking_date\", ...),\n (\"adults\", 1),\n (\"children\", 0),\n (\"room_count\", 1),\n (\"base_price\", ...),\n (\"taxes_fees\", ...),\n (\"total_price\", ...),\n (\"currency\", \"USD\"),\n (\"status\", \"confirmed\"),\n (\"payment_status\", \"paid\"),\n (\"special_requests\", ...),\n (\"arrival_time\", ...),\n (\"purpose_of_trip\", ...),\n (\"guest_first_name\", \"Solomon\"),\n (\"guest_last_name\", \"Kalou\"),\n (\"guest_email\", \"solom.kalour@avb.com\"),\n (\"guest_phone_country_code\", \"+1\"),\n (\"guest_phone_number\", \"9691476385\"),\n (\"guest_country\", ...),\n (\"booking_group_id\", ...),\n (\"rooms_close_together\", ...),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"agreed_to_terms\", 1),\n (\"agreed_to_terms_at\", ...),\n (\"agreed_to_marketing\", ...),\n (\"agreed_to_marketing_at\", ...),\n (\"discount_percentage\", ...),\n (\"discount_amount\", ...),\n (\"original_price\", ...),\n (\"rewards_tier\", ...),\n (\"paperless_confirmation\", ...),\n (\"update_account_with_guest_details\", ...),\n (\"rate_plan_id\", ...),\n ]\n },\n {\n \"table\": \"payment_methods\",\n \"pk\": new_payment_method[\"id\"],\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", new_payment_method[\"id\"]),\n (\"user_id\", 1),\n (\"card_type\", \"Visa\"),\n (\"last_four\", \"6306\"),\n (\"expiry_month\", 2),\n (\"expiry_year\", 2030),\n (\"billing_address\", ...),\n (\"billing_postal_code\", ...),\n (\"is_default\", ...),\n (\"is_saved_for_reuse\", ...),\n (\"card_number\", \"4268719951436306\"),\n (\"cvv\", \"777\"),\n (\"cardholder_name\", ...),\n ]\n },\n {\n \"table\": \"transactions\",\n \"pk\": new_transactions[0][\"id\"] if new_transactions else 0,\n \"type\": \"insert\",\n \"fields\": [\n (\"id\", ...),\n (\"booking_id\", new_booking[\"id\"]),\n (\"payment_method_id\", new_payment_method[\"id\"]),\n (\"amount\", ...),\n (\"currency\", \"USD\"),\n (\"transaction_type\", \"Payment\"),\n (\"status\", \"Success\"),\n (\"gateway_response\", ...),\n (\"gateway_transaction_id\", ...),\n (\"processed_at\", ...),\n ]\n }\n ]\n\n ignore_config = IgnoreConfig(\n tables={\"order_tokens\"},\n table_fields={\n 'booking_checkpoints': {'expires_at', 'created_at', 'updated_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'transactions': {'processed_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_fivk48hyxaet_1766206468764_trvh646zb", + "prompt": "I want to create a wishlist of the best 2 hotels currently featured but I don't really agree that average rating or star rating is a better measurement system. I would like to take the average of the two ratings for each location and use that to find the best two locations. Create a wishlist called Best and add these two.", + "env_id": "booking", + "version": "v0.0.74", + "data_id": "kinesis", + "data_version": "v0.0.15", + "verifier_func": "def validate_task_fivk48hyxaet_1766206468764_trvh646zb(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly\n \n Task: Create a wishlist called \"Best\" with the top 2 hotels based on the \n average of (average_rating + star_rating) / 2.\n \"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA ####\n\n # 1. Check that a wishlist called \"Best\" was created\n # Note: wishlist_lists.user_id is TEXT type based on schema\n wishlist = current.table(\"wishlist_lists\").eq(\"user_id\", \"1\").eq(\"list_name\", \"Best\").first()\n if not wishlist:\n error_accumulator.append(\"[X] Wishlist 'Best' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'Best' was created\")\n\n # 2. Check that exactly 2 hotels were saved to the \"Best\" wishlist\n # Note: saved_properties.user_id is INTEGER type based on schema\n saved_properties = current.table(\"saved_properties\").eq(\"user_id\", 1).eq(\"list_name\", \"Best\").all()\n if len(saved_properties) != 2:\n error_accumulator.append(f\"[X] Expected 2 hotels in 'Best' wishlist, found {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] Exactly 2 hotels were added to 'Best' wishlist\")\n\n # 3. Verify the correct hotel IDs were added (2932 and 6370 based on diff)\n saved_hotel_ids = {prop[\"hotel_id\"] for prop in saved_properties}\n expected_hotel_ids = {2932, 6370}\n \n if saved_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotels {expected_hotel_ids} in wishlist, found {saved_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] Correct hotels {expected_hotel_ids} were added to wishlist\")\n\n # 4. Verify the hotels exist in the database\n hotel_2932 = current.table(\"hotels\").eq(\"id\", 2932).first()\n hotel_6370 = current.table(\"hotels\").eq(\"id\", 6370).first()\n \n if hotel_2932 and hotel_6370:\n success_accumulator.append(\"[C] Both saved hotels exist in the database\")\n else:\n error_accumulator.append(\"[X] Could not find one or both saved hotels in database\")\n\n #### GENERATE THE EXPECTED CHANGES ARRAY ####\n # Note: For composite primary keys, expect_only_v2 needs the pk as a tuple\n expected_changes = [\n {\n \"table\": \"wishlist_lists\",\n \"pk\": (\"1\", \"Best\"), # Composite primary key (user_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", \"1\"),\n (\"list_name\", \"Best\"),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 2932, \"Best\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 2932),\n (\"list_name\", \"Best\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n {\n \"table\": \"saved_properties\",\n \"pk\": (1, 6370, \"Best\"), # Composite primary key (user_id, hotel_id, list_name)\n \"type\": \"insert\",\n \"fields\": [\n (\"user_id\", 1),\n (\"hotel_id\", 6370),\n (\"list_name\", \"Best\"),\n (\"notes\", None),\n (\"saved_at\", ...),\n ]\n },\n ]\n\n ignore_config = IgnoreConfig(\n tables=set(),\n table_fields={\n 'booking_checkpoints': {'created_at', 'updated_at', 'expires_at'},\n 'booking_extras': {'created_at'},\n 'bookings': {'created_at', 'updated_at'},\n 'cancellations': {'created_at'},\n 'facilities': {'created_at'},\n 'generation_checkpoints': {'created_at'},\n 'hosts': {'created_at'},\n 'hotel_images': {'created_at'},\n 'hotels': {'created_at', 'updated_at'},\n 'locations': {'created_at'},\n 'messages': {'created_at'},\n 'nearby_attractions': {'created_at'},\n 'payment_methods': {'created_at'},\n 'property_policies': {'created_at'},\n 'reviews': {'created_at'},\n 'rewards_program': {'updated_at'},\n 'room_images': {'created_at'},\n 'rooms': {'created_at'},\n 'users': {'created_at'},\n 'wishlist_lists': {'created_at'},\n 'saved_properties': {'saved_at'}\n }\n )\n\n try:\n seed.diff(current, ignore_config).expect_only_v2(expected_changes)\n success_accumulator.append(\"[C] Expected changes were found in the database diff\")\n except AssertionError as e:\n error_accumulator.append(f\"[X] Database diff validation failed: {e}\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + }, + { + "key": "task_kebgzwxztwx_1765588433290_c1fi33mda", + "prompt": "Search for the hotel deals. Pick the ones in US that allow bringing along a small dog, and has star ratings higher than 3.5. Add them to a new wishlist called 'US pet friendly deals'.", + "env_id": "booking", + "version": "v0.0.65", + "data_id": "kinesis", + "data_version": "v0.0.4", + "verifier_func": "def validate_task_kebgzwxztwx_1765588433290_c1fi33mda(\n env: Environment, final_answer: str | None = None\n) -> int:\n \"\"\"Validate that the task was completed correctly - adding US pet-friendly hotels with star rating > 3.5 to a wishlist\"\"\"\n error_accumulator = []\n success_accumulator = []\n\n env.instance.load()\n seed = env.db(\"seed\")\n current = env.db(\"current\")\n\n #### VALIDATE THE SOLUTION CRITERIA AGAINST THE DIFF ####\n\n # 1. Verify the wishlist 'US pet friendly deals' was created\n # Note: Based on diff, user_id is stored as string '1' in wishlist_lists\n wishlist_entry = current.table(\"wishlist_lists\").eq(\"list_name\", \"US pet friendly deals\").first()\n \n if wishlist_entry is None:\n error_accumulator.append(\"[X] Wishlist 'US pet friendly deals' was not created\")\n else:\n success_accumulator.append(\"[C] Wishlist 'US pet friendly deals' was created\")\n\n # 2. Verify that saved_properties entries were added to this wishlist\n saved_properties = current.table(\"saved_properties\").eq(\"list_name\", \"US pet friendly deals\").all()\n \n if len(saved_properties) == 0:\n error_accumulator.append(\"[X] No properties saved to the 'US pet friendly deals' wishlist\")\n else:\n success_accumulator.append(f\"[C] {len(saved_properties)} properties were saved to the wishlist\")\n\n # Based on the diff, exactly 3 hotels should be added: hotel_id 13899, 15586, 5369\n expected_hotel_ids = {13899, 15586, 5369}\n actual_hotel_ids = {sp[\"hotel_id\"] for sp in saved_properties}\n \n if actual_hotel_ids != expected_hotel_ids:\n error_accumulator.append(f\"[X] Expected hotels {expected_hotel_ids} but got {actual_hotel_ids}\")\n else:\n success_accumulator.append(f\"[C] All expected hotels ({expected_hotel_ids}) were saved to wishlist\")\n\n # 3. Verify the count is exactly 3\n if len(saved_properties) != 3:\n error_accumulator.append(f\"[X] Expected exactly 3 saved properties but got {len(saved_properties)}\")\n else:\n success_accumulator.append(\"[C] Exactly 3 properties were saved\")\n\n # 4. Verify each saved property has the correct list_name and user_id\n for sp in saved_properties:\n if sp[\"user_id\"] != 1:\n error_accumulator.append(f\"[X] Saved property for hotel {sp['hotel_id']} has wrong user_id: {sp['user_id']}\")\n if sp[\"list_name\"] != \"US pet friendly deals\":\n error_accumulator.append(f\"[X] Saved property for hotel {sp['hotel_id']} has wrong list_name: {sp['list_name']}\")\n\n # 5. Verify the wishlist didn't exist in seed (new creation)\n seed_wishlist = seed.table(\"wishlist_lists\").eq(\"list_name\", \"US pet friendly deals\").first()\n if seed_wishlist is not None:\n error_accumulator.append(\"[X] Wishlist 'US pet friendly deals' already existed in seed - should be new\")\n else:\n success_accumulator.append(\"[C] Wishlist 'US pet friendly deals' is newly created (didn't exist in seed)\")\n\n # 6. Verify these hotel_ids didn't exist in saved_properties for this wishlist in seed\n for hotel_id in expected_hotel_ids:\n seed_saved = seed.table(\"saved_properties\").eq(\"hotel_id\", hotel_id).eq(\"list_name\", \"US pet friendly deals\").first()\n if seed_saved is not None:\n error_accumulator.append(f\"[X] Hotel {hotel_id} was already in 'US pet friendly deals' wishlist in seed\")\n\n if len(error_accumulator) > 0:\n print(f\">>> ERROR_ACCUMULATOR >>>\\n{error_accumulator}\\n<<< ERROR_ACCUMULATOR <<<\")\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_FAILED_SCORE\n\n print(f\">>> SUCCESS_ACCUMULATOR >>>\\n{success_accumulator}\\n<<< SUCCESS_ACCUMULATOR <<<\")\n return TASK_SUCCESSFUL_SCORE\n", + "task_modality": "tool_use" + } +] \ No newline at end of file diff --git a/skyrl-train/docs/README.md b/skyrl-train/docs/README.md new file mode 100644 index 0000000000..cb1d82ae97 --- /dev/null +++ b/skyrl-train/docs/README.md @@ -0,0 +1,11 @@ +To build the docs, run: + +```bash +bash build.sh +``` + +In case the default port is used, you can do: + +```bash +bash build.sh --port 8010 +``` diff --git a/skyrl-train/docs/algorithms/custom_algorithms.rst b/skyrl-train/docs/algorithms/custom_algorithms.rst new file mode 100644 index 0000000000..09030ef796 --- /dev/null +++ b/skyrl-train/docs/algorithms/custom_algorithms.rst @@ -0,0 +1,124 @@ +Implementing Custom Algorithms +============================== + +SkyRL-Train provides a registry system for easily implementing custom algorithms (advantage estimators, policy loss) without modifying the core codebase. +The API for the registry system can be found in the :doc:`registry API <../api/registry>`. +Example scripts of using the registry can be found at :code_link:`examples/algorithms/`. + +Additionally for more control, you can subclass the ``BasePPOExp`` class from :code_link:`skyrl_train/entrypoints/main_base.py` and override the ``BasePPOExp.get_trainer`` method to return a custom trainer class. +This allows you to have full control over the training loop and implementing custom reward functions and output postprocessing. +We provide an example of this for applying custom reward penalties in our :ref:`DAPO example `. + +Registering a Custom Advantage Estimator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can register custom advantage estimators using either a decorator or the registry directly: + +.. code-block:: python + + from skyrl_train.utils.ppo_utils import register_advantage_estimator, AdvantageEstimatorRegistry + import torch + + # Using the decorator + @register_advantage_estimator("simple_baseline") + def compute_simple_baseline_advantage( + token_level_rewards: torch.Tensor, response_mask: torch.Tensor, index: np.ndarray, **kwargs + ): + with torch.no_grad(): + response_rewards = (token_level_rewards * response_mask).sum(dim=-1, keepdim=True) + + # Simple baseline: use the mean reward across the batch + baseline = response_rewards.mean() + advantages = (response_rewards - baseline) * response_mask + returns = advantages.clone() + + return advantages, returns + + # Or register directly + def another_estimator(**kwargs): + # Implementation here + pass + + AdvantageEstimatorRegistry.register("direct_registration", another_estimator) + +Registering a Custom Policy Loss +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Similarly, you can register custom policy loss functions: + +.. code-block:: python + + from skyrl_train.utils.ppo_utils import register_policy_loss, PolicyLossRegistry + + @register_policy_loss("reinforce") + def compute_reinforce_policy_loss(log_probs, old_log_probs, advantages, config, loss_mask=None, rollout_log_probs=None): + # Your custom policy loss implementation (like REINFORCE) + loss = (-log_probs * advantages).mean() + # return loss and clip ratio + return loss, 0.0 + +Registry Ray Distribution +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The registry system handles Ray actor synchronization when Ray is initialized. Functions registered on one process will be available to all Ray actors: + +.. code-block:: python + + import ray + from skyrl_train.utils.ppo_utils import AdvantageEstimatorRegistry, sync_registries + + # Register a function on the main process + def my_function(**kwargs): + # A dummy function for demonstration + pass + AdvantageEstimatorRegistry.register("my_function", my_function) + + # After Ray is initialized, we sync the registries to a named ray actor (in utils/utils.py::initialize_ray) + ray.init() + sync_registries() + + @ray.remote(num_cpus=1) + def skyrl_entrypoint(cfg: DictConfig): + # Function is now available on all Ray processes + available_functions = AdvantageEstimatorRegistry.list_available() # will include "my_function" + + exp = BasePPOExp(cfg) + exp.run() + +.. _custom-trainer: + +Creating a Custom Trainer +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To create a custom trainer for full control of your training loop, you can subclass the ``BasePPOExp`` class from :code_link:`skyrl_train/entrypoints/main_base.py` and override the ``BasePPOExp.get_trainer`` method to return a custom trainer class. +We show the outline of creating a custom trainer below, and you can find a full running example in our :ref:`DAPO example `. + +.. code-block:: python + + class CustomTrainer(RayPPOTrainer): + @torch.no_grad() + def postprocess_generator_output(self, generator_output: GeneratorOutput, uids: List[str]) -> GeneratorOutput: + # apply custom reward penalties + ... + # use base class impl for metrics and per-token reward conversion + return super().postprocess_generator_output(generator_output, uids) + + class CustomExp(BasePPOExp): + def get_trainer(self, *args, **kwargs): + return CustomTrainer(*args, **kwargs) + + @ray.remote(num_cpus=1) + def skyrl_entrypoint(cfg: DictConfig): + exp = CustomExp(cfg) + exp.run() + + @hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) + def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + if __name__ == "__main__": + main() \ No newline at end of file diff --git a/skyrl-train/docs/algorithms/dapo.rst b/skyrl-train/docs/algorithms/dapo.rst new file mode 100644 index 0000000000..7146dab246 --- /dev/null +++ b/skyrl-train/docs/algorithms/dapo.rst @@ -0,0 +1,113 @@ +DAPO +==== + +The `DAPO `_ (Decoupled Clip and Dynamic Sampling Policy Optimization) algorithm consists of the following components on top of a GRPO baseline: + +- **Clip-Higher**: Promotes the diversity of the system and avoids entropy collapse; +- **Dynamic Sampling**: Improves training efficiency and stability; +- **Token-Level Policy Gradient Loss**: Critical in long-CoT RL scenarios; +- **Overlong Reward Shaping**: Reduces reward noise and stabilizes training. + +In this guide, we walk through how to enable each of these components in SkyRL. We provide a simple example script for training DAPO on GSM8K in :code_link:`examples/algorithms/dapo/`. + +Clip-Higher +~~~~~~~~~~~ +To use clip-higher, you can simply configure ``trainer.algorithm.eps_clip_high`` separately from ``trainer.algorithm.eps_clip_low``. + +.. code-block:: yaml + + trainer: + algorithm: + eps_clip_low: 0.2 + eps_clip_high: 0.28 + +Dynamic Sampling +~~~~~~~~~~~~~~~~ +In DAPO style dynamic sampling, we sample rollouts until we have a full batch with non-zero advantages (meaning that we have a non-zero std deviation of rewards for the n rollouts for a given prompt). + +To configure DAPO style dynamic sampling, you can set ``trainer.algorithm.dynamic_sampling.type`` to ``filter`` and configure ``trainer.algorithm.dynamic_sampling.max_sample_batches`` to the maximum number of batches to sample. +If ``max_sample_batches > 0`` and is exceeded, SkyRL-Train will raise an error. If ``max_sample_batches <= 0``, SkyRL-Train will sample until a full batch with non-zero advantages is accumulated. + +.. code-block:: yaml + + trainer: + algorithm: + dynamic_sampling: + type: filter + max_sample_batches: 30 + +Token-Level Policy Gradient Loss +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +DAPO uses token-level policy gradient loss, which can be enabled by setting ``trainer.algorithm.loss_reduction`` to ``token_mean``. This is the default setting in SkyRL-Train. + +.. code-block:: yaml + + trainer: + algorithm: + loss_reduction: "token_mean" + +Overlong Reward Shaping +~~~~~~~~~~~~~~~~~~~~~~~~ +The DAPO paper proposes two methods for overlong reward shaping: + +- **Overlong Filtering**: Sets loss mask to be all zeros for responses that exceed the max response length. +- **Soft Overlong Punishment**: Penalizes responses that exceed the max response length within a punishment interval. Within this interval, the longer the response, the greater the punishment it receives. This penalty is added to the original reward. + +Overlong Filtering +------------------ + +To enable overlong filtering, which sets the loss mask to be all zeros for responses that do not finish with a stop token (i.e. responses that are too long), you can set ``generator.apply_overlong_filtering`` to ``true``. + +.. code-block:: yaml + + generator: + apply_overlong_filtering: true + +.. _dapo-custom-trainer: + +Soft Overlong Punishment +------------------------ + +To enable soft overlong punishment, you can create a custom trainer class and override the ``RayPPOTrainer`` ``postprocess_generator_output`` method to additionally apply soft overlong punishment to rewards. +We provide an example of this in :code_link:`examples/algorithms/dapo/main_dapo.py`, and show an overview of the implementation below: + +.. code-block:: python + :caption: ``skyrl_train/examples/algorithms/dapo/main_dapo.py`` + + class DAPOTrainer(RayPPOTrainer): + @torch.no_grad() + def postprocess_generator_output(self, generator_output: GeneratorOutput, uids: List[str]) -> GeneratorOutput: + # apply soft overlong punishment + overlong_buffer_len = self.cfg.trainer.algorithm.overlong_buffer.len + overlong_buffer_penalty_factor = self.cfg.trainer.algorithm.overlong_buffer.penalty_factor + ... + # use base class impl for metrics and per-token reward conversion + return super().postprocess_generator_output(generator_output, uids) + + class DAPOExp(BasePPOExp): + def get_trainer(self, *args, **kwargs): + return DAPOTrainer(*args, **kwargs) + + @ray.remote(num_cpus=1) + def skyrl_entrypoint(cfg: DictConfig): + exp = DAPOExp(cfg) + exp.run() + +To add the overlong buffer length and penalty factor parameters to the config, you can add the following lines to the ``run_dapo_gsm8k.sh`` script: + +.. code-block:: bash + :caption: ``skyrl_train/examples/algorithms/dapo/run_dapo_gsm8k.sh`` + + +trainer.algorithm.overlong_buffer.len=512 \ + +trainer.algorithm.overlong_buffer.penalty_factor=1.0 \ + +Launching a DAPO Training Run +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An example script with all of the above components enabled for basic GSM8K training can be found at :code_link:`examples/algorithms/dapo/run_dapo_gsm8k.sh`. + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + bash examples/algorithms/dapo/run_dapo_gsm8k.sh + diff --git a/skyrl-train/docs/api/data.rst b/skyrl-train/docs/api/data.rst new file mode 100644 index 0000000000..186c80d25c --- /dev/null +++ b/skyrl-train/docs/api/data.rst @@ -0,0 +1,46 @@ +Data Interface +============== + + +Trainer APIs +------------ +Our interface for training data is modelled after `DataProto `_ in VERL but is much simpler. + +.. autoclass:: skyrl_train.training_batch.TensorBatch + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.training_batch.TrainingInput + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.training_batch.TrainingInputBatch + :members: + :member-order: bysource + :undoc-members: + +.. autoclass:: skyrl_train.training_batch.TrainingOutputBatch + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + + +Generator APIs +-------------- + +.. autoclass:: skyrl_train.generators.GeneratorInput + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.generators.GeneratorOutput + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/skyrl-train/docs/api/entrypoint.rst b/skyrl-train/docs/api/entrypoint.rst new file mode 100644 index 0000000000..5284b60219 --- /dev/null +++ b/skyrl-train/docs/api/entrypoint.rst @@ -0,0 +1,18 @@ +Entrypoint API +============== + +The main entrypoint is the `BasePPOExp` class which runs the main training loop. + +.. autoclass:: skyrl_train.entrypoints.main_base.BasePPOExp + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +The second entrypoint (evaluation-only) is the `EvalOnlyEntrypoint` class which runs evaluation without training. + +.. autoclass:: skyrl_train.entrypoints.main_generate.EvalOnlyEntrypoint + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/skyrl-train/docs/api/env.rst b/skyrl-train/docs/api/env.rst new file mode 100644 index 0000000000..d16e75e2c2 --- /dev/null +++ b/skyrl-train/docs/api/env.rst @@ -0,0 +1,28 @@ +Environment API +=============== + +The core APIs are given below. + +.. autoclass:: skyrl_gym.core.Env + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_gym.core.EnvStepOutput + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_gym.envs.base_text_env.BaseTextEnv + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_gym.envs.base_text_env.BaseTextEnvStepOutput + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/skyrl-train/docs/api/env_vars.rst b/skyrl-train/docs/api/env_vars.rst new file mode 100644 index 0000000000..4ebb2100de --- /dev/null +++ b/skyrl-train/docs/api/env_vars.rst @@ -0,0 +1,5 @@ +Environment Variables +===================== + +.. automodule:: skyrl_train.env_vars + :members: \ No newline at end of file diff --git a/skyrl-train/docs/api/generator.rst b/skyrl-train/docs/api/generator.rst new file mode 100644 index 0000000000..042db74123 --- /dev/null +++ b/skyrl-train/docs/api/generator.rst @@ -0,0 +1,22 @@ +Generator API +============= + +The Generator is responsible for generating trajectories for training. + +Core APIs +--------- + +.. autoclass:: skyrl_train.generators.GeneratorInterface + :members: + :member-order: bysource + :undoc-members: + +.. autoclass:: skyrl_train.inference_engines.base.InferenceEngineInterface + :members: + :member-order: bysource + :undoc-members: + +.. autoclass:: skyrl_train.inference_engines.inference_engine_client.InferenceEngineClient + :members: + :member-order: bysource + :undoc-members: diff --git a/skyrl-train/docs/api/registry.rst b/skyrl-train/docs/api/registry.rst new file mode 100644 index 0000000000..3b9f3fd26c --- /dev/null +++ b/skyrl-train/docs/api/registry.rst @@ -0,0 +1,60 @@ +Algorithm Registry API +===================================== + +The registry system in SkyRL Train provides a way to register and manage custom algorithm functions (like advantage estimators and policy loss functions) across distributed Ray environments. This system allows users to extend the framework with custom implementations without modifying the core codebase. + +Base Registry Classes +--------------------- + +.. autoclass:: skyrl_train.utils.ppo_utils.BaseFunctionRegistry + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.utils.ppo_utils.RegistryActor + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autofunction:: skyrl_train.utils.ppo_utils.sync_registries + +Advantage Estimator Registry +----------------------------- + +The advantage estimator registry manages functions that compute advantages and returns for reinforcement learning algorithms. + +.. autoclass:: skyrl_train.utils.ppo_utils.AdvantageEstimatorRegistry + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.utils.ppo_utils.AdvantageEstimator + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autofunction:: skyrl_train.utils.ppo_utils.register_advantage_estimator + + +Policy Loss Registry +-------------------- + +The policy loss registry manages functions that compute policy losses for PPO and related algorithms. + +.. autoclass:: skyrl_train.utils.ppo_utils.PolicyLossRegistry + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.utils.ppo_utils.PolicyLossType + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autofunction:: skyrl_train.utils.ppo_utils.register_policy_loss diff --git a/skyrl-train/docs/api/tools.rst b/skyrl-train/docs/api/tools.rst new file mode 100644 index 0000000000..d3a2cca05b --- /dev/null +++ b/skyrl-train/docs/api/tools.rst @@ -0,0 +1,14 @@ +Tool API +======== + +.. autoclass:: skyrl_gym.tools.core.ToolGroup + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_gym.tools.core.tool + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/skyrl-train/docs/api/trainer.rst b/skyrl-train/docs/api/trainer.rst new file mode 100644 index 0000000000..cda2a28fa5 --- /dev/null +++ b/skyrl-train/docs/api/trainer.rst @@ -0,0 +1,56 @@ +Trainer API +=========== + +The Trainer drives the training loop. + + +Trainer Class +------------- + +.. autoclass:: skyrl_train.trainer.RayPPOTrainer + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + + +Dispatch APIs +------------- + +.. autoclass:: skyrl_train.distributed.dispatch.Dispatch + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.distributed.dispatch.MeshDispatch + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.distributed.dispatch.PassThroughDispatch + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + + +Actor APIs +----------- + +The base worker abstraction in SkyRL is the + +.. autoclass:: skyrl_train.workers.worker.Worker + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + +.. autoclass:: skyrl_train.workers.worker.PPORayActorGroup + :members: + :member-order: bysource + :undoc-members: + :show-inheritance: + + diff --git a/skyrl-train/docs/build.sh b/skyrl-train/docs/build.sh new file mode 100755 index 0000000000..e0d9727f52 --- /dev/null +++ b/skyrl-train/docs/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +# Ensure a valid UTF-8 locale to avoid "unsupported locale setting" +export LC_ALL=${LC_ALL:-C.UTF-8} +export LANG=${LANG:-C.UTF-8} +export LANGUAGE=${LANGUAGE:-C.UTF-8} + +# Build and serve the documentation with live reload +# Usage: ./build.sh [--build-only] +# --build-only: Build docs without starting the live server + +cd "$(dirname "$0")" # Ensure we're in the docs directory + +if [ "$1" = "--build-only" ]; then + uv run --extra docs --isolated sphinx-build -b html . _build/html $@ +else + uv run --extra docs --isolated sphinx-autobuild . _build/html $@ +fi diff --git a/skyrl-train/docs/checkpointing-logging/checkpointing.rst b/skyrl-train/docs/checkpointing-logging/checkpointing.rst new file mode 100644 index 0000000000..cb84397c1a --- /dev/null +++ b/skyrl-train/docs/checkpointing-logging/checkpointing.rst @@ -0,0 +1,143 @@ +Checkpointing +============= + +SkyRL provides checkpointing features to resume training from a previous state. Training state is saved at regular intervals and provides flexible configuration options for checkpoint management. + +Checkpointed state can be stored in the local file system or uploaded to cloud storage (S3, GCP). + +What State is Saved +------------------- + +SkyRL saves several types of state to enable complete training resumption: + +**Model States** + - **Policy Model**: Model parameters, optimizer state, and learning rate scheduler state + - **Critic Model**: Model parameters, optimizer state, and learning rate scheduler state (if critic is enabled) + - **Reference Model**: Not checkpointed (recreated from policy model) + +**Training State** + - **Global Step**: Current training step counter + - **Configuration**: Complete training configuration used + - **Dataloader State**: Current position in dataset iteration (enables resuming from exact data position) + +Directory Structure +------------------- + +The checkpointing directory structure depends on the training backend used. + +FSDP Checkpointing +~~~~~~~~~~~~~~~~~~~~ + +FSDP checkpoints are organized according to the following directory hierarchy: + +.. code-block:: + + {ckpt_path}/ + ├── latest_ckpt_global_step.txt # Holds the global step of the latest checkpoint + ├── global_step_10/ # Checkpoint at training step 10 + │ ├── policy/ # Policy model checkpoint directory + │ │ ├── fsdp_config.json # stores fsdp version and world size + │ │ ├── huggingface/ # HuggingFace config and tokenizer + │ │ ├── config.json # model config + │ │ ├── tokenizer_config.json # tokenizer config + │ │ ├── generation_config.json # generation config + │ │ ├── ... # other tokenizer config files + │ │ ├── model_state.pt # Model parameters + │ │ ├── optimizer_state.pt # Optimizer state + │ │ └── lr_scheduler_state.pt # Learning rate scheduler state + │ ├── critic/ # Critic model checkpoint (if enabled) + │ │ ├── fsdp_config.json + │ │ ├── huggingface/ + │ │ ├── model_state.pt + │ │ ├── optimizer_state.pt + │ │ └── lr_scheduler_state.pt + │ ├── data.pt # Dataloader state + │ └── trainer_state.pt # High-level trainer state + ├── global_step_20/ # Checkpoint at training step 20 + │ └── ... + └── global_step_30/ # Checkpoint at training step 30 + └── ... + +.. _megatron-checkpointing: + +Megatron Checkpointing +~~~~~~~~~~~~~~~~~~~~~~ + +Megatron checkpoints are handled by the Megatron `dist_checkpointing `_ library to perform checkpointing in parallel across ranks. +This comes with support for reloading checkpoints in a different parallelism scheme than the one they were saved in. + +.. code-block:: + + {ckpt_path}/ + ├── latest_ckpt_global_step.txt # Holds the global step of the latest checkpoint + ├── global_step_10/ # Checkpoint at training step 10 + │ ├── policy/ # Policy model checkpoint directory + │ │ ├── metadata.json # Megatron checkpoint metadata + │ │ ├── huggingface/ # HuggingFace config and tokenizer + │ │ ├── __0_0.distcp # Megatron checkpoint files + │ │ ├── __0_1.distcp + │ │ └── ... + ├── global_step_20/ # Checkpoint at training step 20 + │ └── ... + └── global_step_30/ # Checkpoint at training step 30 + └── ... + +Key Configuration Parameters +---------------------------- + +Checkpointing behavior is controlled by several parameters in the YAML configuration (see :doc:`../configuration/config` for the full training config): + +**Checkpoint Saving** + +``ckpt_interval`` + - **Default**: ``10`` + - **Purpose**: Save checkpoints every N training steps + +``ckpt_path`` + - **Default**: ``"${oc.env:HOME}/ckpts/"`` + - **Purpose**: Base directory where all checkpoints are stored + - **Options**: + - Local file system path (e.g., ``/path/to/ckpts/``) + - Cloud storage path (S3, GCP) (e.g., ``s3://path/to/ckpts/``) + +**Checkpoint Cleanup** + +``max_ckpts_to_keep`` + - **Default**: ``-1`` (keep all checkpoints) + - **Purpose**: Limit number of stored checkpoints to save disk space + - **Options**: + + - ``-1``: Keep all checkpoints indefinitely + - ``N`` (positive integer): Keep only the last N checkpoints, automatically delete older ones + +**Training Resumption** + +``resume_mode`` + - **Default**: ``"latest"`` + - **Purpose**: Controls how training resumption works + - **Options**: + + - ``"none"`` or ``null``: Start training from scratch, ignore existing checkpoints + - ``"latest"``: Automatically resume from the most recent checkpoint + - ``"from_path"``: Resume from a specific checkpoint (requires ``resume_path``) + +``resume_path`` + - **Default**: ``null`` + - **Purpose**: Specific checkpoint directory to resume from (only used when ``resume_mode: "from_path"``) + - **Format**: Must point to a ``global_step_N`` directory + +HuggingFace Model Export +------------------------ + +In addition to checkpointing, users can optionally save the policy model in HuggingFace safetensors format at regular intervals. + +**Configuration Parameters:** + +``hf_save_interval`` + - **Default**: ``-1`` (disabled) + - **Purpose**: Save HuggingFace format policy models every N training steps + +``export_path`` + - **Default**: ``"${oc.env:HOME}/exports/"`` + - **Purpose**: Base directory where HuggingFace models and other artifacts are saved + - **Structure**: Models are saved to ``{export_path}/global_step_{N}/policy/`` diff --git a/skyrl-train/docs/conf.py b/skyrl-train/docs/conf.py new file mode 100644 index 0000000000..e8631e0a89 --- /dev/null +++ b/skyrl-train/docs/conf.py @@ -0,0 +1,51 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("..")) + +project = "SkyRL" +copyright = "2025, NovaSkyAI" +author = "NovaSkyAI" + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", + "sphinx.ext.extlinks", + "sphinx_autodoc_typehints", + "myst_parser", + "sphinx_copybutton", +] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] + +# Intersphinx mapping +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "torch": ("https://pytorch.org/docs/stable/", None), +} + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = True +napoleon_include_private_with_doc = True +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = True +napoleon_use_admonition_for_notes = True +napoleon_use_admonition_for_references = True +napoleon_use_ivar = True +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_type_aliases = None + +# External links configuration +extlinks = { + "code_link": ("https://github.com/NovaSky-AI/skyrl/blob/main/skyrl-train/%s", "%s"), + "skyrl_gym_link": ("https://github.com/NovaSky-AI/skyrl/blob/main/skyrl-gym/%s", "%s"), +} diff --git a/skyrl-train/docs/configuration/config.rst b/skyrl-train/docs/configuration/config.rst new file mode 100644 index 0000000000..ab12315fe7 --- /dev/null +++ b/skyrl-train/docs/configuration/config.rst @@ -0,0 +1,652 @@ +Configuration Overview +====================== + +Data Configuration +------------------ + +.. code-block:: yaml + + data: + train_data: ["${oc.env:HOME}/data/gsm8k/train.parquet"] + val_data: ["${oc.env:HOME}/data/gsm8k/validation.parquet"] + +- ``data.train_data``: A list of files for the training dataset. +- ``data.val_data``: A list of files for the evaluation dataset. + +A dataset file can be a path to a parquet or json file, or the name of a Hugging Face dataset. + +.. note:: + Currently, all datasets are loaded into memory, so the dataset size is limited by available CPU memory on a worker node. + + +Model Placement Configuration +----------------------------- + +.. code-block:: yaml + + placement: + colocate_all: true + colocate_policy_ref: true + colocate_critic_reward: false + policy_num_nodes: 1 + policy_num_gpus_per_node: 4 + critic_num_nodes: 1 + critic_num_gpus_per_node: 4 + ref_num_nodes: 1 + ref_num_gpus_per_node: 4 + reward_num_nodes: 1 + reward_num_gpus_per_node: 4 + +For an in-depth guide on model placement and colocation, please refer to the :doc:`model placement and colocation guide `. + +General Training Configuration +------------------------------ + +.. code-block:: yaml + + epochs: 1 # Number of passes over the full dataset + update_epochs_per_batch: 1 + train_batch_size: 1024 + policy_mini_batch_size: 256 + critic_mini_batch_size: 256 + micro_train_batch_size_per_gpu: 1 + micro_forward_batch_size_per_gpu: 1 + update_ref_every_epoch: false + use_sample_packing: true + max_prompt_length: 512 + gradient_checkpointing: true + seed: 42 + + +- ``epochs``: Number of epochs/ passes over the full dataset (similar to SFT) +- ``update_epochs_per_batch``: Number of gradient update passes over each training batch. This is equivalent to the concept of "PPO epochs" where you iterate over the same experience multiple times. +- ``train_batch_size``: Batch size of prompts used for each dataloader step. +- ``policy_mini_batch_size``: Mini batch size used during RL training step. Each mini batch corresponds to one optimizer step. For example, if the ``train_batch_size`` is 4 and ``policy_mini_batch_size`` is 2, then there will be 2 optimizer steps (i.e., model updates) for a given training batch. Note that is this the global mini batch size. The actual size of the mini batch per worker would be ``policy_mini_batch_size/ number of DP ranks`` +- ``critic_mini_batch_size``: Similar to ``policy_mini_batch_size`` but for the critic model (if applicable). Note that in general, the critic model can tolerate off-policy updates more than the policy. Thus, you would want to set ``critic_mini_batch_size`` to be lower compared ``policy_mini_batch_size`` (i.e., more critic updates). +- ``micro_train_batch_size_per_gpu``: Micro batch size during training step. This is common for both policy and critic models. Each mini batch is split into micro batches of this size, gradients are computed and accumulated over these micro batches. +- ``micro_forward_batch_size_per_gpu``: Micro batch size during forward pass (i.e., for log probability or value computation). This is common for both policy and critic models. Each mini batch is split into micro batches of this size, model forward pass is performed over these micro batches. +- ``update_ref_every_epoch``: Whether to update the reference model every epoch. +- ``use_sample_packing``: Whether to use sample packing during model forward pass (common for all models). +- ``max_prompt_length``: Maximum prompt length during training. Longer prompts will be truncated. +- ``gradient_checkpointing``: Whether to use gradient checkpointing. +- ``seed``: Random seed for training. + + +.. tip:: + If you're facing issues with tuning the right values for ``micro_train_batch_size_per_gpu``, ``policy_mini_batch_size`` and ``micro_forward_batch_size_per_gpu``, see ``utils/utils.py::validate_batch_sizes`` for details on constraints. + +Evaluation Configuration +------------------------------ +.. code-block:: yaml + + eval_batch_size: 1024 + eval_before_train: true + eval_interval: 5 # Set to -1 to disable evaluation. + +- ``eval_batch_size``: Batch size for evaluation. +- ``eval_before_train``: Whether to evaluate the model before training. +- ``eval_interval``: The frequency of evaluating the model with the validation dataset (in terms of number of steps). If set to ``-1``, evaluation will not be performed. + +.. note:: + If multiple validation datasets are provided (e.g. ``data.val_data="['$DATA_DIR/validation1.parquet', '$DATA_DIR/validation2.parquet']" \``), + then the evaluation will be performed on all of them. The metrics for each dataset, and the aggregated metrics, will + all be logged in WandB. If ``dump_eval_results`` is set to ``true``, the per-dataset and aggregated results will be + dumped. + +Checkpoint Configuration +--------------------------------------- + +.. code-block:: yaml + + resume_mode: latest # null/"none", "latest", "from_path" + resume_path: null + ckpt_path: "${oc.env:HOME}/ckpts/" # Local directory path or cloud storage path (S3, GCP) for resumable training checkpoints (model state, optimizer state, etc.) + max_ckpts_to_keep: -1 # -1 to keep all checkpoints, N to keep the last N checkpoints + ckpt_interval: 10 # Save full training checkpoint every `ckpt_interval` steps. + hf_save_interval: -1 # Save HF format model(s)every `hf_save_interval` steps. + export_path: "${oc.env:HOME}/exports/" # Path for exported artifacts (HF models, debug dumps, etc.) + project_name: "skyrl" + run_name: "test_run" + logger: "wandb" + +For an in-depth guide on checkpointing and resumption, please refer to the :doc:`checkpointing guide <../checkpointing-logging/checkpointing>`. + +Logging and Debugging Configuration +----------------------------------- + +.. code-block:: yaml + + logger: "wandb" + project_name: "skyrl" + run_name: "test_run" + dump_data_batch: false + dump_eval_results: true + +- ``logger``: Logger to use. Currently, we support ``wandb``, ``mlflow``, and ``console``. ``console`` will simply log metrics to the console. +- ``project_name``: Name of the project in WandB and MLFlow. +- ``run_name``: Name of the run in WandB and MLFlow. +- ``dump_data_batch``: Whether to dump the data batch to a file. This is useful for debugging. When ``true``, the data batch will be dumped to a file in the ``export_path`` directory. The training batch at global step ``N`` is saved to ``self.cfg.trainer.export_path / "dumped_data" / global_step_N_training_input`` +- ``dump_eval_results``: Whether to dump the evaluation results to a file. When ``true``, the full evaluation results will be dumped to a file in the ``export_path`` directory. The evaluation results at global step ``N`` is saved to ``self.cfg.trainer.export_path / "dumped_eval" / global_step_N_eval_results`` + +Training Backends +----------------- + +We support three backends: FSDP1, FSDP2, and Megatron. The backend can be chosen with ``trainer.strategy`` field. + +.. _fsdp-configurations: + +FSDP Configuration +~~~~~~~~~~~~~~~~~~ + +We use the same configuration group for FSDP1 and FSDP2 + +.. code-block:: yaml + + fsdp_config: + cpu_offload: false # offload params + optimizer state to cpu during fwd pass + reshard_after_forward: true # fsdp2 only, [True, False, int between 1 and fsdp_size] + fsdp_size: -1 + +- ``cpu_offload``: Whether to train with CPU offloading (i.e., offload state during forward pass). This corresponds to `cpu_offload `_ parameter in FSDP1 and `offload_policy `_ in FSDP2. +- ``reshard_after_forward``: Whether to re-shard FSDP model after forward pass. This is a FSDP2 specific configuration, please refer to the `FSDP2 docs `_ for more details. If set to ``false``, this would retain the full model parameters on each worker (similar to DeepSpeed's ZeRO stage 2). +- ``fsdp_size``: The group size within which worker state is sharded with FSDP. This is a parameter to be used for hybrid sharding in multi-node settings. For example, if the number of workers in the actor group is 8, with 4 in each node, and ``fsdp_size`` is 4, then the training state will be fully sharded across 4 ranks in each node, but replicated (DP) across nodes. + +.. note:: + ``cpu_offload`` is different from worker state offloading with model colocation. + + In FSDP, ``cpu_offload`` will offload parameter and optimizer state to CPU memory and only copy over model parameters to GPU during model forward pass. + + In `skyrl-train`, we offload worker state in certain colocation settings - however this happens only after the training step/ log probability computation - thus optimizer step and model forward pass happen as usual with sharded parameters on GPU. For more details, refer to the guide on :doc:`model placement and colocation ` + +.. _megatron-configurations: + +Megatron Configuration +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + megatron_config: + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + context_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: null + + ddp_config: # pass-through config to Megatron's `DistributedDataParallelConfig` object + # https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/distributed/distributed_data_parallel_config.py#L8 + ... + optimizer_config_kwargs: # pass-through kwargs to Megatron's `OptimizerConfig` object + # any overlapping arguments with those we attempt to resolve in trainer.policy.optimizer_config will be overridden by the values here + # https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/optimizer/optimizer_config.py#L12 + ... + model_config_kwargs: # pass-through kwargs to the HuggingFace model config (i.e. for overriding vocab size, etc) + ... + transformer_config_kwargs: # pass-through kwargs to the Megatron's `TransformerConfig` object + # https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/transformer/transformer_config.py#L33 + ... + lora_config: + # see: https://docs.nvidia.com/nemo/megatron-bridge/0.2.0/apidocs/bridge/bridge.peft.lora.html for details - currently "lora" and "canonical_lora" are supported + lora_type: "lora" + # flag to manually empty torch's cuda cache between the forward/backward pass and the optimizer step + # this will free reserved but unallocated memory, and can help avoid OoMs in the optimizer + empty_cuda_cache: true + + +- ``megatron_config.tensor_model_parallel_size``: Tensor model parallel size for reducing memory across model parameters and activations. Sequence parallelism (unrelated to ulysses sequence parallelism) is also enabled by default if tensor parallel size is greater than 1. +- ``megatron_config.pipeline_model_parallel_size``: Pipeline model parallel size for sharding model layers across multiple GPUs. +- ``megatron_config.context_parallel_size``: Context parallel size for reducing activation memory across the sequence length dimension. +- ``megatron_config.expert_model_parallel_size``: The expert parallel size for sharding expert modules across multiple GPUs. +- ``megatron_config.expert_tensor_parallel_size``: The tensor parallel size for each expert module. If set to ``null``, then the value will be resolved to ``tensor_model_parallel_size`` by Megatron. It is recommended to set this to ``1`` when enabling ``expert_model_parallel_size > 1`` for the best performance. + +Some rules for configuring these parameters: + +- ``model_size = pp_size * tp_size * cp_size`` +- ``dp_size = world_size / model_size`` +- ``world_size % (pp_size * ep_size * etp_size) == 0`` + - This means that ``ep_size * etp_size`` can scale independently of ``tp_size * cp_size``, and can go across data parallel ranks. + +.. warning:: + + ``optimizer_config_kwargs.use_precision_aware_optimizer=true`` can cause checkpointing to fail. See: https://github.com/nvidia/megatron-lm/issues/1820. + + We recommend leaving this setting to ``false`` + + +Optimizer Configuration +----------------------- +For both the critic and policy model, we provide a common optimizer configuration + +.. code-block:: yaml + + optimizer_config: + lr: 1.0e-6 + adam_betas: [0.9, 0.999] + weight_decay: 1e-2 + max_grad_norm: 1.0 + offload_after_step: true + num_warmup_steps: 0 + scheduler: "constant_with_warmup" + +- ``optimizer_config.lr``: Learning rate for the optimizer +- ``optimizer_config.adam_betas``: Betas for AdamW optimizer. +- ``optimizer_config.weight_decay``: L2 regularization strength for AdamW. +- ``optimizer_config.max_grad_norm``: Gradient clipping parameter. The total L2 norm of the model gradients will be scaled to this value during training. +- ``optimizer_config.offload_after_step``: Whether to offload optimizer state to CPU after step if colocated. When generation and training workers are colocated, we recommend using the default setting of ``true``. In some cases with non-colocation, it can be desirable to leave optimizer state on GPU memory to avoid offloading costs as well as additional CPU memory usage. +- ``optimizer_config.num_warmup_steps``: Number of mini-batch steps to warmup the optimizer for. +- ``optimizer_config.scheduler``: Which learning rate scheduler to use. Intended to align with ``transformers.SchedulerType`` from `Huggingface `_. + +Policy Configuration +-------------------- + +This section configures the policy model used for training, including optimizer, FSDP, sequence parallelism, and LoRA options. + +.. code-block:: yaml + + policy: + model: + path: "Qwen/Qwen2.5-1.5B-Instruct" # Hugging Face model path for the policy model + lora: + rank: 0 # LoRA rank (0 = disabled) + alpha: 16 # LoRA scaling parameter + dropout: 0 # LoRA dropout rate + lora_sync_path: "/tmp/skyrl_lora_sync" # Path for LoRA adapter sync + target_modules: "all-linear" # Apply to all linear layers OR + # specify specific modules as a list + exclude_modules: null # Modules to exclude from LoRA + # For FSDP, this corresponds to `init_lora_weights` in PEFT. See: https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig + # For Megatron, this is used for `lora_A_init_method`, and "xavier", "normal", "kaiming", and "zero" are supported. + init_method: "kaiming" # Initialization method for LoRA layers + optimizer_config: + lr: 1.0e-6 # Learning rate + adam_betas: [0.9, 0.999] # Betas for Adam optimizer + weight_decay: 1e-2 # L2 regularization strength + max_grad_norm: 1.0 # Gradient clipping + offload_after_step: true # Offload optimizer state to CPU after step (if colocated) + + fsdp_config: + cpu_offload: false # Offload model params to CPU during forward + reshard_after_forward: true # Re-shard FSDP model after forward pass + fsdp_size: -1 # Auto FSDP group sizing + + sequence_parallel_size: 1 # sequence parallel size + + use_torch_compile: false # Enable torch compile for the entropy calculation + record_memory: false # Dump memory snapshot for debugging + + model_config_kwargs: {} # pass through kwargs to the HuggingFace model config for FSDP training backends (i.e. for overriding vocab size, etc) - for megatron, use policy.megatron_config.transformer_config_kwargs instead + +- ``policy.optimizer_config``: Optimizer configuration for the policy model +- ``policy.fsdp_config``: FSDP configuration, applicable if ``trainer.strategy='fsdp'``. +- ``policy.sequence_parallel_size``: Sequence parallel size. We implement `Ulysses sequence parallelism `_ +- ``policy.use_torch_compile``: Whether to enable torch compile for entropy calculation +- ``policy.record_memory``: Whether to record memory usage. If ``True``, this will use PyTorch's `memory snapshotting utility `_ to record memory usage and dump memory snapshots after each policy model training step. + +LoRA Configuration +~~~~~~~~~~~~~~~~~~ + +LoRA (Low-Rank Adaptation) enables parameter-efficient fine-tuning by training only a small number of additional low-rank matrices instead of the full model weights: + +- ``policy.model.lora.rank``: LoRA rank for low-rank decomposition. Set to 0 to disable LoRA. Higher values increase model capacity but also memory usage. Common values include 8, 16, 32, or 64. +- ``policy.model.lora.alpha``: Scaling factor for LoRA updates. +- ``policy.model.lora.dropout``: Dropout probability applied to LoRA layers. Helps prevent overfitting during training. +- ``policy.model.lora.lora_sync_path``: Directory path where LoRA adapter weights are saved and synchronized between training and inference processes. Must be accessible to all workers in distributed setups. +- ``policy.model.lora.init_method``: Initialization method for LoRA layers. For FSDP, this corresponds to `init_lora_weights `_ in PEFT. 'kaiming' is mapped to 'true' by default for PEFT. For Megatron, this is used for `lora_A_init_method`, and "xavier", "normal", "kaiming", and "zero" are supported. + + +Critic Configuration +-------------------- + +We support similar configuration options as the policy model, including LoRA. + +.. code-block:: yaml + + critic: + model: + path: null + lora: + rank: 0 # LoRA rank (0 = disabled) + alpha: 16 # LoRA scaling parameter + dropout: 0 # LoRA dropout rate + target_modules: "all-linear" + exclude_modules: null # Modules to exclude from LoRA + init_method: "kaiming" # Initialization method for LoRA layers + optimizer_config: + lr: 5.0e-6 + adam_betas: [0.9, 0.999] + weight_decay: 1e-2 + max_grad_norm: 1.0 # gradient clipping + offload_after_step: true # offload optimizer state to cpu after each step. Applicable only when `colocate_all=true` + fsdp_config: + cpu_offload: false + reshard_after_forward: true + fsdp_size: -1 + sequence_parallel_size: 1 + model_config_kwargs: {} # pass through kwargs to the HuggingFace model config (i.e. for overriding vocab size, etc) + + +Reference Model Configuration +----------------------------- + + +.. code-block:: yaml + + ref: + model: + path: ${trainer.policy.model.path} + fsdp_config: + cpu_offload: false + reshard_after_forward: true + fsdp_size: -1 + sequence_parallel_size: 1 + model_config_kwargs: {} # pass through kwargs to the HuggingFace model config for FSDP training backends (i.e. for overriding vocab size, etc) - for megatron, use ref.megatron_config.transformer_config_kwargs instead + +- ``ref.model.path``: Path to the reference model. Defaults to the policy model path, but can be separately set (i.e. for distillation based approaches, the reference model can be a different model than the policy model). +- ``ref.fsdp_config``: FSDP configuration, applicable if ``trainer.strategy='fsdp'``. +- ``ref.sequence_parallel_size``: Sequence parallel size. We implement `Ulysses sequence parallelism `_ + +.. note:: + + The reference model is used only if the base model log probabilities are required either as a part of the training loss or as a part of the reward. Thus, ``trainer.algorithm.use_kl_in_reward`` or ``trainer.algorithm.use_kl_loss`` should be set to ``true`` to use the reference model. If both are ``false``, then the reference model is not instantiated. + + +Algorithm Configuration +----------------------- + +.. code-block:: yaml + + algorithm: + advantage_estimator: "grpo" # "grpo", "gae", or customizable with AdvantageEstimatorRegistry + + # KL Penalty Parameters + kl_ctrl: # only used if use_kl_in_reward is true (not applied in the case of use_kl_loss=true) - uses kl_loss_coef as the initial KL coefficient + type: "fixed" # "fixed" or "adaptive" + kl_target: 0.1 # target KL divergence for adaptive KL controller + horizon: 10000 # controls the update rate of the adaptive KL controller + + kl_estimator_type: "k3" # "k1", "k2", "k3", "abs" - see http://joschu.net/blog/kl-approx.html for details + use_kl_estimator_k3: false # to be deprecated, use kl_estimator_type="k3" instead + use_abs_kl: false # to be deprecated, use kl_estimator_type="abs" instead + + # note: use_kl_in_reward and use_kl_loss should be mutually exclusive + use_kl_in_reward: false # apply kl loss to rewards + use_kl_loss: true # used in policy model + kl_loss_coef: 0.001 + # this adds training batch level normalization to advantages + advantage_batch_normalize: false + value_head_prefix: "value_head" + policy_loss_type: "regular" # "regular", "dual_clip", "gspo", "clip_cov", "kl_cov" or customizable with PolicyLossRegistry + loss_reduction: "token_mean" # "token_mean", "sequence_mean", "seq_mean_token_sum_norm" + grpo_norm_by_std: true # set to false to disable normalization by std in GRPO (used in Dr. GRPO) + zero_variance_filter: false # set to true to loss mask out prompts with zero variance rewards. only applicable when rewards are response-level. + + # GAE parameters + lambd: 1.0 + gamma: 1.0 + + # PPO parameters + eps_clip_low: 0.2 + eps_clip_high: 0.2 + # dual clip parameters + clip_ratio_c: 3.0 + + # clip-cov parameters (only used when policy_loss_type: "clip_cov") + clip_cov: + clip_ratio: 0.0002 # fraction of tokens to clip based on covariance + clip_cov_lb: 1.0 # lower bound for covariance clipping + clip_cov_ub: 5.0 # upper bound for covariance clipping + + # kl-cov parameters (only used when policy_loss_type: "kl_cov") + kl_cov: + kl_cov_frac: 0.2 # percentage of tokens to apply KL regularization to (20%) + ppo_kl_coef: 1.0 # coefficient for KL regularization term + + # cispo parameters (only used when policy_loss_type: "cispo") + cispo: + cispo_eps_clip_low: 0 # offset for lower bound of importance sampling ratio clipping (as opposed to PPO token update clipping) + cispo_eps_clip_high: 5 # offset for upper bound of importance sampling ratio clipping (as opposed to PPO token update clipping) + + # value loss parameters + value_clip: 0.2 + + # dynamic sampling parameters + dynamic_sampling: + type: null # filter (DAPO), replace (POLARIS/WebSailor), or null + max_sample_batches: 30 # sample at most this many batches before stopping, -1 to sample forever + min_replace_ratio: 0.3 # minimum proportion of good samples with which to replace bad samples (for replace strategy only) + + # Truncated Importance Sampling as proposed in https://fengyao.notion.site/off-policy-rl + use_tis: false + tis_imp_ratio_cap: -1.0 + + # SAPO parameters (only used when policy_loss_type: "sapo") (https://arxiv.org/pdf/2511.20347) + sapo: + tau_pos: 1.0 + tau_neg: 1.05 # default values used in the paper with Qwen3-30B-A3B-Base + +- ``algorithm.advantage_estimator``: Advantage estimator to use. We currently implement ``grpo``, ``gae``, ``rloo``, ``reinforce++``, and custom advantage estimators can be registered with the ``AdvantageEstimatorRegistry``. +- ``algorithm.kl_ctrl`` Configuration for the KL controller - only used if ``use_kl_in_reward`` is ``true`` (not applied in the case of ``use_kl_loss`` is ``true``). ``kl_loss_coef`` is used as the initial KL coefficient for both ``fixed`` and ``adaptive`` KL controllers. + + - ``type``: Type of KL controller to use. Options include: ``fixed`` or ``adaptive``. + - ``kl_target``: Target KL divergence for adaptive KL controller. + - ``horizon``: Controls the update rate of the adaptive KL controller. + +- ``algorithm.kl_estimator_type``: KL estimator type to use. Options include: ``k1``, ``k2``, ``k3``, ``abs``. See `this blog post `_ for details. We use ``k3`` as the default. +- ``algorithm.use_kl_estimator_k3``: Whether to use the k3 estimator for KL divergence calculation. The k3 estimator is the non negative kl approximation in `this blog post `_. Besides non negative, it is also unbiased and has lower variance. This flag is to be deprecated, use ``kl_estimator_type="k3"`` instead. +- ``algorithm.use_abs_kl``: Whether to use the absolute KL divergence for KL divergence calculation. This flag is to be deprecated, use ``kl_estimator_type="abs"`` instead. +- ``algorithm.use_kl_in_reward``: Whether to apply KL divergence penalty to rewards. The new rewards will be computed as ``rewards - kl * kl_loss_coef``. +- ``algorithm.use_kl_loss``: Whether to add a KL divergence loss to the policy model. The policy loss will be computed as ``policy_loss + kl * kl_loss_coef``. +- ``algorithm.kl_loss_coef``: Coefficient for the KL divergence loss. +- ``algorithm.advantage_batch_normalize``: Whether to normalize advantages by the (global) batch mean and standard deviation. +- ``algorithm.value_head_prefix``: The name used to identify the value head in the critic model. +- ``algorithm.policy_loss_type``: Type of policy loss to use. Options include: + + - ``regular``: Vanilla PPO loss with token-level importance sampling + - ``dual_clip``: Dual clip PPO loss proposed in `this paper `_ + - ``gspo``: `Group Sequence Policy Optimization `_ with sequence-level importance sampling for improved training stability. Implements the "GSPO-token" variant from the paper. + - ``clip_cov``: Clip-Cov combines standard PPO clipping with covariance-based correction masking for improved stability. Based on `this paper `_. + - ``kl_cov``: KL-Cov applies KL regularization to tokens selected based on covariance values. Based on `this paper `_. + - ``cispo``: Clipped Importance Sampling Weight Policy Optimization (CISPO) proposed in `MiniMax-M1 `_. + - Custom policy losses can be registered with the ``PolicyLossRegistry`` + +- ``algorithm.loss_reduction``: Type of loss reduction to use. Options include: + + - ``token_mean``: computes average loss over all valid tokens in the batch. Used in `DAPO `_. + - ``sequence_mean``: computes per-sequence avg token loss, then averages over the batch. + - ``seq_mean_token_sum_norm``: computes the sum of token losses for each sequence, normalizes by the max sequence length (computed as ``cfg.generator.max_input_length + cfg.generator.sampling_params.max_generate_length``), and then averages over the batch. This is used in `Dr. GRPO `_. + +- ``algorithm.grpo_norm_by_std``: Whether to normalize advantages by the standard deviation in GRPO. This is set to ``false`` in `Dr. GRPO `_. +- ``algorithm.zero_variance_filter``: Whether to loss mask out prompts with zero variance rewards. This is only applicable when rewards are response-level. +- ``algorithm.lambd``: Lambda parameter for GAE. +- ``algorithm.gamma``: Gamma parameter for GAE. +- ``algorithm.eps_clip_low``: Lower bound for PPO clipping. +- ``algorithm.eps_clip_high``: Upper bound for PPO clipping. +- ``algorithm.clip_ratio_c``: Clip ratio for dual clip PPO loss. +- ``algorithm.value_clip``: Clip value for value loss. +- ``algorithm.dynamic_sampling``: Dynamic sampling configuration. + - ``algorithm.dynamic_sampling.type``: Type of dynamic sampling to use. Currently, we support ``filter`` (`DAPO `_), ``replace`` (`POLARIS `_ / `WebSailor `_), or ``null`` for no dynamic sampling. + - ``algorithm.dynamic_sampling.max_sample_batches``: Maximum number of batches to sample before stopping. Set to ``-1`` to sample forever. + - ``algorithm.dynamic_sampling.min_replace_ratio``: Minimum proportion of good samples with which to replace bad samples for ``replace`` strategy. +- ``algorithm.use_tis``: Whether to use Truncated Importance Sampling (TIS) as proposed in `this blog `_. +- ``algorithm.tis_imp_ratio_cap``: Cap parameter for the importance ratio in TIS. +- ``algorithm.clip_cov``: Clip-Cov parameters (only used when ``policy_loss_type`` is ``clip_cov``): + + - ``clip_ratio``: Fraction of tokens to clip based on covariance values. + - ``clip_cov_lb``: Lower bound for covariance clipping. + - ``clip_cov_ub``: Upper bound for covariance clipping. + +- ``algorithm.kl_cov``: KL-Cov parameters (only used when ``policy_loss_type`` is ``kl_cov``): + + - ``kl_cov_frac``: Percentage of tokens to apply KL regularization to. + - ``ppo_kl_coef``: Coefficient for KL regularization term. + +- ``algorithm.cispo``: CISPO parameters (only used when ``policy_loss_type`` is ``cispo``): + + - ``cispo_eps_clip_low``: Offset for lower bound of importance sampling ratio clipping. Tokens with importance sampling ratio less than ``1 - cispo_eps_clip_low`` will have their ratio clipped, but can still be updated in the policy gradient update. + - ``cispo_eps_clip_high``: Offset for upper bound of importance sampling ratio clipping. Tokens with importance sampling ratio greater than ``1 + cispo_eps_clip_high`` will have their ratio clipped, but can still be updated in the policy gradient update. + +- ``algorithm.sapo``: SAPO (as proposed in `this paper `) parameters (only used when ``policy_loss_type`` is ``sapo``): + + - ``tau_pos``: Temperature for gating function for tokens with positive advantages. + - ``tau_neg``: Temperature for gating function for tokens with negative (or zero) advantages. + +Policy Loss Formulation +~~~~~~~~~~~~~~~~~~~~~~~ + +It can be helpful to understand the final loss formulation to see how the different configuration options are used. The final loss is computed as below in the ``ppo_policy_loss`` function. + +.. code-block:: python + + def ppo_policy_loss( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, # trainer.algorithm config + loss_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + + ratio = (log_probs - old_log_probs).exp() + surr1 = ratio * advantages + surr2 = ratio.clamp(1 - config.eps_clip_low, 1 + config.eps_clip_high) * advantages + loss = -torch.min(surr1, surr2) + clip_ratio = masked_mean((-surr2 > -surr1).float(), loss_mask).mean().detach().item() + clip_pg_losses1 = loss + if config.policy_loss_type == "dual_clip": + pg_losses3 = -advantages * config.clip_ratio_c + clip_pg_losses2 = torch.min(pg_losses3, clip_pg_losses1) + loss = torch.where(advantages < 0, clip_pg_losses2, clip_pg_losses1) + loss = reduce_loss(loss, loss_mask, config.loss_reduction) + return loss, clip_ratio + + +Generator Configuration +----------------------- + +.. code-block:: yaml + + generator: + model_dtype: "bfloat16" # should match dtype for inference engine + run_engines_locally: true + num_inference_engines: 1 + backend: "vllm" + weight_sync_backend: "nccl" + inference_engine_tensor_parallel_size: 4 + inference_engine_pipeline_parallel_size: 1 + inference_engine_expert_parallel_size: 1 + inference_engine_data_parallel_size: 1 + n_samples_per_prompt: 5 + async_engine: true + batched: true + max_input_length: ${trainer.max_prompt_length} # max generator input length used for multi-turn conversations - for single turn set equal to max_prompt_length + enable_prefix_caching: true + enable_chunked_prefill: true + max_num_batched_tokens: 8192 + enforce_eager: false + gpu_memory_utilization: 0.8 + max_num_seqs: 1024 + remote_inference_engine_urls: ["127.0.0.1:8001"] + max_turns: 1 + + # Custom chat template configuration if needed + chat_template: + source: "name" # "name" or "file" + name_or_path: null # e.g., "qwen3_with_thinking" or "/path/to/template.j2" + + # Chat templating kwargs to pass to `tokenizer.apply_chat_template` + chat_template_kwargs: {} + + engine_init_kwargs: {} + + override_existing_update_group: "auto" # "auto", "enable", "disable" + # sampling params for generation phase + sampling_params: + max_generate_length: 1024 + temperature: 1.0 + top_p: 1.0 + min_p: 0.0 + top_k: -1 + logprobs: 0 + + use_conversation_multi_turn: true + + # sampling params for evaluation + eval_sampling_params: + max_generate_length: ${generator.sampling_params.max_generate_length} + temperature: 1.0 + top_p: 1.0 + min_p: 0.0 + top_k: -1 + logprobs: 0 + + # number of samples per prompt for evaluation + eval_n_samples_per_prompt: 1 + + zero_reward_on_non_stop: false + + apply_overlong_filtering: false + + +Inference Engine Placement Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``generator.run_engines_locally``: Whether to use local inference engines. If ``true``, the inference engine will be initialized during the training run in the current Ray cluster. We use one Ray actor per inference replica and communication will happen via Ray object store. If set to ``false``, then the generator expects a list of remote urls and communication will happen over HTTP. +- ``generator.num_inference_engines``: Number of inference engines to use. If ``run_engines_locally`` is ``false``, then this number should match the number of remote urls. +- ``generator.remote_inference_engine_urls``: List of remote urls to use. Applicable only when ``run_engines_locally`` is ``false``. +- ``generator.enable_http_endpoint``: When ``true``, launch an OpenAI-compatible HTTP endpoint for the inference engine client so that generators can send requests to this server instead of using ``.generate()`` Python calls. +- ``generator.http_endpoint_host``: Host for the inference HTTP endpoint. +- ``generator.http_endpoint_port``: Port for the inference HTTP endpoint. + +For more details on how different placement options work, please refer to the :doc:`placement guide `. + +Weight Transfer Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``generator.weight_sync_backend``: Backend to use for weight synchronization. Currently, we support ``nccl`` and ``gloo``. +- ``generator.override_existing_update_group``: Whether to override the existing update group for the inference engine. This is applicable only for remote inference engines. During training, `skyrl-train` forms a custom process group ("update group") with the rank 0 training worker and all the inference engine ranks. If ``override_existing_update_group=enable``, then during initialization, a previous weight update group will be overriden in the inference engine. For example, if you have a remote server setup and you run training for the same model multiple times, it is helpful to override the previous update group. We recommend leaving this to ``auto`` - since it will automatically determine if the previous update group should be overridden based on ``run_engines_locally``. + +Inference Engine Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``generator.backend``: Backend to use for the inference engine. We support ``vllm`` and ``sglang``. ``sglang`` is supported only for remote inference engines at the moment. +- ``generator.model_dtype``: Dtype used for the inference engine. This is also used during weight transfer - the policy model weights are casted to this dtype before being sent to the inference engine during weight transfer. +- ``generator.async_engine``: Whether to use an asynchronous/ offline inference engine. Applicable only when ``backend="vllm"``. +- ``generator.inference_engine_tensor_parallel_size``: Tensor parallel size for the inference engine. +- ``generator.inference_engine_pipeline_parallel_size``: Pipeline parallel size for the inference engine. Currently, PP is only supported for vLLM backend with async_engine=true. +- ``generator.inference_engine_expert_parallel_size``: Expert parallel size for the inference engine. Currently, EP is only supported for vLLM backend and ep_size must equal dp_size * tp_size. +- ``generator.inference_engine_data_parallel_size``: Data parallel size for the inference engine. Currently, DP is only supported for vLLM backend. +- ``generator.gpu_memory_utilization``: GPU memory utilization for the inference engine. Applicable only for ``run_engines_locally=true``. +- ``generator.vllm_v1_disable_multiproc``: If ``true``, this will set ``VLLM_ENABLE_V1_MULTIPROCESSING=0`` in the environment, which makes the scheduling deterministic. This is useful for reproducibility. +- ``generator.enable_prefix_caching``: Whether to enable prefix caching for the inference engine. Applicable only when ``backend="vllm"``. This can be left to the default ``true`` in most cases. Note that in the case of remote inference engines, you would need to match the setting used when you initialized the remote servers. +- ``generator.enable_chunked_prefill``: Whether to enable chunked prefill for the inference engine. Applicable only when ``backend="vllm"``. With vLLM, this can be left to the default ``true`` in most cases. +- ``generator.max_num_seqs``: Continous batching parameter for vLLM. Maximum number of sequences to pack into a batch. +- ``generator.max_num_batched_tokens``: Continous batching parameter for vLLM. Maximum number of tokens to pack into a batch. + +Generation Parameters +~~~~~~~~~~~~~~~~~~~~~ + +- ``generator.n_samples_per_prompt``: Number of samples to generate per prompt. Note that the total size of the training batch will be ``trainer.train_batch_size * generator.n_samples_per_prompt``. +- ``generator.batched``: Whether to use batched inference. This is applicable only for single turn generation. +- ``generator.max_input_length``: Maximum input length for the inference engine. For single turn generation, this can be same as ``trainer.max_prompt_length`` (i.e., the initial prompt length). For multi-turn generation, this is the maximum input length used for multi-turn conversations at each turn. +- ``generator.sampling_params``: Sampling parameters for the inference engine during trajectory generation phase. + + - ``generator.sampling_params.max_generate_length``: Maximum length of the generated response. + - ``generator.sampling_params.temperature``: Temperature for the inference engine. + - ``generator.sampling_params.top_p``: Top-p sampling parameter for the inference engine. + - ``generator.sampling_params.min_p``: Min-p sampling parameter for the inference engine, as proposed in `this paper `_. + - ``generator.sampling_params.top_k``: Top-k sampling parameter for the inference engine. + - ``generator.sampling_params.logprobs``: Number of logprobs to return from the inference engine. Set to ``0`` to return only the chosen token's logprob. +- ``generator.eval_sampling_params``: Sampling parameters for evaluation. +- ``generator.eval_n_samples_per_prompt``: Number of samples to generate per prompt for evaluation. +- ``generator.max_turns``: Maximum number of turns for generation with multi-turn RL. +- ``generator.use_conversation_multi_turn``: Whether to use conversation format for multi-turn generation. If set to ``true`` then observations are appended to the chat history as a new turn. If set to ``false`` then observations are appended as-is to the assistant response in token space and generation is continued (after removing any EOS token in the response). We've observed some cases where model can be sensitive to chat history format (ex: in SkyRL-SQL), and thus ``false`` can be used for full control over the exact tokens added after environment interaction. +- ``generator.engine_init_kwargs``: Inference engine arguments passed directly to the vLLM or SGLang engine. To specify an engine arg in the CLI override, use the format: +generator.engine_init_kwargs.[arg_name]=value. If duplicate kwargs are passed or kwargs clash with existing generator arguments (e.g., ``tensor_parallel_size``), an error is raised. +- ``generator.chat_template``: Custom chat template configuration if needed. + - ``generator.chat_template.source``: Source of the chat template. Can be either ``name`` or ``file``. + - ``generator.chat_template.name_or_path``: Name or path of the chat template. If the source is ``name``, then it should be one of the supported templates in :code_link:`skyrl_train/generators/utils.py`. If the source is ``file``, then this field should be a path to a Jinja2 template file. +- ``generator.chat_template_kwargs``: Chat templating kwargs to pass to ``tokenizer.apply_chat_template``. Applicable only for non-batched generation with ``generator.batched=false``. + +Misc Configuration +~~~~~~~~~~~~~~~~~~ + +- ``generator.zero_reward_on_non_stop``: Whether to set the reward to 0 if the `stop_reason` is not `stop`. Cases where this is useful: Often, we have format rewards for the LLM to follow, but in cases where the LLM didn't finish the response, we typically don't want to reward it. This is a general setting for all environments. +- ``generator.apply_overlong_filtering``: Whether to apply DAPO Overlong Filtering to the loss masks. For each trajectory that exceeds the max length (i.e., truncated and does not end with an EOS token), this masks out every token in the loss mask. +- ``generator.step_wise_trajectories``: Whether to return outputs in a step-wise fashion. If ``true``, then the generator will return multi-turn generations with the (prompt, response) pair of each turn being a separate trajectory. Advantages are computed based on the last step of each trajectory and propagated to the previous steps. diff --git a/skyrl-train/docs/configuration/placement.rst b/skyrl-train/docs/configuration/placement.rst new file mode 100644 index 0000000000..a3d1572356 --- /dev/null +++ b/skyrl-train/docs/configuration/placement.rst @@ -0,0 +1,97 @@ +Model Placement and Colocation +=============================== + +SkyRL provides flexible control over how to distribute models across available GPU resources. You can either colocate models on the same GPUs or disaggregate them across separate GPUs, depending on your setup and requirements. + +Model Components Overview +------------------------- + +A typical PPO training workflow involves 5 model-based components: + +- **Inference Engines** (handle text generation) +- **Policy model** (learns actions to take) +- **Reference model** (tracks the original policy) +- **Reward model** (optional; scores action quality) +- **Critic model** (estimates future rewards) + +*Note: GRPO training generally uses the first 2-4 components depending on the setup – no critic model needed.* + +Inference Engine Management +---------------------------- + +The ``generator.run_engines_locally`` argument controls inference engine management. + +If ``run_engines_locally=true``, then the inference engines are launched during the training run and managed by SkyRL. + +If ``run_engines_locally=false``, then the user can specify inference engine URLs managed externally (with the ``generator.remote_inference_engine_urls`` parameter). In this case, the user is responsible for setup and teardown. Note that SkyRL expects certain additional endpoints in the inference engine specifically related to weight syncing. We provide scripts for launching remote inference engines `here `_ for convenience. + + +Inference Engine Placement +-------------------------- + +The ``generator.colocate_all`` setting controls inference engine placement. + +**Colocated Engines (colocate_all = true)** + +Inference engines share GPUs with training models: + +- Generation runs on the same hardware as training +- Engines will ``sleep()`` after generation to free GPU memory +- Engines will ``wake_up()`` before the next generation round + +NOTE: As of now, colocated engines are only supported with ``generator.run_engines_locally=true``. + +**Disaggregated Engines (colocate_all = false)** + +Inference engines run on dedicated GPUs: + +- Inference engines do not need to sleep/wake_up +- Updated weights are still efficiently synced to Inference engines (via NCCL, RDMA, etc.) + +Training Model Placement +------------------------ + +The highest-level placement configuration for the training models is ``trainer.placement.colocate_all``: + + +**Full Colocation (colocate_all = true)** + +All training models (policy, critic, reward, reference) share the same GPUs. + +**Granular Control (colocate_all = false)** + +The policy and critic models are not colocated, but fine-grained placement of the reference and reward models can be controlled with two additional parameters: + +- ``trainer.placement.colocate_policy_ref``: Colocate policy and reference models (``true``) or place them on separate GPUs (``false``) +- ``trainer.placement.colocate_critic_reward``: Colocate critic and reward models (``true``) or place them on separate GPUs (``false``) + +Hardware Configuration +---------------------- + +Finally, the configuration for specifying node and GPU counts for each model (along with their default values) is as follows: + +.. code-block:: yaml + + trainer: + # Training model resources + policy_num_nodes: 1 + policy_num_gpus_per_node: 4 + critic_num_nodes: 1 + critic_num_gpus_per_node: 4 + ref_num_nodes: 1 + ref_num_gpus_per_node: 4 + reward_num_nodes: 1 + reward_num_gpus_per_node: 4 + + generator: + # InferenceEngine resources + num_inference_engines: 1 + inference_engine_tensor_parallel_size: 4 + inference_engine_expert_parallel_size: 1 + inference_engine_data_parallel_size: 1 + +.. note:: + **Resource Allocation Guidelines** + + - When ``colocate_all=true``, all training models should have identical node and GPU counts. + - When ``generator.run_engines_locally=true``, the total number of GPUs used for Inference engines should match the total number of GPUs used for training models. diff --git a/skyrl-train/docs/datasets/dataset-preparation.rst b/skyrl-train/docs/datasets/dataset-preparation.rst new file mode 100644 index 0000000000..a7f1bd88e0 --- /dev/null +++ b/skyrl-train/docs/datasets/dataset-preparation.rst @@ -0,0 +1,134 @@ +Dataset Preparation +=================== + +This guide covers: + +1. The dataset format that SkyRL expects for training, and +2. How to prepare and format a new dataset + + +Format Requirements +------------------- + +Each dataset entry is a dictionary with the following required (and some optional) values: + +.. code-block:: python + + data = { + "data_source": data_source, # String: Name/identifier of the data source + "prompt": [ # List: Conversation format + { + "role": "user", + "content": question, + } + ], + "env_class": env_class, # String: Environment class identifier + "reward_spec": { + "method": "rule", # String: Either "rule" or "reward_model" + "ground_truth": solution, # Expected solution + }, + "extra_info": { # Dict: Optional additional metadata + # ... add your own fields here + }, + } + +SkyRL supports loading datasets of this format from a local parquet file, a json file, or by Hugging Face dataset name that SkyRL will download. We load the dataset as a huggingface `DatasetDict `_. + +**Key Requirements:** + +- **data_source**: String identifier for the dataset origin (e.g., "gsm8k", "AIME24", etc.) +- **prompt**: List of dictionaries following standard OpenAI chat format +- **env_class**: Name of environment that the data sample corresponds to. This is used to tell the Generator which environment to instantiate for this prompt. + + - Note: **env_class** can also be specified in the training configuration to apply to all dataset entries. +- **reward_spec**: Dictionary containing the reward specification for the dataset entry (ie, how to get rewards). + + - **method**: Must be either ``"rule"`` or ``"reward_model"`` + - **ground_truth**: If ``method`` is ``"rule"``, this is the expected solution. + +- **extra_info**: Extensible dictionary for additional metadata - you can add custom fields as needed. + + +Data Preparation Scripts +------------------------ + +We provide several example scripts to help you prepare your dataset, including for gsm8k, LiveCodeBench, SearchR1, and the SynSQL text-to-SQL dataset. + +**To use a new dataset for training, you can use the provided scripts as a template to create your own.** + +Generally, only a single method (`make_map_fn`) must be implemented to convert the new dataset into the required format. Below is an example of converting the SynSQL text-to-SQL dataset into the required format: + +.. code-block:: python + + def make_map_fn(split): + def process_fn(example, idx): + """Transform each dataset example into the required format""" + if split == "train": + user_content = ("{db_details}:" + example["schema"] + + ";\n {external_knowledge}: " + example["external_knowledge"] + + ";\n {question}: " + example["question"]) + else: + user_content = ("{db_details}:" + example["schema"] + + "; {question}: " + example["question"]) + + data = { + "data_source": "synsql", + "prompt": [ + {"role": "system", "content": short_system_prompt}, + { + "role": "user", + "content": user_content, + }, + ], + "env_class": "text2sql", + "reward_spec": { + "method": "rule", + "ground_truth": example["sql"], + }, + # Custom fields specific to the SynSQL dataset: + "db_id": example["db_id"], + "data": example["data"], + } + return data + + return process_fn + +Then, the mapping function is called on each sample in the dataset, and the final converted dataset is saved to a parquet file: + +.. code-block:: python + + train_dataset = input_dataset.map(function=make_map_fn("train"), with_indices=True) + train_dataset.to_parquet(os.path.join(args.output, "train.parquet")) + +Note, however, that SkyRL can also load datasets from a local json file or by Hugging Face dataset name. + +Using Dataset to Train +---------------------- + +With your correctly formatted datasets, you can pass the dataset file paths to the training script: + +.. code-block:: bash + + # Dataset file paths + uv run -m skyrl_train.entrypoints.main_base \ + data.train_data="['path/to/train.parquet']" \ + data.val_data="['path/to/validation.parquet']" \ + +or specify HuggingFace dataset(s) prepared in the expected format: + +.. code-block:: bash + + # Huggingface dataset + uv run -m skyrl_train.entrypoints.main_base \ + data.train_data="['username/my_dataset:train']" \ + data.val_data="['username/my_dataset:validation']" \ + + + +Reference Scripts +----------------- + +Use the following scripts as a template to prepare your dataset: + +- `gsm8k_dataset.py `_ +- `multiply_dataset.py `_ diff --git a/skyrl-train/docs/examples/flash_rl.rst b/skyrl-train/docs/examples/flash_rl.rst new file mode 100644 index 0000000000..a34c18ef36 --- /dev/null +++ b/skyrl-train/docs/examples/flash_rl.rst @@ -0,0 +1,120 @@ +FlashRL + SkyRL: Training with Quantized Rollouts +================================================= + +In this example, we walk through how to train a model with quantized rollouts using `FlashRL `_ and SkyRL. + +What is FlashRL? +---------------- + +`FlashRL `_ is a novel method that provides the first open-source RL recipe with quantized (Int8, FP8) rollout generation while preserving downstream performance. FlashRL consists of two main components: + +- Truncated Importance Sampling (TIS): In scalable RL frameworks, policy model and rollout are typically managed by different libraries/ frameworks (FSDP and vLLM, resp.), which leads to a mismatch between the probability distributions. TIS is a technique that solves the rollout and training mismatch problem by applying a token-level correction factor (based on the importance-sampling ratio) to the policy loss. +- Online Quantization Support: While vLLM has support for inference with quantized weights, it is tricky to use this for RL training. FlashRL also has patches for vLLM to support weight updates for FP8 and Int8 during training. + + +FlashRL + SkyRL +--------------- + +SkyRL now has a native integration with FlashRL. Currently, we support training with Int8 quantization as well as `online FP8 quantization `_ in vLLM. + + +.. warning:: + + FlashRL integration only supports single-turn training at the moment. + + +How does it work? +~~~~~~~~~~~~~~~~~~ + +At a high level, we sample generations from the inference engine with quantized weights (Int8, FP8). We then compute advantages and model losses. We apply the TIS correction factor to the policy loss to account for the difference in rollout and training probability distributions. On weight update, we sync the weights (in half precision) with the inference engine layer by layer. These weights are then quantized to the appropriate format (Int8, FP8) before loading. + +Examples +-------- + +We provide examples for training with FP8 and Int8 rollouts for DAPO. The FlashRL related files are in the :code_link:`examples/flash_rl/` folder. + +For FP8, you simply need to specify ``FLASHRL_CONFIG=fp8_vllm`` in your environment variables. + +For Int8, we need to provide calibration data. We leverage the provided calibration data from FlashRL for ``Qwen/Qwen2.5-0.5B-Instruct`` and ``Qwen/Qwen-32B`` models. You can simply specify the appropriate ``FLASHRL_CONFIG`` in your environment variables. See :ref:`flashrl-config` for more details on how this works. + +- 0.5B: ``FLASHRL_CONFIG=LiyuanLucasLiu/Qwen2.5-0.5B-Instruct-quantized.w8a8-RedHatAI/flashrl_config.yaml`` +- 32B: ``FLASHRL_CONFIG=LiyuanLucasLiu/Qwen2.5-32B-quantized.w8a8/flashrl_config.yaml`` + + +Training Qwen2.5-32B with Int8 rollouts on the DAPO recipe +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +First, prepare and save the dataset in a chosen ``DATA_DIR`` by running: + +.. code-block:: bash + # execute from skyrl-train directory + + DATA_DIR="$HOME/data/dapo" bash examples/algorithms/dapo/prepare_dapo_data.sh + +We highlight some important training parameters configured for FlashRL from our example configuration at :code_link:`examples/flash_rl/run_dapo_repro_flashrl_32b_int8.sh`: + +.. code-block:: bash + :caption: Training configuration at ``examples/flash_rl/run_dapo_repro_flashrl_32b_int8.sh`` + + # path for dataset (.parquet files) containing the prompts and metadata for each question + DATA_DIR="$HOME/data/dapo" + + # TIS parameters + USE_TIS=true + TIS_IMP_RATIO_CAP=8.0 + + uv run --isolated --extra flashrl --env-file examples/flash_rl/.env.int8 -m examples.flash_rl.main_dapo_flashrl \ + ... + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + generator.sampling_params.logprobs=0 \ + ... + +Here, we've configured training to use TIS with the importance sampling ratio cap of 8.0. ``generator.sampling_params.logprobs=0`` ensures that logprobs for the chosen tokens are returned by the inference engine, which is required for TIS. Note that for making sure the FlashRL patches are applied for vLLM, we use the ``FLASHRL_CONFIG`` environment variable in ``examples/flash_rl/.env.int8``: + +.. code-block:: bash + :caption: Environment variables at ``examples/flash_rl/.env.int8`` + + FLASHRL_CONFIG=LiyuanLucasLiu/Qwen2.5-32B-quantized.w8a8/flashrl_config.yaml + # FLASHRL_LOGGING_LEVEL=DEBUG <--- optional + ... + + +For a more lightweight example, we also provide scripts for training on Qwen2.5-0.5B-Instruct with Int8 rollouts at :code_link:`examples/flash_rl/run_dapo_repro_flashrl_0.5b_int8.sh`. + + +Training with FP8 +~~~~~~~~~~~~~~~~~~ + +The configuration is similar to the Int8 example. The only difference is the value for ``FLASHRL_CONFIG`` in ``examples/flash_rl/.env.0.5b_fp8``. We provide a script for training Qwen2.5-0.5B-Instruct with FP8 rollouts at :code_link:`examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_fp8.sh`. + + +.. _flashrl-config: + +What does the ``FLASHRL_CONFIG`` do? +------------------------------------ + +We use a custom vLLM wheel (in the ``--flashrl`` extra) to apply some patches from FlashRL. +The ``FLASHRL_CONFIG`` is used to customize vLLM initialization as well as weight syncing behavior. + +For FP8, this is simply a string (``fp8_vllm``) while for Int8, this is a path to a YAML file (either locally, accessible to all nodes in your Ray cluster, or a file path on the HuggingFace Hub). + +For Qwen2.5-0.5B-Instruct, the ``FLASHRL_CONFIG`` is ``LiyuanLucasLiu/Qwen2.5-0.5B-Instruct-quantized.w8a8-RedHatAI/flashrl_config.yaml`` which contains the following: + +.. code-block:: yaml + :caption: ``LiyuanLucasLiu/Qwen2.5-0.5B-Instruct-quantized.w8a8-RedHatAI/flashrl_config.yaml`` + + configs: + - distributed_executor_backend: external_launcher # ignored in SkyRL - We use the ray backend for vLLM + fn: int8 # dictates the quantization type + load_format: auto + model: LiyuanLucasLiu/Qwen2-0.5B-Instruct-quantized.w8a8-RedHatAI # custom model path passed to vLLM at init - weights are loaded directly in int8 + profile: LiyuanLucasLiu/Qwen2-0.5B-Instruct-quantized.w8a8-RedHatAI/profile.pt # calibration profile for Qwen's weights, used during weight syncing + +While most parameters are self-explanatory, the ``profile`` parameter is used to specify the calibration profile for Qwen's weights. This is used during weight syncing, when the policy model sends weights in half precision (bfloat16) to the inference engine. This profile is used to quantize the weights in bfloat16 to int8 before loading. + +.. warning:: + + FlashRL integration is experimental. While generation times can improve for large models with quantization, we've observed that the time spent in weight syncing is much higher with FlashRL for FP8. This negates some of the benefits of FP8 inference. The slowdown is primarily due to slow weight quantization in vLLM's ``process_weights_after_loading`` function and we are working on improving this. + + We recommend to use Int8 quantization over FP8 if possible. diff --git a/skyrl-train/docs/examples/images/openenv-reward.png b/skyrl-train/docs/examples/images/openenv-reward.png new file mode 100644 index 0000000000..1df0e54da8 Binary files /dev/null and b/skyrl-train/docs/examples/images/openenv-reward.png differ diff --git a/skyrl-train/docs/examples/images/parallelism.svg b/skyrl-train/docs/examples/images/parallelism.svg new file mode 100644 index 0000000000..f25644c84e --- /dev/null +++ b/skyrl-train/docs/examples/images/parallelism.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/skyrl-train/docs/examples/images/search-r1-30b.svg b/skyrl-train/docs/examples/images/search-r1-30b.svg new file mode 100644 index 0000000000..b8eb54113b --- /dev/null +++ b/skyrl-train/docs/examples/images/search-r1-30b.svg @@ -0,0 +1,100 @@ +
50100150200250300Step0.40.450.50.55
displayName: megatron_tp2_ep8_qwen30b_4turns_searchr1
 \ No newline at end of file diff --git a/skyrl-train/docs/examples/images/search-r1-3b.svg b/skyrl-train/docs/examples/images/search-r1-3b.svg new file mode 100644 index 0000000000..0e4871696a --- /dev/null +++ b/skyrl-train/docs/examples/images/search-r1-3b.svg @@ -0,0 +1,100 @@ +
20406080Step0.250.30.350.40.45
displayName: megatron_searchr1_dp_8
displayName: fsdp_searchr1
 \ No newline at end of file diff --git a/skyrl-train/docs/examples/llm_as_a_judge.rst b/skyrl-train/docs/examples/llm_as_a_judge.rst new file mode 100644 index 0000000000..feecd32f5a --- /dev/null +++ b/skyrl-train/docs/examples/llm_as_a_judge.rst @@ -0,0 +1,144 @@ +LLM as a Judge for GSM8K +========================================= + +This example demonstrates how to train a model using LLM as a judge for reward computation on the GSM8K dataset. Instead of using rule-based reward functions, this approach leverages an LLM (like GPT-4o-mini) to evaluate the quality of generated solutions. + +The implementation provides a custom environment that uses an LLM judge to compare predicted solutions against ground truth answers, offering more nuanced evaluation than simple exact matching. + +Task Overview +------------- + +In this task, the agent is given a math problem and must generate a solution that ends with the final answer in the format `#### `. The reward is computed by an LLM judge that evaluates: + +1. Whether the predicted solution ends with the correct format (`#### `) +2. Whether the final answer matches the ground truth exactly + +The LLM judge provides a binary reward (0 or 1) based on these criteria. + +Dataset Preparation +------------------- + +To download and prepare the dataset, run the following script: + +.. code-block:: bash + + uv run examples/llm_as_a_judge/gsm8k_dataset_judge.py --output_dir $HOME/data/gsm8k_llm_judge + +This script downloads the GSM8K dataset, extracts ground truth answers, and formats it for the LLM judge environment. + +Environment Implementation +--------------------------- + +The LLM judge environment is implemented in ``examples/llm_as_a_judge/llm_judge_env.py``. We use the OpenAI API to access the LLM judge. + +The environment sends the following prompt to the judge: + +.. code-block:: text + + You are a strict math evaluation assistant. + + Compare the following **gold** and **predicted** math solutions. Your job is to determine if the predicted solution is mathematically correct and if the predicted solution ends with a line of the form: + + #### + + You must only give a score of "1" if: + - The final line of the predicted solution **ends with `#### `**, and + - The number **matches the final answer in the gold solution** exactly. + + Instructions: + - You may provide internal reasoning or explanation before giving your final judgment. + - Your final judgment must appear as a separate line at the end of your response, in the format: + + ### Final Score: 1 + + or + + ### Final Score: 0 + + Do not include any explanation after the final score. + +Installation and Setup +---------------------- + +1. **Set Environment Variables**: Add your API keys to `.env.llm_judge` file. + + .. code-block:: bash + + OPENAI_API_KEY=your_openai_api_key + WANDB_API_KEY=your_wandb_api_key # optional + +2. **Verify Dataset**: Make sure your dataset is properly prepared: + + .. code-block:: bash + + ls $HOME/data/gsm8k_llm_judge/ + # Should show: train.parquet validation.parquet + +Training Configuration +---------------------- + +The training configuration uses GRPO with colocated training and generation. Key parameters include: + +.. code-block:: bash + :caption: Training configuration at ``examples/llm_as_a_judge/run_llm_judge.sh`` + + # Data and model paths + DATA_DIR="$HOME/data/gsm8k_llm_judge" + CKPT_PATH="$HOME/ckpts/llm_judge" + + # Hardware configuration + NUM_GPUS=4 + NUM_INFERENCE_ENGINES=4 + TP_SIZE=1 + + uv run --isolated --extra vllm --env-file .env.llm_judge -m examples.llm_as_a_judge.main_llm_judge \ + # Data configuration + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + + # Algorithm and training + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.epochs=20 \ + trainer.train_batch_size=32 \ + trainer.policy_mini_batch_size=32 \ + + # Placement and strategy + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + + # Generator configuration + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + generator.backend=vllm \ + generator.n_samples_per_prompt=5 \ + + # Environment and LLM judge configuration + environment.env_class=llm_as_a_judge \ + environment.skyrl_gym.llm_as_a_judge.model="gpt-4o-mini" \ + + # Other parameters (see the `examples/llm_as_a_judge/run_llm_judge.sh` for the full script) + ... + + +Launching Your Training Run +--------------------------- + +Now you can launch your training run with the following command: + +.. code-block:: bash + + bash examples/llm_as_a_judge/run_llm_judge.sh + +The training will use the LLM judge to evaluate each generated solution. + +What's Next? +------------ + +Now that you've seen how to use LLM as a judge for reward computation, you might want to explore: + +- :doc:`ppo`: Compare with rule-based PPO training on GSM8K +- :doc:`multi_turn_text2sql`: Explore multi-turn training with async rollouts +- :doc:`search`: Learn about multi-turn search agent training +- :doc:`../tutorials/new_env`: Learn how to build your own custom environments \ No newline at end of file diff --git a/skyrl-train/docs/examples/lora.rst b/skyrl-train/docs/examples/lora.rst new file mode 100644 index 0000000000..21c73a4bac --- /dev/null +++ b/skyrl-train/docs/examples/lora.rst @@ -0,0 +1,141 @@ +LoRA Training in SkyRL +====================== + +This guide demonstrates how to use Low-Rank Adaptation (LoRA) for efficient reinforcement learning training in SkyRL. LoRA allows you to train large language models with significantly reduced memory footprint and computational requirements by only updating a small set of parameters. + +What is LoRA? +------------- + +LoRA (Low-Rank Adaptation) is a parameter-efficient fine-tuning technique that freezes the pre-trained model weights and injects trainable rank decomposition matrices into each layer of the Transformer architecture. + +Configuration +------------- + +LoRA can be configured for both policy and critic models in the training configuration. Here's how to set it up: + +.. code-block:: yaml + + trainer: + policy: + model: + path: "Qwen/Qwen2.5-1.5B-Instruct" + lora: + rank: 32 # LoRA rank (higher = more parameters) + alpha: 32 # LoRA scaling parameter + dropout: 0 # LoRA dropout rate + lora_sync_path: "/tmp/skyrl_lora_sync" # Path for adapter sync + target_modules: "all-linear" # Apply to all linear layers OR + # specify specific modules as a list + exclude_modules: null # Modules to exclude from LoRA + critic: + model: + path: "Qwen/Qwen2.5-1.5B-Instruct" + lora: + rank: 32 + alpha: 32 + dropout: 0 + target_modules: "all-linear" + exclude_modules: null + +Key Parameters +~~~~~~~~~~~~~~ + +- **``rank``**: The rank of the low-rank decomposition. Higher values mean more trainable parameters but also more memory usage. Common values are 8, 16, 32, or 64. +- **``alpha``**: The scaling parameter for LoRA. Often set equal to rank, but can be tuned independently. +- **``dropout``**: Dropout rate applied to LoRA layers. Helps with regularization. +- **``lora_sync_path``**: Directory path where LoRA adapters are saved and synchronized between training and inference engines. +- **``target_modules``**: Specifies which modules to apply LoRA to. By default, LoRA is applied to all linear layers in the model - you can customize which modules to target by providing a list of specific module names. +- **``exclude_modules``**: Specifies which modules to exclude from LoRA application. By default, no modules are excluded. + +Running LoRA Training +--------------------- + +Here's a complete example of running LoRA training on GSM8K: + +Dataset Preparation +~~~~~~~~~~~~~~~~~~~ + +First, prepare your dataset: + +.. code-block:: bash + + uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + +Training Script +~~~~~~~~~~~~~~~ + +Create a training script with LoRA configuration: + +.. code-block:: bash + + #!/bin/bash + set -x + + DATA_DIR="$HOME/data/gsm8k" + NUM_GPUS=4 + LOGGER="wandb" # change to "console" to print to stdout + INFERENCE_BACKEND="vllm" + + CUDA_VISIBLE_DEVICES=0,1,2,3 uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.policy.model.lora.rank=32 \ + trainer.policy.model.lora.alpha=32 \ + trainer.policy.model.lora.lora_sync_path="/tmp/skyrl_lora_sync" \ + trainer.strategy=fsdp2 \ + trainer.placement.colocate_all=true \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=128 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=3.0e-5 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=4 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_0.5b_lora" \ + trainer.run_name="gsm8k_0.5b_lora_test" \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_0.5b_lora_ckpt" + +Launch Training +~~~~~~~~~~~~~~~ + +Set up your WandB API key and run the training: + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + bash examples/lora/run_qwen2_5_0.5b_gsm8k_grpo_lora.sh + +Configuration Tips +------------------ + +We recommend looking at the blog `LoRA Without Regret `_ for more details and practical guidance on setting LoRA hyperparameters. Here are the key takeaways: + +1. **Learning rate:** Use roughly 10× higher learning rate for LoRA than for full fine-tuning; LoRA is more tolerant of large learning rates, especially for shorter runs. + +2. **Rank:** Choose a rank large enough to match dataset complexity (e.g., 32–64 for most RL fine-tuning tasks). LoRA performs best when not capacity-limited. + +3. **Layer coverage:** Apply LoRA to *all* layers, particularly MLP/MoE layers — attention-only LoRA tends to underperform. + + +Current Limitations +------------------- + +SkyRL's LoRA implementation has the following current limitations: + +1. **Disk-based synchronization**: LoRA adapters are saved to disk and reloaded rather than synchronized in-memory. + +4. **Single adapter per model**: Currently, only one LoRA adapter can be active per model at a time. + +These limitations are being addressed in future releases, with plans for in-memory synchronization and improved adapter management. diff --git a/skyrl-train/docs/examples/megatron.rst b/skyrl-train/docs/examples/megatron.rst new file mode 100644 index 0000000000..ad695ad264 --- /dev/null +++ b/skyrl-train/docs/examples/megatron.rst @@ -0,0 +1,144 @@ +Megatron Backend for 5D Parallelism +=================================== + +SkyRL supports NVIDIA's `Megatron-Core `_ library as an RL training backend, inheriting support for 5D parallelism (tensor+sequence, pipeline, context, expert, and data parallelism), and optimized performance for large scale models. + +We provide example scripts for running efficient large scale MoE training with models like ``Qwen3-30B-A3B`` using Megatron in the `examples/megatron `_ directory. +For details on configuring the Megatron backend, and enabling checkpointing, see :ref:`megatron-configurations`, and :ref:`megatron-checkpointing`. + + +When to use the Megatron backend +-------------------------------- + +SkyRL supports efficient data-parallel training with the FSDP backend, with support for Ulysses sequence parallelism for long context training. The Megatron backend is useful to stack additional parallelism strategies (TP, PP, EP) on top of data and sequence/context parallelism. This is helpful both for fitting larger models into memory and for training throughput for MoE models (with EP). The Megatron backend is thus useful for efficient training of small MoE models like ``Qwen3-30B-A3B`` as well as large-scale training with large models such as ``Qwen3-235B-A22B`` and/or large datasets. For resources on understanding different parallelism strategies, see :ref:`parallelism-resources`. + +Comparison to FSDP +------------------ +We show performance comparisons for the Megatron and FSDP2 backends on the Search-R1 task (4K max context length) for various model sizes in the table below. Training speed for small scale dense models with Megatron +is similar to the FSDP2 backend, and Megatron enables high throughput training for larger scale MoE models where FSDP is no longer feasible as the only parallelism strategy. + +.. list-table:: + :header-rows: 1 + :widths: 20 15 20 20 20 10 + + * - Model + - Backend + - Compute + - Policy Training Time (s) + - Forward Pass Time (s) + - Avg Num Tokens + * - Qwen2.5-3B-Instruct + - Megatron + - 8xH100 + - 48 + - 39 + - 658 + * - Qwen2.5-3B-Instruct + - FSDP2 + - 8xH100 + - 42 + - 28 + - 658 + * - Qwen2.5-7B-Instruct + - Megatron + - 8xH100 + - 93 + - 46 + - 819 + * - Qwen2.5-7B-Instruct + - FSDP2 + - 8xH100 + - 100 + - 33 + - 834 + * - Qwen3-30B-A3B + - Megatron + - 4x8xH100 + - 189 + - 145 + - 1158 + +For all experiments, we used a train batch size of 512. For Qwen2.5 3B and 7B Megatron was configured with only data parallel=8. +For Qwen3-30B-A3B Megatron was configured with DP=4, TP=2, and EP=8, and the FSDP2 backend was unable to complete a training step due to memory constraints. +All statistics shown were averaged over the first 10 steps of training. Micro batch sizes were tuned to be the max possible for each backend. + +.. list-table:: + :widths: 50 50 + :header-rows: 0 + + * - .. image:: images/search-r1-3b.svg + :width: 400px + :align: center + - .. image:: images/search-r1-30b.svg + :width: 400px + :align: center + +.. centered:: Left: Matching Qwen2.5-3B-Instruct reward curves for Megatron and FSDP2. Right: Qwen3-30B-A3B reward curve for Megatron (330 steps on 4 8xH100 nodes over 4 days). + +A script for running the Qwen3-30B-A3B experiment can be found `here `_. +Additionally, we provide a script for running basic GSM8K training on Qwen3-235B-A22B with Megatron `here `_. +Note that although training at 100B+ scale using Megatron is currently possible, we are in the process of further optimizing peformance. + +For more details on configuring the Megatron backend, and enabling checkpointing, see :ref:`megatron-configurations`, and :ref:`megatron-checkpointing`. + +.. _megatron-installation: + +Installation +------------ + +Setting up the Docker image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +To get started, you can follow the instructions for installing via Docker in the :doc:`../getting-started/installation` page, but using the ``novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8-megatron`` image instead of the default image. + +This ensures that the necessary dependencies needed for Megatron are installed and don't need to be built on each node for each run, which can be time consuming. Previously, we recommended setting PYTHONPATH to the location of TransformerEngine installation, but this is no longer necessary. + +~~~~~~~~~~~~~~~ +In order to use flash attention with the megatron backend, you must use ``flash_attn`` version ``2.7.4.post1`` or lower for compatibility with ``TransformerEngine==2.7.0``. +This is handled in the ``pyproject.toml`` file for the ``mcore`` extra. + +Configuration +------------- +We provide the following options for fully configuring the Megatron backend, exposing the underlying Megatron optimizer, DDP, and model config objects +for advanced users to fully take advantage of all of Megatron-Core's feature flags. For more details, see the :ref:`megatron-configurations` section. + +.. code-block:: yaml + :caption: ``skyrl_train/config/megatron/policy.yaml`` + + # @package megatron_config.policy + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + context_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: null + + ddp_config: # pass-through config to Megatron's `DistributedDataParallelConfig` object + # https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/distributed/distributed_data_parallel_config.py#L8 + ... + optimizer_config_kwargs: # pass-through kwargs to Megatron's `OptimizerConfig` object + # any overlapping arguments with those we attempt to resolve in trainer.policy.optimizer_config will be overridden by the values here + # https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/optimizer/optimizer_config.py#L12 + ... + transformer_config_kwargs: # pass-through kwargs to the Megatron's `TransformerConfig` object + # https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/transformer/transformer_config.py#L33 + ... + lora_config: + # see: https://docs.nvidia.com/nemo/megatron-bridge/0.2.0/apidocs/bridge/bridge.peft.lora.html for details - currently "lora" and "canonical_lora" are supported + lora_type: "lora" + # flag to manually empty torch's cuda cache between the forward/backward pass and the optimizer step + # this will free reserved but unallocated memory, and can help avoid OoMs in the optimizer + empty_cuda_cache: true + +These default values can be overridden by passing in the corresponding arguments to ``trainer.policy.megatron_config`` in the launch script. + +.. _parallelism-resources: + +Parallelism Resources +---------------------- +Understanding and configuring parallelism strategies for large models can be challenging. +Some helpful resources for understanding and tuning large scale parallelism strategies can be found at the `Huggingface Ultra-Scale Playbook `_, +the `The Mesh Parallelism Zoo `_, and the `Visualizing 6-D Parallelism `_. + +Below, we show a diagram displaying how all 5 parallelism strategies - tensor, pipeline, context, expert, and data parallelism - can be utilized in SkyRL, as well as how dispatching data across these parallel groups works. + +.. image:: images/parallelism.svg + diff --git a/skyrl-train/docs/examples/mini_swe_agent.rst b/skyrl-train/docs/examples/mini_swe_agent.rst new file mode 100644 index 0000000000..afcc79e271 --- /dev/null +++ b/skyrl-train/docs/examples/mini_swe_agent.rst @@ -0,0 +1,112 @@ +SkyRL + Mini-SWE-Agent: Training a SWE-Agent for SWE-Bench +=========================================================== + +In this example, we walk through a simple example on how to train a SWE-Agent on the SWE-Bench task by leveraging `Mini-SWE-Agent `_ + + +How does it work? +------------------ + +The Mini-SWE-Agent integration with SkyRL can be found in :code_link:`examples/mini_swe_agent/`. We implement a custom generator ``MiniSweAgentGenerator`` to use Mini-SWE-Agent to generate trajectories for the SWE-Bench task. + + +.. code-block:: python + + class MiniSweAgentGenerator(SkyRLGymGenerator): + + async def generate_trajectory(self, prompt, ...): + ... + + async def generate(self, generator_input: GeneratorInput) -> GeneratorOutput: + ... + prompts = generator_input["prompts"] + env_extras = generator_input["env_extras"] + tasks = [] + for i in range(len(prompts)): + tasks.append( + self.generate_trajectory( + prompts[i], + env_extras[i], + ) + ) + + all_outputs = await asyncio.gather(*tasks) + ... + +In ``generate_trajectory`` we start a Ray task to generate a trajectory and evaluate it for the given instance. More concretely, this consists of the following: + +1. Generation: + - Initialize a sandbox / environment for the instance + - Generate a trajectory in this environment with Mini-SWE-Agent. For inference, we configure Mini-SWE-Agent to use the HTTP endpoint provided by SkyRL. + - Store the generated git patch after generation completes. +2. Evaluation: + - Initialize a fresh environment for the given instance using the given backend. + - Apply the model's git patch to the working directory in the environment. + - Run the evaluation script for the instance. If the script runs successfully, the instance is considered to be resolved, and unresolved otherwise. + +By running this workflow as a Ray task, we are also able to scale up generation across all the nodes in the Ray cluster. + + + At a high level, the code looks as follows: + +.. code-block:: python + + @ray.remote(num_cpus=0.01) + def init_and_run(instance: dict, litellm_model_name: str, sweagent_config: dict, data_source: str, ...): + model = get_model(litellm_model_name, sweagent_config.get("model", {})) + error = None + try: + env = get_sb_environment(sweagent_config, instance, data_source) + agent = DefaultAgent(model, env, **sweagent_config.get("agent", {})) + exit_status, model_patch = agent.run(instance["problem_statement"]) + eval_result = evaluate_trajectory(instance, model_patch, sweagent_config, data_source) + except Exception as e: + error = str(e) + return agent.messages, eval_result, error + + class MiniSweAgentGenerator(SkyRLGymGenerator): + async def generate_trajectory(self, prompt, env_extras, ...): + messages, eval_result, error = init_and_run.remote(env_extras["instance"], ...) + ... + +Note that the full implementation has some additional logic for configuration and error handling, and can be found in :code_link:`examples/mini_swe_agent/mini_swe_generator.py`. + + +Dataset preparation +------------------- + +For training, we use `SWE-Gym `_, and more specifically the subset of SWE-Gym in `SumanthRH/SWE-Gym-Subset `_. + +Execute the following command: + +.. code-block:: bash + + # execute from skyrl-train directory + uv run --isolated examples/mini_swe_agent/preprocess_swegym.py --output_dir ~/data/swe_gym_subset + + +Training +--------- + +Prerequisites: Ensure that you have the required environment backend installed for generating trajectories with Mini-SWE-Agent. By default, we use `Podman `_. This can be modified in :code_link:`examples/mini_swe_agent/swebench.yaml` + +We provide two example scripts: One for Qwen3-8B model and another for the `Qwen/Qwen3-Coder-30B-A3B-Instruct `_ model. While the first script for Qwen3-8B requires a single 8xH100 node, the script for the 30B model requires 2 8xH100 nodes for training. + +.. code-block:: bash + + # execute from skyrl-train directory + bash examples/mini_swe_agent/run_mini_swe_8B.sh + # or for 30B: + # bash examples/mini_swe_agent/run_mini_swe_30B.sh + + +Tips +~~~~~ + +- If you notice too many errors such as ``ValueError: The decoder prompt (length xxxx) is longer than the maximum model length`` in the logs, this means that the LLM is hitting context length limits. Training can still proceed as usual, but if there are too many such errors per batch, then you should either increase the sequence length (increase ``max_input_length`` and ``max_generate_length``) or reduce the number of steps in the ``swebench.yaml`` file. +- The task can sometimes be too difficult for the base model. For convenience, we log the list of rewards in a batch. If the rewards are all zeros, then the batch is too hard. If you notice too many such batches in your dataset, you should either (1) filter your data to have a better mix of easy and hard samples to promote learning (2) choose a stronger base model or (3) increase ``step_limit`` in ``swebench.yaml``. We've noticed that SWE-Gym can be hard (i.e most 0 rewards) for the Qwen3-8B with the given settings. The choice of the available tools can also affect performance (in Mini-SWE-Agent, agents have one tool - bash commands) +- If you notice errors like "Error during evaluation [Errno 7] Argument list too long: 'podman'" , this is because the evaluation logic currently applies the model's git patch in-line, and for very large git patches, you will hit system ``ARG_MAX`` limits. On modern systems, this maximum is ~ 1 MB, which is very generous. We thus make a simple assumption that large patches that exceed this limit are meant to be incorrect. +- If running podman within a container, you might hit errors due to insufficient UIDs. While the training logs will only have a brief error message: ``Command '['podman', 'run', '-d', '--name', 'minisweagent-e7fbf68e', '-w', '/testbed', '--rm', 'docker://swebench/sweb.eval.x86_64.matplotlib_1776_matplotlib-24026:latest', 'sleep', '2h']' returned non-zero exit status 125.``, more information is available if you run the command with ``--log-level=debug``. Make sure that your user namespace has sufficient UIDs and GIDs. You have two options on Linux-based machines: + + 1. Edit the ``/etc/subuid`` and ``/etc/subgid`` files to use a large range such as ``100000-1100000``. + 2. Set ``ignore_chown_errors=true`` in the containers.conf file for Podman as described `here `_. diff --git a/skyrl-train/docs/examples/multi_turn_text2sql.rst b/skyrl-train/docs/examples/multi_turn_text2sql.rst new file mode 100644 index 0000000000..c5264d5133 --- /dev/null +++ b/skyrl-train/docs/examples/multi_turn_text2sql.rst @@ -0,0 +1,175 @@ +Multi-Turn GRPO Text2SQL with Async Rollouts +===================================================== + +In this example, we walk through how to train an effective multi-turn Text2SQL model (beating GPT-4o) with SkyRL using async rollouts. + +We provide an implementation of a multi-turn Text2SQL environment at :skyrl_gym_link:`skyrl_gym/envs/sql/env.py`. + +You can find the exact recipe for reproducing our prior `SkyRL-SQL-7B `_ release at :doc:`../recipes/skyrl-sql`. + +Task Overview +------------- + +In this task, the agent is given a natural language question and a database schema, and is tasked with generating a SQL query to answer the question. +An abbreviated example prompt is shown below: + +.. code-block:: text + + You are a data science expert. Below, you are provided with a database schema and a natural language question. Your task is to understand the schema and generate a valid SQL query to answer the question within limited turns. You should breakdown the problem, draft your reasoning process, and generate the solution. + + Database Engine: + SQLite + + Database Schema: + {db_details} + This schema describes the database's structure, including tables, columns, primary keys, foreign keys, and any relevant relationships or constraints. + + External Knowledge: + {external_knowledge} + + Question: + {question} + + Instructions: + - Make sure you only output the information that is asked in the question. If the question asks for a specific column, make sure to only include that column in the SELECT clause, nothing more. + - The generated query should return all of the information asked in the question without any missing or extra information. + - Before generating the final SQL query, please think through the steps of how to write the query. It should include detailed considerations such as analyzing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, thinking of how to call SQL tools, and revisiting previous steps. + + Format: + - Conduct thinking inside ... blocks every time you get new observation or information. + - You can use SQL tool written within a single your sql block to explore or verify. SQL tool output will be shown as dataframe inside .... Based on this observation, you can think again and refine. + - The returned dataframe will be truncated in 50 rows if observation is too long. + - If you find no further exploration is needed or reaches max turns, you MUST directly provide the final SQL query solution inside .... + ... + {db_details}: + CREATE TABLE country_aliases ( + alias_id INTEGER, + country_id INTEGER, + alias_name TEXT, + alias_type TEXT, + PRIMARY KEY (alias_id), + FOREIGN KEY (country_id) REFERENCES countries (country_id) + ); + ... + {question}: + Could you please provide me with the English name, current population, average population over the last five years, and population from last year for each country, based on the country's population metrics? I need this information to analyze population trends. + +The agent is given a maximum of 6 turns to generate a valid SQL query with the correct formatting, with negative rewards being assigned for incorrect formatting, 0 rewards for correct formatting but incorrect SQL, and a positive reward for correct SQL. + +Data Preparation +---------------- + +First, follow the :ref:`data preparation instructions ` to download the dataset, as well as the database files (note that +the database files are quite large - around 50GB in total). We reproduce the commands here: + +.. code-block:: bash + + hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset + hf download seeklhy/OmniSQL-datasets data.zip --repo-type dataset --local-dir + unzip + +Training Configuration +---------------------- +Now that we have our dataset and database files, let's walk through the some of the key training configurations. + + +.. code-block:: bash + :caption: Training configuration at ``skyrl_train/examples/text_to_sql/run_skyrl_sql.sh`` + + # path for dataset (.parquet files) containing the prompts and metadata for each question + DATA_DIR="$HOME/data/sql" + # path for .db files for environment interaction + DB_PATH="$HOME/path/to/db_files" + + uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + #### Environment configuration + environment.env_class=text2sql \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + + #### Multi-turn Async Rollouts configuration + # this is used to set the max turns for the environment + generator.max_turns=6 \ + # we need to make sure to set async_engine=true for async rollouts + generator.async_engine=true \ + # we need to make sure to set batched=false for async rollouts + generator.batched=false \ + + #### context length related configurations + # trainer.max_prompt_length is the max length of the initial prompt + trainer.max_prompt_length=6000 \ + # generator.max_input_length is the max length of the input to the model after any number of turns (including the initial prompt) + generator.max_input_length=29000 \ + # generator.sampling_params.max_generate_length is the max length of the generated response for EACH turn + generator.sampling_params.max_generate_length=3000 \ + + #### multi-turn generation format - see `skyrl_train/generators/skyrl_gym_generator.py` for more details + generator.use_conversation_multi_turn=false \ + + #### data configuration + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + + #### Placement configuration - note since we set use_kl_loss=false below, we don't need to use a ref model + # policy placement configuration + trainer.policy.model.path="Qwen/Qwen2.5-Coder-7B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.placement.policy_num_gpus_per_node=8 \ + # inference engine placement configuration + generator.num_inference_engines=2 \ + generator.inference_engine_tensor_parallel_size=4 \ + + #### algorithm configuration + trainer.epochs=30 \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.use_kl_loss=false \ + trainer.algorithm.use_kl_loss=false \ + generator.n_samples_per_prompt=5 \ + + #### generation sampling params (relevant to algorithm correctness) + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.eval_sampling_params.stop='["", ""]' \ + + #### training configuration + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.train_batch_size=256 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + ... # Other parameters (see `examples/text_to_sql/run_skyrl_sql.sh` for the full script) + +- All we have to do to enable multi-turn training with async rollouts is to simply set ``generator.max_turns`` to the maximum number of turns we want the agent to take, + and to make sure ``generator.async_engine=true`` and ``generator.batched=false``. + +- Chat templating and loss masking for multi-turn conversations are handled by the ``SkyRLGymGenerator`` class. + + - In the above example, we set ``use_conversation_multi_turn=false`` to enforce that the multi-turn conversation is formatted as a single assistant response. + - We also set ``stop='["", ""]'`` for both ``sampling_params`` and ``eval_sampling_params`` as a part + of the training recipe. + - If you are using ``generator.use_conversation_multi_turn=true``, you might want to append an EOS token ID to the end of the response after these stop strings to adhere to the model's behavior (i.e. ending generation with an EOS token ID rather than say ````). This can be done by setting ``generator.append_eos_token_after_stop_str_in_multi_turn=true`` in the generator config. The full script is available in `examples/text_to_sql/run_skyrl_sql_conversation_format.sh`. + - If you want to use a conversation-based format, you can set ``use_conversation_multi_turn=true`` and the model will generate a separate assistant response for each turn. This is supported only with ``backend="vllm"`` as of now. + - See :code_link:`skyrl_train/generators/skyrl_gym_generator.py` for more details on both options! + +Launching Your Training Run +--------------------------- + +Let's get our training run started! Make sure to set your WandB API key for logging, and that your database and dataset paths are correctly set. + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + bash examples/text_to_sql/run_skyrl_sql.sh + +And now watch your model start to learn to generate better SQL queries! + +What's Next? +------------ + +Now that you've seen what's possible with multi-turn training with async rollouts, you might want to start building your own multi-turn environments! + +- :doc:`../tutorials/new_env`: Learn how to build your own multi-turn environments! +- :doc:`search`: Learn how to train a multi-turn search agent with SkyRL! \ No newline at end of file diff --git a/skyrl-train/docs/examples/openenv.rst b/skyrl-train/docs/examples/openenv.rst new file mode 100644 index 0000000000..1056e42b50 --- /dev/null +++ b/skyrl-train/docs/examples/openenv.rst @@ -0,0 +1,120 @@ +SkyRL + OpenEnv: Training a RL Agent in OpenEnv +=========================================================== + +In this example, we walk through a simple example on how to train a reinforcement learning agent using SkyRL with `OpenEnv `_ environments. OpenEnv provides isolated execution environments for agentic RL training with Gymnasium-style APIs. + +How does it work? +------------------ + +SkyRL integrates with any Gymnasium API-based environment easily with SkyRL-Gym, which provides a simple interface for text-based environments called ``BaseTextEnv``. We integrate OpenEnv environments through a custom environment wrapper ``OpenEnv`` that implements the ``BaseTextEnv`` interface. This wrapper allows SkyRL to interact with various OpenEnv environments, including: + +- **Echo Environment**: Simple echo environment for testing +- **Coding Environment**: Python code execution in sandboxed environment +- **OpenSpiel Environment**: Game environments using OpenSpiel +- **Atari Environment**: Classic Atari game environments +- **SUMO-RL Environment**: Traffic simulation environments +- **FinRL Environment**: Financial trading environments + +The integration works by: + +1. **Environment Registration**: The OpenEnv environment is registered dynamically in the entrypoint using ``register()`` from ``skyrl_gym.envs`` +2. **Environment Initialization**: SkyRL creates an OpenEnv client using ``from_docker_image()`` to connect to the appropriate Docker container +3. **Action Parsing**: LLM responses are parsed into environment-specific actions (e.g., ``EchoAction``, ``CodeAction``) +4. **Step Execution**: Actions are executed in the isolated environment and observations/rewards are returned +5. **Episode Management**: The environment tracks conversation history and manages episode termination + +At a high level, the integration looks as follows: + +.. code-block:: python + + # The OpenEnv wrapper class + class OpenEnv(BaseTextEnv): + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + self.env_name = extras["env_name"] + self.env_type = self._get_env_class(self.env_name) + self.env = self.env_type.from_docker_image(self.env_name + ":latest") + self.initial_step_result = self.env.reset() + + def step(self, action: str) -> BaseTextEnvStepOutput: + action = self._get_openenv_action(self.env_name, action) + result = self.env.step(action) + # Process result and return observations, reward, done + + +Finally, we also register the new environment in the entrypoint script: + +.. code-block:: python + + # In integrations/openenv/entrypoints/main_openenv.py + from skyrl_gym.envs import register + + register( + id="openenv", + entry_point="integrations.openenv.env:OpenEnv", + ) + + +Environment Setup +----------------- + +Prerequisites: Ensure that you have Docker installed + +First, we need to install the OpenEnv environments: + +.. code-block:: bash + + # Execute from skyrl-train directory + cd SkyRL/skyrl-train + uv run integrations/openenv/install_environment.py echo-env + # Or install all environments: + # uv run integrations/openenv/install_environment.py + +This will pull the necessary Docker images for the OpenEnv environments. + +Dataset Preparation +------------------- + +For training, we use simple example datasets generated by the ``prepare_dummy_dataset.py`` script: + +.. code-block:: bash + + # Execute from skyrl-train directory + cd SkyRL/skyrl-train + uv run integrations/openenv/prepare_dummy_dataset.py --output_dir ~/data/openenv --env_name echo_env + +This creates training and validation datasets with example prompts for the specified environment. We provide dummy train set examples for ``echo_env`` and ``coding_env``. + +Training +-------- + +We provide an example training script for Qwen2.5-1.5B-Instruct on OpenEnv environments: + +.. code-block:: bash + + # Execute from skyrl-train directory + cd SkyRL/skyrl-train + bash integrations/openenv/run_openenv.sh + +You can customize the training by setting environment variables: + +.. code-block:: bash + + ENV_NAME=echo_env NUM_GPUS=2 bash integrations/openenv/run_openenv.sh + + +Supporting environments are: ``echo_env``, ``coding_env``, ``openspiel-env``, ``atari-env``, ``sumo-rl-env``, ``finrl-env``. + +Example Reward Curve +-------- + +Here's how the reward curve for the above example script looks like after a few steps: + +.. image:: images/openenv-reward.png + :scale: 60% + :align: center +Tips +~~~~~ + +- **Docker Resources**: Ensure sufficient Docker resources are available, especially for computationally intensive environments like Atari or OpenSpiel. +- **Generation Format**: The generation format is expected to be a single action wrapped in ``...`` tags for dummy testing. Change `_get_openenv_action` in :code_link:`integrations/openenv/env.py` for custom parsing logic. +- **Multi-Turn Interaction**: Pass ``MAX_TURNS=xx`` to enable multi-turn interaction. \ No newline at end of file diff --git a/skyrl-train/docs/examples/ppo.rst b/skyrl-train/docs/examples/ppo.rst new file mode 100644 index 0000000000..7d05798354 --- /dev/null +++ b/skyrl-train/docs/examples/ppo.rst @@ -0,0 +1,98 @@ +PPO on GSM8K +========================================= + +This example demonstrates how to run PPO training on GSM8K. See :doc:`../getting-started/quickstart` for a similar setup process for GRPO! + +Dataset Preparation +------------------- + +To download and prepare the GSM8K dataset, run the following script. + +.. code-block:: bash + + uv run --isolated examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + +Training Configuration +---------------------- + +Next, let's set up the training configuration. You can find a complete example in ``examples/ppo/run_ppo.sh`` - we highlight some key parameters here: + +.. code-block:: bash + + uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + # Data setup + data.train_data="['$HOME/data/gsm8k/train.parquet']" \ + data.val_data="['$HOME/data/gsm8k/validation.parquet']" \ + + # Trainer and training algorithm + trainer.algorithm.advantage_estimator="gae" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.critic.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + + # Model placement and training strategy (colocate or disaggregate, sharding, etc.) + trainer.strategy=fsdp2 \ + trainer.placement.colocate_all=true \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.critic_num_gpus_per_node=4 \ + + # Batch sizes for critic and policy forward and training passes + trainer.policy_mini_batch_size=256 \ + trainer.critic_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + + # Evaluation and checkpointing + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.ckpt_interval=10 \ + + # Generator setup for spinning up InferenceEngines + generator.backend=vllm \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + generator.weight_sync_backend=nccl \ + + # Environment class for the dataset + # Can be specified here to apply to the full dataset, or at the per-prompt level during preprocessing + environment.env_class=gsm8k \ + + ... # Other parameters (see `examples/ppo/run_ppo.sh` for more) + +Hardware Configuration +---------------------- + +Depending on your hardware setup, you may want to adjust a few parameters: + +1. **GPU Configuration**: Set ``trainer.placement.policy_num_gpus_per_node`` and ``trainer.placement.critic_num_gpus_per_node`` to +match your available GPU count. Note that since we are setting ``trainer.placement.colocate_all`` to ``true``, +the total number of GPUs used for the policy model and critic model should be the same. +Additionally ``generator.num_inference_engines`` * ``generator.inference_engine_tensor_parallel_size`` should +also be equal to the total number of GPUs used for the policy and critic models. + +Launching Your Training Run +--------------------------- + +Let's get our PPO training run started! First, configure your WandB API key for logging: + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + +Then launch your training run: + +.. code-block:: bash + + bash examples/ppo/run_ppo.sh + +Congratulations! You've just launched your first PPO training run! + +What's Next? +------------ + +Now that you've got basic colocated PPO training down, you might want to explore some more advanced features: + +- :doc:`../tutorials/one_step_off_async`: Asynchronous off-by-one training in < 100 lines of code! +- :doc:`../examples/remote_server`: Training with a remote inference engine + + diff --git a/skyrl-train/docs/examples/remote_server.rst b/skyrl-train/docs/examples/remote_server.rst new file mode 100644 index 0000000000..9c86403afb --- /dev/null +++ b/skyrl-train/docs/examples/remote_server.rst @@ -0,0 +1,129 @@ +Remote Inference Engine +========================================= +This example shows how to run GRPO training on GSM8K using a standalone inference engine instance behind an HTTP server. + +In this example, we will be using 4 GPUs for training (policy and ref models) and 4 GPUs for inference (inference engine) on a single node. + +Prerequisites +---------------------- + +First, make sure you are familiar with the standard setup process for running GRPO training. See :doc:`../getting-started/quickstart` for more details. + + +Starting the Remote Inference Engine +------------------------------------ + +We provide an example remote server implementation at :code_link:`skyrl_train/inference_engines/vllm/vllm_server.py`. +In addition to the standard OpenAI-compatible generation endpoints (``/v1/chat/completions`` and ``/v1/completions``), the remote server simply needs to expose endpoints for the following operations: + +- ``/sleep`` and ``/wake_up`` to sleep and wake up the inference engine (only needed if you want to colocate the inference engine with the training process) +- ``/init_weight_update_communicator`` to create a process group for weight updates +- ``/update_weight`` to update the model weights + +That's it! We use a custom ``WorkerWrap`` class at :code_link:`skyrl_train/inference_engines/vllm/vllm_engine.py` with VLLM's worker extension API to enable efficient NCCL weight updates. + +To launch the server, run the following command (the full script is at :code_link:`examples/remote_inference_engine/run_vllm_server.sh`): + +.. code-block:: bash + + uv run --isolated --extra vllm -m skyrl_train.inference_engines.vllm.vllm_server \ + # model and tensor parallel size + --model Qwen/Qwen2.5-1.5B-Instruct \ + --tensor-parallel-size 4 \ + + # host and port for the server + --host 127.0.0.1 \ + --port 8001 \ + + # seed for reproducibility + --seed 42 \ + + # max context length + --max-model-len 4096 \ + + # vllm performance related settings + --enable-prefix-caching \ + --enable-chunked-prefill \ + --dtype bfloat16 \ + --gpu-memory-utilization 0.9 \ + --enable-sleep-mode \ + --max-num_batched_tokens 8192 \ + --max-num-seqs 1024 \ + + # other misc settings + --trust-remote-code \ + --distributed-executor-backend ray \ + + # worker extension class for handling weight updates + --worker-extension-cls skyrl_train.inference_engines.vllm.vllm_engine.WorkerWrap + +Starting Training +---------------------- + +Now that we've started our remote inference engine (located behind ``127.0.0.1:8001``), we can start a training run! + +To start training, we need to set up our training script. You can find a complete example in :code_link:`examples/remote_inference_engine/run_remote.sh`: + +.. code-block:: bash + + uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + # Setup for training with a remote inference engine + generator.run_engines_locally=False \ + generator.remote_inference_engine_urls="['127.0.0.1:8001']" \ + generator.override_existing_update_group=True \ + + # sampling parameters for generation + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + + # Data setup + data.train_data="['$HOME/data/gsm8k/train.parquet']" \ + data.val_data="['$HOME/data/gsm8k/validation.parquet']" \ + + # Policy model - make sure this is the same model used to launch the inference engine server + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.algorithm.advantage_estimator="grpo" \ + + # Whether or not to colocate all models on the same set of GPUs - we set it to false here, + # but you can colocate even with a standalone inference engine! + trainer.placement.colocate_all=False \ + + # Model placement arguments for policy and ref models - make sure that the total number of gpus + # used for training and inference is maximized + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + + # Training batch size and mini/micro batch sizes for logprobs + training passes + trainer.train_batch_size=64 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=20 \ + trainer.micro_train_batch_size_per_gpu=20 \ + + # Evaluation + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + + ... # Other parameters (see `examples/remote_inference_engine/run_remote.sh` for more) + +.. tip:: + +With remote servers, there can be non-trivial HTTP overhead during generation. When running training and inference in the same Ray cluster, it is recommended to use `run_engines_locally=True` to maximize throughput + +Launching Your Training Run +--------------------------- + +You're done setting up! Now let's get our training run started! + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + bash examples/remote_inference_engine/run_remote.sh + +What's Next? +------------ + +Now that you've set up training with a remote inference engine, you might want to explore ways of speeding up training: + +- :doc:`../tutorials/one_step_off_async`: Asynchronous off-by-one training in < 100 lines of code! + diff --git a/skyrl-train/docs/examples/search.rst b/skyrl-train/docs/examples/search.rst new file mode 100644 index 0000000000..0e4346880d --- /dev/null +++ b/skyrl-train/docs/examples/search.rst @@ -0,0 +1,157 @@ +Multi-Turn RL for Search with SkyRL +===================================================== + +In this example, we walk through an example for training a multi-turn search agent with Qwen2.5-3B-Instruct and GRPO (with VLLM async rollouts), using the dataset and recipe +from `Search-R1 `_. + +The full implementation of the search environment can be found in :skyrl_gym_link:`skyrl_gym/envs/search/env.py`. + +You can find the exact step by step commands to reproduce our results in the :doc:`../recipes/searchr1` recipe, and you can find a link to our training runs +with 2, 3, and 4 turns for comparison at our `WandB report `_. + + +Task Overview +------------- + +In this task, the agent is given a natural language question and the ability to query a search engine. The agent can use the search engine to help answer the question. +An example prompt is shown below: + +.. code-block:: text + + You are a helpful and harmless assistant. + + Answer the given question. You must conduct reasoning inside and first every time you get new information. + After reasoning, if you find you lack some knowledge, you can call a search engine by query + and it will return the top searched results between and . You can search as many times as you want. + If you find no further external knowledge needed, you can directly provide the answer inside and , without detailed illustrations. + For example, Beijing . + + + Question: In what year was the company that was founded as Sound of Music added to the S&P 500? + +The agent is given ``n`` turns to output an answer to the question within the ```` and ```` tags, meaning the agent has ``n - 1`` turns to query the search engine by outputting a query inside the ```` and ```` tags. +A reward of 0 is given for incorrect responses, and a reward of 1 is given for correct responses (we do not apply format rewards). + +Training Configuration +---------------------- +Let's walk through configuration for running GRPO to train a 4-turn search agent on the SearchR1 dataset + +.. code-block:: bash + :caption: Training configuration at ``skyrl_train/examples/search/run_search.sh`` + + # path for dataset (.parquet files) containing the prompts and metadata for each question + DATA_DIR="$HOME/data/searchR1" + + uv run --isolated --frozen --extra vllm -m skyrl_train.entrypoints.main_base \ + # - Dataset: train/val data paths + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + + # - Algorithm: GRPO settings, learning rate, KL loss + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.optimizer_config.num_warmup_steps=94 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.kl_loss_coef=0.001 \ + + # - Model: model path, placement, FSDP settings + trainer.policy.model.path="Qwen/Qwen2.5-3B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.placement.policy_num_gpus_per_node=8 \ + trainer.placement.ref_num_gpus_per_node=8 \ + + # - Generator: VLLM backend, GPU settings + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=2 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.gpu_memory_utilization=0.5 \ + + # - Training: epochs, batch sizes + trainer.epochs=1 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=512 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + + # - Length limits: prompt and generation lengths + # trainer.max_prompt_length is the max length of the initial prompt + trainer.max_prompt_length=2048 \ + # generator.max_input_length is the max length of the input to the model after any number of turns (including the initial prompt) + generator.max_input_length=4096 \ + # generator.sampling_params.max_generate_length is the max length of the generated response for EACH turn + generator.sampling_params.max_generate_length=500 \ + + # - Generator multi-turn: async rollouts, batching, sampling settings + # we need to make sure to set async_engine=true for async rollouts + generator.async_engine=true \ + # we need to make sure to set batched=false for async rollouts + generator.batched=false \ + generator.n_samples_per_prompt=5 \ + # this is used to set the max turns for the environment + generator.max_turns=4 \ + # multi-turn generation format - see `skyrl_train/generators/skyrl_gym_generator.py` for more details + generator.use_conversation_multi_turn=false \ + generator.sampling_params.temperature=1.0 \ + generator.sampling_params.top_p=1.0 \ + generator.sampling_params.stop='["", ""]' \ + + # - Environment: environment class, max env workers, search env settings + environment.env_class="search" \ + environment.skyrl_gym.max_env_workers=16 \ + environment.skyrl_gym.search.log_requests=false \ + environment.skyrl_gym.search.search_url="http://127.0.0.1:8000/retrieve" \ + environment.skyrl_gym.search.topk=3 \ + + # - Evaluation: batch size, intervals, sampling params + trainer.eval_batch_size=256 \ + trainer.eval_before_train=false \ + generator.eval_sampling_params.temperature=0 \ + generator.eval_sampling_params.stop='["", ""]' \ + trainer.eval_interval=50 \ + ... # logging + checkpointing configuration (see `examples/search/run_search.sh` for the full script) + +To change the number of turns, you can simply change the ``generator.max_turns`` setting. +For more details on environment implementation, see :skyrl_gym_link:`skyrl_gym/envs/search/env.py`. + +Note we add ``stop='["", ""]'`` for both generation and evaluation sampling parameters +to adhere to the Search-R1 recipe. + +If you are using ``generator.use_conversation_multi_turn=true``, +you might want to append an EOS token ID to the end of the response after these stop strings to adhere +to the model's behavior (i.e. ending generation with an EOS token ID rather than say ````). +This can be done by setting ``generator.append_eos_token_after_stop_str_in_multi_turn=true`` in the generator config. +The full script is available in `examples/search/run_search_conversation_format.sh`. + + +Launching Your Training Run +--------------------------- + +Let's get our training run started! Make sure your WandB API key is set, your dataset paths are correctly set, and that you have launched the local retrieval server, following the :doc:`../recipes/searchr1` recipe instructions. + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + bash examples/search/run_search.sh + +Now just sit back and watch your model learn to search! You can find a link to our training runs with 2, 3, and 4 turns for comparison at our `WandB report `_. + +Attribution +------------- +We thank the authors of Search-R1 for their work: `paper `_, `code `_. +Additionally we thank the SGLang + Verl team for their work reproducing Search-R1 in Verl, which we use to validate our results: `doc `_, +`wandb `_, and `PR `_. + +What's Next? +------------ + +Now that you've trained a multi-turn search agent, you might want to build your own multi-turn environments: + +- :doc:`../tutorials/new_env`: Learn how to build your own multi-turn environments! +- :doc:`../examples/multi_turn_text2sql`: Learn how to train a multi-turn text2sql agent with SkyRL! \ No newline at end of file diff --git a/skyrl-train/docs/examples/training_backends.rst b/skyrl-train/docs/examples/training_backends.rst new file mode 100644 index 0000000000..7eb867df9a --- /dev/null +++ b/skyrl-train/docs/examples/training_backends.rst @@ -0,0 +1,62 @@ +Switching Training Backends +========================================= +In SkyRL, you can switch between different training backends with minimal changes to your training script. + +Currently, we support the following training backends: + +- FSDP +- FSDP2 +- Megatron + +To switch to a different backend, simply set the ``trainer.strategy`` parameter to the desired backend. We use the ``fsdp2`` backend by default. + +Prerequisites +------------- +First, make sure you are familiar with the standard setup process for running GRPO training. See :doc:`../getting-started/quickstart` for more details. + +Running the Examples +--------------------- + +We provide baseline examples for GRPO training on GSM8K for each of these backends starting from the basic quickstart example. The quickstart script is available at :code_link:`skyrl_train/examples/gsm8k/run_gsm8k.sh`. + +.. code-block:: bash + + uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$HOME/data/gsm8k/train.parquet']" \ + data.val_data="['$HOME/data/gsm8k/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + ... # Other parameters (see `examples/gsm8k/run_gsm8k.sh` for more) + +FSDP and FSDP2 +~~~~~~~~~~~~~~ + +To switch to FSDP or FSDP2, set the ``trainer.strategy`` parameter to ``fsdp`` or ``fsdp2`` respectively. + +.. code-block:: bash + + # bash examples/training_backends/fsdp/run_fsdp.sh (or just) + bash examples/gsm8k/run_gsm8k.sh trainer.strategy=fsdp + +.. code-block:: bash + + # bash examples/training_backends/fsdp/run_fsdp2.sh (or just) + bash examples/gsm8k/run_gsm8k.sh trainer.strategy=fsdp2 + +Additionally, you can tune `FSDP specific configurations `_ as shown below: + +.. code-block:: bash + + # enable offloading of model parameters to CPU during the forward pass for the ref model + trainer.ref.fsdp_config.cpu_offload=true \ + +Note that ``cpu_offload`` is distinct from worker state offloading with model colocation. You can find details on this, as well as the full set of FSDP configurations at :ref:`fsdp-configurations`. + +.. note:: + ``cpu_offload`` cannot be enabled for the policy or critic model with FSDP1, since gradient accumulation outside ``no_sync`` mode is not supported with CPU offloading. + See the limitations section in `FSDP docs `_ for more details. + +Megatron +~~~~~~~~ + +Switching to the megatron backend is more involved, requiring additional dependencies and configuration. For more details, see the docs on Megatron :ref:`megatron-installation`. diff --git a/skyrl-train/docs/fleet-training.md b/skyrl-train/docs/fleet-training.md new file mode 100644 index 0000000000..a0c5c710d9 --- /dev/null +++ b/skyrl-train/docs/fleet-training.md @@ -0,0 +1,112 @@ +# SkyRL Fleet Training Guide + +RL training on Fleet-hosted environments using SkyPilot for GPU provisioning. + +## Quick Start + +**Run training via GitHub Actions:** +1. Go to **Actions** → **"Fleet Task Training (SkyPilot)"** → **"Run workflow"** +2. Select `modality` (`tool_use` or `computer_use`), optionally filter by `env_key`, and set `task_name` to point to a YAML file in `skyrl-train/tasks/` +3. Training provisions H100, logs to WandB, uploads checkpoints to S3 + +**Run training via CLI:** +```bash +sky launch skyrl-train/tasks/openenv-fleet-grpo.yaml \ + --env FLEET_API_KEY=$FLEET_API_KEY \ + --env WANDB_API_KEY=$WANDB_API_KEY \ + --env MODALITY=tool_use \ + -d -y +``` + +## Available Tasks (4,881 total) + +| Environment | tool_use | computer_use | Total | +|-------------|----------|--------------|-------| +| github | 1,865 | 0 | 1,865 | +| booking | 1,089 | 0 | 1,089 | +| google-maps | 13 | 739 | 752 | +| reddit | 246 | 24 | 270 | +| ticketmaster | 222 | 25 | 247 | +| amazon | 0 | 145 | 145 | +| rops | 0 | 113 | 113 | +| zillow | 76 | 30 | 106 | +| walmart | 0 | 82 | 82 | +| dmv | 0 | 55 | 55 | +| fira | 54 | 0 | 54 | +| google-flights | 0 | 33 | 33 | +| instacart | 0 | 32 | 32 | +| outlook | 24 | 0 | 24 | +| hubspot | 12 | 0 | 12 | +| dropbox | 2 | 0 | 2 | + +## Dataset Split Strategy + +The `prepare_dataset.py` script creates train/eval splits with the following strategy: + +**Stratified by Environment:** Each environment maintains the same train/eval ratio (default 98/2), ensuring balanced representation across all environments. + +**Hash-based Deterministic Assignment:** Tasks are assigned to splits using MD5 hash of task key, ensuring the same task always goes to the same split across runs. + +**Minimum Eval Threshold:** Environments with fewer than 5 expected eval samples have all tasks go to train (prevents statistically unreliable eval sets). + +**Held-out Eval Environments:** +| Modality | Held-out Env | +|----------|--------------| +| tool_use | outlook | +| computer_use | instacart | + +These environments go entirely to eval (not used in training). + +## GitHub Secrets + +| Secret | Required | Description | +|--------|----------|-------------| +| `FLEET_API_KEY` | Yes | Fleet API access | +| `WANDB_API_KEY_TOOL_USE` | Yes | WandB for tool_use | +| `WANDB_API_KEY_COMPUTER_USE` | Yes | WandB for computer_use | +| `LAMBDA_API_KEY` | Yes | Lambda Labs GPU | +| `RUNPOD_API_KEY` | Yes | RunPod GPU | +| `VAST_API_KEY` | Yes | Vast.ai GPU | +| `PRIME_INTELLECT_API_KEY` | Yes | Prime Intellect GPU | +| `AWS_ACCESS_KEY_ID` | Yes | S3 dataset download & checkpoint upload | +| `AWS_SECRET_ACCESS_KEY` | Yes | S3 dataset download & checkpoint upload | + +## Key Configuration + +Edit `skyrl-train/tasks/openenv-fleet-grpo.yaml`: + +| What to change | Parameter | +|----------------|-----------| +| Base model | `trainer.policy.model.path` (default: `Qwen/Qwen2.5-1.5B-Instruct`) | +| Algorithm | `trainer.algorithm.advantage_estimator` (default: `grpo`) | +| Learning rate | `trainer.policy.optimizer_config.lr` (default: `1e-6`) | +| Batch size | `trainer.train_batch_size` (default: `4`) | +| Epochs | `trainer.epochs` (default: `20`) | +| Max turns | `generator.max_turns` (default: `50`) | +| Checkpoint interval | `trainer.ckpt_interval` (default: `10`) | + +## Checkpoints + +Uploaded to S3 (if AWS credentials set): +``` +s3://skyrl-checkpoints/{project}/{model}/{run_name}/global_step_N/ +``` +Local checkpoints deleted after upload to save disk. + +## Monitoring + +- **WandB:** https://wandb.ai/thefleet/fleet-task-grpo +- **Slack:** `#fleet-training-runs` +- **Logs:** `sky logs --follow ` +- **Status:** `sky status` + +## Files Reference + +| File | Purpose | +|------|---------| +| `.github/workflows/openenv-fleet-train.yaml` | GitHub Actions workflow | +| `skyrl-train/tasks/openenv-fleet-grpo.yaml` | SkyPilot task config | +| `skyrl-train/integrations/fleet/env.py` | Fleet environment (BaseTextEnv) | +| `skyrl-train/integrations/fleet/entrypoints/main_fleet.py` | Training entrypoint | +| `skyrl-train/integrations/fleet/s3_checkpoints.py` | S3 upload module | +| `skyrl-train/integrations/fleet/prepare_dataset.py` | Dataset prep | \ No newline at end of file diff --git a/skyrl-train/docs/getting-started/development.rst b/skyrl-train/docs/getting-started/development.rst new file mode 100644 index 0000000000..d8a7c7e46a --- /dev/null +++ b/skyrl-train/docs/getting-started/development.rst @@ -0,0 +1,116 @@ +================= +Development Guide +================= + + +Pre-requisities +=============== + +Follow the :doc:`installation guide `. Make sure that the installation works with our :doc:`quick start example `. + + +Modifying the code +================== + +- **Adding a new environment or task?** + - Follow the :doc:`new task tutorial <../tutorials/new_env>`. Your custom code can be placed anywhere - building on top of ``skyrl_train`` as a package - but we recommend structuring it as a folder similar to :code_link:`examples/multiply`. + +- **Creating a custom Generator (ex: porting an agent harness, implementing custom trajectory generation logic, etc.)?** + - Same as the above: your custom code can be placed anywhere and we typically use ``skyrl-train/examples/`` for this. See :code_link:`examples/mini_swe_agent` for an example of creating a custom ``Generator`` for `Mini-SWE-Agent `_. + +- **Looking to add a new algorithm by changing your advantage estimator or policy loss?** + - Follow the guide for :doc:`implementing custom algorithms <../algorithms/custom_algorithms>`. See :code_link:`examples/algorithms` for examples of various custom algorithm implementations. + +- **Looking to modify the training loop for full control?** + - Follow the guide for :ref:`creating a custom trainer `. See :code_link:`examples/algorithms/dapo/main_dapo.py` for an example of how to modify the Trainer class for DAPO. + +- **Modifying the existing environment code (ex: adding a custom method for all Env classes, improving the SearchEnv implementation)?** + - You would modify the code in `skyrl-gym `_. Note: you do **not** have to modify the ``skyrl-gym`` package for adding a new environment or task. + +Contributing to SkyRL +===================== + +We welcome all forms of contributions to SkyRL, including bug reports, feature requests, documentation improvements, code contributions etc. + +Testing +------- + +For code contributions, we recommend running unit tests before submitting a pull request. + +SkyRL-train +^^^^^^^^^^^ + +CPU tests +~~~~~~~~~ + +.. code-block:: bash + + cd skyrl-train # make sure you are in the correct directory + uv run --isolated --extra dev pytest tests/cpu + +GPU tests +~~~~~~~~~ + +The GPU tests require a node with atleast 8 GPUs. They have been tested on a 8xH100 node, but should work even on 8xA100 nodes. We are actively working on making these more accessible. + +.. code-block:: bash + + cd skyrl-train # make sure you are in the correct directory + +The tests assume that the GSM8K dataset is downloaded to ``~/data/gsm8k``. If you have not downloaded the dataset, you can do so by running the following command: + +.. code-block:: bash + + uv run --isolated examples/gsm8k/gsm8k_dataset.py --output_dir ~/data/gsm8k + +Finally, you can run the tests by running the following command: + +.. code-block:: bash + + uv run --isolated --extra dev --extra vllm pytest tests/gpu + + +SkyRL-gym +^^^^^^^^^ + +You can run the tests for the ``skyrl-gym`` package by running the following command: + +.. code-block:: bash + + cd skyrl-gym # make sure you are in the correct directory + uv run --isolated --extra dev pytest tests/ + + + +Linting and Formatting +---------------------- + +We use pre-commit hooks to automatically lint and format the code. You can install the pre-commit hooks by running the following command: + +.. code-block:: bash + + uv pip install pre-commit + pre-commit install + +If you wish to manually run linting and formatting: + +.. code-block:: bash + + bash format.sh # run from the root directory + +Documentation +------------- + +We welcome contributions to the documentation. We use `Sphinx `_ to build the documentation. The documentation is stored in the ``skyrl-train/docs/`` folder. + +Developing locally +^^^^^^^^^^^^^^^^^^ + +We recommend building the documentation locally to preview your changes. + +.. code-block:: bash + + cd skyrl-train # make sure you are in the correct directory + bash docs/build.sh + +This will build the documentation and serve it at `http://127.0.0.1:8000 `_. \ No newline at end of file diff --git a/skyrl-train/docs/getting-started/images/system-overview.png b/skyrl-train/docs/getting-started/images/system-overview.png new file mode 100644 index 0000000000..500218303a Binary files /dev/null and b/skyrl-train/docs/getting-started/images/system-overview.png differ diff --git a/skyrl-train/docs/getting-started/installation.rst b/skyrl-train/docs/getting-started/installation.rst new file mode 100644 index 0000000000..42d0239926 --- /dev/null +++ b/skyrl-train/docs/getting-started/installation.rst @@ -0,0 +1,233 @@ +Installation +============ + +Requirements +------------ +- CUDA version 12.8 +- `uv `_ + +We use `uv `_ to manage dependencies. We also make use of the `uv` and `ray` integration to manage dependencies for ray workers. + +If you're :ref:`running on an existing Ray cluster `, we suggest using Ray 2.51.1 and Python 3.12. However, we support Ray versions >= 2.44.0. + +.. warning:: + + ⚠️ We do not recommend using Ray 2.47.0 and 2.47.1 for SkyRL due to known issues in the uv+ray integration. + +.. tip:: + + Interested in easily running SkyRL on a managed platform? See our :doc:`platforms <../platforms/overview>` documentation for more information. + +Docker (recommended) +--------------------- + +We provide a docker image with the base dependencies ``novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8`` for quick setup. + +1. Make sure to have `NVIDIA Container Runtime `_ installed. + +2. You can launch the container using the following command: + +.. code-block:: bash + + docker run -it --runtime=nvidia --gpus all --shm-size=8g --name skyrl-train novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8 /bin/bash + +3. Inside the launched container, setup the latest version of the project: + +.. code-block:: bash + + git clone https://github.com/novasky-ai/SkyRL.git + cd SkyRL/skyrl-train + + +That is it! After initializing the ray cluster as described below, you should now be able to run our :doc:`quick start example `. + +.. warning:: + + ⚠️ The older docker image ``novaskyai/skyrl-train-ray-2.48.0-py3.12-cu12.8`` is compatible with SkyRL only till commit https://github.com/NovaSky-AI/SkyRL/commit/0ee61a70a71344fbf15e0c6a603cdcc8b4d0cad5 + We recommend upgrading to the new docker image ``novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8``. + + If you wish to use SkyRL with ray != 2.51.1, see the guide :ref:`running on an existing Ray cluster ` + + +Install without Dockerfile +-------------------------- + +For installation without the Dockerfile, make sure you meet the pre-requisities: + +- CUDA 12.8 +- `uv `_ +- `ray `_ 2.51.1 + + +.. _system-dependencies: + +System Dependencies +~~~~~~~~~~~~~~~~~~~ + +The only packages required are `build-essential` and `libnuma `_. You can install them using the following command: + +.. code-block:: bash + + sudo apt update && sudo apt-get install build-essential libnuma-dev + +.. note:: + Installing ``libnuma-dev`` will require sudo privileges. If you are running on a machine without sudo access, we recommend using the Dockerfile. However, you can install from source using: + + .. code-block:: bash + + # Get the source + wget https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz + tar xzf numactl-2.0.16.tar.gz + cd numactl-2.0.16 + + # Build to a local prefix + ./configure --prefix=$HOME/.local + make + make install + + # Point compiler and linker to it (add to ~/.bashrc for persistence) + export CPATH=$HOME/.local/include:$CPATH + export LIBRARY_PATH=$HOME/.local/lib:$LIBRARY_PATH + export LD_LIBRARY_PATH=$HOME/.local/lib:$LD_LIBRARY_PATH + + If ``libnuma`` is not installed, you might run into errors such as the following when running SkyRL: + + .. code-block:: bash + + AttributeError: ray::FSDPRefWorkerBase.offload_to_cpu: undefined symbol: numa_parse_nodestring. Did you mean: '_return_value'? + +Installing SkyRL-Train +~~~~~~~~~~~~~~~~~~~~~~ + +All project dependencies are managed by `uv`. + +Clone the repo and `cd` into the `skyrl` directory: + +.. code-block:: bash + + git clone https://github.com/novasky-ai/SkyRL.git + cd SkyRL/skyrl-train + +Base environment +~~~~~~~~~~~~~~~~ + +We recommend having a base virtual environment for the project. + +With ``uv``: + +.. code-block:: bash + + uv venv --python 3.12 + +If ```` is not specified, the virtual environment will be created in the current directory at ``.venv``. + +.. tip:: + Because of how Ray ships content in the `working directory `_, we recommend that the base environment is created *outside* the package directory. For example, ``~/venvs/skyrl-train``. + +Then activate the virtual environment and install the dependencies. + +.. code-block:: bash + + source /bin/activate + uv sync --active --extra vllm + +With ``conda``: + +.. code-block:: bash + + conda create -n skyrl-train python=3.12 + conda activate skyrl-train + +After activating the virtual environment, make sure to configure Ray to use `uv`: + +.. code-block:: bash + + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + # or add to your .bashrc + # echo 'export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook' >> ~/.bashrc + +Initialize Ray cluster +---------------------- + +Finally, you can initialize a Ray cluster using the following command (for single-node): + +.. code-block:: bash + + ray start --head + # sanity check + # ray status + + +.. note:: + For multi-node clusters, please follow the `Ray documentation `_. + +You should now be able to run our :doc:`quick start example `. + +.. _running-on-existing-ray-cluster: + +Running on an existing Ray cluster +---------------------------------- + +For running on an existing Ray cluster, you need to first make sure that the python version used is 3.12. + +Ray >= 2.48.0 +~~~~~~~~~~~~~ + +We recommend using Ray version 2.48.0 and above for the best experience. In this case, you can simply use the ``uv run`` command to get training started. + +.. code-block:: bash + + uv run ... --with ray==2.xx.yy -m skyrl_train.entrypoints.main_base ... + +Ray < 2.51.1 +~~~~~~~~~~~~ +SkyRL-Train is compatible with any Ray version 2.44.0 and above (except 2.47.0 and 2.47.1 -- which we do not recommend due to an issue in the uv + Ray integration). +Since we use a uv lockfile to pin dependencies, the best way to run SkyRL-Train on a custom Ray version (say 2.46.0) would be to override the version at runtime with the ``--with`` flag. +For example, to run with Ray 2.46.0, you can do: + +.. code-block:: bash + + uv run .... --with ray==2.46.0 -m skyrl_train.entrypoints.main_base ... + +For ray versions >= 2.44.0 but < 2.51.1, you additionally need to install vllm in the base pip environment, and then re-install ray to your desired version to ensure that the uv + Ray integration works as expected. +We include these dependencies in the legacy Dockerfile: `Dockerfile.ray244 `_, or you can install them manually: + +.. code-block:: bash + + pip install vllm==0.9.2 --extra-index-url https://download.pytorch.org/whl/cu128 + pip install ray==2.46.0 omegaconf==2.3.0 loguru==0.7.3 jaxtyping==0.3.2 pyarrow==20.0.0 + + +.. warning:: + + ⚠️ We do not recommend using uv versions 0.8.0, 0.8.1, or 0.8.2 due to a `bug `_ in the ``--with`` flag behaviour. + + +Installing SkyRL-Train from PyPI +-------------------------------- + +We publish wheels for SkyRL-Train on PyPI: https://pypi.org/project/skyrl-train/. For using the latest release of SkyRL-Train (0.3.0) as a dependency in your project, you will need to configure your ``pyproject.toml`` to correctly install ``skyrl-train`` and its dependencies that depend on custom indexes/ wheels, such as ``flash-attn`` and ``flashinfer-jit-cache``. Below is an example configuration using ``uv``: + +.. code-block:: toml + + [project] + dependencies = [ + "skyrl-train==0.2.0", + "flashinfer-jit-cache", + # Find the appropriate wheel for your CUDA/PyTorch version from their `releases page `_ + # Alternatively use the match-runtime feature from uv: https://docs.astral.sh/uv/concepts/projects/config/#augmenting-build-dependencies + "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.5.8/flash_attn-2.5.8+cu122torch2.3-cp312-cp312-linux_x86_64.whl" + ] + + [tool.uv.sources] + flashinfer-jit-cache = { index = "flashinfer-cu128" } + + [[tool.uv.index]] + name = "flashinfer-cu128" + url = "https://flashinfer.ai/whl/cu128" + explicit = true + +Development +----------- + +For development, refer to the :doc:`development guide `. diff --git a/skyrl-train/docs/getting-started/overview.rst b/skyrl-train/docs/getting-started/overview.rst new file mode 100644 index 0000000000..8fa4d0bd86 --- /dev/null +++ b/skyrl-train/docs/getting-started/overview.rst @@ -0,0 +1,58 @@ +SkyRL System Overview +===================== + +SkyRL breaks the RL stack into modular components and provides public APIs for each of them. + +Specifically, as shown in figure below, SkyRL separates training into two major components, **Trainer** and **Generator**, and the Generator is further divided into **InferenceEngine** and **Environment**, with a single **Controller** managing setup and execution of each component. + +.. figure:: images/system-overview.png + :alt: SkyRL System Overview + :align: center + :width: 80% + +The components' responsibilities are as follows: + +Trainer +~~~~~~~ +Performs the optimization steps based on configured RL algorithm. Updates model parameters based on generated trajectories and their assigned rewards. + +- `Base Training Worker interface `_ + + - `FSDPWorker `_ + - `MegatronWorker `_ + +- `PPORayActorGroup `_: Our abstraction for a group of training workers (as Ray actors) that jointly execute operations for a given model (e.g., policy model, critic model, etc.). + +Generator +~~~~~~~~~ +Generates complete trajectories and computes their rewards. The Generator encompasses both the InferenceEngine (to get model completions) and Environment (to execute actions) as well as custom agentic or data generation logic build around model inference, such as context management, sampling methods, or tree search. + +- `Base Generator interface `_ +- `SkyRLGymGenerator built for SkyRL-Gym `_ + +InferenceEngine +~~~~~~~~~~~~~~~ +Executes inference on the policy model to produce model outputs (i.e., the RL agent's actions). Typically, multiple InferenceEngines are deployed to process prompts in parallel. + +- `Base InferenceEngine interface `_ +- `InferenceEngineClient to manage multiple engines `_ +- `vLLM backend `_ +- `SGLang backend `_ + + +Environment +~~~~~~~~~~~ +Presents a task for the policy model to solve, and provides the logic for executing the policy's actions (i.e., model outputs) and computing the resulting observations and rewards. + +- `Base Environment interface `_ +- `SkyRL-Gym `_, our ready-built library of tool-use environments + + - `Example environments `_ + + +Controller +~~~~~~~~~~ +Manages physical placement, initialization, and control flow of training execution for each of the above components. + +- The training control loop currently sits in `trainer.py `_ +- It is a WIP to move the control loop to a separate component for even greater flexibility. diff --git a/skyrl-train/docs/getting-started/quickstart.rst b/skyrl-train/docs/getting-started/quickstart.rst new file mode 100644 index 0000000000..67afdfb335 --- /dev/null +++ b/skyrl-train/docs/getting-started/quickstart.rst @@ -0,0 +1,101 @@ +Quick Start: GRPO on GSM8K +========================== + +First, follow the guide at :doc:`installation` to get your environment set up. + +In this quickstart, we'll walk you through running GRPO training on the GSM8K dataset. You'll prepare the dataset, configure your training parameters, and launch your first SkyRL training run. + +Dataset Preparation +------------------- + +To download and prepare the GSM8K dataset, run the following script. We provide convenience scripts for GSM8K and several other popular datasets, but you can also use your own custom dataset by following the instructions in the :ref:`dataset-preparation` section. + +.. code-block:: bash + + uv run --isolated examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + +Training Configuration +---------------------- + +Next, let's set up the training configuration. You can find a complete example in ``examples/gsm8k/run_gsm8k.sh``, and we highlight some key parameters here: + +.. code-block:: bash + + uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + # Data setup + data.train_data="['$HOME/data/gsm8k/train.parquet']" \ + data.val_data="['$HOME/data/gsm8k/validation.parquet']" \ + + # Trainer and training algorithm + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + + # Model placement and training strategy (colocate or disaggregate, sharding, etc.) + trainer.strategy=fsdp2 \ + trainer.placement.colocate_all=true \ + trainer.placement.policy_num_gpus_per_node=4 \ + + # Evaluation and checkpointing + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.ckpt_interval=10 \ + + # Generator setup for spinning up InferenceEngines + generator.backend=vllm \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + generator.weight_sync_backend=nccl \ + + # Environment class for the dataset + # Can be specified here to apply to the full dataset, or at the per-prompt level during preprocessing + environment.env_class=gsm8k \ + + # WandB logging + trainer.logger="wandb" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_test" \ + + ... # Other parameters (see `examples/gsm8k/run_gsm8k.sh` for more) + +Hardware Configuration +---------------------- + +Depending on your hardware setup, you may want to adjust a few parameters: + +1. **GPU Configuration**: Set ``trainer.placement.policy_num_gpus_per_node`` to match your available GPU count. If you need to change the model size, you can also set ``trainer.policy.model.path`` to ``Qwen/Qwen2.5-0.5B-Instruct`` or ``Qwen/Qwen2.5-7B-Instruct``. + +2. **InferenceEngine Matching**: Ensure that ``num_inference_engines * inference_engine_tensor_parallel_size`` equals the total number of GPUs used for the policy model above. + + +Launching Your Training Run +--------------------------- + +Now for the exciting part! First, configure your WandB API key for logging: + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + +Then launch your training run: + +.. code-block:: bash + + bash examples/gsm8k/run_gsm8k.sh + +Congratulations! You've just launched your first SkyRL training run! + +Monitoring Progress +------------------- + +The training progress will be logged to your terminal, showing you which part of the training loop is executing and how long each step takes. You can monitor detailed metrics and visualizations on WandB, or configure logging to output to the console or your preferred logging backend. + +What's Next? +------------ + +Now that you've got the basics down, you might want to explore: + +- :doc:`../tutorials/new_env`: Creating a new environment without touching the training loop +- :doc:`../tutorials/one_step_off_async`: Asynchronous off-by-one training in < 100 lines of code! +- :doc:`../recipes/overview`: A collection of end-to-end recipes with SkyRL. + diff --git a/skyrl-train/docs/index.rst b/skyrl-train/docs/index.rst new file mode 100644 index 0000000000..78dd735581 --- /dev/null +++ b/skyrl-train/docs/index.rst @@ -0,0 +1,109 @@ +Welcome to SkyRL's documentation! +======================================= + +SkyRL is a full-stack RL library designed for modularity and extensibility. + + +.. toctree:: + :maxdepth: 2 + :caption: Getting Started + + getting-started/installation + getting-started/quickstart + getting-started/overview + getting-started/development + +.. toctree:: + :maxdepth: 2 + :caption: Dataset Preparation + + datasets/dataset-preparation + +.. toctree:: + :maxdepth: 2 + :caption: Tutorials + + tutorials/new_env + tutorials/one_step_off_async + tutorials/fully_async + tutorials/tools_guide + tutorials/skyrl_gym_generator + +.. toctree:: + :maxdepth: 2 + :caption: Examples + + examples/megatron + examples/ppo + examples/lora + examples/llm_as_a_judge + examples/remote_server + examples/training_backends + examples/multi_turn_text2sql + examples/search + examples/flash_rl + examples/mini_swe_agent + examples/openenv + +.. toctree:: + :maxdepth: 2 + :caption: Platforms + + platforms/overview + platforms/anyscale + platforms/runpod + platforms/skypilot + +.. toctree:: + :maxdepth: 2 + :caption: Recipes + + recipes/overview + recipes/skyrl-sql + recipes/searchr1 + +.. toctree:: + :maxdepth: 2 + :caption: Algorithms + + algorithms/dapo + algorithms/custom_algorithms + +.. toctree:: + :maxdepth: 2 + :caption: Configuration + + configuration/config + configuration/placement + +.. toctree:: + :maxdepth: 2 + :caption: Checkpointing and Logging + + checkpointing-logging/checkpointing + + +.. toctree:: + :maxdepth: 2 + :caption: API Reference + + api/data + api/entrypoint + api/trainer + api/generator + api/env + api/registry + api/tools + api/env_vars + +.. toctree:: + :maxdepth: 2 + :caption: Troubleshooting + + troubleshooting/troubleshooting + +.. toctree:: + :maxdepth: 2 + :caption: (NEW) SkyAgent + + skyagent/agent-overview diff --git a/skyrl-train/docs/platforms/anyscale.rst b/skyrl-train/docs/platforms/anyscale.rst new file mode 100644 index 0000000000..e037fba261 --- /dev/null +++ b/skyrl-train/docs/platforms/anyscale.rst @@ -0,0 +1,10 @@ +.. _anyscale: + +Anyscale +========= + +`Anyscale `_ is a managed, multi-cloud platform developed by the creators of Ray. + +Anyscale automates the entire lifecycle of Ray clusters in your AWS, GCP, or Azure account, allowing you to deploy fault-tolerant Ray clusters in any cloud with built-in resilience, auto-scaling, observability and more. + +To get started with SkyRL on Anyscale, you can refer to the `example on the Anyscale console `_. diff --git a/skyrl-train/docs/platforms/overview.rst b/skyrl-train/docs/platforms/overview.rst new file mode 100644 index 0000000000..31ad048cfc --- /dev/null +++ b/skyrl-train/docs/platforms/overview.rst @@ -0,0 +1,4 @@ +SkyRL on Managed Platforms +========================== + +To easily run SkyRL on managed platforms with minimal setup, we provide instructions for several popular platforms such as :doc:`Anyscale `, :doc:`Runpod `, and :doc:`SkyPilot `. diff --git a/skyrl-train/docs/platforms/runpod.rst b/skyrl-train/docs/platforms/runpod.rst new file mode 100644 index 0000000000..6633a46215 --- /dev/null +++ b/skyrl-train/docs/platforms/runpod.rst @@ -0,0 +1,71 @@ +RunPod +====== + +`RunPod `_ is an end-to-end AI cloud that simplifies building and deploying models. You can easily launch a GPU pod in seconds and run training jobs or inference workloads. + + +Use the following steps to run SkyRL-Train on Runpod. You can customize them to your needs. +These instructions were tested end-to-end on a single A40 GPU on Runpod, training on the +GSM8K dataset. It's a quick, minimal path to your first training run. + +First, install Miniconda: + +.. code-block:: bash + + cd $HOME + mkdir -p ~/miniconda3 + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh + bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3 + rm ~/miniconda3/miniconda.sh + source ~/miniconda3/bin/activate + conda init --all + +Close the terminal and reopen it to activate the base conda environment. Then run the following +snippet. The NUMA installation follows :ref:`system-dependencies`. + +.. code-block:: bash + + cd $HOME + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r + + # Optionally, export your WANDB_API_KEY + echo "export WANDB_API_KEY=YOUR_WANDB_API_KEY" >> ~/.bashrc + + # Sometimes Runpod places `.cache` under `/workspace`, which can be slow. + # We set them under `~/` here. + mkdir -p "$HOME/.cache" + echo 'export UV_CACHE_DIR="$HOME/.cache/uv"' >> ~/.bashrc + echo 'export HF_HOME="$HOME/.cache/huggingface"' >> ~/.bashrc + + # --------------- + # Install numactl (libnuma) + # --------------- + # Get the source + cd $HOME + wget https://github.com/numactl/numactl/releases/download/v2.0.16/numactl-2.0.16.tar.gz + tar xzf numactl-2.0.16.tar.gz + cd numactl-2.0.16 + + # Build to a local prefix + ./configure --prefix=$HOME/.local + make + make install + cd $HOME + + # Point the compiler and linker to it + echo "export CPATH=$HOME/.local/include:$CPATH" >> ~/.bashrc + echo "export LIBRARY_PATH=$HOME/.local/lib:$LIBRARY_PATH" >> ~/.bashrc + echo "export LD_LIBRARY_PATH=$HOME/.local/lib:$LD_LIBRARY_PATH" >> ~/.bashrc + +Close the terminal and reopen it. Then launch a basic GSM8K training run with the following +commands. For more, see :doc:`quickstart`. + +.. code-block:: bash + + cd $HOME + git clone https://github.com/NovaSky-AI/SkyRL + cd SkyRL/skyrl-train + pip install uv + uv run --isolated python examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + NUM_GPUS=1 LOGGER=console bash examples/gsm8k/run_gsm8k.sh diff --git a/skyrl-train/docs/platforms/skypilot.rst b/skyrl-train/docs/platforms/skypilot.rst new file mode 100644 index 0000000000..7f76591d13 --- /dev/null +++ b/skyrl-train/docs/platforms/skypilot.rst @@ -0,0 +1,9 @@ +.. _skypilot: + +SkyPilot +========= + +`SkyPilot `_ is a system to run, manage, +and scale AI workloads on any infrastructure. SkyPilot works across AWS, GCP, Azure, OCI, Lambda, Nebius, and more. It handles all of the logic of provisioning, setting up, and tearing down clusters on any cloud. + +You can view a detailed tutorial on running SkyRL on SkyPilot in the SkyPilot documentation `here `_. diff --git a/skyrl-train/docs/recipes/overview.rst b/skyrl-train/docs/recipes/overview.rst new file mode 100644 index 0000000000..f5def4991e --- /dev/null +++ b/skyrl-train/docs/recipes/overview.rst @@ -0,0 +1,408 @@ +E2E Recipes with SkyRL +====================== + +We provide a collection of end-to-end recipes for single and multi-turn RL training with SkyRL. + +We provide reproduction runs for the following recipes: + +1. Simple training on GSM8K +2. `Decoupled Clip and Dynamic Sampling Policy Optimization (DAPO) `_ +3. `SkyRL-SQL `_ +4. `SearchR1 `_ + + + +Simple training on GSM8K +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The scripts for training on GSM8K are available at :code_link:`examples/gsm8k/`. + + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BackendModelEval AccuracyHardwareTraining StepsCommitWandB
FSDP2Qwen/Qwen2.5-1.5B-Instruct0.7964xH100140a95b699Link
DeepSpeedQwen/Qwen2.5-1.5B-Instruct0.7914xH100140a95b699Link
+ + +DAPO Recipes +~~~~~~~~~~~~ + +The code for the DAPO recipe is available at :code_link:`examples/algorithms/dapo/`. + +For evals we report Pass@32 and Mean@32. In the WandB metrics we log "avg_score" - since reward is either -1 or 1 for the AIME task, mean@32 can be computed as mean@32 = (avg_score + 1) / 2. +In the table below we report the peak mean@32 and pass@32 over the course of the run. All runs are DAPO but without Dynamic Sampling enabled (just clip-higher, overlong buffer, overlong filtering, and token level loss aggregation). + +All results can be reproduced with commit `cca7d96741f143ce6aa89a0f2cbb18d528cfcf33 `_, and the WandB report for all runs is available `here `_. + + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
RecipeModelTraining BackendAIME24 Pass@32AIME24 Mean@32HardwareTraining Steps (at peak mean@32)
DAPOQwen/Qwen-2.5-32BFSDP20.7660.3812x8xH100260
DAPOQwen/Qwen-2.5-7B-MathFSDP20.6330.3488xH100320
DAPOQwen/Qwen3-1.7B-BaseFSDP20.3660.1448xH100285
DAPO + 0-Var FilteringQwen/Qwen3-1.7B-BaseFSDP20.4330.1698xH100185
DAPOQwen/Qwen3-4B-BaseFSDP20.60.2548xH100110
DAPOQwen/Qwen3-4B-BaseMegatron (tp=4, pp=2)0.6330.2468xH100210
DAPO + LoRA (rank 32, alpha 64)Qwen/Qwen3-4B-BaseMegatron (tp=4, pp=1)0.5660.2098xH100160
+ +SkyRL-SQL Recipes +~~~~~~~~~~~~~~~~~ + + +For more details, please refer to :doc:`../recipes/skyrl-sql`. + +We provide two reference runs: single-turn and multi-training for `Qwen/Qwen2.5-Coder-7B-Instruct `_ (run on 1 8xH100 node up to convergence), with the WandB report `here `_. + +The evaluation results are shown below (using the evaluation code `here `_): + +.. raw:: html + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Eval Turns (Train)Training MethodSpider-DevSpider-TestSpider-RealisticSpider-DKSpider-SynAvgWandB
1Single-Turn81.283.876.867.970.176.0Link
1Multi-Turn82.4 (+1.2%)83.7 (-0.1%)80.3 (+3.5%)70.5 (+2.6%)71.2 (+1.1%)77.6 (+1.6%)Link
5Single-Turn79.582.277.665.668.474.7Link
5Multi-Turn83.9 (+4.4%)85.2 (+3%)81.1 (+3.5%)72.0 (+6.4%)73.7 (+5.3%)79.2 (+4.5%)Link
+
+ + +SearchR1 Recipes +~~~~~~~~~~~~~~~~ + +For more details, please refer to :doc:`../recipes/searchr1`. + + +The WandB report is available `here `_. + +Qwen/Qwen2.5-3B-Instruct +^^^^^^^^^^^^^^^^^^^^^^^^ +The evaluation results are shown below for `Qwen/Qwen2.5-3B-Instruct `_, with all experiments run on 1 8xH100 node up to convergence (330 training steps). + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DatasetSearch-R1
(3 turns)		SkyRL + SearchR1
(2 turns)		SkyRL + SearchR1
(3 turns)		SkyRL + SearchR1
(4 turns)
NQ†0.3970.4550.4490.449
TriviaQA†0.5650.6130.6160.611
PopQA†0.3910.4470.4440.435
HotpotQA*0.3310.3340.4170.407
2wiki*0.3100.3130.3960.403
Musique*0.1240.0860.1790.163
Bamboogle*0.2320.2420.4480.352
Average0.3360.3560.4210.403
+ +Qwen/Qwen3-30B-A3B +^^^^^^^^^^^^^^^^^^^ +Evaluation results for `Qwen3-30B-A3B `_ on SearchR1, with experiments using 4 8xH100 nodes using the :doc:`Megatron <../examples/megatron>` backend, are shown below. These results can be reproduced with commit `9b878cd `_. + +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DatasetSkyRL + SearchR1
(4 turns)
NQ†0.463
TriviaQA†0.664
PopQA†0.448
HotpotQA*0.412
2wiki*0.361
Musique*0.178
Bamboogle*0.488
Average0.457
diff --git a/skyrl-train/docs/recipes/searchr1.rst b/skyrl-train/docs/recipes/searchr1.rst new file mode 100644 index 0000000000..d19549e944 --- /dev/null +++ b/skyrl-train/docs/recipes/searchr1.rst @@ -0,0 +1,88 @@ +SearchR1 +========= + +We provide scripts to reproduce our results for training a multi-turn search agent using the dataset and recipe from `Search-R1 `_. +For a more detailed explanation of the task and training configuration, see :doc:`../examples/search`. + +Pre-requisites +-------------- + +Make sure to have followed the installation commands in :doc:`../getting-started/installation`. + +Data Preparation +---------------- + +We provide a script to download the dataset from huggingface, and preprocess it to run in SkyRL-Gym. + +.. code-block:: bash + + uv run --isolated examples/search/searchr1_dataset.py --local_dir ~/data/searchR1 + +Start the Search Engine +------------------------ +Since faiss-gpu is not available via pip, we setup a separate conda environment for the local retrieval server. Running this server will +use around 6GB of GPU memory per GPU, so make sure to account for this in your training run configuration. + +Retriever environments +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # Create and activate the retriever environment with Python 3.10 + conda create -n retriever python=3.10 -y + conda activate retriever + + # Install PyTorch (with GPU support) and related libraries + conda install numpy==1.26.4 # needed to stop incompatible version of numpy from being installed via pip + pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu124 + + # Install other Python packages + pip install transformers datasets pyserini huggingface_hub + + # Install the GPU version of faiss + conda install faiss-gpu==1.8.0 -c pytorch -c nvidia -y + + # Install the API service framework + pip install uvicorn fastapi + +Download the Index +~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + conda activate retriever + + local_dir=~/data/searchR1 + python examples/search/searchr1_download.py --local_dir $local_dir + cat $local_dir/part_* > $local_dir/e5_Flat.index + gzip -d $local_dir/wiki-18.jsonl.gz + + +Prepare Datasets +~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + python examples/search/searchr1_dataset.py --local_dir $local_dir + + +Start the Local Flat e5 Retrieval Server +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: bash + + conda activate retriever + + # redirect the output to a file to avoid cluttering the terminal + # we have observed outputting to the terminal causing spikes in server response times + bash examples/search/retriever/retrieval_launch.sh > retrieval_server.log + +Start your Training Run +~~~~~~~~~~~~~~~~~~~~~~~ +Now from your base environment, you can launch your training run (which will use uv to package dependencies, separately from the retriever environment). + +.. code-block:: bash + + export WANDB_API_KEY=your_wandb_api_key + bash examples/search/run_search.sh + +You can find a link to our training runs with 2, 3, and 4 turns for comparison at our `WandB report `_. \ No newline at end of file diff --git a/skyrl-train/docs/recipes/skyrl-sql.rst b/skyrl-train/docs/recipes/skyrl-sql.rst new file mode 100644 index 0000000000..f2d7b3c104 --- /dev/null +++ b/skyrl-train/docs/recipes/skyrl-sql.rst @@ -0,0 +1,65 @@ +SkyRL-SQL +========= + +We provide scripts to reproduce the results for `SkyRL-SQL-7B `_ using SkyRL-train and SkyRL-Gym. + +You can find a WandB run for both single-turn and multi-turn Text2SQL training at `this link `_. + +Pre-requisites +--------------- + +Make sure to have followed the installation commands in :doc:`../getting-started/installation`. + + +Start Ray +--------- + +Start ray in your cluster following the guide: https://docs.ray.io/en/latest/ray-core/starting-ray.html. + + +.. _skyrl-sql-data: + +Data Preparation +---------------- + + +We provide the dataset we used on HuggingFace: https://huggingface.co/datasets/NovaSky-AI/SkyRL-SQL-653-data-newfmt +You can download the dataset by running the following command + +.. code-block:: bash + + hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset + + +DB environment +--------------- + +Make sure to setup the database files needed for training. We use the database files from `OmniSQL `_. + +You can download the datasets from: +- `ModelScope-OmniSQL-datasets `_ +- `HuggingFace-OmniSQL-datasets `_ + + + +The datasets include BIRD, Spider, ScienceBenchmark, EHRSQL, Spider2-SQLite, Spider-DK, Spider-Realistic, Spider-Syn, and SynSQL-2.5M. In our training pipeline, we only need to access databases from SynSQL-2.5M and Spider. + +Unzip `data.zip` in this folder, and set the corresponding `DB_PATH` in the training script below. You can download and unzip the data by running + +.. code-block:: bash + + hf download seeklhy/OmniSQL-datasets data.zip --repo-type dataset --local-dir + unzip + +Running the scripts +------------------- + +We provide a script :code_link:`examples/text_to_sql/run_skyrl_sql.sh` for reproducing the results for SkyRL-SQL-7B. Make sure to substitute the `DB_PATH` and `DATA_PATH` variables with your own. + +.. code-block:: bash + + export WANDB_API_KEY= + bash examples/text_to_sql/run_skyrl_sql.sh + + + diff --git a/skyrl-train/docs/skyagent/agent-overview.rst b/skyrl-train/docs/skyagent/agent-overview.rst new file mode 100644 index 0000000000..2901cbaedf --- /dev/null +++ b/skyrl-train/docs/skyagent/agent-overview.rst @@ -0,0 +1,106 @@ +SkyAgent Overview +==================== + +SkyAgent is a generic agent layer for training and evaluating agents. It is a flexible frontend for building your own agents. + +SkyAgent is designed primarily for researchers to have a unified interface around implementing agentic tasks. A modular design allows researchers to + +1. Bring in their own tasks +2. Use any training backend or simply run evaluation +3. Modify runtime implementation for a given task (docker, etc) +4. Improve dispatching logic for a batch of trajectories easily +5. And more ... + + +.. warning:: + + ⚠️ SkyAgent is still under active development. We welcome any early feedback and contributions. + + +Examples +-------- + +We have a few examples in the ``examples`` folder: + +1. `Evaluation: OpenAI `_: This example shows how to run evaluation with an OpenAI compatible endpoint. +2. `Training: SkyAgent and SkyRL-train `_ : Training a model on the SWEBench task with SkyRL-train. +3. `Training: SkyAgent and VeRL `_ : Training a model on the SWEBench task with VeRL. + + + +Core components +--------------- + +SkyAgent consists of the following components: + +1. `AgentRunner `_ : The main entrypoint for Skyagent is the AgentRunner class - it's responsible for generating trajectories for the given batch of prompts +2. `Trajectory `_ : the trajectory class handles generating a single trajectory for the given instance from the batch. +3. `Agent `_ : This is simply an LLM with the ability to call tools. +4. `Task `_ : The task class contains the task specification such as initial instruction, how the agent's runtime should be setup, how to evaluate results, etc. +5. `Dispatcher `_ : the dispatcher is responsible for efficiently handling trajectory execution for a batch of prompts. +6. `Backend `_ : Backend is the LLM backend for generating responses. For example, vLLM for inference or ``SkyRL-train``'s inference engines for training. + + +Trajectory +---------- + +The trajectory class handles generating a single trajectory for the given instance from the batch. It has three methods: + +1. ``initialize_trajectory``: Initialize the trajectory eg: setup any runtime environment needed for the agent to run. +2. ``generate_trajectory``: Generate the trajectory i.e. run the agent loop and get the final conversation and task results. +3. ``evaluate_trajectory``: Evaluate the trajectory i.e. parse the final result and evaluate it for the given task. + + +The results of both ``generate_trajectory`` and ``evaluate_trajectory`` are stored in a ``.result`` attribute of the trajectory. Each trajectory instance will initialize an ``Agent`` instance to generate responses. + +Here's a high-level diagram of the components involved in generating a trajectory: + +.. figure:: images/generate_trajectory.png + :alt: Generate Trajectory + :align: center + :width: 80% + + +Agent +----- + +The agent class is a simple wrapper around an LLM with the ability to call tools. It mainly has a ``step`` method that generates an assistant response to the current history. The agent class manages history and response parsing. The actual LLM call is handled by the backend. + + +Backend +------- + +The backend is the LLM backend for generating responses. For example, this can be an `OpenAI-compatible webserver `_ for inference or `SkyRL-train `_ for training. + + +Dispatcher +---------- + +The dispatcher handles the actual execution of a batch of trajectories efficiently. It takes in a batch of trajectories and executes ``initialize_trajectory``, ``generate_trajectory``, and ``evaluate_trajectory`` for each trajectory in certain concurrency. + +For example, we provide a pipelined dispatcher that can run multiple trajectories in parallel with a maximum concurrency per stage (initialize, generate, evaluate) of ``max_parallel_agents``. + +.. figure:: images/dispatcher_skyagent.png + :alt: SkyAgent Dispatcher + :align: center + :width: 80% + + + Overview of the pipelined dispatcher with ``max_parallel_agents=3`` + + + +Task +---- + +The task class has the following methods: + +1. ``initialize_runtime``: Initialize the runtime for the task in an asyncio-compatible way +2. ``get_instruction``: Get the initial instruction for the agent in the OpenAI messages format +3. ``complete_runtime``: Complete or finalize the runtime for the task. For example, this can involve extracting the git patch from the runtime for SWEBench. +4. ``evaluate_result``: Evaluate model result for the task in an asyncio-compatible way + +We currently provide two tasks: + +1. `SWEBenchTask `_ : Implements the SWEBench task leveraging `OpenHands `_ . +2. `GeneralReactTask `_ : A general task implementation for many basic reasoning tasks like math, science, simple code generation, etc. diff --git a/skyrl-train/docs/skyagent/images/dispatcher_skyagent.png b/skyrl-train/docs/skyagent/images/dispatcher_skyagent.png new file mode 100644 index 0000000000..edf5f2607a Binary files /dev/null and b/skyrl-train/docs/skyagent/images/dispatcher_skyagent.png differ diff --git a/skyrl-train/docs/skyagent/images/generate_trajectory.png b/skyrl-train/docs/skyagent/images/generate_trajectory.png new file mode 100644 index 0000000000..4c5a2e2131 Binary files /dev/null and b/skyrl-train/docs/skyagent/images/generate_trajectory.png differ diff --git a/skyrl-train/docs/troubleshooting/troubleshooting.rst b/skyrl-train/docs/troubleshooting/troubleshooting.rst new file mode 100644 index 0000000000..fa339097d8 --- /dev/null +++ b/skyrl-train/docs/troubleshooting/troubleshooting.rst @@ -0,0 +1,40 @@ +Troubleshooting +=============== + + +Placement Group Timeouts +------------------------- + +In SkyRL, we use Ray placement groups to request resources for different actors. In Ray clusters that autoscale with KubeRay, placement group creation can take a long time since the cluster might have to add a new node, pull the relevant image and start the container, etc. +You can use the ``SKYRL_RAY_PG_TIMEOUT_IN_S`` environment variable (Used in the ``.env`` file passed to the ``uv run`` command with ``--env-file``) to increase the timeout for placement group creation (By default, this is 180 seconds) + +Multi-node Training +------------------- + +For multi-node training, it is helpful to first confirm that your cluster is properly configured. We provide a script at ``scripts/multi_node_nccl_test.py`` to test multi-node communication. + +To run the script, you can use the following command: + +.. code-block:: bash + + uv run --isolated --env-file .env scripts/multi_node_nccl_test.py --num-nodes 2 + +.env is optional, but it is recommended to use for configuring environment variables. + +Note on ``LD_LIBRARY_PATH`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using RDMA, you may need to customize the ``LD_LIBRARY_PATH`` to include the RDMA libraries (Ex: EFA on AWS). We've seen issues with `uv` where the ``LD_LIBRARY_PATH`` is not exported even if it is set in the ``.env`` file. It is recommended to set the ``SKYRL_LD_LIBRARY_PATH_EXPORT=1`` in the ``.env`` file and set ``LD_LIBRARY_PATH`` directly in the current shell. + + +Illegal Memory Access with vLLM +--------------------------------- + +In some cases, you may encounter "illegal memory access" errors with vLLM >= 0.10.0: https://github.com/vllm-project/vllm/issues/23814. Currently, we recommend a workaround by downgrading to vLLM 0.9.2. + +With SkyRL, this can be done with the following overrides: + + +.. code-block:: bash + + uv run --isolated --extra vllm --with vllm==0.9.2 --with transformers==4.53.0 --with torch==2.7.0 --with "flash-attn@https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.7cxx11abiTRUE-cp312-cp312-linux_x86_64.whl" -- ... diff --git a/skyrl-train/docs/tutorials/fully_async.rst b/skyrl-train/docs/tutorials/fully_async.rst new file mode 100644 index 0000000000..40b2c8cd21 --- /dev/null +++ b/skyrl-train/docs/tutorials/fully_async.rst @@ -0,0 +1,346 @@ +(Experimental) Fully Async Training: In-flight Weight Update / Multi-Turn Partial Rollout +========================================================================================= + +SkyRL supports fully async training for any generator that implements the ``GeneratorInterface``, including your **arbitrary agent harness (both single-turn and multi-turn)**, to address the **straggler issues in long-horizon tasks**. + +We treat "fully async training" synonymous to the terms **"in-flight weight update"** and **"multi-turn partial rollout"**, as discussed in works like AReal, PipelineRL, ScaleRL, etc. + +The core logic of fully async training lives in `fully_async_trainer.py `_, a subclass of the synchronous trainer class in `trainer.py `_. This ``fully_async_trainer.py`` works out-of-the-box with any generator that the user implements -- so users do not need to implement any additional logic for fully async training. + +.. figure:: images/skyrl_fully_async.png + :alt: Systems Diagram of Fully Async Training in SkyRL + :align: center + :width: 90% + + **Overview of Fully Async Training in SkyRL** + +.. contents:: Table of Contents + :local: + :depth: 2 + :backlinks: none + +.. note:: + + The fully async training is currently only supported for Generators that use ``/chat/completions``. We will support ``.generate()`` and ``/completions`` soon. + +I. What is Fully Async Training? +--------------------------------- + +There are three types of RL training we are interested in here, well-depicted by AReal's Figure 1 and Figure 3. + +.. figure:: images/areal_figure1.png + :alt: AReal Figure 1: Synchronous Training vs. One-Step-Off Async Training + :align: center + :width: 90% + + **AReal Figure 1: Synchronous Training vs. One-Step-Off Async Training** + + +a. Synchronous Training +~~~~~~~~~~~~~~~~~~~~~~~~ + +In AReal's figure 1 LHS, we have the traditional synchronous training, colocating training and generation on the same set of GPUs. + +- Pro: easy to implement and set up. Works for most small-scale tasks like gsm8k. Always on-policy (if ``train_batch_size == policy_mini_batch_size``). +- Con: cannot scale training and generation separately. + + +b. One-Step Off-Policy Async Training +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In AReal's figure 1 RHS, we have one-step off-policy pipelining, also supported by SkyRL and discussed in :doc:`one_step_off_async`. + +Here we split the GPUs into two groups, dedicated to training and generation respectively. We first generate a batch of trajectories to start the pipeline, and the trainer always trains on the one-step-stale batch of trajectories (hence one-step off). + +- Pro: still relatively easy to implement and setup. +- Con: as shown in the figure, stragglers in a generation batch will stall both generation and training, decreasing throughput. This is especially an issue for long-horizon tasks. + +c. Fully Async Training (this page) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. figure:: images/areal_figure3.png + :alt: AReal Figure 3: Fully Async Training + :align: center + :width: 90% + + **AReal Figure 3: Fully Async Training** + +Both approaches above suffer from the straggler issues in generation. Fully async training addresses this with in-flight weight update. After the trainer finishes a training step, we: 1) pause the generation of the ongoing trajectories, 2) update the generator weights in-flight, 3) resume generation of the trajectories. + +This effectively addresses the straggler issues. While it may sound unintuitive at first (a trajectory can be generated by multiple model versions), it has worked well in practice according to the literature. + + +II. How to use fully async training in SkyRL? +---------------------------------------------- + +The fully async trainer class is implemented in ``fully_async_trainer.py::FullyAsyncRayPPOTrainer``, subclassing the synchronous trainer class ``trainer.py::RayPPOTrainer``. + +This trainer class is generator-agnostic, meaning it can support any generator that implements the ``GeneratorInterface`` interface, including your arbitrary agent harness. + +We provide an example script in ``examples/fully_async``, where we train Qwen2.5-1.5B-Instruct on GSM8K with the fully async approach. + +We break down the usage into two simple steps. + +Step 1: Define your ``main_async.py`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Following `examples/fully_async/main_async.py `_, subclass the base entrypoint class ``BasePPOExp``: + +- Override ``get_trainer()`` to use fully async trainer class ``FullyAsyncRayPPOTrainer`` +- Override ``get_generator()`` to use your custom generator class. Note that currently we only support generators that use ``/chat/completions`` (hence the ``SkyRLGymHTTPGenerator`` used in the example). + +Step 2: Config knobs to tune for fully async training +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Following `examples/fully_async/async_run_gsm8k.sh `_, update the training configuration to use your new entrypoint ``main_async.py``: + +.. code-block:: bash + + uv run --isolated --extra vllm -m examples.fully_async.main_async \ + ... + +For fully async specifically, the following are the main knobs to tune: + +- ``trainer.policy_mini_batch_size``: The mini-batch size for policy training. The trainer triggers a training step whenever the generation workers have generated this many groups of trajectories. +- ``trainer.fully_async.max_staleness_steps``: The maximum off-policy steps allowed for each training batch, trading off between throughput and off-policy bias. See :ref:`async-staleness-manager` for more details. + + - ``max_staleness_steps = 0`` is equivalent to synchronous training (despite wasting throughput since we cannot overlap training and generation). +- ``trainer.fully_async.num_parallel_generation_workers``: The number of generation workers to spawn, where + each worker works on a group of trajectories. It should be ``>= trainer.policy_mini_batch_size`` to avoid wasted throughput, + and ``<= trainer.policy_mini_batch_size * (trainer.fully_async.max_staleness_steps + 1)`` since it would be wasted due to capacity control. + The larger the number, the more throughput, and likely more staleness (and hence off-policy-ness). + +On GPU placement: first disable colocation of training and generation, then configure how many GPUs to dedicate to training and generation respectively. The following snippet dedicates 4 GPUs to each. + +.. code-block:: bash + + trainer.placement.colocate_all=false \ + trainer.placement.colocate_policy_ref=true \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 + + +Other restrictions (will be validated by the trainer): + +- ``trainer.train_batch_size``: Must be set to the same as ``trainer.policy_mini_batch_size``. +- ``generator.batched``: Must be set to ``false`` to use non-batched vLLM engines. +- ``generator.async_engine``: Must be set to ``true`` to use async vLLM engines. +- ``trainer.algorithm.dynamic_sampling.type``: Must be set to ``"none"`` to disable dynamic sampling (not supported yet). + + +III. Design and Implementation of Fully Async Training in SkyRL +---------------------------------------------------------------- + +0. Overview +~~~~~~~~~~~ + +The high-level control logics of fully async training in SkyRL are shown in the figure below. + +.. figure:: images/skyrl_fully_async.png + :alt: Systems Diagram of Fully Async Training in SkyRL + :align: center + :width: 90% + + **Systems Diagram of Fully Async Training in SkyRL** + + +There are 5 core components: + +1. K ``GenerationWorker``: workers that produce the trajectories. Each is simply an ``asyncio.Task`` that runs ``AnyGenerator.generate()``. K is ``trainer.fully_async.num_parallel_generation_workers``. ``AnyGenerator`` means that it works with any generator. +2. ``TrainingWorker``: simply a single thread that runs the training loop. +3. ``GenerationOutputGroupBuffer``: a buffer that stores the generated groups of output. +4. ``AsyncDataloader``: a thin wrapper of the dataloader, polled by the generation workers to get data. +5. ``AsyncStalenessManager``: a controller that manages the staleness control to ensure no trajectory is dropped for being too stale. + +Note that all the control logics pertain to a single epoch. We do not do any cross-epoch asynchrony. + +Besides, we define ``group`` as the smallest unit of data in the control logic. A group (in GRPO sense) +is a set of ``generator.n_samples_per_prompt`` number of trajectories generated for the same prompt. + +1. K Generation Workers +~~~~~~~~~~~~~~~~~~~~~~~~ + +Generation workers are virtual workers, each is simply an ``asyncio.Task`` that runs ``generator.generate()`` +for a group of trajectories. There is only one ``generator`` instance (e.g. ``SkyRLGymGenerator``) and one +physical ``InferenceEngineClient`` in the back serving the actual LLM inference. + +Each generation worker runs the following steps in a while loop: + +1. Get a single data from the ``AsyncDataloader``. Might await on staleness control based on ``AsyncStalenessManager`` (i.e. too many data accumulated in the buffer and a new generation will result in staleness violation). +2. Generate one group of trajectories. Can be single-turn or multi-turn, ``SkyRLGymGenerator`` or your own generator. +3. Enqueue the generated group to the ``GenerationOutputGroupBuffer``. +4. Repeat from step 1 until the ``AsyncDataloader`` is exhausted for this epoch. + +2. Training Worker (i.e. the training loop) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While we call it a "training worker", it is simply the single thread that runs the training loop. + +It follows the following steps in a for-loop over the number of steps per epoch: + +1. Get precisely ``trainer.policy_mini_batch_size`` groups from the ``GenerationOutputGroupBuffer``. + The staleness controller enforces a global capacity bound that aims to keep batches within ``trainer.fully_async.max_staleness_steps`` steps stale. However, it does not strictly guarantee it. See :ref:`async-staleness-manager` for more details. +2. Train on the generated groups. +3. Mark the data that we used to train as "consumed" to the ``AsyncDataloader``, so that when we resume + training from a checkpoint, we know what data has been trained on and hence can be skipped. +4. Make ``AnyGenerator`` (really the ``InferenceEngineClient`` in the back): pause generation, sync weights, resume generation. Note that this operation is agnostic to what the generator is doing. It can either be in the middle of a ``/chat/completions`` generation, or interacting with the environment. +5. Update the global step and hence the capacity controlled by the ``AsyncStalenessManager``, potentially unblocking + generation workers stuck on step 1 -- as they can now generate new data that will not be as stale as before. +6. Repeat from step 1 until the epoch is done. + + +3. Generation Output Group Buffer +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The generation output group buffer is simply a ``asyncio.Queue`` that stores the generated groups of output. +It is enqueued by the generation workers, and dequeued by the training worker. + +4. Async Dataloader +~~~~~~~~~~~~~~~~~~~ + +The async dataloader is a thin wrapper around the ``torchdata.stateful_dataloader.StatefulDataLoader`` we use in sync training or one-step-off async training. It is implemented to satisfy the following requirements of fully async training: + +- Thread-safe dataloader iteration with a lock, as there are multiple concurrent generation workers polling data. +- Records consumed data UIDs for checkpointing to avoid training on the same data upon resuming. +- Do proper trimming of the dataset to ensure that the dataloader always returns a full mini-batch. + + - We cannot rely on ``drop_last`` because the batch size is 1 in fully async training (since each generation worker works on a single group at a time). + - We also cannot trim the ``train_dataset`` before building the dataloader, because it will affect the random permutations of the dataset, and the data order of async training will be different from synchronous training. However if we keep the datasets the same, even if the batch size of the dataloader is different, the permutation is the same after aggregating the batches of size one to the same size as the mini-batch size. + +The ``AsyncDataloader`` is used in the following cases: + +- The generation workers poll data from the ``AsyncDataloader`` to generate trajectories. +- The trainer marks the data that has been used to train as "consumed" to the ``AsyncDataloader`` +- Upon checkpointing, save the consumed data UIDs along with other checkpointing states, so that when we resume training from a checkpoint, we know what data has been trained on for this epoch and hence can be skipped. +- Reset to the initial state at the end of each epoch. + +.. _async-staleness-manager: + +5. Async Staleness Manager +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The async staleness manager makes sure generation never outruns training beyond the user-specified +bound ``trainer.fully_async.max_staleness_steps``. It enforces a simple, global capacity rule so +that groups waiting in the buffer or currently being generated are never "too stale" relative to the +latest trained model. + +Modeled after AReal's `StalenessManager `_ (paper §5.1), our controller maintains a small set of global counters and enforces a simple capacity inequality. + +Definitions +^^^^^^^^^^^ + +- **Consumed by trainer**: groups that have already been used for updating the model weights. +- ``current_global_step``: the model version currently being worked on; the model has completed ``current_global_step - 1`` updates. +- ``mini_batch_size = B``: number of groups consumed per training step. +- ``max_staleness_steps = S``: the maximum allowed difference between when a group was scheduled to generate, and the step that trains on it. +- ``accepted``: total number of groups that finished generation (either already consumed or waiting in the buffer). This includes all the generation outputs throughout the entire training process. Thus, this number is strictly increasing. +- ``running``: number of groups currently being generated by workers. +- ``submitted``: total number of groups submitted to workers (used for logging). + +Capacity control +^^^^^^^^^^^^^^^^ + +At ``current_global_step``, the maximum number of groups the system may have progressed +since the very beginning of the training (counting both completed and in-flight generation) without violating the staleness bound is:: + + capacity = (S + current_global_step) * B + +To stay within the bound, the manager guarantees:: + + accepted + running <= capacity + +Intuition: + +- Of the capacity, ``(current_global_step - 1) * B`` have already been consumed by training. +- The remainder is the "headroom" the system can be ahead of training: items already in the buffer (``accepted - consumed``) plus items currently being generated (``running``). +- Each time training finishes a step, ``current_global_step`` increases by 1, expanding capacity by ``B`` and unblocking generation. + +Operationally, a generation worker: + +1) Acquires capacity before starting a new group via ``await AsyncStalenessManager.acquire_submission_slot()``. This blocks until ``capacity - (accepted + running) > 0`` is satisfied. +2) When the group finishes generation and is enqueued, it calls ``await AsyncStalenessManager.on_rollout_accepted()`` to convert one running slot into an accepted unit. + +After each training step, the trainer increments the version and calls +``await AsyncStalenessManager.notify_capacity_change(current_global_step)``, which increases capacity and wakes up blocked workers. + +Per-sample staleness vs capacity bound +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The capacity inequality keeps generation from outpacing training by more than ``S`` steps in **aggregate**. It does **NOT** strictly guarantee that every trajectory group will be trained within ``S`` steps of when it was scheduled. In rare cases (e.g., very long rollouts), a trajectory group may complete after more than ``S`` steps have elapsed while the system as a whole still satisfies the capacity constraint. However, in **steady state**, staleness remains within the configured budget of ``max_staleness_steps``. + +Possible ways for handling such rare groups that violate the ``S`` bound: + +- **Current behavior**: Still accept the trajectory group for training, but log the staleness metrics with a warning. + + - Matches `PipelineRL's behavior `_ and `AReal's default behavior `_. +- Drop the trajectory group using ``AsyncStalenessManager.on_rollout_rejected()`` **(not supported yet but should be hackable)** + + - Matches AReal's behavior if the ``should_accept_fn`` is based on per-sample staleness. +- Drop the trajectory group and resample. Would require logics to ``AsyncDataloader`` to requeue the data. **(not supported yet but should be hackable)** + +Checkpointing semantics +^^^^^^^^^^^^^^^^^^^^^^^ + +When saving a checkpoint, only trainer-consumed state is recorded. On resume, we set ``accepted := consumed`` (and do not restore those that were running or about-to-be-consumed in the buffer), which preserves the capacity invariant without reintroducing stale or partially generated work. + +.. note:: + + ``AsyncStalenessManager`` is a deliberate design choice. One could instead over-generate and drop groups that are too stale at training time. We prefer proactive admission control to avoid wasted compute and simplify buffer management. With very small ``S``, the system may stall more often; increasing ``S`` trades off stricter on-policy-ness for higher throughput. + +.. note:: + + Our capacity formula uses ``current_global_step`` directly (no ``+1``) because in SkyRL ``current_global_step`` denotes the version being worked on, whereas AReal's derivation counts completed versions. This matches the implementation in ``fully_async_trainer.py``. + + + +IV. More Implementation Details +-------------------------------- + +1. How is in-flight weight update implemented by the inference engines? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The main reason why we can support fully async training for any generator (single-turn or multi-turn) is that, the in-flight weight update is implemented by the inference engines, not the generator / agent harness that the users use. + +To implement in-flight weight update, we need to change the semantics of ``/chat/completions`` (or ``.generate()``, ``/completions``) from "generate an output for a given input" to "generate an output for a given input, while possibly updating the model weights in the middle". + +That is, when the weight update is happening, the generator / agent harness has no clue that this is happening +underlyingly as the ``/chat/completions`` endpoint is simply blocked and not returning. + +This design makes the implementation of fully async training agnostic to the generator / agent harness as +the trajectory could be doing anything when weight update is happening -- in the middle of a output generation or in the middle of an interaction with the environment (where the next output generation will simply be blocked until the weight update is finished). + +We implement this semantics by implementing SkyRL's ``InferenceEngineClient`` to keep calling the underlying +inference engine's ``/chat/completions`` endpoint in a while loop, until the finish reason is not an ``abort``. + +Upon ``InferenceEngineClient.pause_generation()``, we abort all the in-flight ``/chat/completions`` requests, +and save the partial results, waiting to be re-fed into the underlying inference engine as a new +``/chat/completions`` request when ``InferenceEngineClient.resume_generation()`` is called. + +See the following two PRs for more details: +- https://github.com/NovaSky-AI/SkyRL/pull/537 +- https://github.com/NovaSky-AI/SkyRL/pull/557 + +2. How is checkpointing supported +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The main challenge of performing checkpointing and resuming in fully async training is that, the +dataloader's state is not reliable (which is usually used to save checkpoint in sync training). While some data could be retrieved, upon checkpointing, we have no clue whether a data has already been trained or not. + +To address this, our ``AsyncDataloader`` records the data UIDs that have been consumed by the trainer. Upon resuming from a checkpoint, we load the consumed data UIDs and skip the data that has already been trained. Therefore, the state of the dataloader is not used at all. + +V. Future Work +---------------- + +- Validate with correct algorithmic offset (importance weighting) +- Support fully async training for ``.generate()`` and ``/completions`` as well. +- Validate speedup and results with DAPO. +- How to make fully async training reproducible (at the cost of some performance degradation)? + +References +----------- + +- AReal: https://arxiv.org/abs/2505.24298v3 +- PipelineRL: https://arxiv.org/abs/2509.19128v2 +- ScaleRL: https://arxiv.org/abs/2510.13786 diff --git a/skyrl-train/docs/tutorials/images/areal_figure1.png b/skyrl-train/docs/tutorials/images/areal_figure1.png new file mode 100644 index 0000000000..7ba2fd036c Binary files /dev/null and b/skyrl-train/docs/tutorials/images/areal_figure1.png differ diff --git a/skyrl-train/docs/tutorials/images/areal_figure3.png b/skyrl-train/docs/tutorials/images/areal_figure3.png new file mode 100644 index 0000000000..3bd561750b Binary files /dev/null and b/skyrl-train/docs/tutorials/images/areal_figure3.png differ diff --git a/skyrl-train/docs/tutorials/images/skypilot-dashboard.jpeg b/skyrl-train/docs/tutorials/images/skypilot-dashboard.jpeg new file mode 100644 index 0000000000..19438018ce Binary files /dev/null and b/skyrl-train/docs/tutorials/images/skypilot-dashboard.jpeg differ diff --git a/skyrl-train/docs/tutorials/images/skypilot-ray-logs.png b/skyrl-train/docs/tutorials/images/skypilot-ray-logs.png new file mode 100644 index 0000000000..56d0eb1822 Binary files /dev/null and b/skyrl-train/docs/tutorials/images/skypilot-ray-logs.png differ diff --git a/skyrl-train/docs/tutorials/images/skypilot-wandb.jpeg b/skyrl-train/docs/tutorials/images/skypilot-wandb.jpeg new file mode 100644 index 0000000000..f26614982f Binary files /dev/null and b/skyrl-train/docs/tutorials/images/skypilot-wandb.jpeg differ diff --git a/skyrl-train/docs/tutorials/images/skyrl_fully_async.png b/skyrl-train/docs/tutorials/images/skyrl_fully_async.png new file mode 100644 index 0000000000..f16ed52373 Binary files /dev/null and b/skyrl-train/docs/tutorials/images/skyrl_fully_async.png differ diff --git a/skyrl-train/docs/tutorials/new_env.rst b/skyrl-train/docs/tutorials/new_env.rst new file mode 100644 index 0000000000..7a8e7bb920 --- /dev/null +++ b/skyrl-train/docs/tutorials/new_env.rst @@ -0,0 +1,300 @@ +Creating a New Environment or Task +===================================== + +To demonstrate how to create custom environments in SkyRL-Gym and train with SkyRL, let's build a simple multiplication environment! + +We'll walk through the complete process: implementing the environment, registering it, preparing training data, and running your first training session. + +**What we're building:** An environment that asks the model to multiply numbers and checks if the answer is correct. The completed code is available at `examples/multiply `_. + +Environment Interface +--------------------- + +SkyRL-Gym includes a simple text-in/text-out environment interface for LLM tasks, ``BaseTextEnv``, which looks like this: + +.. code-block:: python + :linenos: + :caption: Base text environment interface at `skyrl_gym/envs/base_text_env.py `_ + + class BaseTextEnv(Env[ConversationType, str]): + def step(self, action: str) -> BaseTextEnvStepOutput: + """ + Runs one environment step. + + Args: + action: The LLM's response as a string + + Returns: + BaseTextEnvStepOutput containing: + - observations: New messages from the environment + - reward: Float reward for the action + - done: Whether the episode is finished + - metadata: Additional info (optional) + """ + pass + + def init(self, prompt: ConversationType) -> Tuple[ConversationType, Dict[str, Any]]: + return prompt, {} + + def close(self): + pass + +This class inherits from ``Env``, which is a generic environment interface (i.e., not specific to text-based tasks). An API reference for ``Env`` can be found in :doc:`../api/env`. + +For our multiplication environment, we only need to implement the ``step`` method above because we don't have any initialization or cleanup to do. + + +Simple Single-Turn Environment +------------------------------- + +Let's start with a basic version that gives the model only one chance to get the answer right. + +The prompt and response format expected by the ``multiply`` environment is as follows: + +- The model prompt will be a multiplication problem of 2 n-digit numbers, such as "123 * 456" or "999 * 999". +- The model output should be in the format of ``\\boxed{answer}``, where ``answer`` is the product of the two numbers. + +So, the environment ``step`` must simply parse the answer out of ``\\boxed{answer}`` and check if it matches the ground truth. + +.. code-block:: python + :linenos: + :caption: Simple multiplication environment + + class MultiplyEnv(BaseTextEnv): + def _parse_action(self, action: str) -> str: + """Extract answer from \\boxed{answer} format""" + match = re.search(r"\\boxed\{([^}]+)\}", action) + return match.group(1) if match else None + + def step(self, action: str) -> BaseTextEnvStepOutput: + answer = self._parse_action(action) + is_correct = answer is not None and answer.strip() == str(self.ground_truth).strip() + + return BaseTextEnvStepOutput( + observations=[], + reward=1.0 if is_correct else 0.0, + done=True, + metadata={"parsed_answer": answer} + ) + +That's it! The environment checks if the model's answer matches the ground truth and gives a reward of 1.0 for correct answers, 0.0 for incorrect ones. + +Multi-Turn Environment +---------------------- + +Want to give the model multiple attempts? Let's extend our environment to allow multiple turns. + +We will make a few simple extensions to our ``step()`` method: + +- Keep track of the number of turns (``self.turns``) and indicate the trajectory is ``done`` after a configured maximum number of turns (``self.max_turns``) +- If the turns expire or the model provides a correct answer, we indicate the trajectory is ``done`` and return a reward as follows: + + - Correct answer: 1.0. + - Incorrect answer, but in format of ``\\boxed{...}``: 0.5. + - Incorrect answer, and not in format of ``\\boxed{...}``: 0.0. +- If the model is incorrect and has more turns remaining, we also provide feedback as a new ``observation``. + +.. code-block:: python + :linenos: + :caption: Multi-turn multiplication environment in `examples/multiply/env.py `_ + + def step(self, action: str) -> BaseTextEnvStepOutput: + self.turns += 1 + answer = self._parse_action(action) + is_correct = answer is not None and answer.strip() == str(self.ground_truth).strip() + found_boxed = answer is not None + + # Episode ends if max turns reached or correct answer found + done = self.turns >= self.max_turns or is_correct + + # Reward structure: + # - Correct answer: 1.0 + # - Wrong answer in correct format: 0.5 + # - No boxed answer: 0.0 + if is_correct: + reward = 1.0 + elif found_boxed: + reward = 0.5 + else: + reward = 0.0 + + if done: + return BaseTextEnvStepOutput( + observations=[], + reward=reward, + done=True, + metadata={"parsed_answer": answer} + ) + + # Give feedback for another attempt + if answer is not None: + feedback = f"Your answer '{answer}' is incorrect. Please try again." + else: + feedback = "Please provide your answer in the format \\boxed{your_answer}." + + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": feedback}], + reward=0.0, + done=False, + metadata={"parsed_answer": answer} + ) + +The multi-turn version gives partial credit for formatting the answer correctly, even if it's wrong. This helps the model learn the expected output format. + +The final implementation is available in `examples/multiply/env.py `_. + +(Turn-level) Rewards And Metrics +-------------------------------- + +In the example above, unless ``done=True``, the reward is ``0.0``. That is, the model only receives a single reward for the entire trajectory. +You can experiment with turn-level rewards by returning a non-zero reward in any turn. Otherwise, if you only want to use outcome rewards, you can simply return ``reward=0.0`` for all intermediate turns. + +SkyRL automatically computes the following metrics for logging purposes: + +- ``pass_at_n``: The ``n`` in ``pass_at_n`` is the number of trajectories we generate for each example. ``pass_at_n`` is 1 if any trajectory succeeded, and 0 otherwise. For each trajectory, we assume that the last turn's reward signifies the entire trajectory's reward, and any positive value is considered a "pass". +- ``mean_raw_reward``: for each trajectory, we sum over all the turns' rewards. We then take the average over all the trajectories. + +Whether you use turn-level rewards or outcome rewards, the rewards used to train the model will be translated to per-token rewards. For example, if there are 3 turns with 4 response tokens each and the turn-level rewards are ``[1.0, 2.0, 3.0]``, the resulting per-token rewards are: + +.. code-block:: python + + [ + 0.0, 0.0, 0.0, 1.0, + 0.0, 0.0, 0.0, 2.0, + 0.0, 0.0, 0.0, 3.0, + ] + +If there is only an outcome reward of ``1.0`` (i.e. intermediate turns' rewards are all ``0.0``), the per-token rewards are: + +.. code-block:: python + + [ + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 1.0, + ] + +Registering Your New Environment +-------------------------------- + +Finally, we need to ``register`` the new environment so the training stack can find it by name (which we refer to as ``env_class``). We will name this environment ``multiply``. + +We will create a new entrypoint for training with the ``multiply`` environment by creating a file at ``examples/multiply/main_multiply.py`` that looks like this: + +.. code-block:: python + :linenos: + :caption: Environment registration at `examples/multiply/main_multiply.py `_ + + @ray.remote(num_cpus=1) + def skyrl_entrypoint(cfg: DictConfig): + # Register the multiply environment + # this needs to be done inside the entrypoint task + register( + id="multiply", # <-- The name of the environment. + entry_point="examples.multiply.env:MultiplyEnv", # <-- The path to the environment class. + ) + + # make sure that the training loop is not run on the head node. + exp = BasePPOExp(cfg) + exp.run() + + @hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) + def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + if __name__ == "__main__": + main() + +Now, the training stack can simply build the new environment with ``skyrl_gym.make("multiply")``! + +.. note:: + All example code written in this document is *outside* of the ``skyrl-train`` and ``skyrl-gym`` packages. There is no need to fork and edit ``skyrl-train`` or ``skyrl-gym`` code -- just implement and register your environment, and the training stack can find the environment seamlessly! + +Preparing Training Data +----------------------- + +Before we can train, we need a dataset of problems to train on. + +We can generate a dataset of multiplication problems using `examples/multiply/multiply_dataset.py `_. See the file for more details, but the core idea is to generate random multiplication problems of n-digit numbers, and ensure the dataset example is in the correct format: + +.. code-block:: python + :linenos: + :caption: Generating a dataset of random multiplication problems. + + for idx in range(num_examples): + question, answer = generate_multiplication_problem(num_digits) + + data = { + "data_source": "synthetic_multiply", + "prompt": [ + system_prompt, + { + "role": "user", + "content": question, + } + ], + "env_class": "multiply", + "reward_spec": { + "method": "rule", + "ground_truth": answer, + }, + "extra_info": { + "num_digits": num_digits, + "split": split_name, + }, + } + examples.append(data) + + +Note that the ``env_class`` here should match the name of the environment we registered. In this case, it is ``multiply``. You can optionally omit the ``env_class`` here and instead set it in the training configuration to apply to all training samples, but setting ``env_class`` per-sample allows for multi-environment training so it is the recommended practice. + +See the doc on :doc:`../datasets/dataset-preparation` for more details on the required dataset format and how to prepare your own dataset. + +Now we can generate the datsaet: + +.. code-block:: bash + :linenos: + :caption: Generate training data + + uv run --isolated examples/multiply/multiply_dataset.py \ + --output_dir $HOME/data/multiply \ + --num_digits 4 \ + --train_size 10000 \ + --test_size 200 + +This creates ``train.parquet`` and ``validation.parquet`` files in the ``$HOME/data/multiply`` directory. + +Training Your Model +------------------- + +Time to train! 🚀 + +We will use the ``run_multiply.sh`` script to train the model. This script is located in `examples/multiply/run_multiply.sh `_, which sets up the training configuration and calls ``main_multiply.py``. + +**Common Configuration Parameters** + +First, ensure sure your config matches your available GPUs. You may need to adjust the following parameters to match your GPU count (which we set via an environment variable `NUM_GPUS`): + +- ``trainer.placement.policy_num_gpus_per_node`` +- ``generator.num_inference_engines`` + +Then, configure how the environment should be executed. For multi-turn environments, we recommend setting ``generator.batched=false`` and ``generator.async_engine=true`` to ensure that each environment is executed asynchronously. If your environment is single-turn, you may get better performance by reversing these settings. + +**Launch Training** + +.. code-block:: bash + :linenos: + :caption: Run training + + export WANDB_API_KEY=your_wandb_api_key # or set trainer.logger="console" to print to stdout + bash examples/multiply/run_multiply.sh + +**Next Steps:** Want to make multiplication easier? Try integrating a calculator tool into your environment! Check out the :doc:`tools_guide` documentation for details. + +That's it! You've created a custom environment, prepared training data, and started training. The same pattern works for any text-based task you want to train on. + +Now watch your model become a multiplication master! diff --git a/skyrl-train/docs/tutorials/one_step_off_async.rst b/skyrl-train/docs/tutorials/one_step_off_async.rst new file mode 100644 index 0000000000..b2ee19daaa --- /dev/null +++ b/skyrl-train/docs/tutorials/one_step_off_async.rst @@ -0,0 +1,122 @@ +Async Training: One-Step Off-Policy Pipelining +============================================== + +This example demonstrates how to implement asynchronous training with one-off pipelining, showcasing the flexibility of the training framework to support different execution plans with minimal code changes. + +The complete code from this example is available at `examples/async `_. + +Overview +-------- + +The one-off pipelining approach separates the generation and training phases into two parallel coroutines, allowing the model to generate new samples while simultaneously training on previously generated data. This can lead to better GPU utilization and greater training throughput. + +.. TODO(tgriggs): Add a diagram here. + +In the SkyRL framework, only minimal code changes are required to modify the synchronous training pipeline to an asynchronous one. + +In particular, we follow three simple steps: + +1. Define a new trainer class (``AsyncRayPPOTrainer``) and override a single method (``train()``) from the the base trainer class to separate and parallelize the training and generation phases +2. Create a new training entrypoint (``main_async.py``) that uses the new trainer class +3. Update the training configuration to disable model colocation + +Implementation Steps +-------------------- + +Step 1: New trainer class with modified training loop +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We first create a new trainer class ``AsyncRayPPOTrainer`` that inherits from ``RayPPOTrainer`` and overrides the ``train()`` method. + +The original ``train()`` method performs generation and training sequentially, so the async trainer splits the traditional synchronous training loop into two parallel components: + +1. **Generation Loop** (``_run_generate_loop``): Continuously generates new samples and places them in a queue +2. **Training Loop** (``_run_training``): Consumes generated samples from the queue and performs training updates + +The generation loop passes the training loop completed trajectories via an ``asyncio.Queue`` and coordinates weight synchronization using asyncio events. + +We include a minimal version here, please see `examples/async/async_trainer.py `_ for full details: + +.. code-block:: python + + class AsyncRayPPOTrainer(RayPPOTrainer): + async def train(self): + # Assert non-colocated training + assert not self.cfg.trainer.placement.colocate_all, "colocate_all is not supported for async training" + + # Create buffer of size 1 for generated trajectories. + generation_buffer = asyncio.Queue(maxsize=1) + + # Start generator task asynchronously. + generator_task = asyncio.create_task(self._run_generate_loop(generation_buffer)) + + # Training loop consumes from buffer + for idx in range(len(self.train_dataloader)): + # Training starts when generation buffer is filled. + status = await self._run_training(generation_buffer) + # Trainer waits for generation to complete before next training step. + if idx != len(self.train_dataloader) - 1: + await self.generation_ack.wait() + # Synchronize weights after training. + await self.async_sync_policy_weights_to_inference_engines() + # Signal that weight sync is done, ready for next round of generation. + self.sync_finished.set() + self.generation_ack.clear() + +.. note:: + **Flexible execution plans:** The underlying implementation of training and generation is unchanged -- only the training loop is modified to parallelize the two phases. In general, SkyRL enables flexibile modifications to the execution plan simply by modifying the high-level training loop. + +Step 2: Create New Training Entrypoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Next we create a new training entrypoint ``main_async.py`` that uses the new trainer class. + +The new entrypoint (``AsyncPPOExp``) overrides a single method (``get_trainer()``) in the base entrypoint class (``BasePPOExp``) that constructs the new trainer class: + +.. code-block:: python + + class AsyncPPOExp(BasePPOExp): + def get_trainer(self, cfg, tracker, tokenizer, train_dataset, eval_dataset, + inference_engine_client, generator, colocate_pg): + # Construct the new trainer class. + return AsyncRayPPOTrainer( + cfg=cfg, tracker=tracker, tokenizer=tokenizer, + train_dataset=train_dataset, eval_dataset=eval_dataset, + inference_engine_client=inference_engine_client, generator=generator, + colocate_pg=colocate_pg + ) + +That's it! The rest of the entrypoint logic for launching the training run remains unchanged. + +Step 3: Update Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Finally, we modify the training configuration to use our new entrypoint and disable model colocation: + +.. code-block:: bash + + uv run --isolated --extra vllm -m examples.async.main_async \ + trainer.placement.colocate_all=false \ + trainer.placement.colocate_policy_ref=true \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 + +Key configuration changes: + +* **examples.async.main_async**: Point the bash script to the new entrypoint +* **colocate_all=false, colocate_policy_ref=true**: Disables colocation of generation and training models (but keeps the policy and reference models colocated). + + +Now we can train! + +.. code-block:: bash + + # Prepare the dataset + uv run --isolated examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + + # Run the training script + export WANDB_API_KEY=your_wandb_api_key # or set trainer.logger="console" to print to stdout + bash examples/async/async_run_gsm8k.sh + diff --git a/skyrl-train/docs/tutorials/skyrl_gym_generator.rst b/skyrl-train/docs/tutorials/skyrl_gym_generator.rst new file mode 100644 index 0000000000..78d20d2801 --- /dev/null +++ b/skyrl-train/docs/tutorials/skyrl_gym_generator.rst @@ -0,0 +1,215 @@ +SkyRLGymGenerator: Multi-turn Tokenization and Token-in-Token-out +================================================================= + +Last updated: 2025-10-08 + +This document explains how ``SkyRLGymGenerator`` manages the chat history and tokens for both +single-turn and multi-turn rollouts, and how token-in-token-out (TI/TO) is enforced. + +Overview +-------- + +``SkyRLGymGenerator`` is an implementation of the ``GeneratorInterface``, where we use SkyRL Gym for +the environment of the rollouts. If you would like to use other environments, you can write your +own Generator by extending ``GeneratorInterface``. + +A ``SkyRLGymGenerator`` uses an ``InferenceEngineClient`` like an LLM endpoint to generate response, +ultimately returning ``GeneratorOutput`` (including ``response_ids`` and ``loss_masks``) to +the training loop for updating the model. + +``SkyRLGymGenerator`` is implemented to enforce token-in-token-out (TI/TO) in most cases. To see +what TI/TO is and why it is important, please refer to `issue #123 `_. + +To implement ``GeneratorInterface.generate()``, ``SkyRLGymGenerator`` implements ``generate_batched()`` +for single-turn generation, and ``agent_loop()`` for multi-turn generation. + +Single-turn generation +---------------------- + +``SkyRLGymGenerator.generate_batched()`` is used when ``config.generator.batched`` is set to ``True``. +In this case, only a single assistant message is generated for each prompt. It is used for tasks +such as math problems, where the model is expected to generate a single response without interacting +with the environment. We pass a list of prompts to each invocation of the underlying LLM engine's +``.generate()`` method (hence "batched"). + +Multi-turn generation +--------------------- + +``SkyRLGymGenerator.agent_loop()`` is used when ``config.generator.batched`` is set to ``False``, where +the model is expected to interact with the environment for multiple turns (though ``agent_loop()`` can +also be used for single-turn generation). We pass a single prompt to each invocation of the underlying +LLM engine's ``.generate()`` method. + +There are three distinct codepaths in ``agent_loop()``, each managing the chat history and tokens +differently. The codepaths are determined by: + +- Config's ``generator.use_conversation_multi_turn``: If ``False``, all turns' observations and assistant + generations are stored in the same assistant message. If ``True``, each observation from the + environment's ``step()`` is a message. +- Config's ``generator.chat_template``: Optional custom chat template (primarily for Qwen3 thinking-token handling). + +The three codepaths are: + +1) (Default) Multi-turn conversation, strictly appending tokens + + - Enabled when ``use_conversation_multi_turn == True`` and ``generator.chat_template`` is not defined. + - These are the default values for these configs, so this is the default codepath. + - Each observation is a turn of message following the model's chat template, appending + LLM-generated response and observations as raw tokens to maintain TI/TO, but requires the + fixed-base approach to ensure the raw tokens follow the model's chat template correctly + (see :ref:`multi-turn-tokenization-and-ti-to`). + - TI/TO: enforced. + - Example with Qwen2.5 chat template: + +.. code-block:: python + + <|im_start|>system + System prompt here<|im_end|> + <|im_start|>user + Question here<|im_end|> + <|im_start|>assistant + Response1<|im_end|> + <|im_start|>user + Observation1<|im_end|> + <|im_start|>assistant + Response2<|im_end|> + <|im_start|>user + Observation2<|im_end|> + ... + +2) Single assistant message for all turns, strictly appending tokens + + - Enabled when ``use_conversation_multi_turn == False``. + - Keep an entire multi-step interaction inside a single assistant message, appending + LLM-generated response and observations as raw tokens to maintain TI/TO. + - TI/TO: enforced. + - Example with Qwen2.5 chat template: + +.. code-block:: python + + <|im_start|>system + System prompt here<|im_end|> + <|im_start|>user + Question here<|im_end|> + <|im_start|>assistant + Response1 + Observation1 + Response2 + Observation2 + ...<|im_end|> + +3) Always re-tokenize full chat history (no TI/TO) + + - Enabled when ``use_conversation_multi_turn == True`` and a ``generator.chat_template`` is defined. + - Mainly to serve models like Qwen3 that require special handling (e.g., strip non-last-turn thinking + tokens). We can also get ``[assistant_masks]`` and ``[input_ids]`` from the final tokenized chat + history with the help of ``{% generation %}`` and ``{% endgeneration %}`` tags in the jinja template. + - Chat history is maintained as string messages and re-tokenized every turn and + at the end to obtain ``assistant_masks`` and final ``response_ids``. + - TI/TO: NOT enforced + +.. note:: + + Qwen3's official chat template strips earlier-turn thinking tokens. + + With code path 1 (the default, no custom chat template), we never retokenize the chat history, so + earlier-turn thinking tokens remain in the strictly appended token sequence for each turn's + inference. When passing the token sequence to the training pipeline, all turns' thinking tokens + are kept as well. + + Alternatively, with code path 3 (set ``generator.chat_template`` as below), we retokenize each + turn and the template strips earlier-turn thinking tokens, keeping only the last turn's thinking + tokens in both inference and training. + + .. code-block:: yaml + + chat_template: + source: "name" + name_or_path: "qwen3_without_thinking" + + It remains an open question which is best for training Qwen3. We will soon add a custom attention mask + to match the official chat template's inference behavior (stripping thinking tokens), + while preserving on-policy training by masking the previous turns' thinking tokens during training. + + +.. _multi-turn-tokenization-and-ti-to: + +Multi-turn Tokenization and TI/TO +--------------------------------- + +In this section, we elaborate how TI/TO is enforced in the multi-turn generation, specifically +for the first codepath. TI/TO for the second codepath is simple since we keep appending the +generated tokens to the same message and hence do not need to worry about the chat templating +between messages. The third codepath does not enforce TI/TO. + +In codepath 1, the agent loop does the following: + 1. Tokenize dataset's prompt to initialize ``input_ids`` + 2. Feed ``input_ids`` to LLM engine, get ``output_ids`` out + 3. ``input_ids += output_ids`` (a.k.a. token-in-token-out) -- the next turn's input IDs are precisely what the LLM generated + 4. Tokenize observations got from SkyRL-Gym's environment output (i.e. ``env.step()``), and append to ``input_ids`` + 5. Repeat 2-4 until ``env.step()`` marks done + +To correctly tokenize the observations in step 4, we follow the fixed-base approach described in +`this blog `_. + +Specifically, we instantiate a ``base_conversation`` that we never change ("fixed base"): + +.. code-block:: python + + self.base_conversation = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "I am a user."}, + ] + self.base_conversation_token_ids = tokenizer.apply_chat_template( + self.base_conversation, + add_generation_prompt=False, + tokenize=True, + ) + +When we get new observations ``new_obs``, which is a list of ``{role: str, content: str}``, we +convert them to token IDs while following the model's chat template by: + +.. code-block:: python + + observation_ids = self.tokenizer.apply_chat_template( + [*self.base_conversation, *new_obs], + add_generation_prompt=True, + tokenize=True, + )[len(self.base_conversation_token_ids) :] + input_ids += observation_ids + loss_mask += [0] * len(observation_ids) + +One tricky part is that, for some models, there are tokens after the last EOS token for a turn of +message. For instance, in Qwen2.5 and Qwen3, the ``base_conversation_token_ids`` are equivalent to: + +.. code-block:: python + + <|im_start|>system\nYou are a helpful assistant.<|im_end|>\n + <|im_start|>user\nI am a user.<|im_end|>\n + +Note that there is a ``\n`` in the assistant's message before the next user's message starts. +If we do token-in-token-out, there is no way for the LLM engine to generate ``\n`` since the +EOS token is ``<|im_end|>``. Therefore, we need to add the ``\n`` back when creating ``observation_ids``. +In order to do this, we cut the ``\n`` out in ``base_conversation_token_ids``: + +.. code-block:: python + + if self.tokenizer.eos_token_id in self.base_conversation_token_ids: + last_eos_token_index = ( + len(self.base_conversation_token_ids) + - 1 + - self.base_conversation_token_ids[::-1].index(self.tokenizer.eos_token_id) + ) + self.base_conversation_token_ids = self.base_conversation_token_ids[: last_eos_token_index + 1] + + +This way, ``observation_ids`` will be ``\n<|im_start|>user\nObservation here<|im_end|>\n`` (note the +very first ``\n`` that makes up the former assistant's ``\n``). The ``\n`` at the **final** assistant +turn will still be missing, but this is fine. +You can see ``tests/cpu/generators/test_skyrl_gym_generator_chat_templating.py`` for more details. + + +References +---------- + +- https://jybsuper.github.io/posts/multiturn_tokenization/#the-breakthrough-fixed-base-approach diff --git a/skyrl-train/docs/tutorials/tools_guide.rst b/skyrl-train/docs/tutorials/tools_guide.rst new file mode 100644 index 0000000000..2ebc440c99 --- /dev/null +++ b/skyrl-train/docs/tutorials/tools_guide.rst @@ -0,0 +1,189 @@ +Using Tools in SkyRL-Gym +========================== + +One of the key features of SkyRL-Gym is `reusable tools`. Often, a single tool (such as Python code execution) is useful in multiple environments (such as math or deep research environments). Reusable tools allow developers to implement a tool once, and use it in multiple environments. You can even build an environment from our library of implemented tools! + +This document describes SkyRL-gym's tool interface and how to build and use tools in SkyRL-gym environments. You can also find the source code for tools in `skyrl_gym/tools `_. + +Core Concepts +------------- + +**tool**: A ``tool`` is a single executable function whose inputs and outputs can be flexibly defined. + +**ToolGroup**: A ``ToolGroup`` is a collection of related tools that share the same context or states. Tool groups enable all tools within the group to access and modify the shared state, such as a shared database connection or cache. + +**Environment**: An ``Environment`` is a class that defines the task for the agent to solve, and can integrate one ore more tool groups for the agent to use. See the following doc for more details on how to build an environment: :doc:`new_env`. + + +ToolGroup and the @tool Decorator +----------------------------------- + +The ``ToolGroup`` class provides utilities for managing tools and tool execution, such as ``get_tool`` and ``get_tool_names``. To create a new tool group, you can inherit from ``ToolGroup`` and add or modify utilities as needed. For example, you can add a shared state to the tool group, or add a custom tool execution function. + +The `@tool` decorator marks methods as executable tools. After creating a custom tool group, you can use the `@tool` decorator to mark methods as tools, which the ``ToolGroup`` will automatically register and be able to execute. For instance, a simple Python code execution tool group might look like the following code block, where the tool group is called ``PythonCodeExecutorToolGroup`` and the tool is called ``python`` marked with the ``@tool`` decorator. + +.. code-block:: python + + from skyrl_gym.tools.core import tool, ToolGroup + import subprocess + + + # Create a new tool group that inherits from ToolGroup + class PythonCodeExecutorToolGroup(ToolGroup): + def __init__(self): + super().__init__(name="PythonCodeExecutorToolGroup") + + # Mark the python method as a tool + @tool + def python(self, code: str) -> str: + result = subprocess.run( + ["python", "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=self.timeout, text=True + ) + return result.stdout + + +See the base interface for tools and tool groups at `skyrl_gym/tools/core.py `_. + +Built-in ToolGroups +------------------- + +SkyRL-Gym provides pre-built ToolGroups that can be used out of the box in any SkyRL-gym environment: + +Python Code Execution +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from skygym.tools import PythonCodeExecutorToolGroup + + python_tools = PythonCodeExecutorToolGroup(timeout=15.0) + result = python_tools.execute_tool("python", "print('Hello, World!')") + +SQL Code Execution +~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from skygym.tools import SQLCodeExecutorToolGroup + + sql_tools = SQLCodeExecutorToolGroup(db_file_path="/path/to/databases") + result = sql_tools.execute_tool("sql", "SELECT * FROM users") + +Search ToolGroup +~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from skygym.tools import SearchToolGroup + + search_tools = SearchToolGroup( + search_url="http://127.0.0.1:8000/retrieve" + ) + result = search_tools.execute_tool("search", "Context to search") + +Environment Integration +------------------------ + +Tools groups can be integrated into any environment in SkyGym-RL. The base environment class for text-based environments is ``BaseTextEnv``, which provides simple utilities for managing and using multiple tool groups in a single envrionment. + +The following sub-sections walk through integrating and using tools in an environment. + +Tool Initialization +~~~~~~~~~~~~~~~~~~~ + +To incorporate tools into an envrionment, first build and initialize the tool groups during environment construction: + + +.. code-block:: python + + from skygym.envs.base_text_env import BaseTextEnv + + class MyEnvironment(BaseTextEnv): + def __init__(self, env_config, extras): + super().__init__() + + # Construct the tool groups + python_tools = PythonCodeExecutorToolGroup(timeout=10.0) + search_tools = SearchToolGroup() + + # Initialize and register tool groups + self.init_tool_groups([python_tools, search_tools]) + +Tool Execution +~~~~~~~~~~~~~~ + +To use a tool and get the result, you can call the ``_execute_tool`` (provided by ``BaseTextEnv``) method with the tool group name, tool name, and the tool input. Tools are most often used in the envrionment ``step`` method. + +.. code-block:: python + + def step(self, action: str): + # Parse action to extract tool call + tool_group_name, tool_name, tool_input = self._parse_action(action) + + # Execute the tool + observation = self._execute_tool(tool_group_name, tool_name, tool_input) + + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": observation}], + reward=reward, + done=done, + metadata=info + ) + +Action Parsing +~~~~~~~~~~~~~~ + +Users can flexibly determine how and when tools are called. The following code block shows a common case, where the model's output (``action``) is parsed to extract the intended tool call and its input. + +.. code-block:: python + + import re + + def _parse_action(self, action: str): + # Parse tool blocks like input + tool_block_match = re.search(r"(.*?)", action, re.DOTALL) + if not tool_block_match: + raise ValueError("No tool block found in action") + + tool_content = tool_block_match.group(1).strip() + inner_tag_match = re.search(r"<(\w+)>(.*?)", tool_content, re.DOTALL) + + # Extract the tool name + tool_name = inner_tag_match.group(1) + # Extract the tool input + tool_input = inner_tag_match.group(2).strip() + + tool_group_name = self.tool_to_toolgroup[tool_name] + + return tool_group_name, tool_name, [tool_input] + +Using Multiple ToolGroups +------------------------- + +An arbitrary number of tool groups can be integrated into an environment, creating a more powerful agent. There is no additional configuration required to use multiple tool groups in an environment, simply construct and initialize all desired tool groups. + +.. code-block:: python + + class AdvancedEnvironment(BaseTextEnv): + def __init__(self, env_config, extras): + super().__init__() + + # Different ToolGroups with shared state + self.db_tools = SQLCodeExecutorToolGroup(db_file_path="/path/to/databases") + self.python_tools = PythonCodeExecutorToolGroup(timeout=10.0) + self.search_tools = SearchToolGroup(search_url="http://127.0.0.1:8000/retrieve") + self.custom_tools = MyCustomToolGroup(shared_config=extras.get("config")) + + # Register all tool groups + self.init_tool_groups([self.db_tools, self.python_tools, self.search_tools, self.custom_tools]) + + +API Reference +------------- + +For a reference of the tool APIs, see the following links: + +- :doc:`../api/tools` +- :doc:`../api/env` + +That's it! You've learned how to use tools in SkyRL-Gym environments. The same pattern works for any tool-based task you want to build. \ No newline at end of file diff --git a/skyrl-train/examples/README.md b/skyrl-train/examples/README.md new file mode 100644 index 0000000000..12e0fd86b1 --- /dev/null +++ b/skyrl-train/examples/README.md @@ -0,0 +1,43 @@ +# SkyRL-Train Examples +Welcome to the SkyRL-Train examples! In this folder you can find the following examples. + +## Algorithms + +- `algorithms/`: Examples for how to configure and run RL with various algorithms and policy-loss variants (e.g., DAPO, SAPO, GRPO, CISPO, GSPO, or your own custom advantage estimators and custom policy losses). +- `ppo/`: Vanilla PPO training (with a critic, ref, and policy model) +- `on_policy_distillation/`: [On-policy distillation recipe](https://novasky-ai.notion.site/on-policy-distillation) that uses a teacher model to provide dense token-level rewards during training, reproducing results from the [Thinking Machines blog](https://thinkingmachines.ai/blog/on-policy-distillation/). +- `tis_correction/`: Applying [Flash-RL TIS](https://fengyao.notion.site/off-policy-rl) correction to improve off-policy stability. +- `turn_level_rewards/`: GSM8K multi-turn environment illustrating turn-level rewards and custom advantage estimators. + +## Async RL + +- `async/`: One-step off-policy GRPO with an asynchronous generator–trainer loop. +- `fully_async/`: Fully asynchronous (PipelineRL/AReal-style) GRPO training with in-flight weight updates. [See docs for full design + details](https://skyrl.readthedocs.io/en/latest/tutorials/one_step_off_async.html). + +## Tasks + +- `gsm8k/`: Basic GSM8K math word-problem dataset utilities and baseline training/generation scripts. +- `llm_as_a_judge/`: GSM8K training with an external LLM as a judge to produce rewards instead of strict exact-match grading. +- `multiply/`: Toy arithmetic environment for multiplying numbers, useful for quick sanity checks and debugging. +- `livecodebench/`: LiveCodeBench code-generation task setup and training scripts. +- `text_to_sql/`: [Text-to-SQL (SkyRL-SQL)](https://skyrl.readthedocs.io/en/latest/examples/multi_turn_text2sql.html) environment and training scripts for mapping natural language questions to SQL queries. +- `step_wise/`: Step-wise training for chat-template agnostic multi-turn RL training. +- `search/`: Multi-turn search agent training with the SearchR1 dataset, backed by a FAISS-based retriever server. + +## Integrations + +- `flash_rl/`: Integration with [FlashRL’s](https://fengyao.notion.site/flash-rl) patched vLLM inference engine for high-throughput RL training. +- `terminal_bench/`: Custom [Terminal-Bench/Harbor](https://harborframework.com/) Generator for training agents to solve command-line tasks. +- `mini_swe_agent/`: Integration with [Mini-SWE-Agent](https://github.com/SWE-agent/mini-swe-agent) to train coding agents on SWE-Bench via SkyRL. +- `../integrations/verifiers/`: Integration with PrimeIntellect's [Verifiers Library](https://github.com/PrimeIntellect-ai/verifiers) + [Environments Hub](https://app.primeintellect.ai/dashboard/environments?_gl=1*1vogwn8*_gcl_au*NjA1ODI2MTMxLjE3NjczOTkwMTM) +- `../integrations/openenv/`: Integration with HuggingFace/Meta [OpenEnv](https://github.com/meta-pytorch/OpenEnv) + +## Large Scale Model Training +- `megatron/`: Examples for running SkyRL with the Megatron Backend for 5D parallelism. +- `moe/`: Work-in-progress MoE training example used for development and testing large-scale multi-node Mixture-of-Experts support. +- `gptoss/`: Training example for the GPT-OSS-20B model using patched attention to support attention sinks. + +## Features and More +- `lora/`: LoRA RL fine-tuning recipes. +- `remote_inference_engine/`: Scripts for running remote vLLM/sglang inference servers and connecting them to SkyRL. +- `training_backends/`: Runner scripts demonstrating how to use different training backends on SkyRL. diff --git a/skyrl-train/examples/algorithms/cispo/run_cispo_gsm8k.sh b/skyrl-train/examples/algorithms/cispo/run_cispo_gsm8k.sh new file mode 100644 index 0000000000..f8af11d4cb --- /dev/null +++ b/skyrl-train/examples/algorithms/cispo/run_cispo_gsm8k.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -x + +# Example of CISPO policy loss training +# Clipped Importance Sampling Weight Policy Optimization (CISPO) for better RL efficiency + +# Run data preparation first: +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/cispo/run_cispo_gsm8k.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# Configure CISPO parameters +POLICY_LOSS="cispo" +CISPO_EPS_CLIP_LOW=0 +CISPO_EPS_CLIP_HIGH=5 +USE_KL_LOSS=false + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.policy_loss_type="$POLICY_LOSS" \ + trainer.algorithm.cispo.cispo_eps_clip_low=$CISPO_EPS_CLIP_LOW \ + trainer.algorithm.cispo.cispo_eps_clip_high=$CISPO_EPS_CLIP_HIGH \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="cispo_gsm8k" \ + trainer.run_name="cispo_gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/cispo_gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/clip_cov_kl_cov/README.md b/skyrl-train/examples/algorithms/clip_cov_kl_cov/README.md new file mode 100644 index 0000000000..7c957f2517 --- /dev/null +++ b/skyrl-train/examples/algorithms/clip_cov_kl_cov/README.md @@ -0,0 +1,74 @@ +# Clip-Cov and KL-Cov Policy Loss Examples + +This directory contains examples for using **Clip-Cov** and **KL-Cov** policy loss functions, based on the implementation from [PRIME-RL/Entropy-Mechanism-of-RL](https://github.com/PRIME-RL/Entropy-Mechanism-of-RL). + +## Overview + +Both methods improve training stability by using covariance-based token selection: + +- **Clip-Cov**: Combines standard PPO clipping with covariance-based correction masking +- **KL-Cov**: Applies KL regularization to tokens selected based on covariance values + +## Usage + +### Prerequisites + +1. Prepare GSM8K data: +```bash +uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +``` + +2. Set up Weights & Biases (optional): +```bash +export WANDB_API_KEY= +``` + +### Running Clip-Cov + +```bash +bash examples/algorithms/clip_cov_kl_cov/run_clip_cov.sh +``` + +**Key parameters:** +- `trainer.algorithm.policy_loss_type="clip_cov"` +- `trainer.algorithm.clip_cov.clip_ratio=0.0002` - fraction of tokens to clip based on covariance +- `trainer.algorithm.clip_cov.clip_cov_lb=1.0` - lower bound for covariance clipping +- `trainer.algorithm.clip_cov.clip_cov_ub=5.0` - upper bound for covariance clipping + +### Running KL-Cov + +```bash +bash examples/algorithms/clip_cov_kl_cov/run_kl_cov.sh +``` + +**Key parameters:** +- `trainer.algorithm.policy_loss_type="kl_cov"` +- `trainer.algorithm.kl_cov.kl_cov_frac=0.2` - percentage of tokens to apply KL regularization to (20%) +- `trainer.algorithm.kl_cov.ppo_kl_coef=1.0` - coefficient for KL regularization term + +## Configuration + +Both methods are configured through the algorithm section of your config: + +```yaml +trainer: + algorithm: + policy_loss_type: "clip_cov" # or "kl_cov" + + # Clip-Cov specific parameters + clip_cov: + clip_ratio: 0.0002 + clip_cov_lb: 1.0 + clip_cov_ub: 5.0 + + # KL-Cov specific parameters + kl_cov: + kl_cov_frac: 0.2 + ppo_kl_coef: 1.0 +``` + + +## Reference + +- Paper: https://arxiv.org/abs/2505.22617 +- Code: https://github.com/PRIME-RL/Entropy-Mechanism-of-RL diff --git a/skyrl-train/examples/algorithms/clip_cov_kl_cov/run_clip_cov.sh b/skyrl-train/examples/algorithms/clip_cov_kl_cov/run_clip_cov.sh new file mode 100644 index 0000000000..3326c81d9e --- /dev/null +++ b/skyrl-train/examples/algorithms/clip_cov_kl_cov/run_clip_cov.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -x + +# Example of Clip-Cov policy loss training +# Covariance-based clipping for improved training stability on GSM8K. +# +# Run data preparation first: +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/clip_cov_kl_cov/run_clip_cov.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# Configure Clip-Cov parameters +POLICY_LOSS="clip_cov" +CLIP_COV_RATIO=0.0002 +CLIP_COV_LB=1.0 +CLIP_COV_UB=5.0 + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.policy_loss_type="$POLICY_LOSS" \ + trainer.algorithm.clip_cov.clip_ratio=$CLIP_COV_RATIO \ + trainer.algorithm.clip_cov.clip_cov_lb=$CLIP_COV_LB \ + trainer.algorithm.clip_cov.clip_cov_ub=$CLIP_COV_UB \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.kl_loss_coef=0.001 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="clip_cov_gsm8k" \ + trainer.run_name="clip_cov_gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/clip_cov_gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/clip_cov_kl_cov/run_kl_cov.sh b/skyrl-train/examples/algorithms/clip_cov_kl_cov/run_kl_cov.sh new file mode 100644 index 0000000000..3be0399147 --- /dev/null +++ b/skyrl-train/examples/algorithms/clip_cov_kl_cov/run_kl_cov.sh @@ -0,0 +1,63 @@ +#!/bin/bash +set -x + +# Example of KL-Cov policy loss training +# Uses covariance-based selection to apply KL regularization to a subset of tokens +# for improved training stability on GSM8K. +# +# Run data preparation first: +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/clip_cov_kl_cov/run_kl_cov.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# Configure KL-Cov parameters +POLICY_LOSS="kl_cov" +KL_COV_FRAC=0.2 +PPO_KL_COEF=1.0 + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.policy_loss_type="$POLICY_LOSS" \ + trainer.algorithm.kl_cov.kl_cov_frac=$KL_COV_FRAC \ + trainer.algorithm.kl_cov.ppo_kl_coef=$PPO_KL_COEF \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.kl_loss_coef=0.001 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="kl_cov_gsm8k" \ + trainer.run_name="kl_cov_gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/kl_cov_gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/custom_advantage_estimator/main_custom_adv_est.py b/skyrl-train/examples/algorithms/custom_advantage_estimator/main_custom_adv_est.py new file mode 100644 index 0000000000..85e1c58743 --- /dev/null +++ b/skyrl-train/examples/algorithms/custom_advantage_estimator/main_custom_adv_est.py @@ -0,0 +1,56 @@ +""" +uv run --isolated --extra vllm -m examples.algorithm.custom_advantage_estimator.main_custom_adv_est +""" + +import ray +import hydra +import torch +import numpy as np +from omegaconf import DictConfig +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils.ppo_utils import AdvantageEstimatorRegistry + + +# Example of custom advantage estimator: "simple_baseline" +def compute_simple_baseline_advantage( + token_level_rewards: torch.Tensor, response_mask: torch.Tensor, index: np.ndarray, **kwargs +): + """ + A simple custom advantage estimator that uses response-level rewards + and computes advantages against a simple baseline. + + This is just an example - replace with your own logic. + """ + with torch.no_grad(): + response_rewards = (token_level_rewards * response_mask).sum(dim=-1, keepdim=True) + + # Simple baseline: use the mean reward across the batch + baseline = response_rewards.mean() + advantages = (response_rewards - baseline) * response_mask + returns = advantages.clone() + + return advantages, returns + + +# Register the custom advantage estimator +AdvantageEstimatorRegistry.register("simple_baseline", compute_simple_baseline_advantage) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = BasePPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/algorithms/custom_advantage_estimator/run_custom_adv_est.sh b/skyrl-train/examples/algorithms/custom_advantage_estimator/run_custom_adv_est.sh new file mode 100644 index 0000000000..c24630f956 --- /dev/null +++ b/skyrl-train/examples/algorithms/custom_advantage_estimator/run_custom_adv_est.sh @@ -0,0 +1,55 @@ +set -x + +# Example of custom advantage estimator: "simple_baseline" +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/custom_advantage_estimator/run_custom_adv_est.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# Configure the advantage estimator to use +ADV_EST="simple_baseline" + +uv run --isolated --extra vllm -m examples.algorithm.custom_advantage_estimator.main_custom_adv_est \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="$ADV_EST" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="custom_adv_est_gsm8k" \ + trainer.run_name="custom_adv_est_gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/custom_policy_loss/main_custom_policy_loss.py b/skyrl-train/examples/algorithms/custom_policy_loss/main_custom_policy_loss.py new file mode 100644 index 0000000000..6b9be8b13f --- /dev/null +++ b/skyrl-train/examples/algorithms/custom_policy_loss/main_custom_policy_loss.py @@ -0,0 +1,54 @@ +""" +uv run --isolated --extra vllm -m examples.algorithm.custom_policy_loss.main_custom_policy_loss +""" + +import ray +import hydra +import torch +from typing import Optional +from omegaconf import DictConfig +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils.ppo_utils import PolicyLossRegistry + + +# Example of custom policy loss: "reinforce" +def compute_reinforce_policy_loss( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, +): + """ + Simple REINFORCE baseline - basic policy gradient that will enable learning. + """ + # Classic REINFORCE: minimize -log_prob * advantage + loss = (-log_probs * advantages).mean() + + # Return loss and dummy clip_ratio (no clipping in REINFORCE) + return loss, 0.0 + + +# Register the custom policy loss +PolicyLossRegistry.register("reinforce", compute_reinforce_policy_loss) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = BasePPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/algorithms/custom_policy_loss/run_custom_policy_loss.sh b/skyrl-train/examples/algorithms/custom_policy_loss/run_custom_policy_loss.sh new file mode 100644 index 0000000000..909bc62f28 --- /dev/null +++ b/skyrl-train/examples/algorithms/custom_policy_loss/run_custom_policy_loss.sh @@ -0,0 +1,55 @@ +set -x + +# Example of custom policy loss: "reinforce" +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/custom_policy_loss/run_custom_policy_loss.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# Configure the policy loss to use +POLICY_LOSS="reinforce" + +uv run --isolated --extra vllm -m examples.algorithm.custom_policy_loss.main_custom_policy_loss \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.policy_loss_type="$POLICY_LOSS" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="custom_policy_loss_gsm8k" \ + trainer.run_name="custom_policy_loss_gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/dapo/README.md b/skyrl-train/examples/algorithms/dapo/README.md new file mode 100644 index 0000000000..333acce4f4 --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/README.md @@ -0,0 +1,100 @@ +# DAPO + +## Reproduction Runs +| Recipe | Model | AIME24 Pass@32 | AIME24 Avg Score | Hardware | Training Steps | WandB | +|---|---|---|---|---|---|---| +| DAPO (w/o Dynamic Sampling) | Qwen/Qwen-2.5-7B-Math | 0.633 | -0.304 | 8xH100 | 320 | [Link](https://api.wandb.ai/links/sky-posttraining-uc-berkeley/ijmo1v6q) | +| DAPO (w/o Dynamic Sampling) | Qwen/Qwen3-1.7B | 0.4 | -0.702 | 8xH100 | 225 | [Link](https://api.wandb.ai/links/sky-posttraining-uc-berkeley/ijmo1v6q) | +| DAPO (w/o Dynamic Sampling) | Qwen/Qwen3-4B | 0.6 | -0.51 | 8xH100 | 90 | [Link](https://api.wandb.ai/links/sky-posttraining-uc-berkeley/ijmo1v6q) | + +## DAPO Settings +The DAPO (Decoupled Clip and Dynamic Sampling Policy Optimization) algorithm consists of the following components on top of a GRPO baseline: + +1. Clip-Higher: Promotes the diversity of the system and avoids entropy collapse; + +2. Dynamic Sampling: Improves training efficiency and stability; + +3. Token-Level Policy Gradient Loss: Critical in long-CoT RL scenarios; + +4. Overlong Reward Shaping: Reduces reward noise and stabilizes training. + +In this README, we walk through how to enable each of these components in SkyRL. + +### Clip-Higher +To use clip-higher, you can simply configure `trainer.algorithm.eps_clip_high` separately from `trainer.algorithm.eps_clip_low`. + +```bash +trainer: + algorithm: + eps_clip_low: 0.2 + eps_clip_high: 0.28 +``` + +### Dynamic Sampling +In DAPO style dynamic sampling, we sample rollouts until we have a full batch with non-zero advantages (meaning that we have a non-zero std deviation of rewards for the n rollouts for a given prompt). + +To configure DAPO style dynamic sampling, you can set `trainer.algorithm.dynamic_sampling.type` to `filter` and configure `trainer.algorithm.dynamic_sampling.max_sample_batches` to the maximum number of batches to sample. If `max_sample_batches > 0` and is exceeded, SkyRL-Train will raise an error. If `max_sample_batches <= 0`, SkyRL-Train will sample until a full batch with non-zero advantages is accumulated. + +```bash +trainer: + algorithm: + dynamic_sampling: + type: filter + max_sample_batches: 30 +``` + +### Token-Level Policy Gradient Loss +DAPO uses token-level policy gradient loss, which can be enabled by setting `trainer.algorithm.loss_reduction` to `token_mean`. This is the default setting in SkyRL-Train. + +```bash +trainer: + algorithm: + loss_reduction: "token_mean" +``` + +### Overlong Reward Shaping +The DAPO paper proposes two methods for overlong reward shaping: + +- **Overlong Filtering**: Sets loss mask to be all zeros for responses that exceed the max response length. + +- **Soft Overlong Punishment**: Penalizes responses that exceed the max response length within a punishment interval. Within this interval, the longer the response, the greater the punishment it receives. This penalty is added to the original reward. + +#### Overlong Filtering +To enable overlong filtering, which sets the loss mask to be all zeros for responses that do not finish with a stop token (i.e. responses that are too long), you can set `generator.apply_overlong_filtering` to `true`. + +```bash +generator: + apply_overlong_filtering: true +``` + +#### Soft Overlong Punishment +To enable soft overlong punishment, you can create a custom trainer class and override the `RayPPOTrainer` `postprocess_generator_output` method to additionally apply soft overlong punishment to rewards. We provide an example of this in `main_dapo.py`, and show an overview of the implementation below: + + +```python +class DAPOTrainer(RayPPOTrainer): + @torch.no_grad() + def postprocess_generator_output(self, generator_output: GeneratorOutput, uids: List[str]) -> GeneratorOutput: + # apply soft overlong punishment + overlong_buffer_len = self.cfg.trainer.algorithm.overlong_buffer.len + overlong_buffer_penalty_factor = self.cfg.trainer.algorithm.overlong_buffer.penalty_factor + ... + # use base class impl for metrics and per-token reward conversion + return super().postprocess_generator_output(generator_output, uids) + +class DAPOExp(BasePPOExp): + def get_trainer(self, *args, **kwargs): + return DAPOTrainer(*args, **kwargs) + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = DAPOExp(cfg) + exp.run() +``` + +To add the overlong buffer length and penalty factor parameters to the config, you can add the following lines to the run_dapo_gsm8k.sh script: + +```bash ++trainer.algorithm.overlong_buffer.len=512 \ ++trainer.algorithm.overlong_buffer.penalty_factor=1.0 \ +``` diff --git a/skyrl-train/examples/algorithms/dapo/data_preprocess_dapo_aime.py b/skyrl-train/examples/algorithms/dapo/data_preprocess_dapo_aime.py new file mode 100644 index 0000000000..9965c4057e --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/data_preprocess_dapo_aime.py @@ -0,0 +1,30 @@ +import pandas as pd +import polars as pl +from pathlib import Path + +# Define input and output files +DATA_DIR = Path.home() / "data/dapo" +FILES = { + "dapo-math-17k": "dapo-math-17k.parquet", + "aime-2024": "aime-2024.parquet", +} + +for name, filename in FILES.items(): + in_path = f"{DATA_DIR}/{filename}" + out_path = f"{DATA_DIR}/{name}-cleaned.parquet" + + # Read using pandas + df = pd.read_parquet(in_path) + + # Convert to Polars for fast deduplication and group operations + pl_df = pl.from_pandas(df).unique(subset=["data_source", "prompt", "ability", "reward_model"]) + + # Count number of reward_models per prompt + pl_df = pl_df.with_columns(pl.col("reward_model").n_unique().over("prompt").alias("n_rm")) + + # Keep only prompts with one reward_model + cleaned = pl_df.filter(pl.col("n_rm") == 1).drop("n_rm") + + # Convert back to pandas and save + cleaned.to_pandas().to_parquet(out_path) + print(f"Cleaned file saved to: {out_path}") diff --git a/skyrl-train/examples/algorithms/dapo/main_dapo.py b/skyrl-train/examples/algorithms/dapo/main_dapo.py new file mode 100644 index 0000000000..546e0db6c4 --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/main_dapo.py @@ -0,0 +1,93 @@ +""" +uv run --isolated --extra vllm -m examples.algorithms.dapo.main_dapo +""" + +import ray +import hydra +import torch +from typing import List +from omegaconf import DictConfig +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg + +from skyrl_train.generators.base import GeneratorOutput + + +class DAPOTrainer(RayPPOTrainer): + """ + Custom trainer for DAPO. + + Overrides the postprocess_generator_output method to additionally apply soft overlong punishment to rewards. + """ + + @torch.no_grad() + def postprocess_generator_output(self, generator_output: GeneratorOutput, uids: List[str]) -> GeneratorOutput: + """ + Overrides the postprocess_generator_output method to additionally apply DAPO specific soft overlong punishment to rewards. + + Args: + generator_output: GeneratorOutput + uids: List[str] + + Returns: + GeneratorOutput + """ + overlong_buffer_len = self.cfg.trainer.algorithm.overlong_buffer.len + overlong_buffer_penalty_factor = self.cfg.trainer.algorithm.overlong_buffer.penalty_factor + # modify rewards here + response_ids = generator_output["response_ids"] + rewards = generator_output["rewards"] + + assert not isinstance(rewards[0], list), "we assume verifiable sequence level rewards here" + + # get the response length + response_lengths = [len(response) for response in response_ids] + + # get the max context length + # NOTE: this is only valid for single turn generation + max_response_length = self.cfg.generator.sampling_params.max_generate_length + + # apply soft overlong punishment + for i, response_length in enumerate(response_lengths): + # max_exceed_length is the beginning of the overlong buffer + max_exceed_length = max_response_length - overlong_buffer_len + # if the response is within the overlong buffer, apply the penalty + if response_length > max_exceed_length and response_length <= max_response_length: + exceed_length = response_length - max_exceed_length + penalty = exceed_length / overlong_buffer_len * overlong_buffer_penalty_factor + + rewards[i] -= penalty + # if the response is outside the overlong buffer, set the reward to 0 + elif response_length > max_response_length: + # if self.cfg.generator.apply_overlong_filtering is true, loss masks are already set to 0 for these responses + rewards[i] = 0.0 + + generator_output["rewards"] = rewards + + # use base class impl for metrics and per-token reward conversion + return super().postprocess_generator_output(generator_output, uids) + + +class DAPOExp(BasePPOExp): + def get_trainer(self, *args, **kwargs): + return DAPOTrainer(*args, **kwargs) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = DAPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/algorithms/dapo/prepare_dapo_data.sh b/skyrl-train/examples/algorithms/dapo/prepare_dapo_data.sh new file mode 100644 index 0000000000..2826f19673 --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/prepare_dapo_data.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Similar to https://github.com/volcengine/verl/blob/recipe/dapo/recipe/dapo/prepare_dapo_data.sh +# Run from `skyrl-train` directory. +# Example: DATA_DIR=/path/to/data/dapo bash examples/algorithms/dapo/prepare_dapo_data.sh +set -uxo pipefail + +export DATA_DIR=${DATA_DIR:-"${HOME}/data/dapo"} +export TRAIN_FILE=${TRAIN_FILE:-"${DATA_DIR}/dapo-math-17k.parquet"} +export TEST_FILE=${TEST_FILE:-"${DATA_DIR}/aime-2024.parquet"} + +mkdir -p "${DATA_DIR}" + +wget -O "${TRAIN_FILE}" "https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k/resolve/main/data/dapo-math-17k.parquet?download=true" + +wget -O "${TEST_FILE}" "https://huggingface.co/datasets/BytedTsinghua-SIA/AIME-2024/resolve/main/data/aime-2024.parquet?download=true" + +# remove duplicate rows in the original dataset +# https://huggingface.co/datasets/BytedTsinghua-SIA/DAPO-Math-17k/discussions/3 +uv run --isolated --extra vllm -m examples.algorithms.dapo.data_preprocess_dapo_aime --data-dir ${DATA_DIR} \ No newline at end of file diff --git a/skyrl-train/examples/algorithms/dapo/run_dapo_aime_qwen3_4b_aime.sh b/skyrl-train/examples/algorithms/dapo/run_dapo_aime_qwen3_4b_aime.sh new file mode 100644 index 0000000000..b9106cb855 --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/run_dapo_aime_qwen3_4b_aime.sh @@ -0,0 +1,103 @@ +set -x + +# Colocated DAPO training+generation for Qwen3-4B-Base on DAPO training data and validate on AIME 2024. +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/algorithms/dapo/run_dapo_aime_qwen3_4b_aime.sh + +MODEL_NAME="Qwen/Qwen3-4B-Base" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=2 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=16 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=1 +LOGGER="wandb" # change to "console" to print to stdout + +CLIP_RATIO_LOW=0.2 +CLIP_RATIO_HIGH=0.28 +# use dr. grpo loss reduction +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +LR=1e-6 + +uv run --isolated --extra vllm -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.policy.fsdp_config.fsdp_size=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.epochs=20 \ + trainer.algorithm.eps_clip_low=$CLIP_RATIO_LOW \ + trainer.algorithm.eps_clip_high=$CLIP_RATIO_HIGH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen3_4b_base" \ + trainer.export_path="$HOME/exports/dapo_qwen3_4b_base" \ + trainer.hf_save_interval=10 \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen3_4b_base" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/algorithms/dapo/run_dapo_gsm8k.sh b/skyrl-train/examples/algorithms/dapo/run_dapo_gsm8k.sh new file mode 100644 index 0000000000..ae00563022 --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/run_dapo_gsm8k.sh @@ -0,0 +1,86 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/dapo/run_dapo_gsm8k.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +DYNAMIC_SAMPLING_TYPE=filter +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=512 +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_RESPONSE_LENGTH=1024 + +uv run --isolated --extra vllm -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_dapo" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/dapo/run_dapo_qwen2.5_32b_aime.sh b/skyrl-train/examples/algorithms/dapo/run_dapo_qwen2.5_32b_aime.sh new file mode 100644 index 0000000000..bce4238616 --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/run_dapo_qwen2.5_32b_aime.sh @@ -0,0 +1,109 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-32B-Instruct on DAPO training data and validate on AIME 2024. +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/algorithms/dapo/run_dapo_qwen2.5_32b_aime.sh + +MODEL_NAME="Qwen/Qwen2.5-32B" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=2 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=4 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=4 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +# dynamic sampling parameters - off by default, since this greatly slows down inference +DYNAMIC_SAMPLING_TYPE=null +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 20)) + +# repro run parameters +SEQUENCE_PARALLEL_SIZE=4 +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +MICRO_FORWARD_BATCH_SIZE_PER_GPU=2 +MICRO_TRAIN_BATCH_SIZE_PER_GPU=2 + + +uv run --isolated --extra vllm -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.policy.sequence_parallel_size=$SEQUENCE_PARALLEL_SIZE \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=10 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=$MICRO_FORWARD_BATCH_SIZE_PER_GPU \ + trainer.micro_train_batch_size_per_gpu=$MICRO_TRAIN_BATCH_SIZE_PER_GPU \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen_2.5_32b" \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen_2.5_32b" \ + $@ diff --git a/skyrl-train/examples/algorithms/dapo/run_dapo_qwen2.5_math_7b_aime.sh b/skyrl-train/examples/algorithms/dapo/run_dapo_qwen2.5_math_7b_aime.sh new file mode 100644 index 0000000000..800647a11a --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/run_dapo_qwen2.5_math_7b_aime.sh @@ -0,0 +1,104 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-1.5B-Instruct on DAPO training data and validate on AIME 2024. +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/algorithms/dapo/run_dapo_qwen2.5_math_7b_aime.sh + +# download the model from huggingface and modify the max_position_embeddings in config.json to 32768 +# hf download Qwen/Qwen2.5-Math-7B --local-dir $HOME/qwen2.5-math +MODEL_NAME="$HOME/qwen2.5-math" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=2 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=4 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +# dynamic sampling parameters - off by default, since this greatly slows down inference +DYNAMIC_SAMPLING_TYPE=null +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability + +uv run --isolated --extra vllm -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=10 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=16 \ + trainer.micro_train_batch_size_per_gpu=16 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen_2.5_math_7b" \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen_2.5_math_7b" \ + $@ diff --git a/skyrl-train/examples/algorithms/dapo/run_dapo_qwen3_1.7b_aime.sh b/skyrl-train/examples/algorithms/dapo/run_dapo_qwen3_1.7b_aime.sh new file mode 100644 index 0000000000..8d7a4290f8 --- /dev/null +++ b/skyrl-train/examples/algorithms/dapo/run_dapo_qwen3_1.7b_aime.sh @@ -0,0 +1,103 @@ +set -x + +# Colocated DAPO training+generation for Qwen3-1.7B-Base on DAPO training data and validate on AIME 2024. +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/algorithms/dapo/run_dapo_qwen3_1.7b_aime.sh + +MODEL_NAME="Qwen/Qwen3-1.7B-Base" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=1 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=8 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=1 +LOGGER="wandb" # change to "console" to print to stdout + +CLIP_RATIO_LOW=0.2 +CLIP_RATIO_HIGH=0.28 +# use dr. grpo loss reduction +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +LR=1e-6 + +uv run --isolated --extra vllm -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.policy.fsdp_config.fsdp_size=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.epochs=20 \ + trainer.algorithm.eps_clip_low=$CLIP_RATIO_LOW \ + trainer.algorithm.eps_clip_high=$CLIP_RATIO_HIGH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen3_1.7b_base" \ + trainer.export_path="$HOME/exports/dapo_qwen3_1.7b_base" \ + trainer.hf_save_interval=10 \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen3_1.7b_base" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/algorithms/drgrpo/run_drgrpo_gsm8k.sh b/skyrl-train/examples/algorithms/drgrpo/run_drgrpo_gsm8k.sh new file mode 100644 index 0000000000..ad4247080b --- /dev/null +++ b/skyrl-train/examples/algorithms/drgrpo/run_drgrpo_gsm8k.sh @@ -0,0 +1,59 @@ +set -x + +# Colocated Dr. GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/drgrpo/run_drgrpo_gsm8k.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# Dr. GRPO parameters + +LOSS_REDUCTION="seq_mean_token_sum_norm" +GRPO_NORM_BY_STD=false +USE_KL_LOSS=false + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + trainer.algorithm.grpo_norm_by_std=$GRPO_NORM_BY_STD \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_drgrpo" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/gspo/run_gspo_gsm8k.sh b/skyrl-train/examples/algorithms/gspo/run_gspo_gsm8k.sh new file mode 100644 index 0000000000..c091cff637 --- /dev/null +++ b/skyrl-train/examples/algorithms/gspo/run_gspo_gsm8k.sh @@ -0,0 +1,11 @@ +# Colocated GSPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# bash examples/algorithms/gspo/run_gspo_gsm8k.sh + +bash examples/gsm8k/run_gsm8k.sh \ + trainer.algorithm.policy_loss_type="gspo" \ + trainer.algorithm.loss_reduction="sequence_mean" \ + trainer.micro_forward_batch_size_per_gpu=16 \ + trainer.micro_train_batch_size_per_gpu=16 \ + trainer.run_name="gspo_gsm8k" diff --git a/skyrl-train/examples/algorithms/reinforce++/run_reinforce++.sh b/skyrl-train/examples/algorithms/reinforce++/run_reinforce++.sh new file mode 100644 index 0000000000..895366a3f5 --- /dev/null +++ b/skyrl-train/examples/algorithms/reinforce++/run_reinforce++.sh @@ -0,0 +1,59 @@ +set -x + +# Colocated REINFORCE++ training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/reinforce++/run_reinforce++.sh + + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# REINFORCE++ with +ADV_ESTIMATOR="reinforce++" +USE_KL_IN_REWARD=true +KL_ESTIMATOR_TYPE="k2" + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="$ADV_ESTIMATOR" \ + trainer.algorithm.use_kl_loss=false \ + trainer.algorithm.use_kl_in_reward=$USE_KL_IN_REWARD \ + trainer.algorithm.kl_estimator_type=$KL_ESTIMATOR_TYPE \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_reinforce++" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/rloo/run_rloo.sh b/skyrl-train/examples/algorithms/rloo/run_rloo.sh new file mode 100644 index 0000000000..3a85f60e8e --- /dev/null +++ b/skyrl-train/examples/algorithms/rloo/run_rloo.sh @@ -0,0 +1,56 @@ +set -x + +# Colocated RLOO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/rloo/run_rloo.sh + + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +ADV_ESTIMATOR="rloo" +USE_KL_IN_REWARD=true + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="$ADV_ESTIMATOR" \ + trainer.algorithm.use_kl_loss=false \ + trainer.algorithm.use_kl_in_reward=$USE_KL_IN_REWARD \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_rloo" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/algorithms/sapo/README.md b/skyrl-train/examples/algorithms/sapo/README.md new file mode 100644 index 0000000000..3705504df7 --- /dev/null +++ b/skyrl-train/examples/algorithms/sapo/README.md @@ -0,0 +1,15 @@ +# SAPO Trainer + +# SAPO Trainer + +`run_sapo_qwen3_4b_aime.sh` is a bash script that launches the Qwen3-4B SAPO job using `uv`. It uses the `examples/algorithms/dapo/main_dapo.py` script, but with the SAPO policy loss and sequence mean loss reduction. + +## Quick Start + +1. Set/Export your `WANDB_API_KEY`. +2. Ensure you have `uv` installed and the environment is set up in `skyrl-train`. +3. Submit the job: + + ```bash + bash examples/algorithms/sapo/run_sapo_qwen3_4b_aime.sh + ``` \ No newline at end of file diff --git a/skyrl-train/examples/algorithms/sapo/run_sapo_gsm8k.sh b/skyrl-train/examples/algorithms/sapo/run_sapo_gsm8k.sh new file mode 100644 index 0000000000..c98210136b --- /dev/null +++ b/skyrl-train/examples/algorithms/sapo/run_sapo_gsm8k.sh @@ -0,0 +1,11 @@ +# Colocated SAPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# bash examples/algorithms/sapo/run_sapo_gsm8k.sh + +bash examples/gsm8k/run_gsm8k.sh \ + trainer.algorithm.policy_loss_type="sapo" \ + trainer.algorithm.sapo.tau_pos=1.0 \ + trainer.algorithm.sapo.tau_neg=1.05 \ + trainer.algorithm.loss_reduction="sequence_mean" \ + trainer.run_name="sapo_gsm8k" \ No newline at end of file diff --git a/skyrl-train/examples/algorithms/sapo/run_sapo_qwen3_4b_aime.sh b/skyrl-train/examples/algorithms/sapo/run_sapo_qwen3_4b_aime.sh new file mode 100644 index 0000000000..2eae188aec --- /dev/null +++ b/skyrl-train/examples/algorithms/sapo/run_sapo_qwen3_4b_aime.sh @@ -0,0 +1,104 @@ +set -x + +# Colocated SAPO training+generation for Qwen3-4B-Base on DAPO training data and validate on AIME 2024. +# Uses examples/algorithms/dapo/main_dapo.py script, but with the SAPO policy loss and sequence mean loss reduction. +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/algorithms/sapo/run_sapo_qwen3_4b_aime.sh + +MODEL_NAME="Qwen/Qwen3-4B-Base" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=1 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=8 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=1 +LOGGER="wandb" # change to "console" to print to stdout + +# SAPO specific parameters +TAU_POS=1.0 +TAU_NEG=1.05 +LOSS_REDUCTION="sequence_mean" + +# parameters carried over from DAPO +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +LR=1e-6 + +uv run --isolated --extra vllm -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="sapo" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.policy.fsdp_config.fsdp_size=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.epochs=20 \ + trainer.algorithm.sapo.tau_pos=$TAU_POS \ + trainer.algorithm.sapo.tau_neg=$TAU_NEG \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=10 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=-1 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=80 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="sapo_aime" \ + trainer.run_name="sapo_qwen3_4b_base" \ + trainer.export_path="$HOME/exports/sapo_qwen3_4b_base" \ + trainer.hf_save_interval=-1 \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/sapo_qwen3_4b_base" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/async/README.md b/skyrl-train/examples/async/README.md new file mode 100644 index 0000000000..da1d329601 --- /dev/null +++ b/skyrl-train/examples/async/README.md @@ -0,0 +1,16 @@ +# Async Training Example + +One-step off-policy GRPO for Qwen2.5-1.5B-Instruct on GSM8K. + +## Usage + +```bash +# prepare the dataset +uv run -- python examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + +export WANDB_API_KEY= + +bash examples/async/async_run_gsm8k.sh +``` + +For more details, refer to the [documentation](https://skyrl.readthedocs.io/en/latest/tutorials/async.html) diff --git a/skyrl-train/examples/async/__init__.py b/skyrl-train/examples/async/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/examples/async/async_run_gsm8k.sh b/skyrl-train/examples/async/async_run_gsm8k.sh new file mode 100644 index 0000000000..c1d612332e --- /dev/null +++ b/skyrl-train/examples/async/async_run_gsm8k.sh @@ -0,0 +1,50 @@ +set -x + +# One-step off-policy GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/async/async_run_gsm8k.sh + +DATA_DIR="$HOME/data/gsm8k" + +uv run --isolated --extra vllm -m examples.async.main_async \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=false \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.critic_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k-async" \ + trainer.run_name="gsm8k_test_async" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/async_gsm8k_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/async/async_trainer.py b/skyrl-train/examples/async/async_trainer.py new file mode 100644 index 0000000000..43268b5f26 --- /dev/null +++ b/skyrl-train/examples/async/async_trainer.py @@ -0,0 +1,201 @@ +import asyncio +import traceback +import sys +from loguru import logger +from skyrl_train.trainer import RayPPOTrainer +from tqdm import tqdm +from skyrl_train.utils import Timer +from skyrl_train.utils.ppo_utils import normalize_advantages_dict +from skyrl_train.training_batch import TrainingInputBatch +from skyrl_train.generators.base import GeneratorOutput +from skyrl_train.utils.trainer_utils import ResumeMode +from skyrl_train.generators.utils import prepare_generator_input +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend + + +class AsyncRayPPOTrainer(RayPPOTrainer): + + async def train(self): + """ + Main training loop for PPO + """ + assert not self.colocate_all, "colocate_all is not supported for async training" + + self.global_step = 0 + + # Load checkpoint state if resumption is enabled + if self.resume_mode != ResumeMode.NONE: + with Timer("load_checkpoints"): + self.global_step, _ = self.load_checkpoints() + logger.info(f"Resumed training from global_step {self.global_step}") + + # Initialize weight sync state + with Timer("init_weight_sync_state"): + self.init_weight_sync_state() + + # sync weights to inference engines + with Timer("sync_weights_to_inference_engines"): + await self.async_sync_policy_weights_to_inference_engines() + + # Eval before training + if self.cfg.trainer.eval_interval > 0 and self.cfg.trainer.eval_before_train: + with Timer("eval", self.all_timings): + eval_metrics = await self.eval() + self.tracker.log(eval_metrics, step=self.global_step) + + # main training loop + pbar = tqdm(total=self.total_training_steps, initial=self.global_step, desc="Training Step Progress") + start_epoch = self.global_step // len(self.train_dataloader) + # Start from step 1 + self.global_step += 1 + for epoch in range(start_epoch, self.cfg.trainer.epochs): + # while this is just off by one, you can image a more general queue based approach + # where the generation buffer holds a list of objects that the trainer can read from + # bit by bit. + generation_buffer = asyncio.Queue(maxsize=1) + self.sync_finished = asyncio.Event() + self.generation_ack = asyncio.Event() + + # start generator task + generator_task = asyncio.create_task(self._run_generate_loop(generation_buffer)) + + for idx in range(len(self.train_dataloader)): + with Timer("step", self.all_timings): + status = await self._run_training(generation_buffer) + + # request the generation loop that we should sync sometime soon. + if idx != len(self.train_dataloader) - 1: + await self.generation_ack.wait() + + # sync weights + async with Timer("sync_weights", self.all_timings): + await self.async_sync_policy_weights_to_inference_engines() + + self.sync_finished.set() + self.generation_ack.clear() + + # 5. set logs + logger.info(status) + # log epoch info + self.all_metrics.update({"trainer/epoch": epoch, "trainer/global_step": self.global_step}) + self.tracker.log(self.all_metrics, step=self.global_step) + self.all_metrics = {} + pbar.update(1) + + if self.cfg.trainer.eval_interval > 0 and ( + self.global_step % self.cfg.trainer.eval_interval == 0 + or self.global_step == self.total_training_steps + ): + with Timer("eval", self.all_timings): + eval_metrics = await self.eval() + self.all_metrics.update(eval_metrics) + if self.cfg.trainer.ckpt_interval > 0 and self.global_step % self.cfg.trainer.ckpt_interval == 0: + with Timer("save_checkpoints", self.all_timings): + self.save_checkpoints() + if self.cfg.trainer.hf_save_interval > 0 and self.global_step % self.cfg.trainer.hf_save_interval == 0: + with Timer("save_hf_model", self.all_timings): + self.save_models() + self.tracker.log({"timing/" + k: v for k, v in self.all_timings.items()}, step=self.global_step) + self.all_timings = {} + self.global_step += 1 + + if self.cfg.trainer.update_ref_every_epoch and self.ref_model is not None: + with Timer("update_ref_with_policy", self.all_timings): + await asyncio.to_thread(self.update_ref_with_policy) + + # cancel generation task for this epoch + generator_task.cancel() + + pbar.close() + if self.cfg.trainer.ckpt_interval > 0: + with Timer("save_checkpoints", self.all_timings): + self.save_checkpoints() + logger.info("Saved final checkpoint.") + if self.cfg.trainer.hf_save_interval > 0: + with Timer("save_hf_model", self.all_timings): + self.save_models() + logger.info("Saved final model.") + logger.info("Training done!") + + async def _run_training(self, generation_buffer): + # Get a generation future and await on the object + generator_output, uids = await generation_buffer.get() # GeneratorOutput, List[str] + + # print example just for debugging + vis = self.tokenizer.decode(generator_output["response_ids"][0]) + logger.info(f"Example generated: {vis}") + + with Timer("convert_to_training_input", self.all_timings): + training_input: TrainingInputBatch = self.convert_to_training_input(generator_output, uids) + + # inference and calculate values, log probs, rewards, kl divergence + with Timer("fwd_logprobs_values_reward", self.all_timings): + training_input = await asyncio.to_thread(self.fwd_logprobs_values_reward, training_input) + + # calculate kl divergence and create experiences + if self.cfg.trainer.algorithm.use_kl_in_reward: + with Timer("apply_reward_kl_penalty", self.all_timings): + training_input = self.apply_reward_kl_penalty(training_input) + + # calculate advantages and returns / along with tensorboard logging + with Timer("compute_advantages_and_returns", self.all_timings): + training_input = self.compute_advantages_and_returns(training_input) + # remove some unwanted keys + for key in ["rewards"]: + training_input.pop(key) + training_input.metadata.pop("uids") + + if self.cfg.trainer.algorithm.advantage_batch_normalize: + training_input = normalize_advantages_dict(training_input) + + if self.cfg.trainer.dump_data_batch: + # dump data to file + with Timer("dump_data_batch"): + self.dump_data(training_input, file_name=f"global_step_{self.global_step}_training_input") + + # train policy/critic model + with Timer("train_critic_and_policy", self.all_timings): + status = await asyncio.to_thread(self.train_critic_and_policy, training_input) + + return status + + async def _run_generate_loop(self, generation_buffer: asyncio.Queue): + try: + for i, rand_prompts in enumerate(self.train_dataloader): + # truncate data to have even shards + rand_prompts = self._remove_tail_data(rand_prompts) + generator_input, uids = prepare_generator_input( + rand_prompts, + self.cfg.generator.n_samples_per_prompt, + get_sampling_params_for_backend(self.cfg.generator.backend, self.cfg.generator.sampling_params), + self.cfg.environment.env_class, + "train", + self.global_step, + ) + + # generation phase + async with Timer("generate", self.all_timings): + generator_output: GeneratorOutput = await self.generate(generator_input) + generator_output = self.postprocess_generator_output(generator_output, uids) + + # Add to generation buffer + await generation_buffer.put((generator_output, uids)) + + # If the buffer is full, start weight sync + # Don't weight sync in the first step, because we let the generator run one step ahead + if generation_buffer.full() and i != 0: + # Signal that generation is done, ready for weight sync + self.generation_ack.set() + await self.sync_finished.wait() + # Clear the sync request for next sync + self.sync_finished.clear() + # We have an explicit try-catch here because asyncio doesn't propagate exceptions to the main thread. + except Exception as e: + logger.error(f"Generator errored out with exception: {e}") + logger.error(f"Traceback: \n{traceback.format_exc()}") + sys.exit(1) + + async def async_sync_policy_weights_to_inference_engines(self): + return await self.policy_model.async_run_method( + "pass_through", "broadcast_to_inference_engines", self.inference_engine_client + ) diff --git a/skyrl-train/examples/async/main_async.py b/skyrl-train/examples/async/main_async.py new file mode 100644 index 0000000000..4a25dbe77a --- /dev/null +++ b/skyrl-train/examples/async/main_async.py @@ -0,0 +1,60 @@ +""" +Main entrypoint for async training. +""" + +import hydra +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from .async_trainer import AsyncRayPPOTrainer +import asyncio +from skyrl_train.utils import initialize_ray +import ray + + +class AsyncPPOExp(BasePPOExp): + def get_trainer( + self, + cfg, + tracker, + tokenizer, + train_dataset, + eval_dataset, + inference_engine_client, + generator, + colocate_pg, + ): + return AsyncRayPPOTrainer( + cfg=cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=colocate_pg, + ) + + def run(self): + trainer = self._setup_trainer() + # Start the async training loop + asyncio.run(trainer.train()) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = AsyncPPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/flash_rl/.env.0.5b_int8 b/skyrl-train/examples/flash_rl/.env.0.5b_int8 new file mode 100644 index 0000000000..aed48cd34d --- /dev/null +++ b/skyrl-train/examples/flash_rl/.env.0.5b_int8 @@ -0,0 +1,4 @@ +# can be a local config path or one on huggingface +# int8 requires a profile.pt calibration file for weight scales +FLASHRL_CONFIG=LiyuanLucasLiu/Qwen2.5-0.5B-Instruct-quantized.w8a8-RedHatAI/flashrl_config.yaml +# FLASHRL_LOGGING_LEVEL=DEBUG # optional \ No newline at end of file diff --git a/skyrl-train/examples/flash_rl/.env.fp8 b/skyrl-train/examples/flash_rl/.env.fp8 new file mode 100644 index 0000000000..5d67e66259 --- /dev/null +++ b/skyrl-train/examples/flash_rl/.env.fp8 @@ -0,0 +1,2 @@ +FLASHRL_CONFIG="fp8_vllm" +# FLASHRL_LOGGING_LEVEL=DEBUG # optional \ No newline at end of file diff --git a/skyrl-train/examples/flash_rl/.env.int8 b/skyrl-train/examples/flash_rl/.env.int8 new file mode 100644 index 0000000000..bdd6cc271d --- /dev/null +++ b/skyrl-train/examples/flash_rl/.env.int8 @@ -0,0 +1,4 @@ +# can be a local config path or one on huggingface +# int8 requires a profile.pt calibration file for weight scales +FLASHRL_CONFIG=LiyuanLucasLiu/Qwen2.5-32B-quantized.w8a8/flashrl_config.yaml +# FLASHRL_LOGGING_LEVEL=DEBUG # optional \ No newline at end of file diff --git a/skyrl-train/examples/flash_rl/flash_rl_engine.py b/skyrl-train/examples/flash_rl/flash_rl_engine.py new file mode 100644 index 0000000000..8684316084 --- /dev/null +++ b/skyrl-train/examples/flash_rl/flash_rl_engine.py @@ -0,0 +1,123 @@ +from typing import List +import ray +import vllm +from skyrl_train.inference_engines.vllm.vllm_engine import VLLMInferenceEngine +from skyrl_train.inference_engines.ray_wrapped_inference_engine import RayWrappedInferenceEngine +from ray.util.placement_group import PlacementGroupSchedulingStrategy, placement_group + +from skyrl_train.inference_engines.base import ( + InferenceEngineInterface, +) + + +class FlashRLVLLMInferenceEngine(VLLMInferenceEngine): + + def _create_engine(self, *args, **kwargs): + # apply flashrl's patch just before init + from vllm.model_executor.layers.patch import apply_patch as apply_flashrl_patch + + apply_flashrl_patch() + + llm = vllm.LLM(*args, **kwargs) + return llm + + +VLLMRayActor = ray.remote(FlashRLVLLMInferenceEngine) + + +def create_ray_wrapped_inference_engines_flashrl( + num_inference_engines: int, + tensor_parallel_size: int, + model_dtype: str, + pretrain: str, + seed: int, + vllm_v1_disable_multiproc: bool, + enable_prefix_caching: bool, + enforce_eager: bool, + shared_pg=None, + gpu_memory_utilization=None, + inference_engine_enable_sleep=False, + async_engine=False, + max_num_batched_tokens=8192, + max_num_seqs=1024, + tokenizer=None, + backend="vllm", +) -> List[InferenceEngineInterface]: + """ + Create a list of RayWrappedInferenceEngine instances wrapping Ray actor handles to InferenceEngineInterface instances. + """ + from skyrl_train.utils import ray_noset_visible_devices, get_all_env_variables, get_ray_pg_ready_with_timeout + from skyrl_train.env_vars import SKYRL_RAY_PG_TIMEOUT_IN_S + + assert not async_engine, "`async_engine` is not supported for FlashRL" + + if backend != "vllm": + raise ValueError(f"Unsupported FlashRL backend: {backend}") + + inference_engine_actors = [] + noset_visible_devices = ray_noset_visible_devices(ray.get(get_all_env_variables.remote())) + # NOTE: we use the ray backend for tensor parallel size > 1 to explicitly manage resource allocation + # TODO: we should be able to support mp backend by allocating resources at engine level + distributed_executor_backend = "uni" if tensor_parallel_size == 1 else "ray" + use_hybrid_engine = shared_pg is not None + num_gpus = int(tensor_parallel_size == 1) + if use_hybrid_engine and tensor_parallel_size == 1: + # every worker will use 0.2 GPU, so that we can schedule + # 2 instances on the same GPUs. + num_gpus = 0.2 + + if not use_hybrid_engine: + # Create a big placement group to ensure that all inference engines are packed + bundles = [{"GPU": 1, "CPU": 1} for _ in range(num_inference_engines * tensor_parallel_size)] + shared_pg = placement_group(bundles, strategy="PACK") + get_ray_pg_ready_with_timeout(shared_pg, timeout=SKYRL_RAY_PG_TIMEOUT_IN_S) + + for i in range(num_inference_engines): + bundle_indices = None + if tensor_parallel_size > 1: + bundle_indices = list(range(i * tensor_parallel_size, (i + 1) * tensor_parallel_size)) + + scheduling_strategy = PlacementGroupSchedulingStrategy( + placement_group=shared_pg, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=i * tensor_parallel_size, + ) + + if backend == "vllm": + + engine = VLLMRayActor.options( + num_cpus=num_gpus, + num_gpus=num_gpus, + scheduling_strategy=scheduling_strategy, + ).remote( + model=pretrain, + enforce_eager=enforce_eager, + worker_extension_cls="skyrl_train.inference_engines.vllm.vllm_engine.WorkerWrap", + tensor_parallel_size=tensor_parallel_size, + seed=seed + i, + distributed_executor_backend=distributed_executor_backend, + enable_prefix_caching=enable_prefix_caching, + dtype=model_dtype, + trust_remote_code=True, + vllm_v1_disable_multiproc=vllm_v1_disable_multiproc, + gpu_memory_utilization=gpu_memory_utilization, + bundle_indices=bundle_indices, + num_gpus=0.2 if use_hybrid_engine else 1, + enable_sleep_mode=inference_engine_enable_sleep, + noset_visible_devices=noset_visible_devices, + max_num_batched_tokens=max_num_batched_tokens, + max_num_seqs=max_num_seqs, + tokenizer=tokenizer, + # only need the logprobs for the chosen token if any + max_logprobs=1, + ) + + inference_engine_actors.append(engine) + + engines = [RayWrappedInferenceEngine(actor_handle) for actor_handle in inference_engine_actors] + + if inference_engine_enable_sleep: + sleep_refs = [engine.inference_engine_actor.sleep.remote() for engine in engines] + ray.get(sleep_refs) + + return engines diff --git a/skyrl-train/examples/flash_rl/main_dapo_flashrl.py b/skyrl-train/examples/flash_rl/main_dapo_flashrl.py new file mode 100644 index 0000000000..dc92d6858f --- /dev/null +++ b/skyrl-train/examples/flash_rl/main_dapo_flashrl.py @@ -0,0 +1,176 @@ +""" +Main entrypoint for DAPO training with FlashRL. +""" + +import ray +import os +import hydra +import torch +from typing import List +from omegaconf import DictConfig +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import ( + BasePPOExp, + config_dir, + validate_cfg, + create_remote_inference_engines_from_config, +) +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.generators.base import GeneratorInterface +from loguru import logger +from skyrl_train.generators.base import GeneratorOutput + + +def create_ray_wrapped_inference_engines_from_config_flashrl(cfg: DictConfig, colocate_pg, tokenizer): + from examples.flash_rl.flash_rl_engine import create_ray_wrapped_inference_engines_flashrl + + return create_ray_wrapped_inference_engines_flashrl( + num_inference_engines=cfg.generator.num_inference_engines, + tensor_parallel_size=cfg.generator.inference_engine_tensor_parallel_size, + model_dtype=cfg.generator.model_dtype, + pretrain=cfg.trainer.policy.model.path, + seed=cfg.trainer.seed, + vllm_v1_disable_multiproc=cfg.generator.vllm_v1_disable_multiproc, + enable_prefix_caching=cfg.generator.enable_prefix_caching, + enforce_eager=cfg.generator.enforce_eager, + shared_pg=colocate_pg, + gpu_memory_utilization=cfg.generator.gpu_memory_utilization, + inference_engine_enable_sleep=cfg.trainer.placement.colocate_all, + async_engine=cfg.generator.async_engine, + max_num_batched_tokens=cfg.generator.max_num_batched_tokens, + max_num_seqs=cfg.generator.max_num_seqs, + tokenizer=tokenizer, + backend=cfg.generator.backend, + ) + + +class DAPOTrainer(RayPPOTrainer): + """ + Custom trainer for DAPO. + + Overrides the postprocess_generator_output method to additionally apply soft overlong punishment to rewards. + """ + + @torch.no_grad() + def postprocess_generator_output(self, generator_output: GeneratorOutput, uids: List[str]) -> GeneratorOutput: + """ + Overrides the postprocess_generator_output method to additionally apply DAPO specific soft overlong punishment to rewards. + + Args: + generator_output: GeneratorOutput + uids: List[str] + + Returns: + GeneratorOutput + """ + overlong_buffer_len = self.cfg.trainer.algorithm.overlong_buffer.len + overlong_buffer_penalty_factor = self.cfg.trainer.algorithm.overlong_buffer.penalty_factor + # modify rewards here + response_ids = generator_output["response_ids"] + rewards = generator_output["rewards"] + + assert not isinstance(rewards[0], list), "we assume verifiable sequence level rewards here" + + # get the response length + response_lengths = [len(response) for response in response_ids] + + # get the max context length + # NOTE: this is only valid for single turn generation + max_response_length = self.cfg.generator.sampling_params.max_generate_length + + # apply soft overlong punishment + for i, response_length in enumerate(response_lengths): + # max_exceed_length is the beginning of the overlong buffer + max_exceed_length = max_response_length - overlong_buffer_len + # if the response is within the overlong buffer, apply the penalty + if response_length > max_exceed_length and response_length <= max_response_length: + exceed_length = response_length - max_exceed_length + penalty = exceed_length / overlong_buffer_len * overlong_buffer_penalty_factor + + rewards[i] -= penalty + # if the response is outside the overlong buffer, set the reward to 0 + elif response_length > max_response_length: + # if self.cfg.generator.apply_overlong_filtering is true, loss masks are already set to 0 for these responses + rewards[i] = 0.0 + + generator_output["rewards"] = rewards + + # use base class impl for metrics and per-token reward conversion + return super().postprocess_generator_output(generator_output, uids) + + +class DAPOExp(BasePPOExp): + def get_trainer(self, *args, **kwargs): + return DAPOTrainer(*args, **kwargs) + + def _setup_trainer(self): + """Setup and return the trainer. + + Instantiates the trainer and all the associated models for training. + + Returns: + RayPPOTrainer: The trainer. + """ + logger.info(self.get_cfg_as_str(self.cfg)) + os.makedirs(self.cfg.trainer.export_path, exist_ok=True) + os.makedirs(self.cfg.trainer.ckpt_path, exist_ok=True) + + if self.cfg.trainer.strategy in ("fsdp", "fsdp2"): + from skyrl_train.workers.fsdp.fsdp_worker import PolicyWorker, CriticWorker, RefWorker + else: + raise ValueError(f"Unknown strategy type: {self.cfg.trainer.strategy}") + + # NOTE (sumanthrh): Instantiate tracker before trainer init. + # We have custom validation before this step to give better error messages. + tracker = self.get_tracker() + + tokenizer = self.tokenizer + if self.cfg.generator.run_engines_locally: + inference_engines = create_ray_wrapped_inference_engines_from_config_flashrl( + self.cfg, self.colocate_pg, tokenizer + ) + else: + inference_engines = create_remote_inference_engines_from_config(self.cfg) + + inference_engine_client = InferenceEngineClient(inference_engines, tokenizer, self.cfg) + + generator: GeneratorInterface = self.get_generator(self.cfg, tokenizer, inference_engine_client) + + trainer = self.get_trainer( + cfg=self.cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=self.train_dataset, + eval_dataset=self.eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=self.colocate_pg, + ) + + # Build the models + trainer.build_models(PolicyWorker, CriticWorker, RefWorker) + return trainer + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + + exp = DAPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + if not cfg.generator.run_engines_locally: + raise ValueError("FlashRL only supports colocated training.") + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_fp8.sh b/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_fp8.sh new file mode 100644 index 0000000000..d0aff21abe --- /dev/null +++ b/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_fp8.sh @@ -0,0 +1,93 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-0.5B-Instruct on GSM8K with FP8 rollouts. +# The configuration is tested on 4 H100 GPUs. + +# uv run --isolated examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_fp8.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +DYNAMIC_SAMPLING_TYPE=filter +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl +OVERLONG_BUFFER_LEN=512 +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_RESPONSE_LENGTH=1024 + +CKPT_PATH="$HOME/ckpts/gsm8k_0.5B_ckpt" + + +uv run --isolated --extra flashrl --env-file examples/flash_rl/.env.fp8 --with vllm@https://github.com/NovaSky-AI/SkyRL/releases/download/skyrl_train-v0.1.0/vllm-0.1.dev7509+gcc487699a.d20250821-cp312-cp312-linux_x86_64.whl --with transformers==4.53.3 -- python -m examples.flash_rl.main_dapo_flashrl \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.sampling_params.logprobs=0 \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.algorithm.use_tis=true \ + trainer.algorithm.tis_imp_ratio_cap=2.0 \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_flashrl" \ + trainer.run_name="gsm8k_dapo_flashrl_0.5B_fp8" \ + trainer.ckpt_path="$CKPT_PATH" \ + generator.enforce_eager=false \ + $@ diff --git a/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_int8.sh b/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_int8.sh new file mode 100644 index 0000000000..f547788f5f --- /dev/null +++ b/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_int8.sh @@ -0,0 +1,92 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-0.5B-Instruct on GSM8K with Int8 rollouts. +# The configuration is tested on 4 H100 GPUs. + +# uv run --isolated examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/flash_rl/run_dapo_gsm8k_flashrl_0.5b_int8.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +DYNAMIC_SAMPLING_TYPE=filter +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl +OVERLONG_BUFFER_LEN=512 +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_RESPONSE_LENGTH=1024 + +CKPT_PATH="$HOME/ckpts/gsm8k_0.5B_ckpt" + +uv run --isolated --extra flashrl --env-file examples/flash_rl/.env.int8 --with vllm@https://github.com/NovaSky-AI/SkyRL/releases/download/skyrl_train-v0.1.0/vllm-0.1.dev7509+gcc487699a.d20250821-cp312-cp312-linux_x86_64.whl --with transformers==4.53.3 -- python -m examples.flash_rl.main_dapo_flashrl \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.sampling_params.logprobs=0 \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.algorithm.use_tis=true \ + trainer.algorithm.tis_imp_ratio_cap=2.0 \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_flashrl" \ + trainer.run_name="gsm8k_dapo_flashrl_0.5B_int8" \ + trainer.ckpt_path="$CKPT_PATH" \ + generator.enforce_eager=false \ + $@ diff --git a/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_32b_int8.sh b/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_32b_int8.sh new file mode 100644 index 0000000000..7c6c7fb4aa --- /dev/null +++ b/skyrl-train/examples/flash_rl/run_dapo_gsm8k_flashrl_32b_int8.sh @@ -0,0 +1,93 @@ +set -x + +# Colocated DAPO training+generation for Qwen3-32B on GSM8K with FP8 rollouts. +# The configuration is tested on 2 8xH100 GPUs. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/flash_rl/run_dapo_gsm8k_flashrl_32b_int8.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=16 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +DYNAMIC_SAMPLING_TYPE=filter +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl +OVERLONG_BUFFER_LEN=512 +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_RESPONSE_LENGTH=1024 + +CKPT_PATH="$HOME/ckpts/gsm8k_32B_ckpt" + +uv run --isolated --extra flashrl --env-file examples/flash_rl/.env.int8 --with vllm@https://github.com/NovaSky-AI/SkyRL/releases/download/skyrl_train-v0.1.0/vllm-0.1.dev7509+gcc487699a.d20250821-cp312-cp312-linux_x86_64.whl --with transformers==4.53.3 -m examples.flash_rl.main_dapo_flashrl \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.sampling_params.logprobs=0 \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.algorithm.use_tis=true \ + trainer.algorithm.tis_imp_ratio_cap=2.0 \ + trainer.policy.model.path="Qwen/Qwen2.5-32B" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=8 \ + generator.inference_engine_tensor_parallel_size=2 \ + trainer.epochs=20 \ + trainer.eval_batch_size=32 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=32 \ + trainer.policy_mini_batch_size=32 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.weight_transfer_threshold_cuda_ipc_GB=4.0 \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_flashrl" \ + trainer.run_name="gsm8k_dapo_tis_32b_int8" \ + trainer.ckpt_path="$CKPT_PATH" \ + generator.enforce_eager=false \ + $@ diff --git a/skyrl-train/examples/flash_rl/run_dapo_repro_flashrl_0.5b_int8.sh b/skyrl-train/examples/flash_rl/run_dapo_repro_flashrl_0.5b_int8.sh new file mode 100644 index 0000000000..97ca62fdec --- /dev/null +++ b/skyrl-train/examples/flash_rl/run_dapo_repro_flashrl_0.5b_int8.sh @@ -0,0 +1,90 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-1.5B-Instruct on the original DAPO dataset with Int8 rollouts. +# The configuration is tested on 2 H100 GPUs. + +# DATA_DIR=$HOME/data/dapo bash examples/algorithms/dapo/prepare_dapo_data.sh +# export WANDB_API_KEY= +# bash examples/flash_rl/run_dapo_repro_flashrl_0.5b_int8.sh + +DATA_DIR="$HOME/data/dapo" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +DYNAMIC_SAMPLING_TYPE=filter +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=512 +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_RESPONSE_LENGTH=4096 + +uv run --isolated --extra flashrl --env-file examples/flash_rl/.env.0.5b_int8 --with vllm@https://github.com/NovaSky-AI/SkyRL/releases/download/skyrl_train-v0.1.0/vllm-0.1.dev7509+gcc487699a.d20250821-cp312-cp312-linux_x86_64.whl --with transformers==4.53.3 -- python -m examples.flash_rl.main_dapo_flashrl \ + data.train_data="['$DATA_DIR/dapo-math-17k-cleaned.parquet']" \ + data.val_data="['$DATA_DIR/aime-2024-cleaned.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.sampling_params.logprobs=0 \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.algorithm.use_tis=true \ + trainer.algorithm.tis_imp_ratio_cap=2.0 \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_flashrl" \ + trainer.run_name="dapo_flashrl_0.5B_int8" \ + trainer.ckpt_path="$CKPT_PATH" \ + generator.enforce_eager=false \ + $@ diff --git a/skyrl-train/examples/flash_rl/run_dapo_repro_flashrl_32b_int8.sh b/skyrl-train/examples/flash_rl/run_dapo_repro_flashrl_32b_int8.sh new file mode 100644 index 0000000000..bf4a033f47 --- /dev/null +++ b/skyrl-train/examples/flash_rl/run_dapo_repro_flashrl_32b_int8.sh @@ -0,0 +1,98 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-32B on the original DAPO dataset with Int8 rollouts. +# The configuration is tested on 2 8xH100 GPUs. + +# DATA_DIR=$HOME/data/dapo bash examples/algorithms/dapo/prepare_dapo_data.sh +# export WANDB_API_KEY= +# bash examples/flash_rl/run_dapo_repro_flashrl_32b_int8.sh + +DATA_DIR="$HOME/data/dapo" +NUM_GPUS=16 +LOGGER="wandb" # change to "console" to print to stdout + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +DYNAMIC_SAMPLING_TYPE=filter +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo_flashrl.py +OVERLONG_BUFFER_LEN=4096 +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_RESPONSE_LENGTH=20480 +MAX_PROMPT_LENGTH=2048 + +TIS_IMP_RATIO_CAP=8.0 +USE_TIS=true +LOGPROBS=0 + +CKPT_PATH="$HOME/ckpts/dapo_32b_ckpt" + +uv run --isolated --extra flashrl --env-file examples/flash_rl/.env.int8 --with vllm@https://github.com/NovaSky-AI/SkyRL/releases/download/skyrl_train-v0.1.0/vllm-0.1.dev7509+gcc487699a.d20250821-cp312-cp312-linux_x86_64.whl --with transformers==4.53.3 -m examples.flash_rl.main_dapo_flashrl \ + data.train_data="['$DATA_DIR/dapo-math-17k-cleaned.parquet']" \ + data.val_data="['$DATA_DIR/aime-2024-cleaned.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.sampling_params.logprobs=$LOGPROBS \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.policy.model.path="Qwen/Qwen2.5-32B" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=8 \ + generator.inference_engine_tensor_parallel_size=2 \ + trainer.epochs=20 \ + trainer.eval_batch_size=128 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=32 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.weight_transfer_threshold_cuda_ipc_GB=4.0 \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=16 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_repro" \ + trainer.run_name="dapo_repro_32b_int8" \ + trainer.ckpt_path="$CKPT_PATH" \ + generator.enforce_eager=false \ + $@ diff --git a/skyrl-train/examples/fully_async/README.md b/skyrl-train/examples/fully_async/README.md new file mode 100644 index 0000000000..a7347b3c43 --- /dev/null +++ b/skyrl-train/examples/fully_async/README.md @@ -0,0 +1,19 @@ +# Fully Async Training Example + +Fully asynchronous (PipelineRL / AReal style) GRPO for Qwen2.5-1.5B-Instruct on GSM8K. + +## Usage + +```bash +# prepare the dataset +uv run -- python examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + +export WANDB_API_KEY= + +bash examples/fully_async/async_run_gsm8k.sh +``` + +For more details, refer to the documentation: https://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html + +Especially, refer to the section on what knobs to tune: http://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html#step-2-config-knobs-to-tune-for-fully-async-training + diff --git a/skyrl-train/examples/fully_async/__init__.py b/skyrl-train/examples/fully_async/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/examples/fully_async/async_run_gsm8k.sh b/skyrl-train/examples/fully_async/async_run_gsm8k.sh new file mode 100644 index 0000000000..5a4404b3bb --- /dev/null +++ b/skyrl-train/examples/fully_async/async_run_gsm8k.sh @@ -0,0 +1,81 @@ +set -x + +# Fully async GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. +# This bash script is copied from examples/async/async_run_gsm8k.sh, except for: +# - running examples.fully_async.main_async +# - setting the generator.batched=false. +# - colocate_all=false +# - the various generator configs at the end (http, chat template, etc.) + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/gsm8k/run_gsm8k.sh + +# NOTE (sumanthrh): `micro_train_batch_size_per_gpu` and `micro_forward_batch_size_per_gpu` can be tuned + +# You can override the default values with e.g.: `NUM_GPUS=1 bash examples/gsm8k/run_gsm8k.sh`. + +: "${DATA_DIR:="$HOME/data/gsm8k"}" +: "${NUM_INFERENCE_GPUS:=2}" +: "${NUM_POLICY_GPUS:=2}" +: "${LOGGER:=wandb}" # change to "console" to print to stdout / or use wandb + +: "${INFERENCE_BACKEND:=vllm}" +# : "${INFERENCE_BACKEND:=sglang}" + +# Fully async specific configuration knobs: +: "${MINI_BATCH_SIZE:=256}" +: "${MAX_STALENESS_STEPS:=4}" +: "${NUM_PARALLEL_GENERATION_WORKERS:=$(( MINI_BATCH_SIZE * (MAX_STALENESS_STEPS + 1) ))}" + +USE_TIS=true +TIS_IMP_RATIO_CAP=2.0 + +RUN_NAME=gsm8k-async-qwen2.5_1.5B-useTIS_${USE_TIS}-maxStale${MAX_STALENESS_STEPS}-numCon${NUM_PARALLEL_GENERATION_WORKERS}-${NUM_POLICY_GPUS}train${NUM_INFERENCE_GPUS}gen + +uv run --isolated --extra $INFERENCE_BACKEND -m examples.fully_async.main_async \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.fully_async.max_staleness_steps=${MAX_STALENESS_STEPS} \ + trainer.fully_async.num_parallel_generation_workers=${NUM_PARALLEL_GENERATION_WORKERS} \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=false \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_POLICY_GPUS \ + trainer.placement.critic_num_gpus_per_node=$NUM_POLICY_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_POLICY_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=4 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=${MINI_BATCH_SIZE} \ + trainer.policy_mini_batch_size=${MINI_BATCH_SIZE} \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k-async" \ + trainer.run_name=${RUN_NAME} \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/${RUN_NAME}" \ + trainer.resume_mode=latest \ + generator.enforce_eager=true \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/fully_async/main_async.py b/skyrl-train/examples/fully_async/main_async.py new file mode 100644 index 0000000000..93120c050a --- /dev/null +++ b/skyrl-train/examples/fully_async/main_async.py @@ -0,0 +1,60 @@ +""" +Main entrypoint for async training. +""" + +import hydra +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.fully_async_trainer import FullyAsyncRayPPOTrainer +import asyncio +from skyrl_train.utils import initialize_ray +import ray + + +class AsyncPPOExp(BasePPOExp): + def get_trainer( + self, + cfg, + tracker, + tokenizer, + train_dataset, + eval_dataset, + inference_engine_client, + generator, + colocate_pg, + ): + return FullyAsyncRayPPOTrainer( + cfg=cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=colocate_pg, + ) + + def run(self): + trainer = self._setup_trainer() + # Start the async training loop + asyncio.run(trainer.train()) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = AsyncPPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/gptoss/bench_flex_attn.py b/skyrl-train/examples/gptoss/bench_flex_attn.py new file mode 100644 index 0000000000..ae879847c5 --- /dev/null +++ b/skyrl-train/examples/gptoss/bench_flex_attn.py @@ -0,0 +1,163 @@ +from transformers.models.gpt_oss.modeling_gpt_oss import GptOssAttention, GptOssRotaryEmbedding +from skyrl_train.patches.gptoss.patch_transformers import patch_GptOssAttention +from transformers import AutoConfig +from skyrl_train.utils.profiler import CudaTimer +import ray +import torch +import random +import numpy as np +from einops import repeat +import argparse + +parser = argparse.ArgumentParser() + +parser.add_argument("--batch_size", type=int, default=4, help="Batch size for processing") +parser.add_argument("--sequence_length", type=int, default=512, help="Prefix length") +parser.add_argument("--layer_idx", type=int, default=5, help="Layer index") +parser.add_argument("--num_warmup_trials", type=int, default=10, help="Number of warmup trials that are ignored") +parser.add_argument("--num_trials", type=int, default=20, help="Number of trials total") +parser.add_argument("--torch_dtype", type=lambda x: getattr(torch, x), default=torch.bfloat16, help="Torch dtype") +parser.add_argument("--with-attention", help="Whether to add attention mask", action="store_true") +args = parser.parse_args() + + +@ray.remote(num_gpus=1) +def run_bench(args): + + BATCH_SIZE = args.batch_size + SEQUENCE_LENGTH = args.sequence_length + MODEL_NAME = "unsloth/gpt-oss-20b-BF16" + LAYER_IDX = args.layer_idx + DEVICE = "cuda:0" + NUM_WARMUP_TRIALS = args.num_warmup_trials + NUM_TRIALS = args.num_trials + TORCH_DTYPE = args.torch_dtype + torch.cuda.set_device(DEVICE) + torch.cuda.init() + + config = AutoConfig.from_pretrained(MODEL_NAME) + config.torch_dtype = TORCH_DTYPE + config._attn_implementation = "eager" + + def generate_inputs(batch_size, sequence_length, hidden_size, device, torch_dtype, with_attention: bool = False): + prompt = torch.randn(batch_size, sequence_length, hidden_size).to(device, dtype=torch_dtype) + attention_mask = None + if with_attention: + # zero out 30% + num_zeros = int(0.3 * sequence_length) + left_padding = random.randint(0, num_zeros) + right_padding = num_zeros - left_padding + attention_mask = torch.cat( + ( + torch.zeros(batch_size, left_padding), + torch.ones(batch_size, sequence_length - num_zeros), + torch.zeros(batch_size, right_padding), + ), + dim=1, + ).to(device, dtype=torch.long) + return prompt, attention_mask + + # use 256 MB tensor to clear L2 cache (same as triton timing approach) + x = torch.empty(int(256 * (1024**2)), dtype=torch.int8, device="cuda") + + def flush_cache(): + x.zero_() + + ## eager + eager_gptoss = GptOssAttention(config, LAYER_IDX) + eager_gptoss.to(DEVICE, dtype=TORCH_DTYPE) + + rotary_emb = GptOssRotaryEmbedding(config).to(DEVICE, dtype=TORCH_DTYPE) + + fwd_times = [] + bwd_times = [] + try: + for i in range(NUM_TRIALS): + + prompt, attention_mask = generate_inputs( + BATCH_SIZE, SEQUENCE_LENGTH, config.hidden_size, DEVICE, TORCH_DTYPE, args.with_attention + ) + full_batch = prompt.contiguous() + position_ids = ( + repeat(torch.arange(SEQUENCE_LENGTH, dtype=TORCH_DTYPE), "L -> B L", B=BATCH_SIZE) + .to(DEVICE, dtype=torch.long) + .contiguous() + ) + position_embeddings = rotary_emb(prompt, position_ids) + grad_out = torch.randn_like(full_batch) + + with CudaTimer(DEVICE) as results: + output, _ = eager_gptoss( + full_batch, position_embeddings=position_embeddings, attention_mask=attention_mask + ) + with CudaTimer(DEVICE) as results_bwd: + output.backward(grad_out) + eager_gptoss.zero_grad() + del output + flush_cache() + if i >= NUM_WARMUP_TRIALS: + fwd_times.append(results.elapsed_time) + bwd_times.append(results_bwd.elapsed_time) + eager_time = np.mean(fwd_times) + np.mean(bwd_times) + print(f"Eager Time Fwd: {np.mean(fwd_times)} ± {np.std(fwd_times)}") + print(f"Eager Time Bwd: {np.mean(bwd_times)} ± {np.std(bwd_times)}") + print(f"Overall mean eager time: {eager_time}") + except torch.OutOfMemoryError as e: + print(f"Eager attention OOM'ed with the following traceback: {e}. Skipping...") + finally: + eager_gptoss.zero_grad() + eager_gptoss.to("cpu") + flush_cache() + + # FLEX ATTENTION + patch_GptOssAttention() + flex_gptoss = GptOssAttention(config, LAYER_IDX) + flex_gptoss.to(DEVICE, dtype=TORCH_DTYPE) + + fwd_times = [] + bwd_times = [] + try: + for i in range(NUM_TRIALS): + + prompt, attention_mask = generate_inputs( + BATCH_SIZE, SEQUENCE_LENGTH, config.hidden_size, DEVICE, TORCH_DTYPE, args.with_attention + ) + full_batch = prompt.contiguous() + # #full_batch = torch.cat((chosen_full, rejected_full), dim=0).contiguous() + position_ids = None + position_ids = ( + repeat(torch.arange(SEQUENCE_LENGTH, dtype=TORCH_DTYPE), "L -> B L", B=BATCH_SIZE) + .to(DEVICE, dtype=TORCH_DTYPE) + .contiguous() + ) + position_embeddings = rotary_emb(prompt, position_ids) + grad_out = torch.randn_like(full_batch) + + with CudaTimer(DEVICE) as results: + output, _ = flex_gptoss( + full_batch, position_embeddings=position_embeddings, attention_mask=attention_mask + ) + with CudaTimer(DEVICE) as results_bwd: + output.backward(grad_out) + flex_gptoss.zero_grad() + del output + flush_cache() + if i >= NUM_WARMUP_TRIALS: + fwd_times.append(results.elapsed_time) + bwd_times.append(results_bwd.elapsed_time) + + flex_attn_time = np.mean(fwd_times) + np.mean(bwd_times) + print(f"Flex Attn Time Fwd: {np.mean(fwd_times)} ± {np.std(fwd_times)}") + print(f"Flex Attn Time Bwd: {np.mean(bwd_times)} ± {np.std(bwd_times)}") + print(f"Flex Attn overall time: {flex_attn_time}") + except torch.OutOfMemoryError as e: + print(f"Flex attention OOM'ed with the following traceback: {e}. Skipping...") + finally: + flex_gptoss.zero_grad() + flex_gptoss.to("cpu") + flush_cache() + # need to ensure that any future torch.compile calls doesn't get messed up by the previous compile (which has differently shaped inputs) + torch.compiler.reset() + + +ray.get(run_bench.remote(args)) diff --git a/skyrl-train/examples/gptoss/run_gsm8k_gptoss.sh b/skyrl-train/examples/gptoss/run_gsm8k_gptoss.sh new file mode 100755 index 0000000000..c61100d3cd --- /dev/null +++ b/skyrl-train/examples/gptoss/run_gsm8k_gptoss.sh @@ -0,0 +1,64 @@ +set -exo pipefail + +# Colocated GRPO training+generation for GPT-OSS-20B on GSM8K. +# NOTE (sumanthrh): Currently, gpt-oss requires flash attention to be disabled since attention sinks are not supported: https://github.com/Dao-AILab/flash-attention/issues/1797 +# We thus disable flash attention as well as sample packing +# We only support GPT-OSS training in BF16 precision and single-turn tasks at the moment, and are actively working on multi-turn support. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/gptoss/run_gsm8k_gptoss.sh + +# NOTE (sumanthrh): `micro_train_batch_size_per_gpu` and `micro_forward_batch_size_per_gpu` can be tuned + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=8 +LOGGER="wandb" # change to "console" to print to stdout + +INFERENCE_BACKEND="vllm" +# INFERENCE_BACKEND="sglang" + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="unsloth/gpt-oss-20b-BF16" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=2 \ + trainer.flash_attn=false \ + trainer.use_sample_packing=false \ + generator.inference_engine_tensor_parallel_size=4 \ + generator.enforce_eager=true \ + trainer.epochs=20 \ + trainer.eval_batch_size=32 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=32 \ + trainer.policy_mini_batch_size=32 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=5 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=4096 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_gptoss" \ + trainer.run_name="gsm8k_test_gptoss_low" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt_gptoss" \ + +generator.chat_template_kwargs={reasoning_effort:'low'} \ + trainer.dump_data_batch=true \ + $@ diff --git a/skyrl-train/examples/gptoss/test_gptoss.py b/skyrl-train/examples/gptoss/test_gptoss.py new file mode 100644 index 0000000000..eb069be2fe --- /dev/null +++ b/skyrl-train/examples/gptoss/test_gptoss.py @@ -0,0 +1,82 @@ +"""Simple script to sanity check flex attention patches for GPTOSS""" + +import ray + + +@ray.remote(num_gpus=1) +def run_task(with_padding: bool = True): + + from skyrl_train.patches.gptoss.patch_transformers import custom_attention_mask, custom_attention + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + from skyrl_train.model_wrapper import logprobs_from_logits + + from transformers import AttentionInterface, AttentionMaskInterface + + AttentionInterface.register("custom_flex", custom_attention) + AttentionMaskInterface.register("custom_flex", custom_attention_mask) + + MODEL = "unsloth/gpt-oss-20b-BF16" + model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).to("cuda") + tokenizer = AutoTokenizer.from_pretrained(MODEL) + model.eval() + + model.gradient_checkpointing_enable() + + input_text = "Hello, how are you?" + + input_ids = tokenizer.encode(input_text, return_tensors="pt") + if with_padding: + input_ids = torch.cat( + [ + torch.tensor( + [ + [ + tokenizer.pad_token_id, + ] + ], + dtype=input_ids.dtype, + ), + input_ids, + ], + dim=1, + ) + attention_mask = torch.cat( + [torch.tensor([0], dtype=torch.long), torch.tensor([1] * (input_ids.shape[1] - 1), dtype=torch.long)] + ).unsqueeze(0) + position_ids = attention_mask.long().cumsum(-1) - 1 + else: + attention_mask = None + position_ids = torch.arange(input_ids.size(1)).unsqueeze(0) + + input_ids = input_ids.to("cuda") + attention_mask = attention_mask.to("cuda") if attention_mask is not None else None + position_ids = position_ids.to("cuda") + + with torch.no_grad(): + print(type(input_ids)) + output1 = model(input_ids, attention_mask=attention_mask, position_ids=position_ids) + logprobs1 = logprobs_from_logits(output1["logits"], input_ids) + + # uncommment this to patch the full attention module + # from skyrl_train.patches.gptoss.patch_transformers import patch_GptOssAttention + # patch_GptOssAttention() + model.set_attn_implementation("custom_flex") + + with torch.no_grad(): + output2 = model(input_ids, attention_mask=attention_mask, position_ids=position_ids) + logprobs2 = logprobs_from_logits(output2["logits"], input_ids) + print("logprobs1: \n", logprobs1, flush=True) + print("-----------") + print("logprobs2: \n", logprobs2, flush=True) + # NOTE: this is actually a high error in logprobs. + # TODO: revisit flex attention patches to see if this can be improved + if with_padding: + torch.testing.assert_close(logprobs1, logprobs2, atol=0.1, rtol=0.02) + else: + torch.testing.assert_close(logprobs1, logprobs2, atol=0.02, rtol=0.02) + + +ray.init() +ray.get(run_task.remote(with_padding=False)) +ray.get(run_task.remote(with_padding=True)) diff --git a/skyrl-train/examples/gsm8k/gsm8k-grpo-skypilot.yaml b/skyrl-train/examples/gsm8k/gsm8k-grpo-skypilot.yaml new file mode 100644 index 0000000000..c4f4d1bc20 --- /dev/null +++ b/skyrl-train/examples/gsm8k/gsm8k-grpo-skypilot.yaml @@ -0,0 +1,127 @@ +# Run this from the repository root after cloning SkyRL. +resources: + infra: aws # replace this with what cloud you want to launch on + accelerators: L40S:4 # 4x 48 GB GPUs; adjust to the SKU that matches your quota + memory: 64+ # every node has at least 64 GB memory + ports: 6479 # expose port for ray dashboard +# network_tier: best # when using multiple nodes, communication can become a bottleneck + +num_nodes: 2 # cluster size + +# Set to a git repository +workdir: + url: https://github.com/NovaSky-AI/SkyRL.git + ref: main + +# Set secrets +secrets: + WANDB_API_KEY: null + +envs: + LOGGER: "wandb" # change to "console" to print to stdout + INFERENCE_BACKEND: "vllm" + # INFERENCE_BACKEND: "sglang" + NUM_EPOCHS: 20 + +# Commands run on each node of the remote cluster to set up the environment (e.g., install dependencies). These are run directly inside Docker. +setup: | + cd skyrl-train + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + uv pip install wandb + uv run -- python examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k + +# If you already have processed the data locally, delete the above line + +# The actual task commands to be executed on the remote cluster. +# This script will first start the Ray cluster (different ray start commands are executed on Head and Worker nodes). +# Then, your training script will only be run on the Head node (SKYPILOT_NODE_RANK == 0). +run: | + set -euo pipefail + + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + read -r head_ip _ <<< "$SKYPILOT_NODE_IPS" + DATA_DIR="$HOME/data/gsm8k" + + # Login to Weights & Biases once the secrets are available. + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + wait_for_ray() { + local address=$1 + for _ in $(seq 1 24); do + if ray status --address "$address" >/dev/null 2>&1; then + return 0 + fi + sleep 5 + done + echo "Ray cluster at $address failed to become ready" >&2 + return 1 + } + + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + if [ "$SKYPILOT_NODE_RANK" = "0" ]; then + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 + fi + wait_for_ray 127.0.0.1:6479 + # Ensure rollout GPUs match policy GPUs when colocating all models. + # With colocate_all=true, we must have: + # policy_num_gpus_per_node * policy_num_nodes == + # num_inference_engines * inference_engine_tensor_parallel_size * inference_engine_data_parallel_size + # Given TP=1 and DP=1, set num_inference_engines to total policy GPUs across the cluster. + TOTAL_POLICY_GPUS=$((SKYPILOT_NUM_GPUS_PER_NODE * SKYPILOT_NUM_NODES)) + uv run --isolated --extra "$INFERENCE_BACKEND" -m skyrl_train.entrypoints.main_base \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node="$SKYPILOT_NUM_GPUS_PER_NODE" \ + trainer.placement.ref_num_gpus_per_node="$SKYPILOT_NUM_GPUS_PER_NODE" \ + trainer.placement.ref_num_nodes="$SKYPILOT_NUM_NODES" \ + trainer.placement.policy_num_nodes="$SKYPILOT_NUM_NODES" \ + generator.num_inference_engines="$TOTAL_POLICY_GPUS" \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=40 \ + trainer.micro_train_batch_size_per_gpu=40 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend="$INFERENCE_BACKEND" \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" + else + if ! ray status --address "$head_ip:6479" >/dev/null 2>&1; then + ray start --address "$head_ip:6479" --disable-usage-stats + fi + wait_for_ray "$head_ip:6479" + fi + + echo "Node setup and Ray start script finished for rank ${SKYPILOT_NODE_RANK}." diff --git a/skyrl-train/examples/gsm8k/gsm8k_dataset.py b/skyrl-train/examples/gsm8k/gsm8k_dataset.py new file mode 100644 index 0000000000..f1015446f3 --- /dev/null +++ b/skyrl-train/examples/gsm8k/gsm8k_dataset.py @@ -0,0 +1,99 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the GSM8k dataset to parquet format +""" + +import argparse +import re +import os + +import datasets + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", default="~/data/gsm8k") + parser.add_argument( + "--max_train_dataset_length", + type=int, + default=None, + help="If set, truncate the training split to this many examples.", + ) + + args = parser.parse_args() + + args.output_dir = os.path.expanduser(args.output_dir) + + data_source = "openai/gsm8k" + + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + val_dataset = dataset["test"] + + if args.max_train_dataset_length is not None: + max_len = min(args.max_train_dataset_length, len(train_dataset)) + train_dataset = train_dataset.select(range(max_len)) + + instruction_following = 'Let\'s think step by step and output the final answer after "####".' + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + solution = extract_solution(answer_raw) + data = { + "data_source": data_source, + "prompt": [ + { + "role": "user", + "content": question, + } + ], + "env_class": "gsm8k", + "reward_spec": { + "method": "rule", + "ground_truth": solution, + }, + "extra_info": { + "split": split, + "index": idx, + "answer": answer_raw, + "question": question_raw, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + val_dataset = val_dataset.map(function=make_map_fn("test"), with_indices=True) + + output_dir = args.output_dir + os.makedirs(output_dir, exist_ok=True) + train_dataset.to_parquet(os.path.join(output_dir, "train.parquet")) + val_dataset.to_parquet(os.path.join(output_dir, "validation.parquet")) diff --git a/skyrl-train/examples/gsm8k/run_32b_gsm8k.sh b/skyrl-train/examples/gsm8k/run_32b_gsm8k.sh new file mode 100755 index 0000000000..ecb9453bb8 --- /dev/null +++ b/skyrl-train/examples/gsm8k/run_32b_gsm8k.sh @@ -0,0 +1,52 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-32B-Instruct on GSM8K. +# export WANDB_API_KEY= +# bash examples/gsm8k/run_32b_gsm8k.sh + + +DATA_DIR="$HOME/data/gsm8k" + +# NOTE: the batch size can likely be increased for more throughput. +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-32B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=8 \ + trainer.placement.ref_num_gpus_per_node=8 \ + trainer.placement.ref_num_nodes=2 \ + trainer.placement.policy_num_nodes=2 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=4 \ + trainer.epochs=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=32 \ + trainer.policy_mini_batch_size=32 \ + trainer.critic_mini_batch_size=32 \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=false \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k-32b" \ + trainer.run_name="gsm8k_test_32b" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_32B_ckpt" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/gsm8k/run_generation_gsm8k.sh b/skyrl-train/examples/gsm8k/run_generation_gsm8k.sh new file mode 100644 index 0000000000..e765efbf3c --- /dev/null +++ b/skyrl-train/examples/gsm8k/run_generation_gsm8k.sh @@ -0,0 +1,28 @@ +set -x + +# Generation only for for Qwen2.5-0.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/gsm8k/run_generation_gsm8k.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=1 +LOGGER="wandb" # change to "console" to print to stdout + +INFERENCE_BACKEND="vllm" # or "sglang" + +uv run --isolated --extra $INFERENCE_BACKEND \ + -m skyrl_train.entrypoints.main_generate \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.logger="$LOGGER" \ + trainer.placement.colocate_all=false \ + generator.backend=$INFERENCE_BACKEND \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + generator.gpu_memory_utilization=0.9 \ + generator.eval_sampling_params.max_generate_length=1024 \ + generator.eval_sampling_params.temperature=0.7 \ + environment.env_class=gsm8k \ + "$@" diff --git a/skyrl-train/examples/gsm8k/run_gsm8k.sh b/skyrl-train/examples/gsm8k/run_gsm8k.sh new file mode 100755 index 0000000000..6b558da62a --- /dev/null +++ b/skyrl-train/examples/gsm8k/run_gsm8k.sh @@ -0,0 +1,59 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/gsm8k/run_gsm8k.sh + +# NOTE (sumanthrh): `micro_train_batch_size_per_gpu` and `micro_forward_batch_size_per_gpu` can be tuned + +# You can override the default values with e.g.: `NUM_GPUS=1 bash examples/gsm8k/run_gsm8k.sh`. + +: "${DATA_DIR:="$HOME/data/gsm8k"}" +: "${NUM_GPUS:=4}" +: "${LOGGER:=wandb}" # change to "console" to print to stdout + +: "${INFERENCE_BACKEND:=vllm}" +# : "${INFERENCE_BACKEND:=sglang}" + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.critic_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/gsm8k/run_gsm8k_modal.sh b/skyrl-train/examples/gsm8k/run_gsm8k_modal.sh new file mode 100644 index 0000000000..2210e852ce --- /dev/null +++ b/skyrl-train/examples/gsm8k/run_gsm8k_modal.sh @@ -0,0 +1,60 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# Intended to be used with Modal (SkyRL/skyrl-train/integration/modal) +# modal run main.py --command "WANDB_API_KEY=... bash examples/gsm8k/run_gsm8k_modal.sh" + +# NOTE (sumanthrh): `micro_train_batch_size_per_gpu` and `micro_forward_batch_size_per_gpu` can be tuned + +# You can override the default values with e.g.: `NUM_GPUS=1 bash examples/gsm8k/run_gsm8k.sh`. + +DATA_DIR="/root/data/gsm8k" +CHECK_POINT_DIR="/root/data/ckpts/gsm8k_1.5B_ckpt" +EXPORT_PATH="/root/data/export" +: "${NUM_GPUS:=4}" +: "${LOGGER:=wandb}" # change to "console" to print to stdout + +: "${INFERENCE_BACKEND:=vllm}" +# : "${INFERENCE_BACKEND:=sglang}" + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path=$CHECK_POINT_DIR \ + trainer.export_path=$EXPORT_PATH \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/harbor/README.md b/skyrl-train/examples/harbor/README.md new file mode 100644 index 0000000000..4b32a2deee --- /dev/null +++ b/skyrl-train/examples/harbor/README.md @@ -0,0 +1,44 @@ +## Harbor Integration + +> [!IMPORTANT] +> Please refer to [`SkyRL/examples/train_integrations/harbor`](/examples/train_integrations/harbor) instead, as SkyRL is undergoing a repo reorganization into the `SkyRL/skyrl` folder, which unifies the skyrl libraries into a single package. The existing packages (e.g. `skyrl-train`) are fully functional but will be migrated to new paths shortly. See issue: https://github.com/NovaSky-AI/SkyRL/issues/1145 + +RL training with [Harbor](https://github.com/laude-institute/harbor) as the environment and reward source. See the [full documentation](https://docs.skyrl.ai/docs/harbor) for details. + +### Structure + +``` +examples/harbor/ + harbor_generator.py # HarborGenerator: bridges SkyRL <-> Harbor + dataset.py # HarborTaskDataset: loads task directory paths + prepare_harbor_dataset.py # Downloads + extracts datasets from HuggingFace + harbor_trial_config/ + default.yaml # Harbor TrialConfig template + entrypoints/ + main_harbor.py # Full training entrypoint + main_harbor_generate.py # Generation-only debug entrypoint + run_codecontest.sh # Code contest training (Qwen3-8B) + run_harbor_gen.sh # Debug generation-only +``` + +### Quick Start + +```bash +cd SkyRL/skyrl-train + +# 1. Set credentials +export WANDB_API_KEY=your_wandb_api_key +# Pick your sandbox provider: +export DAYTONA_API_KEY=your_daytona_api_key +# export MODAL_TOKEN_ID=your_modal_token_id +# export MODAL_TOKEN_SECRET=your_modal_token_secret + +# 2. Prepare dataset +python examples/harbor/prepare_harbor_dataset.py \ + --dataset DCAgent/code-contests-sandboxes-with-tests +python examples/harbor/prepare_harbor_dataset.py \ + --dataset open-thoughts/OpenThoughts-TB-dev + +# 3. Launch training +bash examples/harbor/run_codecontest.sh +``` diff --git a/skyrl-train/examples/harbor/dataset.py b/skyrl-train/examples/harbor/dataset.py new file mode 100644 index 0000000000..1d9b5468ed --- /dev/null +++ b/skyrl-train/examples/harbor/dataset.py @@ -0,0 +1,104 @@ +from loguru import logger +from typing import List +from pathlib import Path + + +class HarborTaskDataset: + """ + A dataset that loads Harbor task data from direct file/directory paths. + Each dataset item is a path to a task directory. + """ + + def __init__( + self, + data_files: List[str], + ): + """ + Initialize the HarborTaskDataset. + + Args: + data_files: List of direct file/directory paths pointing to Harbor task data + """ + self.data_files = data_files + + # Load all data files + self.task_paths = self._load_data_files() + + logger.info(f"HarborTaskDataset initialized with {len(self.task_paths)} task paths") + + def _load_data_files(self) -> List[Path]: + """Load all data files from direct paths and return list of task paths.""" + task_paths = [] + + for data_source in self.data_files: + source_path = Path(data_source) + + if not source_path.exists(): + logger.warning(f"Path does not exist: {data_source}") + continue + + logger.info(f"Loading data from: {data_source}") + + # If the path is a directory, find all valid task subdirectories + if source_path.is_dir(): + # Look for task subdirectories and validate them + all_dirs = [d for d in source_path.iterdir() if d.is_dir()] + valid_task_dirs = [d for d in all_dirs if self._is_valid_task_directory(d)] + + if valid_task_dirs: + task_paths.extend(valid_task_dirs) + logger.info( + f"Found {len(valid_task_dirs)} valid task directories out of {len(all_dirs)} total directories" + ) + elif self._is_valid_task_directory(source_path): + # If no subdirectories but the main directory is valid, treat it as a task + task_paths.append(source_path) + logger.info("Using main directory as valid task") + else: + logger.warning(f"No valid task directories found in {source_path}") + else: + # If it's a file, treat it as a single task (files can't be valid task directories) + logger.warning(f"File {source_path} cannot be a valid task directory (missing instruction.md)") + + return task_paths + + def _is_valid_task_directory(self, task_path: Path) -> bool: + """Check if a directory is a valid task directory (has instruction.md file).""" + if not task_path.is_dir(): + return False + + instruction_file = task_path / "instruction.md" + return instruction_file.exists() and instruction_file.is_file() + + def __getitem__(self, index: int) -> dict: + """Get a task path by index as a dictionary with 'prompt', 'env_class', and 'env_extras' keys.""" + if index >= len(self.task_paths): + raise IndexError(f"Index {index} out of range for dataset of size {len(self.task_paths)}") + return { + "prompt": str(self.task_paths[index]), + "env_class": None, + "env_extras": {"data_source": str(self.task_paths[index])}, + "uid": str(index), + } + + def __len__(self) -> int: + """Return the number of tasks in the dataset.""" + return len(self.task_paths) + + def __iter__(self): + """Iterate over all task paths as dictionaries.""" + for index, task_path in enumerate(self.task_paths): + yield { + "prompt": str(task_path), + "env_class": None, + "env_extras": {"data_source": str(task_path)}, + "uid": str(index), + } + + def get_task_paths(self) -> List[Path]: + """Return all task paths as a list.""" + return self.task_paths.copy() + + def collate_fn(self, item_list): + """Collate function for batching task dictionaries.""" + return item_list diff --git a/skyrl-train/examples/harbor/entrypoints/main_harbor.py b/skyrl-train/examples/harbor/entrypoints/main_harbor.py new file mode 100644 index 0000000000..4def3b4fe6 --- /dev/null +++ b/skyrl-train/examples/harbor/entrypoints/main_harbor.py @@ -0,0 +1,74 @@ +""" +Main entrypoint for training on Harbor tasks. +""" + +import ray +import hydra +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir +from skyrl_train.utils import validate_cfg +from skyrl_train.utils.utils import initialize_ray +from examples.harbor.harbor_generator import HarborGenerator +from examples.harbor.dataset import HarborTaskDataset + + +class HarborExp(BasePPOExp): + def get_generator(self, cfg, tokenizer, inference_engine_client): + """ + Initializes the HarborGenerator. + """ + return HarborGenerator( + generator_cfg=cfg.generator, + harbor_cfg=cfg.harbor_trial_config, # Pass harbor config to the generator + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + max_seq_len=cfg.trainer.algorithm.max_seq_len, + ) + + def get_train_dataset(self): + """Initializes the training dataset. + + Returns: + HarborTaskDataset: The training dataset. + """ + prompts_dataset = HarborTaskDataset( + data_files=self.cfg.data.train_data, + ) + # make sure the dataset is large enough to train on + assert ( + len(prompts_dataset) >= self.cfg.trainer.train_batch_size + ), f"dataset should be atleast as large as `train_batch_size` {self.cfg.trainer.train_batch_size}, got size {len(prompts_dataset)}" + return prompts_dataset + + def get_eval_dataset(self): + """Initializes the evaluation dataset. + + Returns: + HarborTaskDataset: The evaluation dataset. + """ + if self.cfg.trainer.eval_interval > 0 and self.cfg.data.val_data: + prompts_dataset = HarborTaskDataset( + data_files=self.cfg.data.val_data, + ) + return prompts_dataset + return None + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = HarborExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/harbor/entrypoints/main_harbor_generate.py b/skyrl-train/examples/harbor/entrypoints/main_harbor_generate.py new file mode 100644 index 0000000000..8a0f804531 --- /dev/null +++ b/skyrl-train/examples/harbor/entrypoints/main_harbor_generate.py @@ -0,0 +1,100 @@ +""" +Main entrypoint for generating rollouts on Harbor tasks. For debugging purposes. +""" + +import ray +import asyncio +import hydra +from loguru import logger +from omegaconf import DictConfig + +from skyrl_train.utils import validate_cfg +from skyrl_train.utils.utils import initialize_ray +from skyrl_train.entrypoints.main_base import ( + BasePPOExp, + config_dir, +) +from skyrl_train.generators.base import GeneratorInput, TrajectoryID +from examples.harbor.harbor_generator import HarborGenerator +from examples.harbor.dataset import HarborTaskDataset + + +# For debugging purposes, we only generate a few samples. +NUM_SAMPLES_TO_TEST = 10 + + +class HarborGenerateExp(BasePPOExp): + def get_generator(self, cfg, tokenizer, inference_engine_client): + """ + Initializes the HarborGenerator. + """ + return HarborGenerator( + generator_cfg=cfg.generator, + harbor_cfg=cfg.harbor_trial_config, # Pass harbor config to the generator + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + max_seq_len=cfg.trainer.algorithm.max_seq_len, + ) + + def _setup_generator(self): + logger.info(self.get_cfg_as_str(self.cfg)) + + inference_engine_client = self.get_inference_client() + asyncio.run(inference_engine_client.wake_up()) + + return self.get_generator(self.cfg, self.tokenizer, inference_engine_client) + + def get_train_dataset(self): + """Initializes the training dataset. + + Returns: + HarborTaskDataset: The training dataset. + """ + prompts_dataset = HarborTaskDataset( + data_files=self.cfg.data.train_data, + ) + assert ( + len(prompts_dataset) >= self.cfg.trainer.train_batch_size + ), f"dataset should be atleast as large as `train_batch_size` {self.cfg.trainer.train_batch_size}, got size {len(prompts_dataset)}" + return prompts_dataset + + def run(self): + generator = self._setup_generator() + + prompts = [] + trajectory_ids = [] + for item in self.train_dataset: + prompts.append(item["prompt"]) + trajectory_ids.append(TrajectoryID(instance_id=item["uid"], repetition_id=0)) + + # Build input from the training dataset + input_batch = GeneratorInput( + prompts=prompts[:NUM_SAMPLES_TO_TEST], + trajectory_ids=trajectory_ids[:NUM_SAMPLES_TO_TEST], + env_classes=None, + env_extras=None, + sampling_params=None, + ) + + # Start generation + asyncio.run(generator.generate(input_batch)) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = HarborGenerateExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/harbor/harbor_generator.py b/skyrl-train/examples/harbor/harbor_generator.py new file mode 100644 index 0000000000..d1353015e0 --- /dev/null +++ b/skyrl-train/examples/harbor/harbor_generator.py @@ -0,0 +1,348 @@ +import asyncio +from copy import deepcopy +from dataclasses import dataclass +from typing import List, Optional +from loguru import logger +from uuid import uuid4 +from skyrl_train.generators.base import GeneratorInterface, GeneratorInput, GeneratorOutput, TrajectoryID +from skyrl_train.generators.utils import get_rollout_metrics, get_response_ids_and_loss_mask_from_messages +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.base import ConversationType +from skyrl_train.utils.rate_limiter import create_rate_limiter +from tqdm import tqdm +from omegaconf import DictConfig, OmegaConf +from harbor.trial.trial import Trial +from harbor.models.trial.config import TrialConfig + +# Suppress LiteLLM verbose logging + +import litellm +import logging + +litellm.suppress_debug_info = True # Suppress the "Provider List" output +litellm.set_verbose = False +logging.getLogger("LiteLLM").setLevel(logging.WARNING) + +# We have N retries for each trial, if one of the rollout (out of n_samples_per_prompt) fails +# after N attemptes, we skip this prompt altogether. +MAX_NUM_RETRIES_PER_TRIAL = 2 + + +@dataclass +class HarborAgentOutput: + response_ids: List[int] + reward: float + stop_reason: str + loss_mask: List[int] + prompt_ids: List[int] + trajectory_id: TrajectoryID + summarization_count: Optional[int] = None + num_turns: Optional[int] = None + + +class HarborGenerator(GeneratorInterface): + def __init__( + self, + generator_cfg: DictConfig, + harbor_cfg: DictConfig, + inference_engine_client: InferenceEngineClient, + tokenizer, + max_seq_len: int, + ): + """ + Args: + generator_cfg: DictConfig object containing the generator configuration + harbor_cfg: DictConfig object containing the Harbor configuration + inference_engine_client: InferenceEngineClient object for interacting with the inference engines + tokenizer: tokenizer object for encoding and decoding text + max_seq_len: Maximum total sequence length (prompt + response). Used to truncate responses. + """ + self.base_url = f"http://{generator_cfg.http_endpoint_host}:{generator_cfg.http_endpoint_port}" + self.generator_cfg = generator_cfg + self.tokenizer = tokenizer + self.max_seq_len = max_seq_len + self.model_name = generator_cfg.model_name + + # Harbor config template - users can specify any Harbor TrialConfig options in YAML or command line. + # SkyRL injects: model_name and api_base (once at init), task.path and session_id (per trial) + self._harbor_trial_config_template = OmegaConf.to_container(harbor_cfg, resolve=True) + + # Set model_name and api_base once (constant across all trials) + assert generator_cfg.served_model_name is not None, "served_model_name must be set" + assert ( + "/" not in generator_cfg.served_model_name + ), "served_model_name must not contain '/', Harbor expects hosted_vllm/{model_name}" + self._harbor_trial_config_template.setdefault("agent", {})[ + "model_name" + ] = f"hosted_vllm/{generator_cfg.served_model_name}" + self._harbor_trial_config_template["agent"].setdefault("kwargs", {})["api_base"] = f"{self.base_url}/v1" + + logger.info( + f"HarborGenerator initialized with Harbor config. " + f"Agent: {self._harbor_trial_config_template.get('agent', {}).get('name')}, " + f"Trials dir: {self._harbor_trial_config_template.get('trials_dir', 'trials')}" + ) + + # Read custom chat template + custom_chat_template_path = generator_cfg.engine_init_kwargs.get("chat_template", None) + if custom_chat_template_path: + with open(custom_chat_template_path, "r") as f: + self.custom_chat_template_content = f.read() + logger.info(f"HarborGenerator initialized with custom chat template read from: {custom_chat_template_path}") + else: + self.custom_chat_template_content = None + + # Initialize rate limiter from generator config (not part of Harbor TrialConfig) + rate_limit_config = generator_cfg.get("rate_limit", None) + self._rate_limiter = create_rate_limiter(rate_limit_config) + + async def generate(self, input_batch: GeneratorInput) -> GeneratorOutput: + prompts = input_batch["prompts"] + trajectory_ids = input_batch["trajectory_ids"] + + if trajectory_ids is None: + raise ValueError("`trajectory_ids` is required in the input batch") + if len(prompts) != len(trajectory_ids): + raise ValueError( + f"Prompt count ({len(prompts)}) doesn't match " f"trajectory_ids count ({len(trajectory_ids)})" + ) + + all_outputs: List[HarborAgentOutput] = [None] * len(prompts) # type: ignore[list-item] + progress = tqdm( + total=len(prompts), + desc="Generating Trajectories", + miniters=max(1, len(prompts) // 10), + mininterval=5, + ) + + async def _worker(idx, prompt, trajectory_id): + result = await self.harbor_agent_loop(prompt=prompt, trajectory_id=trajectory_id) + all_outputs[idx] = result + progress.update(1) + + try: + async with asyncio.TaskGroup() as tg: + for idx, (prompt, trajectory_id) in enumerate(zip(prompts, trajectory_ids)): + tg.create_task(_worker(idx, prompt, trajectory_id)) + finally: + progress.close() + all_outputs, rollout_metrics = self._mask_failed_instances_and_compute_metrics(all_outputs) + + generator_output: GeneratorOutput = { + "prompt_token_ids": [output.prompt_ids for output in all_outputs], + "response_ids": [output.response_ids for output in all_outputs], + "rewards": [output.reward for output in all_outputs], + "loss_masks": [output.loss_mask for output in all_outputs], + "stop_reasons": [output.stop_reason for output in all_outputs], + "rollout_metrics": rollout_metrics, + "rollout_logprobs": None, + } + + return generator_output + + @staticmethod + def _mask_failed_instances_and_compute_metrics( + all_outputs: List[HarborAgentOutput], + ) -> tuple[List[HarborAgentOutput], dict]: + """Mutates all_outputs in-place: zeros out every output belonging to a failed instance. + + For a group of trajectories (n_samples_per_prompt for the same prompt), + if one trajectory fails we skip training the entire group. + + Returns: + all_outputs: The same list, with failed-instance outputs zeroed out. + rollout_metrics: Dict of rollout metrics for logging. + """ + # Count failures by type before grouping overwrites stop_reason. + num_timeout_trajectories = 0 + num_error_trajectories = 0 + timeout_instance_ids = set() + error_instance_ids = set() + all_instance_ids = set() + for output in all_outputs: + cur_instance_id = output.trajectory_id.instance_id + all_instance_ids.add(cur_instance_id) + if output.stop_reason == "agent_timeout": + num_timeout_trajectories += 1 + timeout_instance_ids.add(cur_instance_id) + elif output.stop_reason == "error": + num_error_trajectories += 1 + error_instance_ids.add(cur_instance_id) + + masked_instance_ids = timeout_instance_ids | error_instance_ids + + # Zero-out all outputs belonging to any timeout or error instance so we skip training on them. + successful_outputs: List[HarborAgentOutput] = [] + for output in all_outputs: + if output.trajectory_id.instance_id in masked_instance_ids: + output.response_ids = [0] + output.stop_reason = "error" + output.loss_mask = [0] + output.prompt_ids = [0] + output.reward = 0 + else: + successful_outputs.append(output) + + # Rollout metrics for successful outputs. + if len(successful_outputs) > 0: + rollout_metrics = get_rollout_metrics( + [output.response_ids for output in successful_outputs], + [output.reward for output in successful_outputs], + ) + rollout_metrics["generate/trajectories_summarized"] = sum( + 1 for output in successful_outputs if output.summarization_count > 0 + ) + rollout_metrics["generate/trajectories_context_length_exceeded"] = sum( + 1 for output in successful_outputs if output.stop_reason == "context_length" + ) + rollout_metrics["generate/avg_num_turns"] = sum(output.num_turns for output in successful_outputs) / len( + successful_outputs + ) + else: + rollout_metrics = {} + + # Failure metrics: timeout vs unknown error trajectories, and masked instances. + rollout_metrics["generate/num_timeout_trajectories"] = num_timeout_trajectories + rollout_metrics["generate/num_error_trajectories"] = num_error_trajectories + rollout_metrics["generate/num_masked_instances"] = len(masked_instance_ids) + + logger.info( + f"\n# of masked instances: {len(masked_instance_ids)} / {len(all_instance_ids)}\n" + f"# of timeout trajectories: {num_timeout_trajectories}\n" + f"# of error trajectories: {num_error_trajectories}" + ) + + return all_outputs, rollout_metrics + + async def harbor_agent_loop( + self, + prompt: ConversationType, + trajectory_id: TrajectoryID, + ) -> HarborAgentOutput: + """ + Run a single harbor agent. + """ + # Run the trial to get `reward`, `chat_history`, `summarization_count`, and `num_turns` + reward = None + chat_history = None + summarization_count = None + num_turns = None + successful = False + is_context_length_error = False + is_agent_timeout_error = False + for i in range(MAX_NUM_RETRIES_PER_TRIAL): + prefix = f"Trajectory {trajectory_id} attempt {i+1}/{MAX_NUM_RETRIES_PER_TRIAL}" + results = None + try: + # Create a fresh Trial each attempt so agent state is clean on retry. + config = deepcopy(self._harbor_trial_config_template) + config["task"] = {"path": prompt} + config["agent"]["kwargs"]["session_id"] = uuid4().hex + trial_config = TrialConfig.model_validate(config) + trial = Trial(trial_config) + + async with self._rate_limiter: + results = await trial.run() + + # Parse exception type + exc_type = results.exception_info.exception_type if results.exception_info else None + is_context_length_error = exc_type == "ContextLengthExceededError" + is_agent_timeout_error = exc_type == "AgentTimeoutError" + + # --- Determine reward --- + if is_agent_timeout_error: + # AgentTimeoutError: not successful, no retry, loss-masked + logger.debug(f"{prefix} hit AgentTimeoutError (no retry). Results: {results}") + break + elif is_context_length_error: + # ContextLengthExceededError: always train with reward=0. + logger.debug( + f"{prefix} hit ContextLengthExceededError, will train with reward=0. Results: {results}" + ) + reward = 0 + elif not results.verifier_result: + # Does not have a verifier result, so it's not successful, will retry + logger.warning(f"{prefix} failed: Exception info: {results.exception_info}. Results: {results}") + continue + else: + reward = results.verifier_result.rewards["reward"] + + # --- Extract chat history and check for success --- + chat_history = results.agent_result.metadata["all_messages"] + summarization_count = results.agent_result.metadata["summarization_count"] + num_turns = results.agent_result.metadata["n_episodes"] + if len(chat_history) > 1 and chat_history[0]["role"] == "user": + successful = True + logger.debug(f"{prefix} successful: reward={reward}. Results: {results}") + break + else: + logger.warning( + f"{prefix} failed: Did not return a chat history with a user message. chat_history: {chat_history}\nResults: {results}" + ) + except Exception as e: + logger.warning(f"{prefix} failed: Error running trial: {e}. Results: {results}") + continue + + if not successful: + # We make loss mask 0 so it does not contribute to model updates + stop_reason = "agent_timeout" if is_agent_timeout_error else "error" + error_message = f"Trajectory {trajectory_id} failed (stop_reason={stop_reason}), will set loss mask to [0]." + if stop_reason == "error": + error_message += f" Results: {results}" + logger.warning(error_message) + return HarborAgentOutput( + response_ids=[0], + reward=0, + stop_reason=stop_reason, + loss_mask=[0], + prompt_ids=[0], + trajectory_id=trajectory_id, + ) + + # Use the first message as the prompt. We assume to be no systems messages. + assert chat_history[0]["role"] == "user", "The first message should be a user message" + prompt = [chat_history[0]] + prompt_ids = self.tokenizer.apply_chat_template( + prompt, + add_generation_prompt=False, # the message below will add it themselves + tokenize=True, + chat_template=self.custom_chat_template_content, + ) + initial_prompt_length = len(prompt_ids) + + # Process response messages (everything after the first message) + response_messages = chat_history[1:] + assistant_logprobs = getattr(results.agent_result, "output_logprobs", None) + response_ids, loss_mask, rollout_logprobs = get_response_ids_and_loss_mask_from_messages( + response_messages, self.tokenizer, assistant_logprobs, chat_template=self.custom_chat_template_content + ) + + # Determine stop reason + max_response_tokens = max(0, self.max_seq_len - initial_prompt_length) + if is_context_length_error or len(response_ids) > max_response_tokens: + stop_reason = "context_length" + else: + stop_reason = "complete" + + # Apply overlong filtering. + # TODO(Charlie): should this also apply when the end reason is max_turns in Harbor? + # Revisit. We would like to reuse `utils.py`'s implementation for overlong filtering. + if self.generator_cfg.apply_overlong_filtering and stop_reason == "context_length": + loss_mask = [0] * len(loss_mask) + + # Truncate to maximum allowed length. + # NOTE(Charlie): though it shouldn't happen since it'd reach `ContextLengthExceededError` + # from Harbor first. We do it anyway to be safe. + response_ids = response_ids[:max_response_tokens] + loss_mask = loss_mask[:max_response_tokens] + + return HarborAgentOutput( + response_ids=response_ids, + reward=reward, + stop_reason=stop_reason, + loss_mask=loss_mask, + prompt_ids=prompt_ids, + trajectory_id=trajectory_id, + summarization_count=summarization_count, + num_turns=num_turns, + ) diff --git a/skyrl-train/examples/harbor/harbor_trial_config/default.yaml b/skyrl-train/examples/harbor/harbor_trial_config/default.yaml new file mode 100644 index 0000000000..9a78b19b15 --- /dev/null +++ b/skyrl-train/examples/harbor/harbor_trial_config/default.yaml @@ -0,0 +1,103 @@ +# @package harbor_trial_config +# +# Default Harbor Configuration for SkyRL training +# +# This config is passed directly to Harbor's TrialConfig. You can use any +# Harbor-supported options here. SkyRL only injects a few runtime values: +# - task.path (from the prompt) +# - agent.model_name (constructed from served_model_name) +# - agent.kwargs.api_base (from SkyRL's HTTP endpoint) +# - agent.kwargs.session_id (generated per-trial) +# +# See Harbor documentation for all available options: +# https://harborframework.com/docs +# And a full list of fields that TrialConfig supports: +# https://github.com/laude-institute/harbor/blob/5921cc40e8f6b5b0df0a3dc6354e0e679447eb54/src/harbor/models/trial/config.py + +# Harbor TrialConfig fields below +# -------------------------------- + +# Where to store trial logging outputs +trials_dir: ~/trials + +# Trial timeout multiplier (applied to default timeouts) +timeout_multiplier: 1.0 + +# Agent configuration +# See: harbor.models.trial.config.AgentConfig +agent: + # Agent name. Options: terminus-2, terminus-1, terminus, oracle, claude-code, + # aider, swe-agent, openhands, etc. + name: terminus-2 + + # The time given a single Trial to run. If fails, the Trial raises an AgentTimeoutError. + override_timeout_sec: 1200 + + # Agent-specific settings passed to the agent constructor + kwargs: + # Maximum number of agent episodes/iterations + max_turns: 32 + + # Suppress Harbor override warnings for max_turns + suppress_max_turns_warning: true + + # Whether to enable context summarization when approaching token limits + enable_summarize: false + + # Store all messages in the trial output (required for SkyRL training) + store_all_messages: true + + # The only sampling param that directly gets passed to Terminus + temperature: 1.0 + + # Model info for token budgeting + # NOTE: max_input_tokens should match +generator.engine_init_kwargs.max_model_len + model_info: + max_input_tokens: 32768 + max_output_tokens: 32768 + # Set costs to 0 for local training + input_cost_per_token: 0.0 + output_cost_per_token: 0.0 + + # Additional kwargs passed through to LiteLLM's acompletion() call. + # These are forwarded via Terminus2 -> LiteLLM -> litellm.acompletion(**kwargs). + # Any parameter that litellm.acompletion() accepts can be set here, including: + # - timeout: LLM request timeout in seconds (default: litellm's 600s) + # - max_retries: Number of retries at the OpenAI SDK level (default: 2) + # - top_p, top_k, frequency_penalty, presence_penalty, etc. + llm_kwargs: + # LLM request timeout in seconds. Higher values prevent premature retries. + timeout: 900 + # Set to 0 to disable OpenAI SDK retries from LiteLLM. + max_retries: 0 + # Sampling params that directly get passed to LiteLLM's acompletion() call. + top_p: 1.0 + top_k: -1 + min_p: 0.0 + +# Environment configuration +# See: harbor.models.trial.config.EnvironmentConfig +environment: + # Environment type. Options: daytona, docker, e2b, modal, runloop, gke + type: daytona + + # Resource overrides (optional) + override_cpus: 1 + override_memory_mb: 1024 + override_storage_mb: 1024 + suppress_override_warnings: true + + # Environment-specific settings + kwargs: + # sandbox_timeout_secs: 1800 # Modal-specific. Timeout of the sandbox after which it is automatically stopped. + auto_stop_interval_mins: 30 # Daytona-specific. Minutes of inactivity before the sandbox is automatically stopped. + + +# Verifier configuration +# See: harbor.models.trial.config.VerifierConfig +verifier: + # Set to true to skip verification (useful for debugging) + disable: false + + # Override the verifier timeout (seconds) + # override_timeout_sec: 300 diff --git a/skyrl-train/examples/harbor/prepare_harbor_dataset.py b/skyrl-train/examples/harbor/prepare_harbor_dataset.py new file mode 100644 index 0000000000..88c44b1d8b --- /dev/null +++ b/skyrl-train/examples/harbor/prepare_harbor_dataset.py @@ -0,0 +1,157 @@ +""" +Prepare Harbor task datasets from HuggingFace Hub. + +Downloads a dataset and extracts Harbor tasks from parquet files containing +tar-archived task directories (columns: path, task_binary). + +Output directory defaults to ~/data/harbor/. + +Usage: + + python examples/harbor/prepare_harbor_dataset.py \ + --dataset open-thoughts/CodeContests +""" + +import argparse +import io +import os +import shutil +import tarfile +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path, PurePosixPath + +import pyarrow.parquet as pq + + +def _is_within(base: Path, target: Path) -> bool: + try: + return os.path.commonpath([str(base.resolve()), str(target.resolve())]) == str(base.resolve()) + except Exception: + return False + + +def _sanitize_tar_member_name(name: str) -> str: + p = PurePosixPath(name) + parts = [part for part in p.parts if part not in ("..", ".", "")] + while parts and parts[0] == "/": + parts.pop(0) + return str(PurePosixPath(*parts)) if parts else "" + + +def _safe_extract_tar(archive_bytes: bytes, dest_dir: Path) -> None: + dest_dir.mkdir(parents=True, exist_ok=True) + buf = io.BytesIO(archive_bytes) + with tarfile.open(fileobj=buf, mode="r:*") as tf: + for member in tf.getmembers(): + member_name = _sanitize_tar_member_name(member.name) + if not member_name or member_name.endswith("/"): + (dest_dir / member_name).mkdir(parents=True, exist_ok=True) + continue + if ".snapshot" in PurePosixPath(member_name).parts: + continue + target = (dest_dir / member_name).resolve() + if not _is_within(dest_dir, target): + raise RuntimeError(f"Unsafe path in archive: {member.name}") + target.parent.mkdir(parents=True, exist_ok=True) + if member.isfile(): + with tf.extractfile(member) as src: + if src is None: + continue + with open(target, "wb") as dst: + dst.write(src.read()) + elif member.isdir(): + target.mkdir(parents=True, exist_ok=True) + + +def _extract_one(args: tuple) -> bool: + """Extract a single task from its tar archive. Runs in a worker process.""" + rel_path, data, output_dir_str = args + if not isinstance(rel_path, str) or not isinstance(data, (bytes, bytearray, memoryview)): + return False + output_dir = Path(output_dir_str) + + safe_rel = PurePosixPath(rel_path) + parts = [p for p in safe_rel.parts if p not in ("..", "")] + rel_norm = Path(*parts) if parts else Path("task_unknown") + target_dir = (output_dir / rel_norm).resolve() + + if not _is_within(output_dir, target_dir): + return False + + if target_dir.exists() and (target_dir / "instruction.md").exists(): + return True + + try: + _safe_extract_tar(bytes(data), target_dir) + return True + except Exception as e: + print(f" Warning: Failed to extract {rel_path}: {e}") + return False + + +def extract_parquet(parquet_path: Path, output_dir: Path) -> int: + """Extract tasks from a parquet file with path + task_binary columns.""" + table = pq.read_table(parquet_path) + path_col = table.column("path").to_pylist() + data_col = table.column("task_binary").to_pylist() + + output_dir.mkdir(parents=True, exist_ok=True) + args = [(p, d, str(output_dir)) for p, d in zip(path_col, data_col)] + + with ProcessPoolExecutor(max_workers=8) as pool: + results = list(pool.map(_extract_one, args, chunksize=64)) + + return sum(results) + + +def prepare(dataset_name: str, output_dir: str | None = None) -> str: + from huggingface_hub import snapshot_download + + repo_name = dataset_name.split("/")[-1] if "/" in dataset_name else dataset_name + if output_dir is None: + output_dir = os.path.join("~/data/harbor", repo_name) + output_path = Path(os.path.expanduser(output_dir)).resolve() + + print(f"Downloading {dataset_name}...") + snapshot_dir = Path(snapshot_download(repo_id=dataset_name, repo_type="dataset")) + print(f"Downloaded to {snapshot_dir}") + + # Find parquet files with path + task_binary columns + parquets = [] + for f in snapshot_dir.glob("**/*.parquet"): + try: + schema = pq.read_schema(f) + if "path" in schema.names and "task_binary" in schema.names: + parquets.append(f) + except Exception as e: + print(f" Warning: Could not read schema from {f}: {e}") + continue + + if not parquets: + # No parquet files — dataset already contains task directories directly. + # Just symlink to the snapshot. + print("No parquet files found, symlinking snapshot directly...") + output_path.parent.mkdir(parents=True, exist_ok=True) + if output_path.is_symlink(): + output_path.unlink() + elif output_path.exists(): + shutil.rmtree(output_path) + output_path.symlink_to(snapshot_dir) + print(f"Done! Symlinked {output_path} -> {snapshot_dir}") + return str(output_path) + + total = 0 + for pq_file in parquets: + print(f"Extracting {pq_file.name}...") + total += extract_parquet(pq_file, output_path) + + print(f"Done! {total} tasks extracted to {output_path}") + return str(output_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Prepare Harbor task dataset from HuggingFace Hub") + parser.add_argument("--dataset", required=True, help="HuggingFace dataset (e.g. open-thoughts/CodeContests)") + parser.add_argument("--output_dir", default=None, help="Output directory (default: ~/data/harbor/)") + args = parser.parse_args() + prepare(args.dataset, args.output_dir) diff --git a/skyrl-train/examples/harbor/run_codecontest.sh b/skyrl-train/examples/harbor/run_codecontest.sh new file mode 100644 index 0000000000..be4d8ba0c6 --- /dev/null +++ b/skyrl-train/examples/harbor/run_codecontest.sh @@ -0,0 +1,113 @@ +set -ex + +# wandb api key. +# export WANDB_API_KEY=YOUR_KEY_HERE + +# Pick the sandbox provider and provide the credentials. +# export DAYTONA_API_KEY=YOUR_KEY_HERE +# export MODAL_TOKEN_ID=YOUR_KEY_HERE +# export MODAL_TOKEN_SECRET=YOUR_KEY_HERE + +#----------------------- +# Dataset setup +#----------------------- +# Prepare datasets first (downloads from HuggingFace and extracts tasks): +# python examples/harbor/prepare_harbor_dataset.py --dataset open-thoughts/CodeContests +# python examples/harbor/prepare_harbor_dataset.py --dataset open-thoughts/OpenThoughts-TB-dev + +DATA_DIR="$HOME/data/harbor" +TRAIN_DATA="['$DATA_DIR/CodeContests']" +EVAL_DATA="['$DATA_DIR/OpenThoughts-TB-dev']" + +#----------------------- +# Directory setup +#----------------------- +RUN_NAME="codecontest" +TRIALS_DIR="$HOME/$RUN_NAME/trials_run" +CKPTS_DIR="$HOME/$RUN_NAME/ckpts" +EXPORTS_DIR="$HOME/$RUN_NAME/exports" +LOG_DIR="/tmp/skyrl-logs/$RUN_NAME" + +#----------------------- +# Training setup +#----------------------- +MINI_BATCH_SIZE=32 +MAX_MODEL_LEN=32768 +APPLY_OVERLONG_FILTERING=true + +# Dr. GRPO parameters +LOSS_REDUCTION="seq_mean_token_sum_norm" +GRPO_NORM_BY_STD=false +USE_KL_LOSS=false + +# Essentially achieves interleaved thinking and hence on-policy training without step-wise training. +CHAT_TEMPLATE_PATH="$(dirname "$0")/../../skyrl_train/utils/templates/qwen3_acc_thinking.jinja2" + +#---------------- +# Infrastructure setup +#---------------- +NUM_GPUS=4 +ENABLE_RATE_LIMITING=true # Enable rate/concurrency limiting for trajectory submissions +TRAJECTORIES_PER_SECOND=5 # Maximum trajectories per second (must be >= 1.0, fractional values like 1.5 are supported). null or omit to disable rate limiting +MAX_CONCURRENCY=512 # Maximum concurrent trial.run() calls allowed (must be >= 1). null or omit to disable concurrency limiting + +# Run SkyRL command +uv run --isolated --extra vllm --extra harbor -m examples.harbor.entrypoints.main_harbor \ + data.train_data=$TRAIN_DATA \ + data.val_data=$EVAL_DATA \ + trainer.policy.model.path=Qwen/Qwen3-8B \ + generator.served_model_name=Qwen3-8B \ + hydra.searchpath=['file://examples/harbor'] \ + +harbor_trial_config=default \ + ++harbor_trial_config.trials_dir=$TRIALS_DIR \ + trainer.export_path=$EXPORTS_DIR \ + trainer.ckpt_path=$CKPTS_DIR \ + trainer.log_path=$LOG_DIR \ + trainer.algorithm.advantage_estimator=grpo \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + trainer.algorithm.grpo_norm_by_std=$GRPO_NORM_BY_STD \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_nodes=1 \ + trainer.placement.ref_num_nodes=1 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + +generator.engine_init_kwargs.chat_template=$CHAT_TEMPLATE_PATH \ + +generator.engine_init_kwargs.max_model_len=$MAX_MODEL_LEN \ + +generator.engine_init_kwargs.enable_log_requests=false \ + trainer.epochs=3 \ + trainer.eval_batch_size=128 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$MINI_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=5 \ + trainer.hf_save_interval=5 \ + trainer.algorithm.max_seq_len=$MAX_MODEL_LEN \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + generator.n_samples_per_prompt=8 \ + generator.eval_n_samples_per_prompt=4 \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger=wandb \ + trainer.project_name=harbor \ + trainer.run_name=$RUN_NAME \ + trainer.resume_mode=latest \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.enforce_eager=false \ + generator.enable_http_endpoint=true \ + generator.http_endpoint_host=127.0.0.1 \ + generator.http_endpoint_port=8000 \ + +generator.rate_limit.enabled=$ENABLE_RATE_LIMITING \ + +generator.rate_limit.trajectories_per_second=$TRAJECTORIES_PER_SECOND \ + +generator.rate_limit.max_concurrency=$MAX_CONCURRENCY diff --git a/skyrl-train/examples/harbor/run_harbor_gen.sh b/skyrl-train/examples/harbor/run_harbor_gen.sh new file mode 100644 index 0000000000..ce7d1f0b53 --- /dev/null +++ b/skyrl-train/examples/harbor/run_harbor_gen.sh @@ -0,0 +1,59 @@ +set -ex + +# wandb api key. +# export WANDB_API_KEY=YOUR_KEY_HERE + +# Pick the sandbox provider and provide the credentials. +# export DAYTONA_API_KEY=YOUR_KEY_HERE +# export MODAL_TOKEN_ID=YOUR_KEY_HERE +# export MODAL_TOKEN_SECRET=YOUR_KEY_HERE + +# Prepare dataset first (downloads from HuggingFace and extracts tasks): +# python examples/harbor/prepare_harbor_dataset.py --dataset open-thoughts/CodeContests + +DATA_DIR="$HOME/data/harbor" +TRAIN_DATA="['$DATA_DIR/CodeContests']" + +CHAT_TEMPLATE_PATH="$(dirname "$0")/../../skyrl_train/utils/templates/qwen3_acc_thinking.jinja2" +TRIALS_DIR="$HOME/trials_run" + +#---------------- +# Infrastructure setup +#---------------- +NUM_GPUS=4 +ENABLE_RATE_LIMITING=true # Enable rate/concurrency limiting for trajectory submissions +TRAJECTORIES_PER_SECOND=5 # Maximum trajectories per second (must be >= 1.0, fractional values like 1.5 are supported). null or omit to disable rate limiting +MAX_CONCURRENCY=512 # Maximum concurrent trial.run() calls allowed (must be >= 1). null or omit to disable concurrency limiting + + +uv run --isolated --extra vllm --extra harbor -m examples.harbor.entrypoints.main_harbor_generate \ + data.train_data=$TRAIN_DATA \ + hydra.searchpath=['file://examples/harbor'] \ + +harbor_trial_config=default \ + ++harbor_trial_config.trials_dir=$TRIALS_DIR \ + ++harbor_trial_config.trial_name="dummy" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + generator.served_model_name="Qwen2.5-1.5B-Instruct" \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + generator.enable_http_endpoint=true \ + generator.http_endpoint_host="127.0.0.1" \ + generator.http_endpoint_port=8000 \ + generator.sampling_params.max_generate_length=4096 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.gpu_memory_utilization=0.8 \ + +generator.engine_init_kwargs.chat_template=$CHAT_TEMPLATE_PATH \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.placement.colocate_all=false \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + trainer.train_batch_size=$NUM_GPUS \ + trainer.policy_mini_batch_size=$NUM_GPUS \ + trainer.logger=console \ + +generator.rate_limit.enabled=$ENABLE_RATE_LIMITING \ + +generator.rate_limit.trajectories_per_second=$TRAJECTORIES_PER_SECOND \ + +generator.rate_limit.max_concurrency=$MAX_CONCURRENCY \ + $@ diff --git a/skyrl-train/examples/livecodebench/lcb.md b/skyrl-train/examples/livecodebench/lcb.md new file mode 100644 index 0000000000..d0ee01d2b1 --- /dev/null +++ b/skyrl-train/examples/livecodebench/lcb.md @@ -0,0 +1,14 @@ +# DeepCoder + LCB Run + +## Download Dataset +``` +pip install gdown + +python examples/livecodebench/lcb_download.py --local_dir ~/data/lcb/download + +python examples/livecodebench/lcb_dataset.py --dataset_dir ~/data/lcb/download --local_dir ~/data/lcb/ +``` + +## Note +* Read from the json file instead of parquet +* Need to truncate the json file otherwise it is too large with normal `datasets.load_dataset()`, need to use streaming or load with PyArrow directly \ No newline at end of file diff --git a/skyrl-train/examples/livecodebench/lcb_dataset.py b/skyrl-train/examples/livecodebench/lcb_dataset.py new file mode 100644 index 0000000000..396d5a8ec8 --- /dev/null +++ b/skyrl-train/examples/livecodebench/lcb_dataset.py @@ -0,0 +1,200 @@ +"""Script to prepare code datasets for training and testing. + +This script processes code problem datasets into a standardized format for training +and testing models. It loads problems from various code datasets (APPS, CodeForces, +LiveCodeBench etc.), adds appropriate instruction prompts, and saves the processed +data as parquet files. +""" + +import argparse +import json +import os +from typing import Any, Dict, List, Optional + +import pandas as pd + +"""Utility functions for loading and processing datasets.""" + +LCB_SYSTEM_MESSAGE_GENERIC = "You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests." + +LCB_FORMATTING_MESSAGE_WITH_STARTER_CODE = "You will use the following starter code to write the solution to the problem and enclose your code within delimiters." + +LCB_FORMATTING_WITHOUT_STARTER_CODE = "Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows. Ensure that when the python program runs, it reads the inputs, runs the algorithm and writes output to STDOUT." + + +# Dataset constants +LIVECODEBENCH = "livecodebench" + + +def load_dataset(dataset_name: str, split: str, base_dir: Optional[str] = None) -> List[Dict[str, Any]]: + """Load a dataset from a JSON file. + + Args: + dataset_name: Name of the dataset (e.g., "livecodebench") + split: Either "train" or "test" + base_dir: Base directory containing the dataset files + + Returns: + List[Dict[str, Any]]: A list of dictionaries containing the dataset items. + + Raises: + ValueError: If the dataset file cannot be found or contains invalid JSON. + """ + if base_dir is None: + current_dir = os.path.dirname(os.path.realpath(__file__)) + else: + current_dir = base_dir + + file_path = os.path.join(current_dir, split, "code", f"{dataset_name}.json") + + if not os.path.exists(file_path): + raise ValueError(f"Dataset file not found: {file_path}") + + try: + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + return data + except json.JSONDecodeError: + raise ValueError(f"Invalid JSON format in {file_path}") + except Exception as e: + raise ValueError(f"Error loading dataset: {str(e)}") + + +def fetch_live_code_bench_system_prompt(prompt: str, starter_code: str = None): + # https://github.com/LiveCodeBench/LiveCodeBench/blob/main/lcb_runner/prompts/code_generation.py + prompt = LCB_SYSTEM_MESSAGE_GENERIC + "\n\n" + prompt + if starter_code: + prompt += f"### Format: {LCB_FORMATTING_MESSAGE_WITH_STARTER_CODE}\n" + prompt += f"```python\n{starter_code}\n```\n\n" + else: + prompt += f"### Format: {LCB_FORMATTING_WITHOUT_STARTER_CODE}\n" + prompt += "```python\n# YOUR CODE HERE\n```\n\n" + prompt += "### Answer: (use the provided format with backticks)\n\n" + return prompt + + +def process_example(example: Dict[str, Any], idx: int, dataset_name: str, split: str) -> Optional[Dict[str, Any]]: + """Process a single dataset example. + + Args: + example: Raw dataset example + idx: Index of the example + dataset_name: Name of the dataset + split: Dataset split ('train' or 'test') + + Returns: + Processed example dictionary or None if processing fails + """ + question = example.pop("problem") + tests = example.pop("tests") + + if example.get("metadata", {}): + assert ( + "func_name" in example["metadata"] + ), f"Function name is not found, check if your LCB data is preprocessed correctly: {example['metadata']}" + if isinstance(tests, dict): + tests["metadata"] = example["metadata"] + else: + for test in tests: + assert isinstance(test, dict), "Test is not a dict" + test["metadata"] = example["metadata"] + + tests = json.dumps(tests) + + if dataset_name == LIVECODEBENCH: + starter_code = example.get("starter_code", None) + question = fetch_live_code_bench_system_prompt(question, starter_code) + if isinstance(question, dict): + question = json.dumps(question) + data = { + "data_source": dataset_name, + "prompt": [{"role": "user", "content": question}], + "ability": "code", + "reward_spec": {"method": "rule", "ground_truth": tests}, + "extra_info": { + "split": split, + "index": idx, + "reference": example.get("completion", None), # For leetcode + }, + } + return data + + +def process_dataset(dataset_name: str, split: str, dataset_dir: str, local_dir: str, max_rows: Optional[int] = None): + """Process a dataset for a given split. + + Args: + dataset_name: Name of the dataset + split: Either "train" or "test" + dataset_dir: Directory containing raw datasets + local_dir: Directory to save processed datasets + max_rows: Maximum number of rows to process + """ + # Load dataset + raw_data = load_dataset(dataset_name, split, base_dir=dataset_dir) + print(f"{split.capitalize()} dataset {dataset_name}: {len(raw_data)} examples") + + # Process examples + processed_data = [] + for idx, example in enumerate(raw_data): + processed_example = process_example(example, idx, dataset_name, split) + if processed_example is not None: + processed_data.append(processed_example) + + # Apply max_rows limit if specified + if max_rows: + processed_data = processed_data[:max_rows] + + # Save individual dataset files + df = pd.DataFrame(processed_data) + parquet_path = os.path.join(local_dir, f"{split}_{dataset_name}.parquet") + json_path = os.path.join(local_dir, f"{split}_{dataset_name}.json") + + print(f"Writing {len(df)} rows to {split}_{dataset_name}.parquet") + df.to_parquet(parquet_path) + + if split == "test": + print(f"Writing {len(df)} rows to {split}_{dataset_name}.json") + df.to_json(json_path, orient="records") + + return processed_data + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process datasets for DeepScaler training") + parser.add_argument( + "--local_dir", + default=os.path.expanduser("~/data/lcb"), + help="Local directory to save processed datasets", + ) + parser.add_argument( + "--dataset_dir", + default=os.path.expanduser("~/data/lcb"), + help="Directory containing raw input datasets", + ) + parser.add_argument( + "--max_rows", + type=int, + default=None, + help="Maximum number of rows to include in output files (truncate if set).", + ) + args = parser.parse_args() + + local_dir = args.local_dir + print(f"Local_dir:{local_dir}") + + # Make local directory if it doesn't exist + if not os.path.exists(local_dir): + os.makedirs(local_dir, exist_ok=True) + + # Process train dataset + train_data = process_dataset(LIVECODEBENCH, "train", args.dataset_dir, local_dir, args.max_rows) + + # Process test dataset + val_data = process_dataset(LIVECODEBENCH, "test", args.dataset_dir, local_dir, args.max_rows) + + # Save combined train dataset + all_train_df = pd.DataFrame(train_data) + all_train_df.to_parquet(os.path.join(local_dir, "deepcoder_train.parquet")) + all_train_df.to_json(os.path.join(local_dir, "deepcoder_train.json"), orient="records") + print(f"Writing {len(all_train_df)} rows to deepcoder_train.parquet and deepcoder_train.json") diff --git a/skyrl-train/examples/livecodebench/lcb_download.py b/skyrl-train/examples/livecodebench/lcb_download.py new file mode 100644 index 0000000000..5ba0a29b76 --- /dev/null +++ b/skyrl-train/examples/livecodebench/lcb_download.py @@ -0,0 +1,44 @@ +import os +import gdown +import shutil +import argparse + +# Define the Google Drive file IDs for the JSON files +FILE_IDS = { + "train_livecodebench.json": "1-lKdRfRjytdTltgLyAxTqVRoksI2cJfU", + "test_livecodebench.json": "1B0sotl48BLd4gqlitL5HVJf1cy3RxpEV", +} + + +def main(): + parser = argparse.ArgumentParser(description="Download LCB dataset from Google Drive") + parser.add_argument( + "--local_dir", type=str, required=True, help="Base destination directory to store downloaded JSON files" + ) + args = parser.parse_args() + + dest_dir = os.path.expanduser(args.local_dir) + + # Ensure destination subdirectories exist + for split in ["train", "test"]: + path = os.path.join(dest_dir, split, "code") + os.makedirs(path, exist_ok=True) + + # Download and move each file + for filename, file_id in FILE_IDS.items(): + split = "train" if "train" in filename else "test" + dest_dir_path = os.path.join(dest_dir, split, "code") + temp_path = os.path.join(dest_dir_path, f"temp_{filename}") # Temporary download location in dest directory + dest_path = os.path.join(dest_dir_path, "livecodebench.json") + + print(f"Downloading {filename}...") + gdown.download(f"https://drive.google.com/uc?id={file_id}", temp_path, quiet=False) + + print(f"Moving {filename} to {dest_path}...") + shutil.move(temp_path, dest_path) + + print("All files downloaded and moved successfully.") + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/livecodebench/run_lcb.sh b/skyrl-train/examples/livecodebench/run_lcb.sh new file mode 100644 index 0000000000..06891578cb --- /dev/null +++ b/skyrl-train/examples/livecodebench/run_lcb.sh @@ -0,0 +1,53 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-3B-Instruct on SearchR1 data. +# export WANDB_API_KEY= +# bash examples/livecodebench/run_lcb.sh + +DATA_DIR="$HOME/data/lcb" +train_data="['${DATA_DIR}/deepcoder_train.json']" +val_data="['${DATA_DIR}/test_livecodebench.json']" + +# NOTE (sumanthrh): micro_train_batch_size and micro_forward_batch_size can be tuned +uv run --isolated --frozen --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data=$train_data \ + data.val_data=$val_data \ + trainer.policy.model.path="Qwen/Qwen2.5-3B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.placement.policy_num_gpus_per_node=8 \ + trainer.placement.ref_num_gpus_per_node=8 \ + generator.num_inference_engines=2 \ + generator.inference_engine_tensor_parallel_size=4 \ + trainer.policy_mini_batch_size=4 \ + trainer.train_batch_size=16 \ + trainer.micro_forward_batch_size_per_gpu=16 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.max_prompt_length=29000 \ + generator.max_input_length=29000 \ + generator.sampling_params.max_generate_length=3000 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.kl_loss_coef=0.001 \ + trainer.ckpt_interval=100000 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=lcb \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + trainer.logger="wandb" \ + trainer.project_name="skyrl" \ + trainer.run_name="skyrlcode_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/lcb_3B_ckpt" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ diff --git a/skyrl-train/examples/llm_as_a_judge/gsm8k_dataset_judge.py b/skyrl-train/examples/llm_as_a_judge/gsm8k_dataset_judge.py new file mode 100644 index 0000000000..a50d72fdce --- /dev/null +++ b/skyrl-train/examples/llm_as_a_judge/gsm8k_dataset_judge.py @@ -0,0 +1,90 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the GSM8k dataset to parquet format +""" + +import argparse +import re +import os + +import datasets + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", default="~/data/gsm8k_llm_judge") + + args = parser.parse_args() + + args.output_dir = os.path.expanduser(args.output_dir) + + data_source = "openai/gsm8k" + + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + val_dataset = dataset["test"] + + instruction_following = 'Let\'s think step by step and output the final answer after "####".' + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + solution = extract_solution(answer_raw) + data = { + "data_source": data_source, + "prompt": [ + { + "role": "user", + "content": question, + } + ], + # TODO: just repeating the full data preprocess script for a single env change isn't very convenient. + "env_class": "llm_as_a_judge", + "reward_spec": { + "method": "rule", + "ground_truth": solution, + }, + "extra_info": { + "split": split, + "index": idx, + "answer": answer_raw, + "question": question_raw, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + val_dataset = val_dataset.map(function=make_map_fn("test"), with_indices=True) + + output_dir = args.output_dir + os.makedirs(output_dir, exist_ok=True) + train_dataset.to_parquet(os.path.join(output_dir, "train.parquet")) + val_dataset.to_parquet(os.path.join(output_dir, "validation.parquet")) diff --git a/skyrl-train/examples/llm_as_a_judge/llm_judge_env.py b/skyrl-train/examples/llm_as_a_judge/llm_judge_env.py new file mode 100644 index 0000000000..a006f9c7a9 --- /dev/null +++ b/skyrl-train/examples/llm_as_a_judge/llm_judge_env.py @@ -0,0 +1,84 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from typing import Any +from typing import Dict +from omegaconf import DictConfig +from openai import OpenAI +import os +import re + +PROMPT = """ +You are a strict math evaluation assistant. + +Compare the following **gold** and **predicted** math solutions. Your job is to determine if the predicted solution is mathematically correct and if the predicted solution ends with a line of the form: + +#### + +You must only give a score of "1" if: +- The final line of the predicted solution **ends with `#### `**, and +- The number **matches the final answer in the gold solution** exactly. + +Instructions: +- You may provide internal reasoning or explanation before giving your final judgment. +- Your final judgment must appear as a separate line at the end of your response, in the format: + +### Final Score: 1 + +or + +### Final Score: 0 + +Do not include any explanation after the final score. +""" + + +class GSM8kLLMJudgeEnv(BaseTextEnv): + """ + Example implementtion of GSM8k environment with LLM as judge. + + Use LLM as judge to evaluate the answer similarity with the ground truth. + """ + + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + + assert "reward_spec" in extras, "reward_spec field is required" + assert "ground_truth" in extras["reward_spec"], "ground_truth is required in reward_spec field" + self.ground_truth = extras["reward_spec"]["ground_truth"] + + # Set up OpenAI client + openai_api_key = os.getenv("OPENAI_API_KEY") + if openai_api_key is None: + raise ValueError("`OPENAI_API_KEY` must be set for Llm as a judge env") + self.llm_judge_client = OpenAI(base_url=env_config.base_url, api_key=openai_api_key) + self.model = env_config.model + + def _get_reward(self, action: str) -> float: + message = PROMPT + f"\n\nGOLD SOLUTION:\n{self.ground_truth}\n\nPREDICTED SOLUTION:\n{action}\n\nAnswer:" + + try: + response = self.llm_judge_client.chat.completions.create( + model=self.model, messages=[{"role": "user", "content": message}] + ) + reply = response.choices[0].message.content.strip() + + # Try to parse score from "### Final Score: x" + match = re.search(r"### Final Score:\s*([01](?:\.0)?)", reply) + if match: + return float(match.group(1)) + + # Fallback: raw "1" or "0" + if reply.strip() in {"1", "0"}: + return float(reply.strip()) + + print(f"Unrecognized reward output: {reply}") + return 0.0 + + except Exception as e: + print(f"LLM Judge error: {type(e).__name__}: {e}") + return 0.0 + + def step(self, action: str) -> BaseTextEnvStepOutput: + done = True + reward = self._get_reward(action) + + return BaseTextEnvStepOutput(observations=[], reward=reward, done=done, metadata={}) diff --git a/skyrl-train/examples/llm_as_a_judge/main_llm_judge.py b/skyrl-train/examples/llm_as_a_judge/main_llm_judge.py new file mode 100644 index 0000000000..dee1e52dd5 --- /dev/null +++ b/skyrl-train/examples/llm_as_a_judge/main_llm_judge.py @@ -0,0 +1,36 @@ +""" +Main entrypoint for the LLM-as-a-judge example. +""" + +import ray +import hydra +from omegaconf import DictConfig +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_gym.envs import register + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # Register the llm_as_a_judge environment inside the entrypoint task (no need to modify the skyrl-gym package). + register( + id="llm_as_a_judge", + entry_point="examples.llm_as_a_judge.llm_judge_env:GSM8kLLMJudgeEnv", + ) + + # make sure that the training loop is not run on the head node. + exp = BasePPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/llm_as_a_judge/run_llm_judge.sh b/skyrl-train/examples/llm_as_a_judge/run_llm_judge.sh new file mode 100644 index 0000000000..d3a468cbc6 --- /dev/null +++ b/skyrl-train/examples/llm_as_a_judge/run_llm_judge.sh @@ -0,0 +1,57 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-1.5B-Instruct on GSM8k dataset. +# Uses 1 node with 4 GPUs. +# uv run examples/llm_as_a_judge/gsm8k_dataset_judge.py --output_dir $HOME/data/gsm8k_llm_judge +# add OPENAI_API_KEY and WANDB_API_KEY to .env.llm_judge +# bash examples/llm_as_a_judge/run_llm_judge.sh + +DATA_DIR="$HOME/data/gsm8k_llm_judge" +CKPT_PATH="$HOME/ckpts/llm_judge" + +NUM_GPUS=4 +NUM_INFERENCE_ENGINES=4 +TP_SIZE=1 +LOGGER=wandb + +# We use a smaller batch size here for demonstration +uv run --isolated --extra vllm --env-file .env.llm_judge -m examples.llm_as_a_judge.main_llm_judge \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=32 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=32 \ + trainer.policy_mini_batch_size=32 \ + trainer.micro_forward_batch_size_per_gpu=40 \ + trainer.micro_train_batch_size_per_gpu=40 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_llm_as_a_judge" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + environment.env_class=llm_as_a_judge \ + environment.skyrl_gym.llm_as_a_judge.model="gpt-4o-mini" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/lora/run_qwen2_5_0.5b_gsm8k_grpo_lora.sh b/skyrl-train/examples/lora/run_qwen2_5_0.5b_gsm8k_grpo_lora.sh new file mode 100644 index 0000000000..9aa25b7866 --- /dev/null +++ b/skyrl-train/examples/lora/run_qwen2_5_0.5b_gsm8k_grpo_lora.sh @@ -0,0 +1,60 @@ +set -x + +set -x + +# Colocated GRPO LoRA training + generation for Qwen2.5-0.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/lora/run_qwen2_5_0.5b_gsm8k_grpo_lora.sh + +# NOTE (sumanthrh): `micro_train_batch_size_per_gpu` and `micro_forward_batch_size_per_gpu` can be tuned + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +INFERENCE_BACKEND="vllm" + + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.policy.model.lora.rank=32 \ + trainer.policy.model.lora.alpha=32 \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=3.0e-5 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_0.5b_lora" \ + trainer.run_name="gsm8k_0.5b_lora_grpo" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_0.5b_lora_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/lora/run_qwen2_5_0.5b_gsm8k_ppo_lora.sh b/skyrl-train/examples/lora/run_qwen2_5_0.5b_gsm8k_ppo_lora.sh new file mode 100644 index 0000000000..4405b32300 --- /dev/null +++ b/skyrl-train/examples/lora/run_qwen2_5_0.5b_gsm8k_ppo_lora.sh @@ -0,0 +1,61 @@ +set -x + +# Colocated PPO LoRA training + generation for Qwen2.5-0.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/lora/run_qwen2_5_0.5b_gsm8k_ppo_lora.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +INFERENCE_BACKEND="vllm" + + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="gae" \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.critic.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.policy.model.lora.rank=32 \ + trainer.policy.model.lora.alpha=32 \ + trainer.critic.model.lora.rank=32 \ + trainer.critic.model.lora.alpha=32 \ + trainer.strategy=fsdp2 \ + trainer.placement.colocate_all=true \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.critic_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=3.0e-5 \ + trainer.critic.optimizer_config.lr=3.0e-5 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_0.5b_lora" \ + trainer.run_name="gsm8k_0.5b_lora_ppo" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_0.5b_lora_ckpt" \ + $@ diff --git a/skyrl-train/examples/megatron/run_fsdp_baseline.sh b/skyrl-train/examples/megatron/run_fsdp_baseline.sh new file mode 100644 index 0000000000..5291b5d960 --- /dev/null +++ b/skyrl-train/examples/megatron/run_fsdp_baseline.sh @@ -0,0 +1,53 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-0.6B on GSM8K with FSDP. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/megatron/run_fsdp_baseline.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +INFERENCE_BACKEND="vllm" + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-0.6B" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=4.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_megatron" \ + trainer.run_name="gsm8k_fsdp1_4gpus" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_fsdp_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron.sh b/skyrl-train/examples/megatron/run_megatron.sh new file mode 100644 index 0000000000..cf0d9f9eda --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron.sh @@ -0,0 +1,73 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-0.6B on GSM8K with Megatron. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/megatron/run_megatron.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout +MODEL_NAME="Qwen/Qwen3-0.6B" + +INFERENCE_BACKEND="vllm" # currently only vllm is supported for megatron + +MEGATRON_TP=2 +MEGATRON_PP=2 +MEGATRON_CP=1 + +# torch profiler config +ENABLE_TORCH_PROFILER=false +RANKS_TO_PROFILE="[0]" +SAVE_PATH="$HOME/megatron_prof/tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_${MODEL_NAME}" + +uv run --isolated --extra mcore -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.policy.megatron_config.torch_profiler_config.enable=$ENABLE_TORCH_PROFILER \ + trainer.policy.megatron_config.torch_profiler_config.ranks=$RANKS_TO_PROFILE \ + trainer.policy.megatron_config.torch_profiler_config.save_path=$SAVE_PATH \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.ref.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.use_sample_packing=true \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_megatron" \ + trainer.run_name="gsm8k_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_${MODEL_NAME}" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_megatron_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_30b_a3b.sh b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_30b_a3b.sh new file mode 100644 index 0000000000..d2af35b5b8 --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_30b_a3b.sh @@ -0,0 +1,123 @@ +set -x + +# Colocated DAPO training+generation for Qwen3-30B-A3B-Base on DAPO with Megatron. +# Should run on 2 node of 8xH100s + +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/megatron/run_megatron_dapo_qwen3_30b_a3b.sh + +MODEL_NAME="Qwen/Qwen3-30B-A3B-Base" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=2 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=2 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=8 +LOGGER="wandb" # change to "console" to print to stdout + +CLIP_RATIO_LOW=0.2 +CLIP_RATIO_HIGH=0.28 +# use token mean loss reduction +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +LR=1e-6 + +# megatron config +MEGATRON_TP=4 +MEGATRON_PP=1 +MEGATRON_CP=1 +MEGATRON_EP=8 +MEGATRON_ETP=1 + + +# TIS parameters +TIS_IMP_RATIO_CAP=2.0 +USE_TIS=true + +uv run --isolated --extra mcore -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.epochs=20 \ + trainer.algorithm.eps_clip_low=$CLIP_RATIO_LOW \ + trainer.algorithm.eps_clip_high=$CLIP_RATIO_HIGH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.7 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen3_30b_a3b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}" \ + trainer.export_path="$HOME/exports/dapo_qwen3_30b_a3b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}" \ + trainer.hf_save_interval=300 \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen3_30b_a3b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_30b_a3b_lora.sh b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_30b_a3b_lora.sh new file mode 100644 index 0000000000..7ae14bc4df --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_30b_a3b_lora.sh @@ -0,0 +1,128 @@ +set -x + +# Colocated DAPO training+generation for Qwen3-30B-A3B-Base on DAPO with Megatron and LoRA. +# Should run on 2 node of 8xH100s + +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/megatron/run_megatron_dapo_qwen3_30b_a3b_lora.sh + +MODEL_NAME="Qwen/Qwen3-30B-A3B-Base" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=2 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=2 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=8 +LOGGER="wandb" # change to "console" to print to stdout + +CLIP_RATIO_LOW=0.2 +CLIP_RATIO_HIGH=0.28 +# use token mean loss reduction +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +LR=3e-5 + +# megatron config +MEGATRON_TP=4 +MEGATRON_PP=1 +MEGATRON_CP=1 +MEGATRON_EP=8 +MEGATRON_ETP=1 + +# lora config +LORA_RANK=32 +LORA_ALPHA=64 + +# TIS parameters +TIS_IMP_RATIO_CAP=2.0 +USE_TIS=true + +uv run --isolated --extra mcore -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.policy.model.lora.rank=$LORA_RANK \ + trainer.policy.model.lora.alpha=$LORA_ALPHA \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.epochs=20 \ + trainer.algorithm.eps_clip_low=$CLIP_RATIO_LOW \ + trainer.algorithm.eps_clip_high=$CLIP_RATIO_HIGH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.7 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen3_30b_a3b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + trainer.export_path="$HOME/exports/dapo_qwen3_30b_a3b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + trainer.hf_save_interval=300 \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen3_30b_a3b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_4b.sh b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_4b.sh new file mode 100644 index 0000000000..014ee567fe --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_4b.sh @@ -0,0 +1,120 @@ +set -x + +# Colocated DAPO training+generation for Qwen3-4B-Base on DAPO training data with Megatron. +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/megatron/run_megatron_dapo_qwen3_4b.sh + +MODEL_NAME="Qwen/Qwen3-4B-Base" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=1 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=4 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=2 +LOGGER="wandb" # change to "console" to print to stdout + +CLIP_RATIO_LOW=0.2 +CLIP_RATIO_HIGH=0.28 +# use token mean loss reduction +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +LR=1e-6 + +# megatron config +MEGATRON_TP=4 +MEGATRON_PP=2 +MEGATRON_CP=1 +MEGATRON_EP=1 +MEGATRON_ETP=null + +# TIS parameters +TIS_IMP_RATIO_CAP=2.0 +USE_TIS=true + +uv run --isolated --extra mcore -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.epochs=20 \ + trainer.algorithm.eps_clip_low=$CLIP_RATIO_LOW \ + trainer.algorithm.eps_clip_high=$CLIP_RATIO_HIGH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen3_4b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}" \ + trainer.export_path="$HOME/exports/dapo_qwen3_4b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}" \ + trainer.hf_save_interval=25 \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen3_4b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_4b_lora.sh b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_4b_lora.sh new file mode 100644 index 0000000000..25dc8e6d30 --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_dapo_qwen3_4b_lora.sh @@ -0,0 +1,128 @@ +set -x + +# Colocated DAPO training+generation for Qwen3-4B-Base on DAPO training data with Megatron and LoRA. +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/megatron/run_megatron_dapo_qwen3_4b_lora.sh + +MODEL_NAME="Qwen/Qwen3-4B-Base" +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +NUM_NODES=1 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=4 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=2 +LOGGER="wandb" # change to "console" to print to stdout + +CLIP_RATIO_LOW=0.2 +CLIP_RATIO_HIGH=0.28 +# use token mean loss reduction +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=$((1024 * 4)) +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_PROMPT_LENGTH=$((1024 * 2)) +MAX_RESPONSE_LENGTH=$((1024 * 8)) + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=32 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true # cuda graphs can cause some instability +LR=3e-5 + +# megatron config +MEGATRON_TP=4 +MEGATRON_PP=1 +MEGATRON_CP=1 +MEGATRON_EP=1 +MEGATRON_ETP=null + +# lora config +LORA_RANK=32 +LORA_ALPHA=64 +LORA_A_INIT_METHOD="kaiming" + +# TIS parameters +TIS_IMP_RATIO_CAP=2.0 +USE_TIS=true + +uv run --isolated --extra mcore -m examples.algorithms.dapo.main_dapo \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.policy.model.path="$MODEL_NAME" \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.policy.model.lora.rank=$LORA_RANK \ + trainer.policy.model.lora.alpha=$LORA_ALPHA \ + trainer.policy.model.lora.init_method=$LORA_A_INIT_METHOD \ + trainer.epochs=20 \ + trainer.algorithm.eps_clip_low=$CLIP_RATIO_LOW \ + trainer.algorithm.eps_clip_high=$CLIP_RATIO_HIGH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=160 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="dapo_aime" \ + trainer.run_name="dapo_qwen3_4b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + trainer.export_path="$HOME/exports/dapo_qwen3_4b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + trainer.hf_save_interval=300 \ + trainer.resume_mode=latest \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_path="$HOME/ckpts/dapo_qwen3_4b_base_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_lora_qwen3-0.6b.sh b/skyrl-train/examples/megatron/run_megatron_lora_qwen3-0.6b.sh new file mode 100644 index 0000000000..02cf41c70d --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_lora_qwen3-0.6b.sh @@ -0,0 +1,76 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-0.6B on GSM8K with Megatron and LoRA. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/megatron/run_megatron_lora_qwen3-0.6b.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=8 +LOGGER="wandb" # change to "console" to print to stdout +MODEL_NAME="Qwen/Qwen3-0.6B" + +INFERENCE_BACKEND="vllm" # currently only vllm is supported for megatron + +MEGATRON_TP=1 +MEGATRON_PP=1 +MEGATRON_CP=1 + +# LoRA configuration +LORA_RANK=32 +LORA_ALPHA=64 +LORA_A_INIT_METHOD="kaiming" + + +uv run --isolated --extra mcore -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.ref.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.model.lora.rank=$LORA_RANK \ + trainer.policy.model.lora.alpha=$LORA_ALPHA \ + trainer.policy.model.lora.init_method=$LORA_A_INIT_METHOD \ + trainer.gradient_checkpointing=true \ + trainer.policy.model.lora.target_modules="all-linear" \ + trainer.use_sample_packing=true \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-5 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_megatron" \ + trainer.run_name="gsm8k_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_${MODEL_NAME}_lora_r${LORA_RANK}_a${LORA_ALPHA}" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_megatron_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_lora_qwen3-30b-a3b.sh b/skyrl-train/examples/megatron/run_megatron_lora_qwen3-30b-a3b.sh new file mode 100644 index 0000000000..6c2f3a899b --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_lora_qwen3-30b-a3b.sh @@ -0,0 +1,103 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-30B-A3B on GSM8K with Megatron. +# Should run on 1 node of 8xH100s + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/megatron/run_megatron_lora_qwen3-30b-a3b.sh + +DATA_DIR="$HOME/data/gsm8k" +LOGGER="wandb" # change to "console" to print to stdout +MODEL_NAME="Qwen/Qwen3-30B-A3B" + +INFERENCE_BACKEND="vllm" # currently only vllm is supported for megatron + +NUM_NODES=1 +NUM_GPUS=8 + +MEGATRON_TP=4 +MEGATRON_PP=1 +MEGATRON_CP=1 +MEGATRON_EP=8 +MEGATRON_ETP=1 + +NUM_INFERENCE_ENGINES=1 +INFERENCE_ENGINE_TP=8 +FLASH_ATTN=true + +# Megatron gradient checkpointing config +RECOMPUTE_GRANULARITY="full" +RECOMPUTE_METHOD="uniform" +RECOMPUTE_NUM_LAYERS=1 + +# LoRA configuration +LORA_RANK=64 +LORA_ALPHA=64 +LORA_A_INIT_METHOD="kaiming" + +# TIS parameters +TIS_IMP_RATIO_CAP=2.0 +USE_TIS=true + +uv run --isolated --extra mcore -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.ref_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TP \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.ref.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.ref.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.ref.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.policy.model.lora.rank=$LORA_RANK \ + trainer.policy.model.lora.alpha=$LORA_ALPHA \ + trainer.policy.model.lora.init_method=$LORA_A_INIT_METHOD \ + trainer.policy.megatron_config.transformer_config_kwargs.recompute_granularity=$RECOMPUTE_GRANULARITY \ + trainer.policy.megatron_config.transformer_config_kwargs.recompute_method=$RECOMPUTE_METHOD \ + trainer.policy.megatron_config.transformer_config_kwargs.recompute_num_layers=$RECOMPUTE_NUM_LAYERS \ + trainer.use_sample_packing=true \ + trainer.flash_attn=$FLASH_ATTN \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-5 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_megatron" \ + trainer.run_name="gsm8k_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}_qwen3_30b_a3b" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_megatron_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_moonlight.sh b/skyrl-train/examples/megatron/run_megatron_moonlight.sh new file mode 100644 index 0000000000..ff1b909a0d --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_moonlight.sh @@ -0,0 +1,89 @@ +set -x + +# Colocated GRPO training+generation for Moonlight-16B-A3B-Instruct on GSM8K with Megatron. +# Runs on 2 nodes of 8xH100s + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/megatron/run_megatron_moonlight.sh + +# running moonlight16b +# hf download moonshotai/Moonlight-16B-A3B-Instruct --local-dir ~/moonlight16b + +DATA_DIR="$HOME/data/gsm8k" +LOGGER="wandb" # change to "console" to print to stdout +MODEL_NAME="$HOME/moonlight16b" + +INFERENCE_BACKEND="vllm" # currently only vllm is supported for megatron + +NUM_NODES=2 +NUM_GPUS=8 + +MEGATRON_TP=2 +MEGATRON_PP=1 +MEGATRON_CP=1 +MEGATRON_EP=8 +MEGATRON_ETP=1 + +NUM_INFERENCE_ENGINES=2 +INFERENCE_ENGINE_TP=8 + +# flash attn is not supported for moonlight16b since it is a DeepSeekV3 like model, and uses Multi-Head Latent Attention (MLA) +# https://github.com/NVIDIA/TransformerEngine/blob/483d9594fb070f62966f6a12ed6c90942310b48e/transformer_engine/pytorch/attention/dot_product_attention/utils.py#L483 +FLASH_ATTN=false + +uv run --isolated --extra mcore --with blobfile -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.ref_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TP \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.ref.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.ref.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.ref.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + +trainer.policy.megatron_config.transformer_config_kwargs.num_layers_in_last_pipeline_stage=13 \ + +trainer.ref.megatron_config.transformer_config_kwargs.num_layers_in_last_pipeline_stage=13 \ + trainer.use_sample_packing=true \ + trainer.flash_attn=$FLASH_ATTN \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_megatron" \ + trainer.run_name="gsm8k_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}_moonlight16b-a3b" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_megatron_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_qwen3-235b-a22b.sh b/skyrl-train/examples/megatron/run_megatron_qwen3-235b-a22b.sh new file mode 100644 index 0000000000..ddfa37f48b --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_qwen3-235b-a22b.sh @@ -0,0 +1,101 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-235B-A22B on GSM8K with Megatron. +# Runs on 8 nodes of 8xH100s + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/megatron/run_megatron_qwen3-235b-a22b.sh + +LOGGER="wandb" # change to "console" to print to stdout + +# Make sure these paths are accessible by or present on all nodes +DATA_DIR="$HOME/data/gsm8k" +# download Qwen/Qwen3-235B-A22B-Instruct-2507 from huggingface +# `pip install huggingface_hub hf_transfer` +# `HF_HUB_ENABLE_HF_TRANSFER=1 hf download Qwen/Qwen3-235B-A22B-Instruct-2507 --local-dir ~/qwen235b` +MODEL_NAME="$HOME/qwen235b" + +NUM_NODES=8 +NUM_GPUS=8 + +### Megatron configuration +# the max TP that can be used is 4, since Qwen3-235B-A22B uses Grouped Query Attention with 4 groups +MEGATRON_TP=4 +MEGATRON_PP=16 +MEGATRON_CP=1 +MEGATRON_EP=4 +MEGATRON_ETP=1 +# Qwen3-235B-A22B has 94 blocks, so we need to set the last pipeline stage layer to use 4 blocks +MEGATRON_LAST_PIPELINE_STAGE_LAYER=4 +FLASH_ATTN=true +# configure optimizer offloading +OPTIMIZER_OFFLOAD=true +OPTIMIZER_OFFLOAD_FRACTION=1.0 + +### Inference engine configuration +INFERENCE_BACKEND="vllm" # currently only vllm is supported for megatron +NUM_INFERENCE_ENGINES=4 +# this is not ideal at the moment - enable inference engine pp in order to avoid this +# https://github.com/NovaSky-AI/SkyRL/issues/353 +INFERENCE_ENGINE_TP=16 +# the default max model len for Qwen3-235B-A22B-Instruct-2507 is 262K, and VLLM checks that +# the KV cache memory allocated is enough to serve 1 request with max model len. Lowering to the actual +# max model len value for this script. +INFERENCE_ENGINE_MAX_MODEL_LEN=2048 + +# no kl loss, so just use the policy model +USE_KL_LOSS=false + +uv run --isolated --extra mcore -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TP \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.policy.megatron_config.optimizer_config_kwargs.overlap_cpu_optimizer_d2h_h2d=$OPTIMIZER_OFFLOAD \ + trainer.policy.megatron_config.optimizer_config_kwargs.use_precision_aware_optimizer=$OPTIMIZER_OFFLOAD \ + trainer.policy.megatron_config.optimizer_config_kwargs.optimizer_cpu_offload=$OPTIMIZER_OFFLOAD \ + trainer.policy.megatron_config.optimizer_config_kwargs.optimizer_offload_fraction=$OPTIMIZER_OFFLOAD_FRACTION \ + +trainer.policy.megatron_config.transformer_config_kwargs.num_layers_in_last_pipeline_stage=$MEGATRON_LAST_PIPELINE_STAGE_LAYER \ + +generator.engine_init_kwargs.max_model_len=$INFERENCE_ENGINE_MAX_MODEL_LEN \ + trainer.use_sample_packing=true \ + trainer.flash_attn=$FLASH_ATTN \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.80 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_megatron" \ + trainer.run_name="gsm8k_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}_qwen3-235b-a22b" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_megatron_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_megatron_qwen3-30b-a3b.sh b/skyrl-train/examples/megatron/run_megatron_qwen3-30b-a3b.sh new file mode 100644 index 0000000000..cb1d81cd37 --- /dev/null +++ b/skyrl-train/examples/megatron/run_megatron_qwen3-30b-a3b.sh @@ -0,0 +1,89 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-30B-A3B on GSM8K with Megatron. +# Runs on 2 nodes of 8xH100s + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/megatron/run_megatron_qwen3-30b-a3b.sh + +DATA_DIR="$HOME/data/gsm8k" +LOGGER="wandb" # change to "console" to print to stdout +MODEL_NAME="Qwen/Qwen3-30B-A3B" + +INFERENCE_BACKEND="vllm" # currently only vllm is supported for megatron + +NUM_NODES=2 +NUM_GPUS=8 + +MEGATRON_TP=4 +MEGATRON_PP=2 +MEGATRON_CP=1 +MEGATRON_EP=8 +MEGATRON_ETP=1 + +NUM_INFERENCE_ENGINES=2 +INFERENCE_ENGINE_TP=8 +FLASH_ATTN=true + +# Megatron gradient checkpointing config +RECOMPUTE_GRANULARITY="full" +RECOMPUTE_METHOD="uniform" +RECOMPUTE_NUM_LAYERS=1 + +uv run --isolated --extra mcore -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.ref_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TP \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.ref.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.ref.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.ref.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.policy.megatron_config.transformer_config_kwargs.recompute_granularity=$RECOMPUTE_GRANULARITY \ + trainer.policy.megatron_config.transformer_config_kwargs.recompute_method=$RECOMPUTE_METHOD \ + trainer.policy.megatron_config.transformer_config_kwargs.recompute_num_layers=$RECOMPUTE_NUM_LAYERS \ + trainer.use_sample_packing=true \ + trainer.flash_attn=$FLASH_ATTN \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.6 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k_megatron" \ + trainer.run_name="gsm8k_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_ep${MEGATRON_EP}_etp${MEGATRON_ETP}_qwen3_30b_a3b" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_megatron_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/megatron/run_search_megatron.sh b/skyrl-train/examples/megatron/run_search_megatron.sh new file mode 100644 index 0000000000..e7e6a5970e --- /dev/null +++ b/skyrl-train/examples/megatron/run_search_megatron.sh @@ -0,0 +1,98 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-30B-A3B on SearchR1 data. +# follow the instructions in examples/search/README.md for setting up the dataset +# and for starting the local search server +# export WANDB_API_KEY= +# bash examples/megatron/run_search_megatron.sh +# Runs on 4 nodes of 8xH100s + +# path for dataset (.parquet files) containing the prompts and metadata for each question +MODEL_NAME="Qwen/Qwen3-30B-A3B" + +DATA_DIR="$HOME/data/searchR1" # save to shared storage across nodes or use local storage on each node +NUM_NODES=4 +NUM_GPUS_PER_NODE=8 + +MEGATRON_TP=2 +MEGATRON_PP=1 +MEGATRON_CP=1 +MEGATRON_EP=8 +MEGATRON_ETP=1 + +MICRO_TRAIN_BATCH_SIZE_PER_GPU=1 +MICRO_FORWARD_BATCH_SIZE_PER_GPU=2 + +NUM_INFERENCE_ENGINES=4 +INFERENCE_ENGINE_TP=8 + +uv run --isolated --frozen --extra mcore -m skyrl_train.entrypoints.main_base \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.optimizer_config.num_warmup_steps=94 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.kl_loss_coef=0.001 \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.ref_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.ref.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.ref.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.ref.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.ref.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TP \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.gpu_memory_utilization=0.6 \ + trainer.epochs=1 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=512 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=$MICRO_FORWARD_BATCH_SIZE_PER_GPU \ + trainer.micro_train_batch_size_per_gpu=$MICRO_TRAIN_BATCH_SIZE_PER_GPU \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=4096 \ + generator.sampling_params.max_generate_length=500 \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=false \ + generator.n_samples_per_prompt=5 \ + generator.max_turns=4 \ + generator.sampling_params.temperature=1.0 \ + generator.sampling_params.top_p=1.0 \ + generator.sampling_params.stop='["", ""]' \ + environment.env_class="search" \ + environment.skyrl_gym.max_env_workers=16 \ + environment.skyrl_gym.search.log_requests=false \ + environment.skyrl_gym.search.search_url="http://172.25.102.175:8000/retrieve" \ + environment.skyrl_gym.search.topk=3 \ + trainer.logger="wandb" \ + trainer.project_name="skyrl-search" \ + trainer.run_name="skyrl-search_4turns_maxgeneratelen_500_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_qwen30b" \ + trainer.ckpt_interval=20 \ + trainer.hf_save_interval=100 \ + trainer.max_ckpts_to_keep=5 \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/skyrl-search_4turns_maxgeneratelen_500_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_qwen30b" \ + trainer.eval_batch_size=256 \ + trainer.eval_before_train=false \ + generator.eval_sampling_params.temperature=0 \ + generator.eval_sampling_params.stop='["", ""]' \ + trainer.export_path="$HOME/skyrl-search_4turns_maxgeneratelen_500_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_qwen30b/exports" \ + trainer.eval_interval=50 \ + $@ + \ No newline at end of file diff --git a/skyrl-train/examples/mini_swe_agent/.env.miniswe b/skyrl-train/examples/mini_swe_agent/.env.miniswe new file mode 100644 index 0000000000..972abfc37f --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/.env.miniswe @@ -0,0 +1,5 @@ +OPENAI_API_KEY=dummy +# needs to match the host and port provided in the run script +OPENAI_BASE_URL=http://127.0.0.1:8001/v1 +# relative to the `skyrl-train` directory, or an absolute path accessible to all nodes in the Ray cluster +LITELLM_MODEL_REGISTRY_PATH=examples/mini_swe_agent/litellm.json \ No newline at end of file diff --git a/skyrl-train/examples/mini_swe_agent/README.md b/skyrl-train/examples/mini_swe_agent/README.md new file mode 100644 index 0000000000..c1f37bf882 --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/README.md @@ -0,0 +1,81 @@ +## Guide: Mini-SWE-Agent + SkyRL + +This directory contains an integration to train a coding agent on the SWE-Bench task using [Mini-SWE-Agent](https://github.com/SWE-agent/mini-swe-agent) and SkyRL. + +To start training, follow three simple steps: +1) Prepare the SWE-Gym dataset. +2) Configure your environment backend (Podman). +3) Launch training! + +Start by following the SkyRL [installation instructions](https://skyrl.readthedocs.io/en/latest/getting-started/installation.html), then enter the `skyrl-train` directory: +```bash +cd SkyRL/skyrl-train +``` + +## How it works + +The Mini-SWE-Agent integration implements a custom `MiniSweAgentGenerator` that uses Mini-SWE-Agent to generate trajectories for SWE-Bench instances. The workflow consists of: + +1. **Generation**: Initialize a sandbox environment and generate a trajectory using Mini-SWE-Agent configured with SkyRL's HTTP endpoint, producing a git patch. +2. **Evaluation**: Apply the generated patch to a fresh environment and run the evaluation script to determine if the instance was resolved. + +We launch a Ray task per trajectory to scale this across all nodes in the cluster. + +### 1) Prepare the dataset + +We use [SWE-Gym](https://huggingface.co/SWE-Gym), specifically the subset from [SumanthRH/SWE-Gym-Subset](https://huggingface.co/datasets/SumanthRH/SWE-Gym-Subset). + +Execute the following command: +```bash +uv run --isolated examples/mini_swe_agent/preprocess_swegym.py --output_dir ~/data/swe_gym_subset # or modify to your desired path +``` + +### 2) Configure environment backend + +**Prerequisites**: Install the required environment backend. By default, we use [Podman](https://podman.io/docs). This can be modified in `examples/mini_swe_agent/swebench.yaml`. + +### 3) Launch training + +We provide example scripts for different model sizes: + +**Qwen3-8B** (requires 1x 8xH100 node): +```bash +bash examples/mini_swe_agent/run_mini_swe_8B.sh +``` + +**Qwen3-Coder-30B** (requires 2x 8xH100 nodes): +```bash +bash examples/mini_swe_agent/run_mini_swe_30B.sh +``` + +Make sure to update the `DATA_DIR` variable in the bash script if you saved the data to a custom path. + +All training parameters can be modified in the run scripts, such as model choice, GRPO group size, or training batch size. + +## Troubleshooting + +For issues with SkyRL or the Mini-SWE-Agent integration, please [open an Issue](https://github.com/NovaSky-AI/SkyRL/issues/new). + +### Common Issues + +- **Context length errors**: If you see `ValueError: The decoder prompt (length xxxx) is longer than the maximum model length`, increase `max_input_length` and `max_generate_length` or reduce steps in `swebench.yaml`. + +- **All zero rewards**: If rewards are consistently zero, the task may be too difficult. Consider: + - Filtering data for a better mix of easy/hard samples + - Using a stronger base model + - Increasing `step_limit` in `swebench.yaml` + +- **Argument list too long**: For very large git patches, you might notice evaluation errors such as `Argument list too long: 'podman'`. This is because we apply the model's git patch by passing it as a CLI argument, and for large patches, you can hit the system's `ARG_MAX` limits. On modern systems, this limit is about ~1MB. We make a simple assumption that such large patches are meant to be incorrect. + +- **Podman UID errors**: If running podman within a container, you might hit errors due to insufficient UIDs. To resolve this, you have two options on Linux-based machines: + 1. Edit the `/etc/subuid` and `/etc/subgid` files to use a larger range of UIDs, like `100000-1100000` + 2. Set `ignore_chown_errors=true` in Podman's containers.conf + +## Configuration + +Beyond the configuration for SkyRL in the training script, the task-specific configuration file is `examples/mini_swe_agent/swebench.yaml`, which controls: +- Environment backend settings +- Step limits for agent execution +- Tool configurations for Mini-SWE-Agent + +For more details, refer to the [documentation](https://skyrl.readthedocs.io/en/latest/examples/mini_swe_agent.html). diff --git a/skyrl-train/examples/mini_swe_agent/litellm.json b/skyrl-train/examples/mini_swe_agent/litellm.json new file mode 100644 index 0000000000..177ee589ff --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/litellm.json @@ -0,0 +1,62 @@ +{ + "together_ai/qwen/qwen3-coder-480b-a35b-instruct-fp8": { + "input_cost_per_token": 2e-09, + "output_cost_per_token": 2e-09, + "mode": "chat", + "max_tokens": 256000 + }, + "hosted_vllm/Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 64000 + }, + "hosted_vllm/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 64000 + }, + "hosted_vllm/Qwen/Qwen3-Coder-30B-A3B-Instruct": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 64000 + }, + "hosted_vllm/Qwen/Qwen2.5-0.5B-Instruct": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 32000 + }, + "hosted_vllm/Qwen/Qwen3-4B": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 32000 + }, + "openai/Qwen/Qwen3-4B": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 32768 + }, + "openai/Qwen/Qwen3-8B": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 32768 + }, + "hosted_vllm/Qwen/Qwen3-8B": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 32768 + }, + "openai/Qwen/Qwen3-Coder-30B-A3B-Instruct": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "mode": "chat", + "max_tokens": 64000 + } +} \ No newline at end of file diff --git a/skyrl-train/examples/mini_swe_agent/main_mini_swe.py b/skyrl-train/examples/mini_swe_agent/main_mini_swe.py new file mode 100644 index 0000000000..430928c17c --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/main_mini_swe.py @@ -0,0 +1,40 @@ +import hydra +from omegaconf import DictConfig, OmegaConf +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils import initialize_ray +import ray + +from .mini_swe_generator import MiniSweAgentGenerator + + +class MiniSWEPPOExp(BasePPOExp): + def get_generator(self, cfg, tokenizer, inference_engine_client): + generator = MiniSweAgentGenerator( + generator_cfg=cfg.generator, + skyrl_gym_cfg=OmegaConf.create({"max_env_workers": 0}), + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + model_name=self.cfg.trainer.policy.model.path, + ) + return generator + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = MiniSWEPPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/mini_swe_agent/mini_swe_generator.py b/skyrl-train/examples/mini_swe_agent/mini_swe_generator.py new file mode 100644 index 0000000000..52611d9079 --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/mini_swe_generator.py @@ -0,0 +1,266 @@ +import asyncio +from typing import Dict, List, Optional, Any, Tuple +from omegaconf import DictConfig +import yaml +import traceback +import ray +from pathlib import Path + +from minisweagent.models import get_model +from minisweagent.agents.default import DefaultAgent +from minisweagent.run.utils.save import save_traj +from minisweagent.config import get_config_path +from .mini_swe_utils import evaluate_trajectory, get_sb_environment + +from skyrl_train.generators.skyrl_gym_generator import SkyRLGymGenerator, GeneratorOutput, GeneratorInput +from skyrl_train.generators.base import TrajectoryID, TrainingPhase, BatchMetadata +from skyrl_train.inference_engines.base import ConversationType +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.generators.utils import ( + get_rollout_metrics, + get_response_ids_and_loss_mask_from_messages, +) + + +class DefaultAgentWithReminder(DefaultAgent): + def get_observation(self, response: dict) -> dict: + """Execute the action and return the output.""" + output = self.execute_action(self.parse_action(response)) + observation = self.render_template(self.config.action_observation_template, output=output) + remaining = self.config.step_limit - self.model.n_calls + + if remaining == 1: + observation = f"{observation}\nREMINDER: You only have 1 turn left. Please provide the final answer" + elif remaining > 1: + observation = f"{observation}\nREMINDER: You have {remaining} turns left to arrive at the solution." + + self.add_message("user", observation) + return output + + +@ray.remote(num_cpus=0.01) +def init_and_run( + instance: dict, + litellm_model_name: str, + sweagent_config: dict, + generator_cfg: DictConfig, + data_source: str, + sampling_params: dict, + trajectory_id: TrajectoryID, + global_step: int, + training_phase: TrainingPhase, +): + from loguru import logger + + model_config = sweagent_config.get("model", {}) + # Use new sampling parameters + # Can also have custom sampling parameters per trajectory (ex: custom max tokens) + model_config.setdefault("model_kwargs", {}).update(sampling_params) + model = get_model(litellm_model_name, model_config) + + agent = None + env = None + extra_info = None + result = None + reward = 0 + error = None + try: + env = get_sb_environment(sweagent_config, instance, data_source) + agent = DefaultAgentWithReminder(model, env, **sweagent_config.get("agent", {})) + exit_status, result = agent.run(instance["problem_statement"]) # type: ignore[arg-type] + except Exception as e: + logger.error(f"Error processing instance {instance['instance_id']}: {e}", exc_info=True) + exit_status, result = type(e).__name__, str(e) + error = str(e) + extra_info = {"traceback": traceback.format_exc()} + finally: + # Create trajectory directory with proper structure: step_{global_step}/{train/eval} + path = Path(generator_cfg.miniswe_traj_dir) / f"step_{global_step}" / training_phase + path.mkdir(parents=True, exist_ok=True) + # Use instance_id and repetition_id for meaningful filename: {instance_id}_{repetition_id}.json + instance_id = instance["instance_id"] + filename = f"{instance_id}_{trajectory_id.repetition_id}.json" + path = path / filename + if agent is not None: + eval_error = None + try: + result = evaluate_trajectory(instance, result, sweagent_config, data_source) + reward = int(result["resolved"]) + eval_error = result["eval_error"] + if eval_error: + error = eval_error + logger.debug(f"Error during evaluation {eval_error}") + except Exception as e: + logger.debug(f"Error during evaluation {e}") + logger.debug(f"traceback: {traceback.format_exc()}") + eval_error = str(e) + error = str(e) + + save_traj(agent, path, exit_status=exit_status, result=result, extra_info=extra_info, reward=reward, eval_error=eval_error) # type: ignore[arg-type] + + return (agent.messages if agent is not None else [], reward, error) + + +class MiniSweAgentGenerator(SkyRLGymGenerator): + def __init__( + self, + generator_cfg: DictConfig, + skyrl_gym_cfg: DictConfig, + inference_engine_client: InferenceEngineClient, + tokenizer, + model_name: str, + ): + + # Call parent constructor first + super().__init__(generator_cfg, skyrl_gym_cfg, inference_engine_client, tokenizer, model_name) + + self.http_server_inference_engine_client_host = generator_cfg.get( + "http_server_inference_engine_client_host", "127.0.0.1" + ) + self.http_server_inference_engine_client_port = generator_cfg.get( + "http_server_inference_engine_client_port", 8000 + ) + self.base_url = ( + f"http://{self.http_server_inference_engine_client_host}:{self.http_server_inference_engine_client_port}" + ) + self.generator_cfg = generator_cfg + self.tokenizer = tokenizer + self.model_name = model_name + self.litellm_model_name = "openai/" + self.model_name + + if self.generator_cfg.chat_template.name_or_path is not None: + raise NotImplementedError("MiniSWEAgentGenerator doesn't support custom chat template") + + async def minisweagent_agent_loop( + self, + prompt: ConversationType, + env_extras: Dict[str, Any], + max_tokens: int, + max_input_length: int, + sampling_params: Dict[str, Any], + trajectory_id: TrajectoryID, + batch_metadata: BatchMetadata, + ) -> Tuple[List[int], float, str, List[int], List[int], Optional[List[int]]]: + + sweagent_config = yaml.safe_load(get_config_path(self.generator_cfg.miniswe_config_path).read_text()) + # NOTE (sumanthrh): Input `prompt` is not used here because mini-swe-agent uses a similar entry from the `instance` obj + messages, reward, error = await init_and_run.remote( + env_extras["instance"], + self.litellm_model_name, + sweagent_config, + self.generator_cfg, + env_extras["data_source"], + sampling_params, + trajectory_id, + batch_metadata.global_step, + batch_metadata.training_phase, + ) + if not len(messages): + return None, None, None, None, None, None + + # TODO (sumanthrh): This is currently hardcoded for SWEBench with 2 initial messages (system and user). + response_messages = messages[2:] + + for message in messages[:2]: + assert message["role"] in ( + "system", + "user", + ), "Expected the first two messages to be system and user messages" + + initial_input_ids = self.tokenizer.apply_chat_template(messages[:2], add_generation_prompt=False, tokenize=True) + initial_prompt_length = len(initial_input_ids) + + # We remove trailing `user` messages - this is added by Mini-SWE-Agent to capture the final git diff for the trajectory + last_idx = len(response_messages) - 1 + while response_messages[last_idx]["role"] == "user": + last_idx -= 1 + if last_idx < 0: + raise ValueError( + "Found no assistant messages. Please ensure that your environment is configured correctly and the `OPENAI_BASE_URL` points to the HTTP server from the inference engine client" + ) + response_messages = response_messages[: last_idx + 1] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages( + response_messages, + self.tokenizer, + assistant_logprobs=None, + ) + + # Extract prompt ids + prompt_ids = initial_input_ids + + # Calculate maximum response tokens allowed + max_response_tokens = max_tokens + max_input_length - initial_prompt_length + + # Determine stop reason + stop_reason = "complete" # Default for trial completion + if len(response_ids) > max_response_tokens: + stop_reason = "length" + + # Truncate to maximum allowed length + response_ids = response_ids[:max_response_tokens] + loss_mask = loss_mask[:max_response_tokens] + + return (response_ids, reward, stop_reason, loss_mask, prompt_ids, None) + + async def generate(self, input_batch: GeneratorInput) -> GeneratorOutput: + """ + Generate trajectories for the input batch. + + Returns outputs in the same order as the input batch. + Args: + input_batch: GeneratorInput + Returns: + GeneratorOutput + """ + prompts = input_batch["prompts"] + env_extras = input_batch["env_extras"] + trajectory_ids = input_batch["trajectory_ids"] + batch_metadata = input_batch["batch_metadata"] + max_tokens = self.generator_cfg.sampling_params.max_generate_length + max_input_length = self.generator_cfg.max_input_length + sampling_params = get_sampling_params_for_backend( + self.generator_cfg.backend, self.generator_cfg.sampling_params + ) + + tasks = [] + + for i in range(len(prompts)): + tasks.append( + self.minisweagent_agent_loop( + prompts[i], + env_extras[i], + max_tokens=max_tokens, + max_input_length=max_input_length, + sampling_params=sampling_params, + trajectory_id=trajectory_ids[i], + batch_metadata=batch_metadata, + ) + ) + + all_outputs = await asyncio.gather(*tasks) + + # Filter out the `None` entries, which means that trajectory generation failed + responses = [output[0] for output in all_outputs if output[0] is not None] + rewards = [output[1] for output in all_outputs if output[0] is not None] + stop_reasons = [output[2] for output in all_outputs if output[0] is not None] + loss_masks = [output[3] for output in all_outputs if output[0] is not None] + prompt_token_ids = [output[4] for output in all_outputs if output[0] is not None] + if not len(responses): + raise ValueError( + "Found no valid responses for this step. This means that generation failed for all trajectories, likely due to errors in environment setup." + ) + rollout_metrics = get_rollout_metrics(responses, rewards) + + generator_output: GeneratorOutput = { + "prompt_token_ids": prompt_token_ids, + "response_ids": responses, + "rewards": rewards, + "loss_masks": loss_masks, + "stop_reasons": stop_reasons, + "rollout_metrics": rollout_metrics, + "rollout_logprobs": None, + } + + return generator_output diff --git a/skyrl-train/examples/mini_swe_agent/mini_swe_utils.py b/skyrl-train/examples/mini_swe_agent/mini_swe_utils.py new file mode 100644 index 0000000000..bff7355662 --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/mini_swe_utils.py @@ -0,0 +1,90 @@ +from typing import TypedDict, Optional +import traceback +import uuid + +from typing import Dict, Any +from loguru import logger + +from jinja2 import Template + +from minisweagent.environments import Environment, get_environment + + +class MiniSWEEvaluationResult(TypedDict): + instance_id: str + resolved: bool + eval_error: Optional[str] + + +def get_sb_environment(config: dict, instance: dict, data_source: str) -> Environment: + env_config = config.setdefault("environment", {}) + env_config["environment_class"] = env_config.get("environment_class", "docker") + image_name = get_docker_image_name(instance, data_source) + if env_config["environment_class"] == "docker": + env_config["image"] = image_name + elif env_config["environment_class"] == "singularity": + env_config["image"] = f"docker://{image_name}" + env = get_environment(env_config) + if startup_command := config.get("run", {}).get("env_startup_command"): + startup_command = Template(startup_command).render(**instance) + out = env.execute(startup_command) + if out["returncode"] != 0: + raise RuntimeError(f"Error executing startup command: {out}") + return env + + +def get_docker_image_name(instance: dict, data_source: str) -> str: + """Get the image name for a SWEBench/SWE-Gym instance.""" + image_name = instance.get("image_name", None) + if image_name is None: + iid = instance["instance_id"] + if "swe-gym" in data_source.lower(): + id_docker_compatible = iid.replace("__", "_s_") # to comply with docker image naming convention + image_name = f"docker.io/xingyaoww/sweb.eval.x86_64.{id_docker_compatible}:latest".lower() + elif "swe-bench" in data_source.lower(): + # Docker doesn't allow double underscore, so we replace them with a magic token + id_docker_compatible = iid.replace("__", "_1776_") + image_name = f"docker.io/swebench/sweb.eval.x86_64.{id_docker_compatible}:latest".lower() + else: + raise NotImplementedError(f"Data source: {data_source} is not supported") + return image_name + + +def evaluate_trajectory( + instance: Dict[str, Any], model_patch: str, sweagent_config: dict, data_source: str +) -> MiniSWEEvaluationResult: + + ret = MiniSWEEvaluationResult(instance_id=instance["instance_id"], resolved=False, eval_error=None) + + env = None + try: + env = get_sb_environment(sweagent_config, instance, data_source) + except Exception as e: + ret["eval_error"] = f"Env creation failed with {e}" + logger.info(f"Starting environment failed with exception: {e}\n, {traceback.format_exc()}") + return ret + + # apply git patch + # NOTE (sumanthrh): This applies patch in-line, and the maximum patch size is limited by the OS limits for `ARG_MAX`. + # In modern systems, this is typically ~ 1 MB, which is pretty generous. + # For simplicity, we assume that large patches greater than `ARG_MAX` are meant to fail + delimiter = f"PATCH_{uuid.uuid4().hex}" # unlikely to collide with symbols in the patch + command = f"git apply <<'{delimiter}'\n{model_patch}\n{delimiter}" + + obs = env.execute(command) + + if obs["returncode"] != 0: + ret["eval_error"] = obs["output"] + else: + # run eval script in-line + eval_script = instance["eval_script"] + eval_cmd = f"bash <<'EOF'\n{eval_script}\nEOF" + # add longer timeout for evaluation + obs = env.execute(eval_cmd, timeout=3600) + # use the return value + ret["resolved"] = obs["returncode"] == 0 + # truncate to last 1000 characters for brevity + ret["eval_error"] = ( + f"(truncated to last 1000 characters)\n{obs["output"][-1000:]}" if not ret["resolved"] else None + ) + return ret diff --git a/skyrl-train/examples/mini_swe_agent/preprocess_swegym.py b/skyrl-train/examples/mini_swe_agent/preprocess_swegym.py new file mode 100644 index 0000000000..11595ef2c2 --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/preprocess_swegym.py @@ -0,0 +1,48 @@ +""" +Preprocess the SWEBench dataset to SkyRL format +""" + +import argparse +import os + +import datasets + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", default="~/data/swe_gym_subset") + + args = parser.parse_args() + + args.output_dir = os.path.expanduser(args.output_dir) + + data_source = "SumanthRH/SWE-Gym-Subset" + eval_data_source = "SumanthRH/SWE-bench_Verified" + + train_dataset = datasets.load_dataset(data_source, "default")["train"] + val_dataset = datasets.load_dataset(eval_data_source, "default")["test"] + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + data = { + "data_source": data_source if split == "train" else eval_data_source, + "prompt": [ + { + "role": "user", + "content": example["problem_statement"], + } + ], + "env_class": "null", + "instance": example, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + val_dataset = val_dataset.map(function=make_map_fn("test"), with_indices=True) + + output_dir = args.output_dir + os.makedirs(output_dir, exist_ok=True) + train_dataset.to_parquet(os.path.join(output_dir, "train.parquet")) + val_dataset.to_parquet(os.path.join(output_dir, "validation.parquet")) diff --git a/skyrl-train/examples/mini_swe_agent/run_mini_swe_30B.sh b/skyrl-train/examples/mini_swe_agent/run_mini_swe_30B.sh new file mode 100644 index 0000000000..94bea6ea75 --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/run_mini_swe_30B.sh @@ -0,0 +1,73 @@ +set -x + +# Colocated GRPO training+generation for Qwen/Qwen3-Coder-30B-A3B-Instruct on the SWE-Bench task. +# Uses 2 node with 8 GPUs each. +# uv run --isolated examples/mini_swe_agent/preprocess_swegym.py --output_dir ~/data/swe_gym_subset +# bash examples/mini_swe_agent/run_mini_swe_30B.sh + +# ensure that all worker nodes can access this data directory +DATA_DIR="$DATA/data/swe_gym_subset" + +CKPT_PATH="$DATA/ckpts/llm_mini_swe" + +# Save trajectories here for debugging. +# NOTE: For a multi-node cluster, ensure that this is on NFS so that you can save all trajectories in the same path +MINISWE_TRAJ_DIR="$HOME/mini_swe_agent_trajs_32B" + +NUM_GPUS=8 +NNODES=2 +NUM_INFERENCE_ENGINES=4 +TP_SIZE=4 +LOGGER=wandb + +# We use a small batch size here for demonstration +# NOTE (sumanthrh): The `generator.max_turns` here is actually unused, and we use the `step_limit` from the `swebench.yaml` file. +# This simply has to be a value > 1 +uv run --isolated --extra vllm --extra miniswe --env-file examples/mini_swe_agent/.env.miniswe -m examples.mini_swe_agent.main_mini_swe \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-Coder-30B-A3B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.policy_num_nodes=$NNODES \ + trainer.placement.ref_num_nodes=$NNODES \ + trainer.policy.sequence_parallel_size=4 \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=16 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=16 \ + trainer.policy_mini_batch_size=16 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=4096 \ + generator.sampling_params.max_generate_length=4096 \ + generator.max_input_length=30720 \ + generator.max_turns=50 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=True \ + generator.enable_http_endpoint=True \ + generator.http_endpoint_host='127.0.0.1' \ + generator.http_endpoint_port=8001 \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="mini_swe" \ + trainer.run_name="mini_swe_32B_swe_gym" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$CKPT_PATH" \ + +generator.miniswe_config_path="examples/mini_swe_agent/swebench.yaml" \ + +generator.miniswe_traj_dir=$MINISWE_TRAJ_DIR + $@ diff --git a/skyrl-train/examples/mini_swe_agent/run_mini_swe_8B.sh b/skyrl-train/examples/mini_swe_agent/run_mini_swe_8B.sh new file mode 100644 index 0000000000..bd6e65b280 --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/run_mini_swe_8B.sh @@ -0,0 +1,72 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-8B on the SWE-Bench task. +# Uses 1 node with 8 GPUs. +# uv run --isolated examples/mini_swe_agent/preprocess_swegym.py --output_dir ~/data/swe_gym_subset +# bash examples/mini_swe_agent/run_mini_swe_8B.sh + +DATA_DIR="$HOME/data/swe_gym_subset" +CKPT_PATH="$HOME/ckpts/llm_mini_swe" + +# Save trajectories here for debugging +# NOTE: For a multi-node cluster, ensure that this is on NFS so that you can save all trajectories in the same path +MINISWE_TRAJ_DIR="$HOME/mini_swe_agent_trajs" + +NUM_GPUS=8 +NNODES=1 +NUM_INFERENCE_ENGINES=4 +TP_SIZE=2 +LOGGER=wandb + +# We use a small batch size here for demonstration +# NOTE (sumanthrh): The `generator.max_turns` here is actually unused, and we use the `step_limit` from the `swebench.yaml` file. +# This simply has to be a value > 1 +uv run --isolated --extra vllm --extra miniswe --env-file examples/mini_swe_agent/.env.miniswe -m examples.mini_swe_agent.main_mini_swe \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-8B" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.policy_num_nodes=$NNODES \ + trainer.placement.ref_num_nodes=$NNODES \ + trainer.policy.sequence_parallel_size=2 \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=50 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=16 \ + trainer.policy_mini_batch_size=16 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.dump_data_batch=true \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=4096 \ + generator.sampling_params.max_generate_length=4096 \ + generator.max_input_length=30720 \ + generator.max_turns=20 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=True \ + generator.enable_http_endpoint=True \ + generator.http_endpoint_host='127.0.0.1' \ + generator.http_endpoint_port=8001 \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="mini_swe" \ + trainer.run_name="mini_swe_8B_swe_gym" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$CKPT_PATH" \ + +generator.miniswe_config_path="examples/mini_swe_agent/swebench.yaml" \ + +generator.miniswe_traj_dir=$MINISWE_TRAJ_DIR + $@ diff --git a/skyrl-train/examples/mini_swe_agent/swebench.yaml b/skyrl-train/examples/mini_swe_agent/swebench.yaml new file mode 100644 index 0000000000..cfe79a427a --- /dev/null +++ b/skyrl-train/examples/mini_swe_agent/swebench.yaml @@ -0,0 +1,231 @@ +agent: + system_template: | + You are a helpful assistant that can interact multiple times with a computer shell to solve programming tasks. + Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||). + + Include a THOUGHT section before your command where you explain your reasoning process. + Format your response as shown in . + + + THOUGHT: Your reasoning and analysis here + + ```bash + your_command_here + ``` + + + Failure to follow these rules will cause your response to be rejected. + instance_template: | + + Consider the following PR description: + {{task}} + + + + # Task Instructions + + ## Overview + You're a software engineer interacting continuously with a computer by submitting commands. + You'll be helping implement necessary changes to meet requirements in the PR description. + Your task is specifically to make changes to non-test files in the current directory in order to fix the issue described in the PR description in a way that is general and consistent with the codebase. + + IMPORTANT: This is an interactive process where you will think and issue ONE command, see its result, then think and issue your next command. + + For each response: + 1. Include a THOUGHT section explaining your reasoning and what you're trying to accomplish + 2. Provide exactly ONE bash command to execute + + ## Important Boundaries + - MODIFY: Regular source code files in /testbed (this is the working directory for all your subsequent commands) + - DO NOT MODIFY: Tests, configuration files (pyproject.toml, setup.cfg, etc.) + + ## Recommended Workflow + 1. Analyze the codebase by finding and reading relevant files + 2. Create a script to reproduce the issue + 3. Edit the source code to resolve the issue + 4. Verify your fix works by running your script again + 5. Test edge cases to ensure your fix is robust + + ## Command Execution Rules + You are operating in an environment where + 1. You write a single command + 2. The system executes that command in a subshell + 3. You see the result + 4. You write your next command + + Each response should include: + 1. A **THOUGHT** section where you explain your reasoning and plan + 2. A single bash code block with your command + + Format your responses like this: + + + THOUGHT: Here I explain my reasoning process, analysis of the current situation, + and what I'm trying to accomplish with the command below. + + ```bash + your_command_here + ``` + + + Commands must be specified in a single bash code block: + + ```bash + your_command_here + ``` + + **CRITICAL REQUIREMENTS:** + - Your response SHOULD include a THOUGHT section explaining your reasoning + - Your response MUST include EXACTLY ONE bash code block + - This bash block MUST contain EXACTLY ONE command (or a set of commands connected with && or ||) + - If you include zero or multiple bash blocks, or no command at all, YOUR RESPONSE WILL FAIL + - Do NOT try to run multiple independent commands in separate blocks in one response + - Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + - However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files + + Example of a CORRECT response: + + THOUGHT: I need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase. + + ```bash + ls -la + ``` + + + Example of an INCORRECT response: + + THOUGHT: I need to examine the codebase and then look at a specific file. I'll run multiple commands to do this. + + ```bash + ls -la + ``` + + Now I'll read the file: + + ```bash + cat file.txt + ``` + + + If you need to run multiple commands, either: + 1. Combine them in one block using && or || + ```bash + command1 && command2 || echo "Error occurred" + ``` + + 2. Wait for the first command to complete, see its output, then issue the next command in your following response. + + ## Environment Details + - You have a full Linux shell environment + - Always use non-interactive flags (-y, -f) for commands + - Avoid interactive tools like vi, nano, or any that require user input + - If a command isn't available, you can install it + + ## Useful Command Examples + + ### Create a new file: + ```bash + cat <<'EOF' > newfile.py + import numpy as np + hello = "world" + print(hello) + EOF + ``` + + ### Edit files with sed: + ```bash + # Replace all occurrences + sed -i 's/old_string/new_string/g' filename.py + + # Replace only first occurrence + sed -i 's/old_string/new_string/' filename.py + + # Replace first occurrence on line 1 + sed -i '1s/old_string/new_string/' filename.py + + # Replace all occurrences in lines 1-10 + sed -i '1,10s/old_string/new_string/g' filename.py + ``` + + ### View file content: + ```bash + # View specific lines with numbers + nl -ba filename.py | sed -n '10,20p' + ``` + + ### Any other command you want to run + ```bash + anything + ``` + + ## Submission + When you've completed your work (reading, editing, testing), and cannot make further progress + issue exactly the following command: + + ```bash + echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && git add -A && git diff --cached + ``` + + This command will submit your work. + You cannot continue working (reading, editing, testing) in any way on this task after submitting. + + action_observation_template: | + {{output.returncode}} + {% if output.output | length < 10000 -%} + + {{ output.output -}} + + {%- else -%} + + The output of your last command was too long. + Please try a different command that produces less output. + If you're looking at a file you can try use head, tail or sed to view a smaller number of lines selectively. + If you're using grep or find and it produced too much output, you can use a more selective search pattern. + If you really need to see something from the full command's output, you can redirect output to a file and then search in that file. + + {%- set elided_chars = output.output | length - 10000 -%} + + {{ output.output[:5000] }} + + + {{ elided_chars }} characters elided + + + {{ output.output[-5000:] }} + + {%- endif -%} + format_error_template: | + Please always provide EXACTLY ONE action in triple backticks, found {{actions|length}} actions. + + Please format your action in triple backticks as shown in . + + + Here are some thoughts about why you want to perform the action. + + ```bash + + ``` + + + If you have completed your assignment, please consult the first message about how to + submit your solution (you will not be able to continue working on this task after that). + step_limit: 20 + cost_limit: 3. + +environment: + cwd: "/testbed" + timeout: 180 + pull_timeout: 1200 + env: + PAGER: cat + MANPAGER: cat + LESS: -R + PIP_PROGRESS_BAR: 'off' + TQDM_DISABLE: '1' + environment_class: docker + executable: podman + +model: + model_name: "" + model_kwargs: + drop_params: true diff --git a/skyrl-train/examples/moe/README.md b/skyrl-train/examples/moe/README.md new file mode 100644 index 0000000000..974d3c1324 --- /dev/null +++ b/skyrl-train/examples/moe/README.md @@ -0,0 +1,6 @@ +## MoE Training +MoE training is not feature complete and still under active development. + +The `examples/moe` directory serves only for development and testing as we continue to improve support for large-scale multi-node MoE training. + +MoE training support is tracked in [this issue](https://github.com/NovaSky-AI/SkyRL/issues/203) and inference support is tracked in [this issue](https://github.com/NovaSky-AI/SkyRL/issues/202). \ No newline at end of file diff --git a/skyrl-train/examples/moe/run_qwen1_5_MoE_A2_7B.sh b/skyrl-train/examples/moe/run_qwen1_5_MoE_A2_7B.sh new file mode 100644 index 0000000000..ebedeca157 --- /dev/null +++ b/skyrl-train/examples/moe/run_qwen1_5_MoE_A2_7B.sh @@ -0,0 +1,54 @@ +set -x + +# Colocated GRPO training+generation for Qwen1.5-MoE-A2.7B-Chat on GSM8K. + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +INFERENCE_BACKEND="vllm" +TP_SIZE=$NUM_GPUS +EP_SIZE=$NUM_GPUS +DP_SIZE=1 + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen1.5-MoE-A2.7B-Chat" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=1 \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + generator.inference_engine_expert_parallel_size=$EP_SIZE \ + generator.inference_engine_data_parallel_size=$DP_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="moe" \ + trainer.run_name="moe_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/moe_qwen_a2_7b_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/multiply/env.py b/skyrl-train/examples/multiply/env.py new file mode 100644 index 0000000000..95c6f8cfdd --- /dev/null +++ b/skyrl-train/examples/multiply/env.py @@ -0,0 +1,56 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from typing import Dict, Any +import re + + +class MultiplyEnv(BaseTextEnv): + """ + Environment for multiplication. + """ + + def __init__( + self, + env_config: Dict[str, Any] = {}, + extras: Dict[str, Any] = {}, + ): + super().__init__() + + assert "reward_spec" in extras, "reward_spec field is required" + assert "ground_truth" in extras["reward_spec"], "ground_truth is required in reward_spec field" + self.ground_truth = extras["reward_spec"]["ground_truth"] + + self.max_turns = extras["max_turns"] if "max_turns" in extras else 5 + + def _parse_action(self, action: str) -> str: + match = re.search(r"\\boxed\{([^}]+)\}", action) + return match.group(1) if match else None + + def step(self, action: str) -> BaseTextEnvStepOutput: + self.turns += 1 + + answer = self._parse_action(action) + + is_correct = answer is not None and answer.strip() == str(self.ground_truth).strip() + found_boxed = answer is not None + + done = self.turns >= self.max_turns or is_correct + if is_correct: + reward = 1.0 + elif found_boxed: + reward = 0.5 + else: + reward = 0.0 + + if done: + return BaseTextEnvStepOutput(observations=[], reward=reward, done=True, metadata={"parsed_answer": answer}) + + if answer is not None: + # Agent provided an answer but it's incorrect + feedback = f"Your answer '{answer}' is incorrect. Please try again." + else: + # Agent didn't provide a boxed answer + feedback = "Please provide your answer in the format \\boxed{your_answer}." + + new_obs = {"role": "user", "content": feedback} + + return BaseTextEnvStepOutput(observations=[new_obs], reward=0.0, done=False, metadata={"parsed_answer": answer}) diff --git a/skyrl-train/examples/multiply/main_multiply.py b/skyrl-train/examples/multiply/main_multiply.py new file mode 100644 index 0000000000..2d4ef134fd --- /dev/null +++ b/skyrl-train/examples/multiply/main_multiply.py @@ -0,0 +1,36 @@ +""" +uv run --isolated --extra vllm -m examples.multiply.main_multiply +""" + +import ray +import hydra +from omegaconf import DictConfig +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_gym.envs import register + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # Register the multiply environment inside the entrypoint task (no need to modify the skyrl-gym package). + register( + id="multiply", + entry_point="examples.multiply.env:MultiplyEnv", + ) + + # make sure that the training loop is not run on the head node. + exp = BasePPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/multiply/multiply_dataset.py b/skyrl-train/examples/multiply/multiply_dataset.py new file mode 100644 index 0000000000..4a9e409ef8 --- /dev/null +++ b/skyrl-train/examples/multiply/multiply_dataset.py @@ -0,0 +1,83 @@ +""" +Preprocess the dataset for the 'multiply' environment in parquet format. +""" + +import argparse +import random +import os +from datasets import Dataset + + +def generate_multiplication_problem(num_digits): + """Generate a random multiplication problem with n-digit numbers.""" + # Generate random n-digit numbers + min_val = 10 ** (num_digits - 1) + max_val = 10**num_digits - 1 + + num1 = random.randint(min_val, max_val) + num2 = random.randint(min_val, max_val) + + question = f"{num1} * {num2}" + answer = num1 * num2 + + return question, str(answer) + + +def create_dataset(num_examples, num_digits, split_name): + """Create a dataset of multiplication problems.""" + examples = [] + + system_prompt = { + "role": "system", + "content": "You are a helpful assistant that solves multiplication problems. Please solve the given multiplication problem step by step. Put your final answer in \\boxed{answer} format.", + } + + for idx in range(num_examples): + question, answer = generate_multiplication_problem(num_digits) + + data = { + "data_source": "synthetic_multiply", + "prompt": [ + system_prompt, + { + "role": "user", + "content": question, + }, + ], + "env_class": "multiply", + "reward_spec": { + "method": "rule", + "ground_truth": answer, + }, + "extra_info": { + "num_digits": num_digits, + "split": split_name, + }, + } + examples.append(data) + + return Dataset.from_list(examples) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", default="~/data/multiply") + parser.add_argument("--num_digits", type=int, default=2, help="Number of digits for each number in multiplication") + parser.add_argument("--train_size", type=int, default=10000, help="Number of training examples") + parser.add_argument("--test_size", type=int, default=200, help="Number of test examples") + + args = parser.parse_args() + + # Generate datasets + train_dataset = create_dataset(args.train_size, args.num_digits, "train") + val_dataset = create_dataset(args.test_size, args.num_digits, "test") + + # Save datasets + output_dir = args.output_dir + os.makedirs(output_dir, exist_ok=True) + train_dataset.to_parquet(os.path.join(output_dir, "train.parquet")) + val_dataset.to_parquet(os.path.join(output_dir, "validation.parquet")) + + print(f"Generated {args.train_size} training examples and {args.test_size} test examples") + print(f"Using {args.num_digits}-digit numbers") + print(f"Saved to {output_dir}") diff --git a/skyrl-train/examples/multiply/run_multiply.sh b/skyrl-train/examples/multiply/run_multiply.sh new file mode 100644 index 0000000000..2bccc64240 --- /dev/null +++ b/skyrl-train/examples/multiply/run_multiply.sh @@ -0,0 +1,48 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on a simple multiplication environment. +# uv run examples/multiply/multiply_dataset.py --output_dir $HOME/data/multiply +# export WANDB_API_KEY= +# bash examples/multiply/run_multiply.sh + +DATA_DIR="$HOME/data/multiply" +NUM_GPUS=4 + +uv run --isolated --extra vllm -m examples.multiply.main_multiply \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.critic_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=multiply \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="multiply" \ + trainer.run_name="multiply_test" \ + $@ diff --git a/skyrl-train/examples/on_policy_distillation/README.md b/skyrl-train/examples/on_policy_distillation/README.md new file mode 100644 index 0000000000..5265c659a5 --- /dev/null +++ b/skyrl-train/examples/on_policy_distillation/README.md @@ -0,0 +1,29 @@ +# On-Policy Distillation +This folder contains scripts for running On-Policy Distillation on SkyRL. + +

+ + +

+ +On-Policy Distillation is a technique recently highlighted by [Thinking Machines](https://thinkingmachines.ai/blog/on-policy-distillation), which combines the benefits of on-policy RL style training, with the dense reward signal of distillation. The main idea is to collect on-policy samples from a student model, use a teacher to grade each token from the student samples, update the student policy accordingly, and repeat. On-Policy Distillation has previously been shown to be a more compute efficient approach than RL for efficiently post-training models ([Agarwal et al](https://arxiv.org/abs/2306.13649), [Gu et al](https://arxiv.org/abs/2306.08543), [Qwen3 team](https://arxiv.org/abs/2505.09388)). + +In `main_on_policy_distill.py` we provide a simple example for modifying SkyRL to implement On-Policy Distillation by replacing the ref model with a teacher model, and modifying the reward/advantage computation logic to use the reverse KL loss. +image + +We provide a full writeup at https://novasky-ai.notion.site/on-policy-distillation. + +## Quickstart +To get started, first set up the dataset from the DAPO example: + +```bash +# Run from the `skyrl-train` directory +bash examples/algorithms/dapo/prepare_dapo_data.sh +``` + +Then, just make sure to set the path to your desired teacher model, and you're ready to kick off training! + +```bash +TEACHER_MODEL= +bash examples/on_policy_distillation/run_on_policy_distill_math_qwen3_4b.sh trainer.ref.mode.path=$TEACHER_MODEL +``` diff --git a/skyrl-train/examples/on_policy_distillation/main_on_policy_distill.py b/skyrl-train/examples/on_policy_distillation/main_on_policy_distill.py new file mode 100644 index 0000000000..c7030cbccd --- /dev/null +++ b/skyrl-train/examples/on_policy_distillation/main_on_policy_distill.py @@ -0,0 +1,80 @@ +import torch +import ray +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp +import hydra +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import config_dir, validate_cfg +from skyrl_train.utils.ppo_utils import ( + register_advantage_estimator, + register_policy_loss, + reduce_loss, +) +from skyrl_train.training_batch import TrainingInputBatch + + +class OnPolicyDistillationTrainer(RayPPOTrainer): + """ + Custom trainer for On Policy Distillation. + + Overrides the apply_reward_kl_penalty method to set the rewards just to the kl penalty + """ + + def apply_reward_kl_penalty( + self, + data: TrainingInputBatch, + ) -> TrainingInputBatch: + """Computes the KL penalty and sets the rewards to the KL penalty""" + loss_masks_all: torch.Tensor = data["loss_mask"] + teacher_action_log_probs: torch.Tensor = data["base_action_log_probs"] + action_log_probs: torch.Tensor = data["action_log_probs"] + + # set rewards to the KL penalty + # note: tinker seems to use k1 or k2: https://github.com/thinking-machines-lab/tinker-cookbook/blob/3dd0463472dda5847efee80010b50514fa3068ef/tinker_cookbook/rl/metrics.py#L40 + rewards = -(action_log_probs - teacher_action_log_probs) * loss_masks_all + data["rewards"] = rewards + return data + + +# Using the decorator +@register_advantage_estimator("no_op") +def compute_no_op_advantage(token_level_rewards: torch.Tensor, **kwargs): + # just pass through the rewards + return token_level_rewards, token_level_rewards + + +@register_policy_loss("importance_sampling") +def compute_importance_sampling_policy_loss( + log_probs, old_log_probs, advantages, config, loss_mask=None, rollout_logprobs=None, **kwargs +): + # as defined here: https://tinker-docs.thinkingmachines.ai/losses#policy-gradient-importance_sampling + loss = -torch.exp(log_probs - old_log_probs) * advantages + + loss = reduce_loss(loss, loss_mask, "seq_mean_token_sum_norm", config.max_seq_len) + # return loss and a dummy clip ratio value as we aren't clipping here + return loss, 0.0 + + +class OnPolicyDistillationExp(BasePPOExp): + def get_trainer(self, *args, **kwargs): + return OnPolicyDistillationTrainer(*args, **kwargs) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = OnPolicyDistillationExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/on_policy_distillation/run_on_policy_distill_math_qwen3_1.7b.sh b/skyrl-train/examples/on_policy_distillation/run_on_policy_distill_math_qwen3_1.7b.sh new file mode 100644 index 0000000000..250e8252af --- /dev/null +++ b/skyrl-train/examples/on_policy_distillation/run_on_policy_distill_math_qwen3_1.7b.sh @@ -0,0 +1,93 @@ +set -x + +# Running on policy distillation for Math on the DAPO math dataset, with eval on AIME 2024. +# Uses Qwen-3-1.7B-Base as the student model and an RL trained Qwen-3-4B as the teacher model +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/on_policy_distillation/run_on_policy_distill_math_qwen3_1.7b.sh + +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +LOGGER=wandb + +# On Policy Distillation args +# set this to the huggingface/local path of your teacher model +TEACHER_MODEL="$HOME/ckpts/dapo_qwen3_4b_base/global_step_90/" +STUDENT_MODEL="Qwen/Qwen3-1.7B-Base" +ADVANTAGE_ESTIMATOR="no_op" +POLICY_LOSS="importance_sampling" +USE_KL_IN_REWARD=true # this adds the kl penalty to the advantage +USE_KL_LOSS=false # turns off kl loss in the loss since we are using it directly in the reward + +# Placement args +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=8 +INFERENCE_ENGINE_TP_SIZE=1 + +# sampling params +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=512 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true +LR=1e-5 + +uv run --isolated --extra vllm -m examples.on_policy_distillation.main_on_policy_distill \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator=$ADVANTAGE_ESTIMATOR \ + trainer.algorithm.policy_loss_type=$POLICY_LOSS \ + trainer.policy.model.path=$STUDENT_MODEL \ + trainer.ref.model.path=$TEACHER_MODEL \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TP_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=2 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.sampling_params.max_generate_length=8192 \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.max_generate_length=8192 \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=0 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.use_kl_in_reward=$USE_KL_IN_REWARD \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="aime_on_policy_distillation" \ + trainer.run_name="on_policy_distillation_aime_qwen3_1.7b_base_from_4b" \ + trainer.resume_mode=latest \ + trainer.export_path="$HOME/exports/aime_on_policy_distill_1.7b_base_from_4b" \ + trainer.hf_save_interval=10 \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_interval=10 \ + trainer.ckpt_path="$HOME/ckpts/aime_on_policy_distill_1.7b_base_from_4b" \ + $@ diff --git a/skyrl-train/examples/on_policy_distillation/run_on_policy_distill_math_qwen3_4b.sh b/skyrl-train/examples/on_policy_distillation/run_on_policy_distill_math_qwen3_4b.sh new file mode 100644 index 0000000000..670a351520 --- /dev/null +++ b/skyrl-train/examples/on_policy_distillation/run_on_policy_distill_math_qwen3_4b.sh @@ -0,0 +1,93 @@ +set -x + +# Running on policy distillation for Math on the DAPO math dataset, with eval on AIME 2024. +# Uses Qwen-3-4B-Base as the student model and an RL trained Qwen-3-4B as the teacher model +# bash examples/algorithms/dapo/prepare_dapo_data.sh +# bash examples/on_policy_distillation/run_on_policy_distill_math_qwen3_4b.sh + +DATA_DIR="$HOME/data/dapo" +TRAIN_FILE="$DATA_DIR/dapo-math-17k-cleaned.parquet" +TEST_FILE="$DATA_DIR/aime-2024-cleaned.parquet" +LOGGER=wandb + +# On Policy Distillation args +# set this to the huggingface/local path of your teacher model +TEACHER_MODEL="$HOME/ckpts/dapo_qwen3_4b_base/global_step_90/" +STUDENT_MODEL="Qwen/Qwen3-4B-Base" +ADVANTAGE_ESTIMATOR="no_op" +POLICY_LOSS="importance_sampling" +USE_KL_IN_REWARD=true # this adds the kl penalty to the advantage +USE_KL_LOSS=false # turns off kl loss in the loss since we are using it directly in the reward + +# Placement args +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=8 +INFERENCE_ENGINE_TP_SIZE=1 + +# sampling params +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 + +# repro run parameters +TRAIN_BATCH_SIZE=512 +MINI_BATCH_SIZE=512 +N_SAMPLES_PER_PROMPT=16 +EVAL_N_SAMPLES_PER_PROMPT=32 +ENFORCE_EAGER=true +LR=1e-5 + +uv run --isolated --extra vllm -m examples.on_policy_distillation.main_on_policy_distill \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator=$ADVANTAGE_ESTIMATOR \ + trainer.algorithm.policy_loss_type=$POLICY_LOSS \ + trainer.policy.model.path=$STUDENT_MODEL \ + trainer.ref.model.path=$TEACHER_MODEL \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TP_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=2 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.enforce_eager=$ENFORCE_EAGER \ + generator.sampling_params.max_generate_length=8192 \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.eval_sampling_params.temperature=$TEMPERATURE \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + generator.eval_sampling_params.max_generate_length=8192 \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + trainer.policy.optimizer_config.lr=$LR \ + trainer.policy.optimizer_config.num_warmup_steps=0 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.use_kl_in_reward=$USE_KL_IN_REWARD \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=aime \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="aime_on_policy_distillation" \ + trainer.run_name="on_policy_distillation_aime_qwen3_4b_base_from_4b" \ + trainer.resume_mode=latest \ + trainer.export_path="$HOME/exports/aime_on_policy_distill_4b_base_from_4b" \ + trainer.hf_save_interval=10 \ + trainer.max_ckpts_to_keep=3 \ + trainer.ckpt_interval=10 \ + trainer.ckpt_path="$HOME/ckpts/aime_on_policy_distill_4b_base_from_4b" \ + $@ diff --git a/skyrl-train/examples/ppo/run_ppo.sh b/skyrl-train/examples/ppo/run_ppo.sh new file mode 100644 index 0000000000..353d28bf31 --- /dev/null +++ b/skyrl-train/examples/ppo/run_ppo.sh @@ -0,0 +1,52 @@ +set -x + +# PPO training with colocated training and generation for Qwen2.5-1.5B-Instruct on GSM8K. +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/ppo/run_ppo.sh + +DATA_DIR="$HOME/data/gsm8k" + + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="gae" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.critic.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + trainer.placement.critic_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.critic_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_ppo" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt_ppo" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ diff --git a/skyrl-train/examples/remote_inference_engine/run_remote.sh b/skyrl-train/examples/remote_inference_engine/run_remote.sh new file mode 100644 index 0000000000..73c66f0d60 --- /dev/null +++ b/skyrl-train/examples/remote_inference_engine/run_remote.sh @@ -0,0 +1,44 @@ +set -x + +# GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K using a standalone vllm server (at 127.0.0.1:8001) +# First run `uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k` to setup the dataset. +# then to launch the server, first run +# bash examples/remote_inference_engine/run_vllm_server.sh +# then to start training, run +# bash examples/remote_inference_engine/run_remote.sh + +DATA_DIR="$HOME/data/gsm8k" + +BACKEND="vllm" # or "sglang" +INF_ENGINE_TP=4 + +NUM_TRAINING_GPUS=4 + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + generator.run_engines_locally=False \ + generator.remote_inference_engine_urls="['127.0.0.1:8001']" \ + generator.inference_engine_tensor_parallel_size="$INF_ENGINE_TP" \ + generator.backend="$BACKEND" \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.placement.colocate_all=False \ + trainer.placement.policy_num_gpus_per_node="$NUM_TRAINING_GPUS" \ + trainer.placement.ref_num_gpus_per_node="$NUM_TRAINING_GPUS" \ + trainer.strategy=fsdp2 \ + trainer.train_batch_size=64 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=20 \ + trainer.micro_train_batch_size_per_gpu=20 \ + trainer.logger="wandb" \ + trainer.project_name="remote_inference_engine" \ + trainer.run_name="remote_inference_engine_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ diff --git a/skyrl-train/examples/remote_inference_engine/run_sglang_server.sh b/skyrl-train/examples/remote_inference_engine/run_sglang_server.sh new file mode 100644 index 0000000000..ab638e6cdb --- /dev/null +++ b/skyrl-train/examples/remote_inference_engine/run_sglang_server.sh @@ -0,0 +1,12 @@ +# Launches sglang server for Qwen2.5-1.5B-Instruct on 4 GPUs. +# bash examples/remote_inference_engine/run_sglang_server.sh +set -x + +CUDA_VISIBLE_DEVICES=4,5,6,7 uv run --isolated --extra sglang -m \ + skyrl_train.inference_engines.sglang.sglang_server \ + --model-path Qwen/Qwen2.5-1.5B-Instruct \ + --tp 4 \ + --host 127.0.0.1 \ + --port 8001 \ + --context-length 4096 \ + --dtype bfloat16 \ No newline at end of file diff --git a/skyrl-train/examples/remote_inference_engine/run_vllm_server.sh b/skyrl-train/examples/remote_inference_engine/run_vllm_server.sh new file mode 100644 index 0000000000..2cf02b1159 --- /dev/null +++ b/skyrl-train/examples/remote_inference_engine/run_vllm_server.sh @@ -0,0 +1,21 @@ +# Launches vllm server for Qwen2.5-1.5B-Instruct on 4 GPUs. +# bash examples/remote_inference_engine/run_vllm_server.sh +set -x + +CUDA_VISIBLE_DEVICES=4,5,6,7 uv run --isolated --extra vllm -m skyrl_train.inference_engines.vllm.vllm_server \ + --model Qwen/Qwen2.5-1.5B-Instruct \ + --tensor-parallel-size 4 \ + --host 127.0.0.1 \ + --port 8001 \ + --seed 42 \ + --max-model-len 4096 \ + --enable-prefix-caching \ + --enable-chunked-prefill \ + --dtype bfloat16 \ + --gpu-memory-utilization 0.9 \ + --enable-sleep-mode \ + --max-num_batched_tokens 8192 \ + --max-num-seqs 1024 \ + --trust-remote-code \ + --distributed-executor-backend mp \ + --worker-extension-cls skyrl_train.inference_engines.vllm.vllm_engine.WorkerWrap \ No newline at end of file diff --git a/skyrl-train/examples/search/README.md b/skyrl-train/examples/search/README.md new file mode 100644 index 0000000000..17cd5f6870 --- /dev/null +++ b/skyrl-train/examples/search/README.md @@ -0,0 +1,61 @@ +# SearchR1 Replication Setup Instructions + +We provide scripts to reproduce our results for training a multi-turn search agent using the dataset and recipe from [SearchR1](https://raw.githubusercontent.com/PeterGriffinJin/Search-R1/refs/heads/main/docs/retriever.md). + +Additional Reference: [Verl+Sglang Instructions](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/multi-turn/tool_examples/verl-multiturn-searchR1-like.md). + +## Prepare Datasets +```bash +local_dir=~/data/searchR1 +uv run --isolated examples/search/searchr1_dataset.py --local_dir $local_dir +``` + +# Start the Search Engine +Since faiss-gpu is not available via pip, we setup a separate conda environment for the local retrieval server. Running this server will use around 6GB of GPU memory per GPU, so make sure to account for this in your training run configuration. + +## Retriever environments +```bash +# Create and activate the retriever environment with Python 3.10 +conda create -n retriever python=3.10 -y +conda activate retriever + +# Install PyTorch (with GPU support) and related libraries +conda install numpy==1.26.4 # needed to stop incompatible version of numpy from being installed via pip +pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu124 + +# Install other Python packages +pip install transformers datasets pyserini huggingface_hub + +# Install the GPU version of faiss +conda install faiss-gpu==1.8.0 -c pytorch -c nvidia -y + +# Install the API service framework +pip install uvicorn fastapi +``` + +## Download the Index +```bash +conda activate retriever + +local_dir=~/data/searchR1 +python examples/search/searchr1_download.py --local_dir $local_dir +cat $local_dir/part_* > $local_dir/e5_Flat.index +gzip -d $local_dir/wiki-18.jsonl.gz +``` + +## Start the Local Flat e5 Retrieval Server +```bash +conda activate retriever + +# redirect the output to a file to avoid cluttering the terminal +# we have observed outputting to the terminal causing spikes in server response times +bash examples/search/retriever/retrieval_launch.sh > retrieval_server.log +``` + +## Launch your Training Job +Now from your base environment, you can launch your training run (which will use uv to package dependencies, separately from the retriever environment). + +```bash + export WANDB_API_KEY=your_wandb_api_key + bash examples/search/run_search.sh +``` diff --git a/skyrl-train/examples/search/retriever/retrieval_launch.sh b/skyrl-train/examples/search/retriever/retrieval_launch.sh new file mode 100644 index 0000000000..201625a02b --- /dev/null +++ b/skyrl-train/examples/search/retriever/retrieval_launch.sh @@ -0,0 +1,14 @@ +save_path=$HOME/data/searchR1 + +index_file=$save_path/e5_Flat.index +corpus_file=$save_path/wiki-18.jsonl +retriever_name=e5 +retriever_path=intfloat/e5-base-v2 + +python examples/search/retriever/retrieval_server.py \ + --index_path $index_file \ + --corpus_path $corpus_file \ + --topk 3 \ + --retriever_name $retriever_name \ + --retriever_model $retriever_path \ + --faiss_gpu \ No newline at end of file diff --git a/skyrl-train/examples/search/retriever/retrieval_server.py b/skyrl-train/examples/search/retriever/retrieval_server.py new file mode 100644 index 0000000000..847efc8be6 --- /dev/null +++ b/skyrl-train/examples/search/retriever/retrieval_server.py @@ -0,0 +1,388 @@ +import json +import warnings +from typing import List, Optional +import argparse + +import faiss +import torch +import numpy as np +from transformers import AutoTokenizer, AutoModel +import datasets + +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel + + +def load_corpus(corpus_path: str): + corpus = datasets.load_dataset("json", data_files=corpus_path, split="train", num_proc=4) + return corpus + + +def read_jsonl(file_path): + data = [] + with open(file_path, "r") as f: + for line in f: + data.append(json.loads(line)) + return data + + +def load_docs(corpus, doc_idxs): + results = [corpus[int(idx)] for idx in doc_idxs] + return results + + +def load_model(model_path: str, use_fp16: bool = False): + # model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + model = AutoModel.from_pretrained(model_path, trust_remote_code=True) + model.eval() + model.cuda() + if use_fp16: + model = model.half() + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, trust_remote_code=True) + return model, tokenizer + + +def pooling(pooler_output, last_hidden_state, attention_mask=None, pooling_method="mean"): + if pooling_method == "mean": + last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0) + return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] + elif pooling_method == "cls": + return last_hidden_state[:, 0] + elif pooling_method == "pooler": + return pooler_output + else: + raise NotImplementedError("Pooling method not implemented!") + + +class Encoder: + def __init__(self, model_name, model_path, pooling_method, max_length, use_fp16): + self.model_name = model_name + self.model_path = model_path + self.pooling_method = pooling_method + self.max_length = max_length + self.use_fp16 = use_fp16 + + self.model, self.tokenizer = load_model(model_path=model_path, use_fp16=use_fp16) + self.model.eval() + + @torch.no_grad() + def encode(self, query_list: List[str], is_query=True) -> np.ndarray: + # processing query for different encoders + if isinstance(query_list, str): + query_list = [query_list] + + if "e5" in self.model_name.lower(): + if is_query: + query_list = [f"query: {query}" for query in query_list] + else: + query_list = [f"passage: {query}" for query in query_list] + + if "bge" in self.model_name.lower(): + if is_query: + query_list = [ + f"Represent this sentence for searching relevant passages: {query}" for query in query_list + ] + + inputs = self.tokenizer( + query_list, max_length=self.max_length, padding=True, truncation=True, return_tensors="pt" + ) + inputs = {k: v.cuda() for k, v in inputs.items()} + + if "T5" in type(self.model).__name__: + # T5-based retrieval model + decoder_input_ids = torch.zeros((inputs["input_ids"].shape[0], 1), dtype=torch.long).to( + inputs["input_ids"].device + ) + output = self.model(**inputs, decoder_input_ids=decoder_input_ids, return_dict=True) + query_emb = output.last_hidden_state[:, 0, :] + else: + output = self.model(**inputs, return_dict=True) + query_emb = pooling( + output.pooler_output, output.last_hidden_state, inputs["attention_mask"], self.pooling_method + ) + if "dpr" not in self.model_name.lower(): + query_emb = torch.nn.functional.normalize(query_emb, dim=-1) + + query_emb = query_emb.detach().cpu().numpy() + query_emb = query_emb.astype(np.float32, order="C") + + return query_emb + + +class BaseRetriever: + def __init__(self, config): + self.config = config + self.retrieval_method = config.retrieval_method + self.topk = config.retrieval_topk + + self.index_path = config.index_path + self.corpus_path = config.corpus_path + + def _search(self, query: str, num: int, return_score: bool): + raise NotImplementedError + + def _batch_search(self, query_list: List[str], num: int, return_score: bool): + raise NotImplementedError + + def search(self, query: str, num: int = None, return_score: bool = False): + return self._search(query, num, return_score) + + def batch_search(self, query_list: List[str], num: int = None, return_score: bool = False): + return self._batch_search(query_list, num, return_score) + + +class BM25Retriever(BaseRetriever): + def __init__(self, config): + super().__init__(config) + from pyserini.search.lucene import LuceneSearcher + + self.searcher = LuceneSearcher(self.index_path) + self.contain_doc = self._check_contain_doc() + if not self.contain_doc: + self.corpus = load_corpus(self.corpus_path) + self.max_process_num = 8 + + def _check_contain_doc(self): + return self.searcher.doc(0).raw() is not None + + def _search(self, query: str, num: int = None, return_score: bool = False): + if num is None: + num = self.topk + hits = self.searcher.search(query, num) + if len(hits) < 1: + if return_score: + return [], [] + else: + return [] + scores = [hit.score for hit in hits] + if len(hits) < num: + warnings.warn("Not enough documents retrieved!") + else: + hits = hits[:num] + + if self.contain_doc: + all_contents = [json.loads(self.searcher.doc(hit.docid).raw())["contents"] for hit in hits] + results = [ + { + "title": content.split("\n")[0].strip('"'), + "text": "\n".join(content.split("\n")[1:]), + "contents": content, + } + for content in all_contents + ] + else: + results = load_docs(self.corpus, [hit.docid for hit in hits]) + + if return_score: + return results, scores + else: + return results + + def _batch_search(self, query_list: List[str], num: int = None, return_score: bool = False): + results = [] + scores = [] + for query in query_list: + item_result, item_score = self._search(query, num, True) + results.append(item_result) + scores.append(item_score) + if return_score: + return results, scores + else: + return results + + +class DenseRetriever(BaseRetriever): + def __init__(self, config): + super().__init__(config) + self.index = faiss.read_index(self.index_path) + if config.faiss_gpu: + co = faiss.GpuMultipleClonerOptions() + co.useFloat16 = True + co.shard = True + self.index = faiss.index_cpu_to_all_gpus(self.index, co=co) + + self.corpus = load_corpus(self.corpus_path) + self.encoder = Encoder( + model_name=self.retrieval_method, + model_path=config.retrieval_model_path, + pooling_method=config.retrieval_pooling_method, + max_length=config.retrieval_query_max_length, + use_fp16=config.retrieval_use_fp16, + ) + self.topk = config.retrieval_topk + self.batch_size = config.retrieval_batch_size + + def _search(self, query: str, num: int = None, return_score: bool = False): + if num is None: + num = self.topk + query_emb = self.encoder.encode(query) + scores, idxs = self.index.search(query_emb, k=num) + idxs = idxs[0] + scores = scores[0] + results = load_docs(self.corpus, idxs) + if return_score: + return results, scores + else: + return results + + def _batch_search(self, query_list: List[str], num: int = None, return_score: bool = False): + if isinstance(query_list, str): + query_list = [query_list] + if num is None: + num = self.topk + + results = [] + scores = [] + for start_idx in range(0, len(query_list), self.batch_size): + query_batch = query_list[start_idx : start_idx + self.batch_size] + batch_emb = self.encoder.encode(query_batch) + batch_scores, batch_idxs = self.index.search(batch_emb, k=num) + + batch_scores = batch_scores.tolist() + batch_idxs = batch_idxs.tolist() + # load_docs is not vectorized, but is a python list approach + flat_idxs = sum(batch_idxs, []) + batch_results = load_docs(self.corpus, flat_idxs) + # chunk them back + batch_results = [batch_results[i * num : (i + 1) * num] for i in range(len(batch_idxs))] + results.extend(batch_results) + scores.extend(batch_scores) + if return_score: + return results, scores + else: + return results + + +def get_retriever(config): + if config.retrieval_method == "bm25": + return BM25Retriever(config) + else: + return DenseRetriever(config) + + +##################################### +# FastAPI server below +##################################### + + +class Config: + """ + Minimal config class (simulating your argparse) + Replace this with your real arguments or load them dynamically. + """ + + def __init__( + self, + retrieval_method: str = "bm25", + retrieval_topk: int = 10, + index_path: str = "./index/bm25", + corpus_path: str = "./data/corpus.jsonl", + dataset_path: str = "./data", + data_split: str = "train", + faiss_gpu: bool = True, + retrieval_model_path: str = "./model", + retrieval_pooling_method: str = "mean", + retrieval_query_max_length: int = 256, + retrieval_use_fp16: bool = False, + retrieval_batch_size: int = 128, + ): + self.retrieval_method = retrieval_method + self.retrieval_topk = retrieval_topk + self.index_path = index_path + self.corpus_path = corpus_path + self.dataset_path = dataset_path + self.data_split = data_split + self.faiss_gpu = faiss_gpu + self.retrieval_model_path = retrieval_model_path + self.retrieval_pooling_method = retrieval_pooling_method + self.retrieval_query_max_length = retrieval_query_max_length + self.retrieval_use_fp16 = retrieval_use_fp16 + self.retrieval_batch_size = retrieval_batch_size + + +class QueryRequest(BaseModel): + query: str + topk: Optional[int] = None + return_scores: bool = False + + +app = FastAPI() + + +@app.post("/retrieve") +def retrieve_endpoint(request: QueryRequest): + """ + Endpoint that accepts a single query and performs retrieval. + Input format: + { + "query": "What is Python?", + "topk": 3, + "return_scores": true + } + """ + if not request.topk: + request.topk = config.retrieval_topk # fallback to default + + # Perform retrieval + if request.return_scores: + results, scores = retriever.search(query=request.query, num=request.topk, return_score=True) + else: + results = retriever.search(query=request.query, num=request.topk, return_score=False) + scores = None + + # Format response + resp = [] + if request.return_scores and scores is not None: + # If scores are returned, combine them with results + combined = [] + for doc, score in zip(results, scores): + # Convert numpy float32 to regular Python float for JSON serialization + combined.append({"document": doc, "score": float(score)}) + resp.append(combined) + else: + resp.append(results) + return {"result": resp} + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="Launch the local faiss retriever.") + parser.add_argument( + "--index_path", type=str, default="/home/peterjin/mnt/index/wiki-18/e5_Flat.index", help="Corpus indexing file." + ) + parser.add_argument( + "--corpus_path", + type=str, + default="/home/peterjin/mnt/data/retrieval-corpus/wiki-18.jsonl", + help="Local corpus file.", + ) + parser.add_argument("--topk", type=int, default=3, help="Number of retrieved passages for one query.") + parser.add_argument("--retriever_name", type=str, default="e5", help="Name of the retriever model.") + parser.add_argument( + "--retriever_model", type=str, default="intfloat/e5-base-v2", help="Path of the retriever model." + ) + parser.add_argument("--faiss_gpu", action="store_true", help="Use GPU for computation") + + args = parser.parse_args() + + # 1) Build a config (could also parse from arguments). + # In real usage, you'd parse your CLI arguments or environment variables. + config = Config( + retrieval_method=args.retriever_name, # or "dense" + index_path=args.index_path, + corpus_path=args.corpus_path, + retrieval_topk=args.topk, + faiss_gpu=args.faiss_gpu, + retrieval_model_path=args.retriever_model, + retrieval_pooling_method="mean", + retrieval_query_max_length=256, + retrieval_use_fp16=True, + retrieval_batch_size=512, # this is unused in the current retrieval implementation, which only supports single query + ) + + # 2) Instantiate a global retriever so it is loaded once and reused. + retriever = get_retriever(config) + + # 3) Launch the server. By default, it listens on http://127.0.0.1:8000 + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/skyrl-train/examples/search/run_search.sh b/skyrl-train/examples/search/run_search.sh new file mode 100755 index 0000000000..1703e455ea --- /dev/null +++ b/skyrl-train/examples/search/run_search.sh @@ -0,0 +1,77 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-3B-Instruct on SearchR1 data. +# follow the instructions in examples/search/README.md for setting up the dataset +# and for starting the local search server +# export WANDB_API_KEY= +# bash examples/search/run_search.sh + +# path for dataset (.parquet files) containing the prompts and metadata for each question +DATA_DIR="$HOME/data/searchR1" + +RUN_NAME="skyrl-search_4turns_maxgeneratelen_500-multiturn-sync-TIS_2.0" + +USE_TIS=true +TIS_IMP_RATIO_CAP=2.0 + +uv run --isolated --frozen --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.optimizer_config.num_warmup_steps=94 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.kl_loss_coef=0.001 \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.policy.model.path="Qwen/Qwen2.5-3B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.placement.policy_num_gpus_per_node=8 \ + trainer.placement.ref_num_gpus_per_node=8 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=2 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.gpu_memory_utilization=0.5 \ + trainer.epochs=1 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=512 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=4096 \ + generator.sampling_params.max_generate_length=500 \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=false \ + generator.n_samples_per_prompt=5 \ + generator.max_turns=4 \ + generator.sampling_params.temperature=1.0 \ + generator.sampling_params.top_p=1.0 \ + generator.sampling_params.stop='["", ""]' \ + environment.env_class="search" \ + environment.skyrl_gym.max_env_workers=16 \ + environment.skyrl_gym.search.log_requests=false \ + environment.skyrl_gym.search.search_url="http://127.0.0.1:8000/retrieve" \ + environment.skyrl_gym.search.topk=3 \ + trainer.logger="wandb" \ + trainer.project_name="skyrl-search" \ + trainer.run_name="${RUN_NAME}" \ + trainer.ckpt_interval=20 \ + trainer.hf_save_interval=100 \ + trainer.max_ckpts_to_keep=5 \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/${RUN_NAME}" \ + trainer.eval_batch_size=256 \ + trainer.eval_before_train=false \ + generator.eval_sampling_params.temperature=0 \ + generator.eval_sampling_params.stop='["", ""]' \ + trainer.export_path="$HOME/${RUN_NAME}/exports" \ + trainer.eval_interval=50 \ + $@ diff --git a/skyrl-train/examples/search/run_search_conversation_format.sh b/skyrl-train/examples/search/run_search_conversation_format.sh new file mode 100755 index 0000000000..9346efc13d --- /dev/null +++ b/skyrl-train/examples/search/run_search_conversation_format.sh @@ -0,0 +1,86 @@ +set -x + +# The exact same script as `run_search.sh` but with `use_conversation_multi_turn=true` +# and hence `append_eos_token_after_stop_str_in_multi_turn=true` +# See https://skyrl.readthedocs.io/en/latest/tutorials/skyrl_gym_generator.html on the +# difference between the two options. You might want to change the data generation prompt +# to let the model know that we are doing multi-turn conversations (i.e. user will provide +# the search result for each turn). + +# Colocated GRPO training+generation for Qwen2.5-Coder-3B-Instruct on SearchR1 data. +# follow the instructions in examples/search/README.md for setting up the dataset +# and for starting the local search server +# export WANDB_API_KEY= +# bash examples/search/run_search_conversation_format.sh + +# path for dataset (.parquet files) containing the prompts and metadata for each question +DATA_DIR="$HOME/data/searchR1" + +RUN_NAME="skyrl-search_4turns_maxgeneratelen_500" + +USE_TIS=true +TIS_IMP_RATIO_CAP=2.0 + +uv run --isolated --frozen --extra vllm -m skyrl_train.entrypoints.main_base \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.optimizer_config.num_warmup_steps=94 \ + trainer.algorithm.use_kl_loss=true \ + trainer.algorithm.kl_loss_coef=0.001 \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.policy.model.path="Qwen/Qwen2.5-3B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.placement.policy_num_gpus_per_node=8 \ + trainer.placement.ref_num_gpus_per_node=8 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=2 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.gpu_memory_utilization=0.5 \ + trainer.epochs=1 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=512 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=4096 \ + generator.sampling_params.max_generate_length=500 \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=5 \ + generator.max_turns=4 \ + generator.sampling_params.temperature=1.0 \ + generator.sampling_params.top_p=1.0 \ + generator.sampling_params.stop='["", ""]' \ + generator.append_eos_token_after_stop_str_in_multi_turn=true \ + environment.env_class="search" \ + environment.skyrl_gym.max_env_workers=16 \ + environment.skyrl_gym.search.log_requests=false \ + environment.skyrl_gym.search.search_url="http://127.0.0.1:8000/retrieve" \ + environment.skyrl_gym.search.topk=3 \ + trainer.logger="wandb" \ + trainer.project_name="skyrl-search" \ + trainer.run_name="${RUN_NAME}" \ + trainer.ckpt_interval=20 \ + trainer.hf_save_interval=100 \ + trainer.max_ckpts_to_keep=5 \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/${RUN_NAME}" \ + trainer.eval_batch_size=256 \ + trainer.eval_before_train=false \ + generator.eval_sampling_params.temperature=0 \ + generator.eval_sampling_params.stop='["", ""]' \ + trainer.export_path="$HOME/${RUN_NAME}/exports" \ + trainer.eval_interval=50 \ + $@ + \ No newline at end of file diff --git a/skyrl-train/examples/search/searchr1_dataset.py b/skyrl-train/examples/search/searchr1_dataset.py new file mode 100644 index 0000000000..4eacc32d5d --- /dev/null +++ b/skyrl-train/examples/search/searchr1_dataset.py @@ -0,0 +1,326 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2023-2024 SGLang Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import tempfile + +import pandas as pd +from huggingface_hub import hf_hub_download +from huggingface_hub.utils import EntryNotFoundError +import shutil + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_SFT_LOGGING_LEVEL", "WARN")) + +_HDFS_PREFIX = "hdfs://" + +_HDFS_BIN_PATH = shutil.which("hdfs") + + +def exists(path: str, **kwargs) -> bool: + r"""Works like os.path.exists() but supports hdfs. + + Test whether a path exists. Returns False for broken symbolic links. + + Args: + path (str): path to test + + Returns: + bool: True if the path exists, False otherwise + """ + if _is_non_local(path): + return _exists(path, **kwargs) + return os.path.exists(path) + + +def _exists(file_path: str): + """hdfs capable to check whether a file_path is exists""" + if file_path.startswith("hdfs"): + return _run_cmd(_hdfs_cmd(f"-test -e {file_path}")) == 0 + return os.path.exists(file_path) + + +def makedirs(name, mode=0o777, exist_ok=False, **kwargs) -> None: + r"""Works like os.makedirs() but supports hdfs. + + Super-mkdir; create a leaf directory and all intermediate ones. Works like + mkdir, except that any intermediate path segment (not just the rightmost) + will be created if it does not exist. If the target directory already + exists, raise an OSError if exist_ok is False. Otherwise no exception is + raised. This is recursive. + + Args: + name (str): directory to create + mode (int): file mode bits + exist_ok (bool): if True, do not raise an exception if the directory already exists + kwargs: keyword arguments for hdfs + + """ + if _is_non_local(name): + # TODO(haibin.lin): + # - handle OSError for hdfs(?) + # - support exist_ok for hdfs(?) + _mkdir(name, **kwargs) + else: + os.makedirs(name, mode=mode, exist_ok=exist_ok) + + +def _mkdir(file_path: str) -> bool: + """hdfs mkdir""" + if file_path.startswith("hdfs"): + _run_cmd(_hdfs_cmd(f"-mkdir -p {file_path}")) + else: + os.makedirs(file_path, exist_ok=True) + return True + + +def copy(src: str, dst: str, **kwargs) -> bool: + r"""Works like shutil.copy() for file, and shutil.copytree for dir, and supports hdfs. + + Copy data and mode bits ("cp src dst"). Return the file's destination. + The destination may be a directory. + If source and destination are the same file, a SameFileError will be + raised. + + Arg: + src (str): source file path + dst (str): destination file path + kwargs: keyword arguments for hdfs copy + + Returns: + str: destination file path + + """ + if _is_non_local(src) or _is_non_local(dst): + # TODO(haibin.lin): + # - handle SameFileError for hdfs files(?) + # - return file destination for hdfs files + return _copy(src, dst) + else: + if os.path.isdir(src): + return shutil.copytree(src, dst, **kwargs) + else: + return shutil.copy(src, dst, **kwargs) + + +def _copy(from_path: str, to_path: str, timeout: int = None) -> bool: + if to_path.startswith("hdfs"): + if from_path.startswith("hdfs"): + returncode = _run_cmd(_hdfs_cmd(f"-cp -f {from_path} {to_path}"), timeout=timeout) + else: + returncode = _run_cmd(_hdfs_cmd(f"-put -f {from_path} {to_path}"), timeout=timeout) + else: + if from_path.startswith("hdfs"): + returncode = _run_cmd( + _hdfs_cmd( + f"-get \ + {from_path} {to_path}" + ), + timeout=timeout, + ) + else: + try: + shutil.copy(from_path, to_path) + returncode = 0 + except shutil.SameFileError: + returncode = 0 + except Exception as e: + logger.warning(f"copy {from_path} {to_path} failed: {e}") + returncode = -1 + return returncode == 0 + + +def _run_cmd(cmd: str, timeout=None): + return os.system(cmd) + + +def _hdfs_cmd(cmd: str) -> str: + return f"{_HDFS_BIN_PATH} dfs {cmd}" + + +def _is_non_local(path: str): + return path.startswith(_HDFS_PREFIX) + + +# Setup logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + +# Configuration constants +DEFAULT_SYSTEM_CONTENT = "You are a helpful and harmless assistant." +DEFAULT_USER_CONTENT_PREFIX = ( + "Answer the given question. You must conduct reasoning inside and " + "first every time you get new information. After reasoning, if you find you lack " + "some knowledge, you can call a search engine by query " + "and it will return the top searched results between and " + ". You can search as many times as you want. If you find no " + "further external knowledge needed, you can directly provide the answer inside " + " and , without detailed illustrations. For example, " + " Beijing . Question: " +) + + +def process_single_row(row, current_split_name, row_index): + """ + Process a single row of data for SearchR1-like format. + + Args: + row: DataFrame row containing the original data + current_split_name: Name of the current split (train/test) + row_index: Index of the row in the DataFrame + + Returns: + pd.Series: Processed row data in the required format + """ + question = row.get("question", "") + + # Build prompt structure + user_content = user_content_prefix.rstrip("\n") + question + prompt = [{"role": "system", "content": system_content}, {"role": "user", "content": user_content}] + + # Extract ground truth from the "reward" field or fallback to golden_answers + reward_data = row.get("reward_model") + if isinstance(reward_data, dict) and "ground_truth" in reward_data: + ground_truth = reward_data.get("ground_truth") + else: + ground_truth = row.get("golden_answers", []) + + # Process data source + data_source_tagged = "searchR1_" + str(row.get("data_source", "")) + + # Build tools kwargs structure + tools_kwargs = { + "search": { + "create_kwargs": {"ground_truth": ground_truth, "question": question, "data_source": data_source_tagged} + } + } + + # Build complete extra_info structure + extra_info = { + "index": row_index, + "need_tools_kwargs": True, + "question": question, + "split": current_split_name, + "tools_kwargs": tools_kwargs, + } + + return pd.Series( + { + "data_source": data_source_tagged, + "prompt": prompt, + "ability": row.get("ability"), + "env_class": "search", + "reward_spec": reward_data, + "extra_info": extra_info, + "metadata": row.get("metadata"), + } + ) + + +def main(): + local_save_dir = os.path.expanduser(args.local_dir) + os.makedirs(local_save_dir, exist_ok=True) + + processed_files = [] + + # Determine splits to process + splits = [args.split] if args.split else ["train", "test"] + + # Download and process files using temporary directory + with tempfile.TemporaryDirectory() as tmp_download_dir: + for split in splits: + parquet_filename = f"{split}.parquet" + logger.info(f"Processing {split} split...") + + try: + # Download Parquet file from HuggingFace + logger.info(f"Downloading {parquet_filename} from {args.hf_repo_id}") + local_parquet_filepath = hf_hub_download( + repo_id=args.hf_repo_id, + filename=parquet_filename, + repo_type="dataset", + local_dir=tmp_download_dir, + local_dir_use_symlinks=False, + ) + + # Load and process Parquet file + df_raw = pd.read_parquet(local_parquet_filepath) + if args.max_rows is not None: + if split == "train": + df_raw = df_raw.iloc[: args.max_rows] + else: + df_raw = df_raw.iloc[: min(args.max_rows, 10)] + + logger.info(f"Loaded {len(df_raw)} rows from {parquet_filename}") + + def apply_process_row(row, split_name=split): + return process_single_row(row, current_split_name=split_name, row_index=row.name) + + df_processed = df_raw.apply(apply_process_row, axis=1) + + # Save processed DataFrame + if split == "test": + logger.info("Saving `test.parquet` from HF as `validation.parquet` locally") + output_file_path = os.path.join(local_save_dir, "validation.parquet") + else: + output_file_path = os.path.join(local_save_dir, f"{split}.parquet") + df_processed.to_parquet(output_file_path, index=False) + logger.info(f"Saved {len(df_processed)} processed rows to {output_file_path}") + processed_files.append(output_file_path) + + except EntryNotFoundError: + logger.warning(f"{parquet_filename} not found in repository {args.hf_repo_id}") + except Exception as e: + logger.error(f"Error processing {split} split: {e}") + + if not processed_files: + logger.warning("No data was processed or saved") + return + + logger.info(f"Successfully processed {len(processed_files)} files to {local_save_dir}") + + # Copy to HDFS if specified + if args.hdfs_dir: + try: + makedirs(args.hdfs_dir) + copy(src=local_save_dir, dst=args.hdfs_dir) + logger.info(f"Successfully copied files to HDFS: {args.hdfs_dir}") + except Exception as e: + logger.error(f"Error copying files to HDFS: {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Download Search-R1 from HuggingFace, process, and save to Parquet.") + parser.add_argument( + "--hf_repo_id", default="PeterJinGo/nq_hotpotqa_train", help="HuggingFace dataset repository ID." + ) + parser.add_argument( + "--local_dir", default="./data/searchR1", help="Local directory to save the processed Parquet files." + ) + parser.add_argument("--hdfs_dir", default=None, help="Optional HDFS directory to copy the Parquet files to.") + parser.add_argument("--max_rows", type=int, default=None, help="Maximum number of rows to process from each split.") + parser.add_argument( + "--split", choices=["train", "test"], default=None, help="If set, only download this split (train or test)." + ) + + args = parser.parse_args() + + # System and user content configuration + system_content = DEFAULT_SYSTEM_CONTENT + user_content_prefix = DEFAULT_USER_CONTENT_PREFIX + + main() diff --git a/skyrl-train/examples/search/searchr1_download.py b/skyrl-train/examples/search/searchr1_download.py new file mode 100644 index 0000000000..79dc4650bb --- /dev/null +++ b/skyrl-train/examples/search/searchr1_download.py @@ -0,0 +1,25 @@ +import argparse +from huggingface_hub import hf_hub_download + +parser = argparse.ArgumentParser(description="Download files from a Hugging Face dataset repository.") +parser.add_argument("--repo_id", type=str, default="PeterJinGo/wiki-18-e5-index", help="Hugging Face repository ID") +parser.add_argument("--local_dir", type=str, required=True, help="Local directory to save files") + +args = parser.parse_args() + +repo_id = "PeterJinGo/wiki-18-e5-index" +for file in ["part_aa", "part_ab"]: + hf_hub_download( + repo_id=repo_id, + filename=file, # e.g., "e5_Flat.index" + repo_type="dataset", + local_dir=args.local_dir, + ) + +repo_id = "PeterJinGo/wiki-18-corpus" +hf_hub_download( + repo_id=repo_id, + filename="wiki-18.jsonl.gz", + repo_type="dataset", + local_dir=args.local_dir, +) diff --git a/skyrl-train/examples/step_wise/run_skyrl_sql_step_wise.sh b/skyrl-train/examples/step_wise/run_skyrl_sql_step_wise.sh new file mode 100644 index 0000000000..1d41211729 --- /dev/null +++ b/skyrl-train/examples/step_wise/run_skyrl_sql_step_wise.sh @@ -0,0 +1,75 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-7B-Instruct on SkyRL-SQL-653 data. +# Uses 1 node with 8 GPUs. +# hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +# export WANDB_API_KEY= +# bash examples/step_wise/run_skyrl_sql_step_wise.sh + +# change these paths to your own +DATA_DIR="$HOME/data/sql" +DB_PATH="$HOME/data/sql/db_files/data" +CKPT_PATH="$HOME/ckpts/skyrl_sql_7B_ckpt_new" + +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=4 +TP_SIZE=2 +MAX_INPUT_LENGTH=29000 +MAX_GENERATE_LENGTH=3000 +TRAIN_BATCH_SIZE=256 +MAX_TURNS=6 + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-Coder-7B-Instruct" \ + trainer.epochs=30 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.sequence_parallel_size=1 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.max_prompt_length=6000 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=$TRAIN_BATCH_SIZE \ + trainer.algorithm.use_kl_loss=false \ + trainer.ckpt_interval=60 \ + trainer.hf_save_interval=30 \ + trainer.dump_data_batch=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=text2sql \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.max_turns=$MAX_TURNS \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.eval_sampling_params.stop='["", ""]' \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + trainer.logger="wandb" \ + trainer.project_name="stepwise_multiturn" \ + trainer.run_name="skyrlsql_multiturn_test_7b" \ + trainer.resume_mode=null \ + trainer.ckpt_path=$CKPT_PATH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.algorithm.policy_loss_type="dual_clip" \ + generator.step_wise_trajectories=true \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/step_wise/run_skyrl_sql_step_wise_qwen3.sh b/skyrl-train/examples/step_wise/run_skyrl_sql_step_wise_qwen3.sh new file mode 100644 index 0000000000..26613d0cfe --- /dev/null +++ b/skyrl-train/examples/step_wise/run_skyrl_sql_step_wise_qwen3.sh @@ -0,0 +1,79 @@ +set -x + +# Colocated GRPO training+generation for Qwen3-4B on SkyRL-SQL-653 data with step-wise training +# Uses 1 node with 8 GPUs. +# hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +# export WANDB_API_KEY= +# bash examples/step_wise/run_skyrl_sql_step_wise_qwen3.sh + +# change these paths to your own +DATA_DIR="$HOME/data/sql" +DB_PATH="$HOME/data/sql/db_files/data" +CKPT_PATH="$HOME/ckpts/skyrl_sql_qwen3_4b_stepwise" + +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=4 +TP_SIZE=2 +MAX_INPUT_LENGTH=29000 +MAX_GENERATE_LENGTH=3000 +TRAIN_BATCH_SIZE=64 +MAX_TURNS=6 + +# NOTE: we set `generator.retokenize_chat_history` to true so that +# chat template is applied to the input each time - this ensures +# that previous think tokens are removed +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen3-4B" \ + trainer.epochs=30 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.sequence_parallel_size=1 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.max_prompt_length=6000 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=$TRAIN_BATCH_SIZE \ + trainer.algorithm.use_kl_loss=false \ + trainer.ckpt_interval=10 \ + trainer.hf_save_interval=30 \ + trainer.dump_data_batch=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=text2sql \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.max_turns=$MAX_TURNS \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.eval_sampling_params.stop='["", ""]' \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + trainer.logger="wandb" \ + trainer.project_name="stepwise_multiturn" \ + trainer.run_name="skyrlsql_multiturn_qwen3" \ + trainer.resume_mode=null \ + trainer.ckpt_path=$CKPT_PATH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.algorithm.policy_loss_type="dual_clip" \ + generator.step_wise_trajectories=true \ + +generator.retokenize_chat_history=true \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/terminal_bench/README.md b/skyrl-train/examples/terminal_bench/README.md new file mode 100644 index 0000000000..bcea5712cd --- /dev/null +++ b/skyrl-train/examples/terminal_bench/README.md @@ -0,0 +1,20 @@ +### Terminal-Bench integration (WIP) + +Integration with Terminal-Bench is a work in progress. For now, training tasks are hard-coded as "hello-world" in the prototype. The next TODO is to support specifying a training set of Terminal-Bench tasks. + +This integration requires the `harbor` repo (ie, the new and improved terminal bench): +```bash +cd SkyRL/skyrl-train +git clone https://github.com/laude-institute/harbor.git +``` + + +- **Training**: +```bash +bash examples/terminal_bench/run_tbench.sh +``` + +- **Generation only**: launch the generator/serving process. This entrypoint is primarily for rapid debugging to avoid the trainer setup overhead. +```bash +bash examples/terminal_bench/run_tbench_gen.sh +``` \ No newline at end of file diff --git a/skyrl-train/examples/terminal_bench/dataset.py b/skyrl-train/examples/terminal_bench/dataset.py new file mode 100644 index 0000000000..438793a7ec --- /dev/null +++ b/skyrl-train/examples/terminal_bench/dataset.py @@ -0,0 +1,104 @@ +from loguru import logger +from typing import List +from pathlib import Path + + +class TerminalBenchTaskDataset: + """ + A dataset that loads terminal bench task data from direct file/directory paths. + Each dataset item is a path to a task directory. + """ + + def __init__( + self, + data_files: List[str], + ): + """ + Initialize the TerminalBenchTaskDataset. + + Args: + data_files: List of direct file/directory paths pointing to terminal bench task data + """ + self.data_files = data_files + + # Load all data files + self.task_paths = self._load_data_files() + + logger.info(f"TerminalBenchTaskDataset initialized with {len(self.task_paths)} task paths") + + def _load_data_files(self) -> List[Path]: + """Load all data files from direct paths and return list of task paths.""" + task_paths = [] + + for data_source in self.data_files: + source_path = Path(data_source) + + if not source_path.exists(): + logger.warning(f"Path does not exist: {data_source}") + continue + + logger.info(f"Loading data from: {data_source}") + + # If the path is a directory, find all valid task subdirectories + if source_path.is_dir(): + # Look for task subdirectories and validate them + all_dirs = [d for d in source_path.iterdir() if d.is_dir()] + valid_task_dirs = [d for d in all_dirs if self._is_valid_task_directory(d)] + + if valid_task_dirs: + task_paths.extend(valid_task_dirs) + logger.info( + f"Found {len(valid_task_dirs)} valid task directories out of {len(all_dirs)} total directories" + ) + elif self._is_valid_task_directory(source_path): + # If no subdirectories but the main directory is valid, treat it as a task + task_paths.append(source_path) + logger.info("Using main directory as valid task") + else: + logger.warning(f"No valid task directories found in {source_path}") + else: + # If it's a file, treat it as a single task (files can't be valid task directories) + logger.warning(f"File {source_path} cannot be a valid task directory (missing instruction.md)") + + return task_paths + + def _is_valid_task_directory(self, task_path: Path) -> bool: + """Check if a directory is a valid task directory (has instruction.md file).""" + if not task_path.is_dir(): + return False + + instruction_file = task_path / "instruction.md" + return instruction_file.exists() and instruction_file.is_file() + + def __getitem__(self, index: int) -> dict: + """Get a task path by index as a dictionary with 'prompt', 'env_class', and 'env_extras' keys.""" + if index >= len(self.task_paths): + raise IndexError(f"Index {index} out of range for dataset of size {len(self.task_paths)}") + return { + "prompt": str(self.task_paths[index]), + "env_class": None, + "env_extras": {"data_source": str(self.task_paths[index])}, + "uid": str(index), + } + + def __len__(self) -> int: + """Return the number of tasks in the dataset.""" + return len(self.task_paths) + + def __iter__(self): + """Iterate over all task paths as dictionaries.""" + for index, task_path in enumerate(self.task_paths): + yield { + "prompt": str(task_path), + "env_class": None, + "env_extras": {"data_source": str(task_path)}, + "uid": str(index), + } + + def get_task_paths(self) -> List[Path]: + """Return all task paths as a list.""" + return self.task_paths.copy() + + def collate_fn(self, item_list): + """Collate function for batching task dictionaries.""" + return item_list diff --git a/skyrl-train/examples/terminal_bench/entrypoints/main_tbench.py b/skyrl-train/examples/terminal_bench/entrypoints/main_tbench.py new file mode 100644 index 0000000000..db922953ea --- /dev/null +++ b/skyrl-train/examples/terminal_bench/entrypoints/main_tbench.py @@ -0,0 +1,73 @@ +""" +Main entrypoint for training on terminal bench tasks. +""" + +import ray +import hydra +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir +from skyrl_train.utils import validate_cfg +from skyrl_train.utils.utils import initialize_ray +from examples.terminal_bench.terminal_bench_generator import TerminalBenchGenerator +from examples.terminal_bench.dataset import TerminalBenchTaskDataset + + +class TerminalBenchExp(BasePPOExp): + def get_generator(self, cfg, tokenizer, inference_engine_client): + """ + Initializes the TerminalBenchGenerator. + """ + return TerminalBenchGenerator( + generator_cfg=cfg.generator, + terminal_bench_cfg=cfg.terminal_bench_config, # Pass terminal_bench config to the generator + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + ) + + def get_train_dataset(self): + """Initializes the training dataset. + + Returns: + TerminalBenchTaskDataset: The training dataset. + """ + prompts_dataset = TerminalBenchTaskDataset( + data_files=self.cfg.data.train_data, + ) + # make sure the dataset is large enough to train on + assert ( + len(prompts_dataset) >= self.cfg.trainer.train_batch_size + ), f"dataset should be atleast as large as `train_batch_size` {self.cfg.trainer.train_batch_size}, got size {len(prompts_dataset)}" + return prompts_dataset + + def get_eval_dataset(self): + """Initializes the evaluation dataset. + + Returns: + TerminalBenchTaskDataset: The evaluation dataset. + """ + if self.cfg.trainer.eval_interval > 0 and self.cfg.data.val_data: + prompts_dataset = TerminalBenchTaskDataset( + data_files=self.cfg.data.val_data, + ) + return prompts_dataset + return None + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = TerminalBenchExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/terminal_bench/entrypoints/main_tbench_generate.py b/skyrl-train/examples/terminal_bench/entrypoints/main_tbench_generate.py new file mode 100644 index 0000000000..cd8f478fb8 --- /dev/null +++ b/skyrl-train/examples/terminal_bench/entrypoints/main_tbench_generate.py @@ -0,0 +1,97 @@ +""" +Main entrypoint for generating rollouts on terminal bench tasks. +""" + +import ray +import asyncio +import hydra +from loguru import logger +from omegaconf import DictConfig + +from skyrl_train.utils import validate_cfg +from skyrl_train.utils.utils import initialize_ray +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.entrypoints.main_base import ( + create_ray_wrapped_inference_engines_from_config, + create_remote_inference_engines_from_config, + BasePPOExp, + config_dir, +) +from skyrl_train.generators.base import GeneratorInput +from examples.terminal_bench.generator.terminal_bench_generator import TerminalBenchGenerator +from examples.terminal_bench.dataset import TerminalBenchTaskDataset + + +class TerminalBenchGenerateExp(BasePPOExp): + def get_generator(self, cfg, tokenizer, inference_engine_client): + """ + Initializes the TerminalBenchGenerator. + """ + return TerminalBenchGenerator( + generator_cfg=cfg.generator, + terminal_bench_cfg=cfg.terminal_bench_config, # Pass terminal_bench config to the generator + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + ) + + def _setup_generator(self): + logger.info(self.get_cfg_as_str(self.cfg)) + + tokenizer = self.tokenizer + if self.cfg.generator.run_engines_locally: + inference_engines = create_ray_wrapped_inference_engines_from_config(self.cfg, self.colocate_pg, tokenizer) + else: + inference_engines = create_remote_inference_engines_from_config(self.cfg, tokenizer) + + inference_engine_client = InferenceEngineClient(inference_engines, tokenizer, self.cfg) + asyncio.run(inference_engine_client.wake_up()) + + return self.get_generator(self.cfg, tokenizer, inference_engine_client) + + def get_train_dataset(self): + """Initializes the training dataset. + + Returns: + TerminalBenchTaskDataset: The training dataset. + """ + prompts_dataset = TerminalBenchTaskDataset( + data_files=self.cfg.data.train_data, + ) + assert ( + len(prompts_dataset) >= self.cfg.trainer.train_batch_size + ), f"dataset should be atleast as large as `train_batch_size` {self.cfg.trainer.train_batch_size}, got size {len(prompts_dataset)}" + return prompts_dataset + + def run(self): + generator = self._setup_generator() + + # Build input from the training dataset + input_batch = GeneratorInput( + prompts=[item["prompt"] for item in self.train_dataset], + env_classes=None, + env_extras=None, + sampling_params=None, + ) + + # Start generation + asyncio.run(generator.generate(input_batch)) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = TerminalBenchGenerateExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/terminal_bench/generator/terminal_bench_generator.py b/skyrl-train/examples/terminal_bench/generator/terminal_bench_generator.py new file mode 100644 index 0000000000..57495d093f --- /dev/null +++ b/skyrl-train/examples/terminal_bench/generator/terminal_bench_generator.py @@ -0,0 +1,183 @@ +import asyncio +from dataclasses import dataclass +from typing import List, Optional +from uuid import uuid4 +from skyrl_train.generators.base import GeneratorInterface, GeneratorInput, GeneratorOutput +from skyrl_train.generators.utils import get_rollout_metrics, get_response_ids_and_loss_mask_from_messages +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.base import ConversationType +from omegaconf import DictConfig +from pathlib import Path +from harbor.models.trial.config import TrialConfig, AgentConfig, TaskConfig, EnvironmentConfig +from harbor.models.environment_type import EnvironmentType +from harbor.models.agent.name import AgentName +from harbor.trial.trial import Trial + + +@dataclass +class TerminalBenchAgentOutput: + response_ids: List[int] + reward: float + stop_reason: str + loss_mask: List[int] + prompt_ids: List[int] + rollout_logprobs: Optional[List[float]] + + +class TerminalBenchGenerator(GeneratorInterface): + def __init__( + self, + generator_cfg: DictConfig, + terminal_bench_cfg: DictConfig, + inference_engine_client: InferenceEngineClient, + tokenizer, + ): + """ + Args: + generator_cfg: DictConfig object containing the generator configuration + terminal_bench_cfg: DictConfig object containing the terminal bench configuration + inference_engine_client: InferenceEngineClient object for interacting with the inference engines + tokenizer: tokenizer object for encoding and decoding text + """ + self.base_url = f"http://{generator_cfg.http_endpoint_host}:{generator_cfg.http_endpoint_port}" + self.generator_cfg = generator_cfg + self.tokenizer = tokenizer + self.model_name = generator_cfg.model_name + + # TerminalBench config + self.trials_dir = terminal_bench_cfg.trials_dir + self.agent_name = terminal_bench_cfg.agent_name + self.max_episodes = terminal_bench_cfg.max_episodes + + if self.generator_cfg.chat_template.name_or_path is not None: + raise NotImplementedError("TerminalBenchGenerator doesn't support custom chat template") + + async def generate(self, input_batch: GeneratorInput) -> GeneratorOutput: + prompts = input_batch["prompts"] + tasks = [] + for prompt in prompts: + tasks.append( + self.terminal_bench_agent_loop( + prompt=prompt, + ) + ) + + all_outputs = await asyncio.gather(*tasks) + + responses = [output.response_ids for output in all_outputs] + rewards = [output.reward for output in all_outputs] + rollout_metrics = get_rollout_metrics(responses, rewards) + + generator_output: GeneratorOutput = { + "prompt_token_ids": [output.prompt_ids for output in all_outputs], + "response_ids": responses, + "rewards": rewards, + "loss_masks": [output.loss_mask for output in all_outputs], + "stop_reasons": [output.stop_reason for output in all_outputs], + "rollout_metrics": rollout_metrics, + "rollout_logprobs": [output.rollout_logprobs for output in all_outputs], + } + + return generator_output + + async def terminal_bench_agent_loop( + self, + prompt: ConversationType, + ) -> TerminalBenchAgentOutput: + """ + Run a single terminal_bench agent. + """ + # Generate session_id for sticky routing to inference engines + # All LLM requests in this trial will share the same session_id + session_id = uuid4().hex + + if self.agent_name == "terminus": + trial_config = TrialConfig( + task=TaskConfig(path=prompt), + trials_dir=Path(self.trials_dir), + environment=EnvironmentConfig(type=EnvironmentType.DAYTONA), + agent=AgentConfig( + name=AgentName.TERMINUS_2.value, + model_name=f"hosted_vllm/{self.model_name}", + kwargs={ + "api_base": f"{self.base_url}/v1", + "key": "fake_key", + "session_id": session_id, + "max_episodes": self.max_episodes, + }, + ), + ) + elif self.agent_name == "oracle": + trial_config = TrialConfig( + task=TaskConfig(path=prompt), + trials_dir=Path(self.trials_dir), + environment=EnvironmentConfig(type=EnvironmentType.DAYTONA), + agent=AgentConfig( + name=AgentName.ORACLE, + model_name=f"hosted_vllm/{self.model_name}", + ), + ) + else: + raise ValueError(f"Invalid agent name: {self.agent_name}") + + trial = Trial(trial_config) + # Run the trial + while True: + try: + results = await trial.run() + print(f"Results: {results}") + if not results.verifier_result: + print(f"[WARNING] Exception info: {results.exception_info}") + continue + reward = results.verifier_result.reward + chat_history = results.agent_result.all_messages + if len(chat_history) > 0: + break + else: + print(f"[WARNING] Agent {self.agent_name} did not return a response") + except Exception as e: + print(f"Error running trial: {e}") + continue + + # Use the first message as the prompt. We assume to be no systems messages. + assert chat_history[0]["role"] == "user", "The first message should be a user message" + prompt = [chat_history[0]] + prompt_ids = self.tokenizer.apply_chat_template( + prompt, + add_generation_prompt=False, # the message below will add it themselves + tokenize=True, + ) + initial_prompt_length = len(prompt_ids) + + # Process response messages (everything after the first message) + response_messages = chat_history[1:] + assistant_logprobs = getattr(results.agent_result, "output_logprobs", None) + response_ids, loss_mask, rollout_logprobs = get_response_ids_and_loss_mask_from_messages( + response_messages, self.tokenizer, assistant_logprobs + ) + + # Determine stop reason + max_response_tokens = ( + self.generator_cfg.sampling_params.max_generate_length + + self.generator_cfg.max_input_length + - initial_prompt_length + ) + stop_reason = "complete" # Default for trial completion + if len(response_ids) > max_response_tokens: + stop_reason = "length" + # TODO(Charlie): should we do rewards = self._zero_reward_if_not_stop(rewards, stop_reasons)? + + # Truncate to maximum allowed length + response_ids = response_ids[:max_response_tokens] + loss_mask = loss_mask[:max_response_tokens] + rollout_logprobs = rollout_logprobs[:max_response_tokens] + + return TerminalBenchAgentOutput( + response_ids=response_ids, + reward=reward, + stop_reason=stop_reason, + loss_mask=loss_mask, + prompt_ids=prompt_ids, + # in case harbor doesn't return logprobs, use None + rollout_logprobs=rollout_logprobs if assistant_logprobs is not None else None, + ) diff --git a/skyrl-train/examples/terminal_bench/run_tbench.sh b/skyrl-train/examples/terminal_bench/run_tbench.sh new file mode 100644 index 0000000000..a446273241 --- /dev/null +++ b/skyrl-train/examples/terminal_bench/run_tbench.sh @@ -0,0 +1,60 @@ +set -x + +# WORK IN PROGRESS +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on TerminalBench tasks. + +# export WANDB_API_KEY= +# bash examples/terminal_bench/run_tbench.sh + +DATA_DIR="$(pwd)/harbor/examples/tasks" +NUM_GPUS=1 +LOGGER="console" # change to "console" to print to stdout +TBENCH_CONFIG_DIR="examples/terminal_bench" +SANDBOXES_DIR="sandboxes" # TODO: For now, `sandboxes` is cloned into SkyRL/skyrl-train. + +uv run --isolated --extra vllm --extra sandboxes --with "harbor@./harbor" -m examples.terminal_bench.entrypoints.main_tbench \ + data.train_data="['$DATA_DIR']" \ + data.val_data="['$DATA_DIR']" \ + hydra.searchpath=[file://$TBENCH_CONFIG_DIR] \ + +terminal_bench_config=terminal_bench \ + terminal_bench_config.max_episodes=16 \ + terminal_bench_config.sandboxes_dir=$SANDBOXES_DIR \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=-1 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=8 \ + trainer.policy_mini_batch_size=2 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=-1 \ + trainer.max_prompt_length=4096 \ + generator.enable_http_endpoint=true \ + generator.http_endpoint_host="127.0.0.1" \ + generator.http_endpoint_port=8000 \ + generator.sampling_params.max_generate_length=26000 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="terminal_bench" \ + trainer.run_name="terminal_bench_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_0.5B_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/terminal_bench/run_tbench_gen.sh b/skyrl-train/examples/terminal_bench/run_tbench_gen.sh new file mode 100644 index 0000000000..82dffcdc3e --- /dev/null +++ b/skyrl-train/examples/terminal_bench/run_tbench_gen.sh @@ -0,0 +1,39 @@ +set -x + +# WORK IN PROGRESS +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on TerminalBench tasks. + +# export WANDB_API_KEY= +# bash examples/terminal_bench/run_tbench.sh + +DATA_DIR="$(pwd)/harbor/examples/tasks" +NUM_GPUS=1 +LOGGER="console" # change to "console" to print to stdout +TBENCH_CONFIG_DIR="examples/terminal_bench" +SANDBOXES_DIR="sandboxes" # TODO: For now, `sandboxes` is cloned into SkyRL/skyrl-train. + +uv run --isolated --extra vllm --extra sandboxes --with "harbor@./harbor" -m examples.terminal_bench.entrypoints.main_tbench_generate \ + data.train_data="['$DATA_DIR']" \ + hydra.searchpath=[file://$TBENCH_CONFIG_DIR] \ + +terminal_bench_config=terminal_bench \ + terminal_bench_config.max_episodes=16 \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + generator.enable_http_endpoint=true \ + generator.http_endpoint_host="127.0.0.1" \ + generator.http_endpoint_port=8010 \ + generator.sampling_params.max_generate_length=4096 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.gpu_memory_utilization=0.8 \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.placement.colocate_all=true \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + trainer.train_batch_size=1 \ + trainer.policy_mini_batch_size=1 \ + trainer.logger="$LOGGER" \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/terminal_bench/terminal_bench_config/terminal_bench.yaml b/skyrl-train/examples/terminal_bench/terminal_bench_config/terminal_bench.yaml new file mode 100644 index 0000000000..f95d7adb8c --- /dev/null +++ b/skyrl-train/examples/terminal_bench/terminal_bench_config/terminal_bench.yaml @@ -0,0 +1,5 @@ +# @package terminal_bench_config + +trials_dir: "~/trials" +agent_name: "terminus" +max_episodes: 16 \ No newline at end of file diff --git a/skyrl-train/examples/text_to_sql/run_skyrl_sql.sh b/skyrl-train/examples/text_to_sql/run_skyrl_sql.sh new file mode 100644 index 0000000000..a7449e8ae7 --- /dev/null +++ b/skyrl-train/examples/text_to_sql/run_skyrl_sql.sh @@ -0,0 +1,72 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-7B-Instruct on SkyRL-SQL-653 data. +# Uses 1 node with 8 GPUs. +# hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +# export WANDB_API_KEY= +# bash examples/text_to_sql/run_skyrl_sql.sh + +# change these paths to your own +DATA_DIR="$HOME/data/sql" +DB_PATH="$HOME/data/sql/db_files/data" +CKPT_PATH="$HOME/ckpts/skyrl_sql_7B_ckpt" + +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=2 +TP_SIZE=4 +MAX_INPUT_LENGTH=29000 +MAX_GENERATE_LENGTH=3000 +TRAIN_BATCH_SIZE=256 + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-Coder-7B-Instruct" \ + trainer.epochs=30 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.sequence_parallel_size=1 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.max_prompt_length=6000 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=256 \ + trainer.algorithm.use_kl_loss=false \ + trainer.ckpt_interval=60 \ + trainer.hf_save_interval=30 \ + trainer.dump_data_batch=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=text2sql \ + generator.use_conversation_multi_turn=false \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.max_turns=6 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.eval_sampling_params.stop='["", ""]' \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + trainer.logger="wandb" \ + trainer.project_name="skyrlsql" \ + trainer.run_name="skyrlsql_repro" \ + trainer.resume_mode=latest \ + trainer.ckpt_path=$CKPT_PATH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/text_to_sql/run_skyrl_sql_conversation_format.sh b/skyrl-train/examples/text_to_sql/run_skyrl_sql_conversation_format.sh new file mode 100644 index 0000000000..5194596475 --- /dev/null +++ b/skyrl-train/examples/text_to_sql/run_skyrl_sql_conversation_format.sh @@ -0,0 +1,80 @@ +set -x + +# The exact same script as `run_skyrl_sql.sh` but with `use_conversation_multi_turn=true` +# and hence `append_eos_token_after_stop_str_in_multi_turn=true` +# See https://skyrl.readthedocs.io/en/latest/tutorials/skyrl_gym_generator.html on what behavior +# use_conversation_multi_turn corresponds to. You might want to change the data generation prompt +# to let the model know that we are doing multi-turn conversations (i.e. user will provide +# the search result for each turn). + +# Colocated GRPO training+generation for Qwen2.5-Coder-7B-Instruct on SkyRL-SQL-653 data. +# Uses 1 node with 8 GPUs. +# hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +# export WANDB_API_KEY= +# bash examples/text_to_sql/run_skyrl_sql_conversation_format.sh + +# change these paths to your own +DATA_DIR="$HOME/data/sql" +DB_PATH="$HOME/data/sql/db_files/data" +CKPT_PATH="$HOME/ckpts/skyrl_sql_7B_ckpt" + +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=2 +TP_SIZE=4 +MAX_INPUT_LENGTH=29000 +MAX_GENERATE_LENGTH=3000 +TRAIN_BATCH_SIZE=256 + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-Coder-7B-Instruct" \ + trainer.epochs=30 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.sequence_parallel_size=1 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.max_prompt_length=6000 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=256 \ + trainer.algorithm.use_kl_loss=false \ + trainer.ckpt_interval=60 \ + trainer.hf_save_interval=30 \ + trainer.dump_data_batch=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=text2sql \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.max_turns=6 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.append_eos_token_after_stop_str_in_multi_turn=true \ + generator.eval_sampling_params.stop='["", ""]' \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + trainer.logger="wandb" \ + trainer.project_name="skyrlsql" \ + trainer.run_name="skyrlsql_repro" \ + trainer.resume_mode=latest \ + trainer.ckpt_path=$CKPT_PATH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/text_to_sql/run_skyrl_sql_megatron_lora.sh b/skyrl-train/examples/text_to_sql/run_skyrl_sql_megatron_lora.sh new file mode 100644 index 0000000000..f33cb7e65c --- /dev/null +++ b/skyrl-train/examples/text_to_sql/run_skyrl_sql_megatron_lora.sh @@ -0,0 +1,94 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-7B-Instruct on SkyRL-SQL-653 data using Megatron and LoRA. +# Uses 1 node with 8 GPUs. +# huggingface-cli download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +# export WANDB_API_KEY= +# bash examples/text_to_sql/run_skyrl_sql_megatron_lora.sh + +# change these paths to your own +DATA_DIR="$HOME/data/sql" +DB_PATH="$HOME/data/sql/db_files/data" +CKPT_PATH="$HOME/ckpts/skyrl_sql_7B_ckpt" + +LORA_RANK=32 +LORA_ALPHA=64 +LORA_A_INIT_METHOD="kaiming" + +NUM_NODES=1 +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=2 +TP_SIZE=4 +MAX_INPUT_LENGTH=29000 +MAX_GENERATE_LENGTH=3000 +TRAIN_BATCH_SIZE=256 + +# Megatron parameters +MEGATRON_TP=4 +MEGATRON_PP=1 +MEGATRON_CP=1 +MEGATRON_EP=1 +MEGATRON_ETP=null + +# TIS parameters +TIS_IMP_RATIO_CAP=2.0 +USE_TIS=true + +uv run --isolated --extra mcore -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-Coder-7B-Instruct" \ + trainer.policy.model.lora.rank=$LORA_RANK \ + trainer.policy.model.lora.alpha=$LORA_ALPHA \ + trainer.policy.model.lora.init_method=$LORA_A_INIT_METHOD \ + trainer.epochs=30 \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_nodes=$NUM_NODES \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.policy.megatron_config.expert_model_parallel_size=$MEGATRON_EP \ + trainer.policy.megatron_config.expert_tensor_parallel_size=$MEGATRON_ETP \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=2 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.max_prompt_length=6000 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + trainer.policy.optimizer_config.lr=3.0e-5 \ + trainer.policy_mini_batch_size=256 \ + trainer.algorithm.use_kl_loss=false \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.ckpt_interval=60 \ + trainer.hf_save_interval=30 \ + trainer.dump_data_batch=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=text2sql \ + generator.use_conversation_multi_turn=false \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.max_turns=6 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.eval_sampling_params.stop='["", ""]' \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + trainer.logger="wandb" \ + trainer.project_name="skyrlsql" \ + trainer.run_name="skyrlsql_repro_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path=$CKPT_PATH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/text_to_sql/run_sql_fsdp.sh b/skyrl-train/examples/text_to_sql/run_sql_fsdp.sh new file mode 100755 index 0000000000..c62a12bd34 --- /dev/null +++ b/skyrl-train/examples/text_to_sql/run_sql_fsdp.sh @@ -0,0 +1,74 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-7B-Instruct on SkyRL-SQL-653 data. +# Uses 1 node with 8 GPUs. +# hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +# export WANDB_API_KEY= +# bash examples/text_to_sql/run_sql_fsdp.sh + +DATA_DIR="$HOME/data/sql" +DB_PATH="$HOME/data/sql/db_files/data" +CKPT_PATH="$HOME/ckpts/skyrl_sql_7B_ckpt" + +NUM_GPUS=8 +NUM_INFERENCE_ENGINES=2 +TP_SIZE=4 +MAX_INPUT_LENGTH=29000 +MAX_GENERATE_LENGTH=3000 +TRAIN_BATCH_SIZE=256 + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-Coder-7B-Instruct" \ + trainer.epochs=50 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=false \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.sequence_parallel_size=1 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.train_batch_size=$TRAIN_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.max_prompt_length=6000 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=256 \ + trainer.algorithm.use_kl_loss=false \ + trainer.algorithm.policy_loss_type="dual_clip" \ + trainer.ckpt_interval=5 \ + trainer.hf_save_interval=5 \ + trainer.export_path=$HOME/export/ \ + trainer.dump_data_batch=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=text2sql \ + generator.use_conversation_multi_turn=false \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.max_turns=6 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.eval_sampling_params.stop='["", ""]' \ + trainer.seed=1234 \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + trainer.logger="wandb" \ + trainer.project_name="skyrlsql" \ + trainer.run_name="skyrlsql_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path=$CKPT_PATH \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ \ No newline at end of file diff --git a/skyrl-train/examples/text_to_sql/run_sql_fsdp_2node.sh b/skyrl-train/examples/text_to_sql/run_sql_fsdp_2node.sh new file mode 100644 index 0000000000..147e5eb92f --- /dev/null +++ b/skyrl-train/examples/text_to_sql/run_sql_fsdp_2node.sh @@ -0,0 +1,65 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-Coder-32B-Instruct on SkyRL-SQL-653 data. +# Uses 2 nodes with 8 GPUs each. +# NOTE: you may need to download the DB files for env interaction onto each node for multi-node training, since the driver worker +# will be scheduled on a worker node +# hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +# export WANDB_API_KEY= +# bash examples/text_to_sql/run_sql_fsdp_2node.sh + +DATA_DIR="$HOME/data/sql" +DB_PATH="$HOME/data/sql/db_files/data" + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-Coder-32B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.policy.fsdp_config.cpu_offload=true \ + trainer.ref.fsdp_config.cpu_offload=true \ + trainer.policy.optimizer_config.max_grad_norm=0.5 \ + trainer.policy.sequence_parallel_size=2 \ + trainer.placement.policy_num_nodes=2 \ + trainer.placement.critic_num_nodes=2 \ + trainer.placement.ref_num_nodes=2 \ + trainer.placement.reward_num_nodes=2 \ + trainer.placement.policy_num_gpus_per_node=8 \ + trainer.placement.ref_num_gpus_per_node=8 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=4 \ + trainer.train_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.max_prompt_length=6000 \ + generator.max_input_length=29000 \ + generator.sampling_params.max_generate_length=3000 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=64 \ + trainer.algorithm.use_kl_loss=true \ + trainer.ckpt_interval=10 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=text2sql \ + generator.use_conversation_multi_turn=false \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.7 \ + generator.max_turns=5 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='["", ""]' \ + generator.eval_sampling_params.stop='["", ""]' \ + environment.skyrl_gym.text2sql.db_path=$DB_PATH \ + trainer.logger="wandb" \ + trainer.project_name="skyrlsql" \ + trainer.run_name="skyrlsql_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/sql_32B_ckpt" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 diff --git a/skyrl-train/examples/text_to_sql/sql.md b/skyrl-train/examples/text_to_sql/sql.md new file mode 100644 index 0000000000..eb53fb1fba --- /dev/null +++ b/skyrl-train/examples/text_to_sql/sql.md @@ -0,0 +1,40 @@ +## Downloading SQL Data + +We provide the training set we used on HuggingFace: https://huggingface.co/datasets/NovaSky-AI/SkyRL-SQL-653-data-newfmt + +You can download the dataset by running the following command: + +```bash +hf download NovaSky-AI/SkyRL-SQL-653-data-newfmt --local-dir $HOME/data/sql --repo-type dataset +``` + +## DB environment + +Next, download the data used to populate the database. We use the database files from [OmniSQL](https://github.com/RUCKBReasoning/OmniSQL/blob/main/train_and_evaluate/README.md): + + +- [ModelScope-OmniSQL-datasets](https://modelscope.cn/datasets/seeklhy/OmniSQL-datasets/summary) +- [HuggingFace-OmniSQL-datasets](https://huggingface.co/datasets/seeklhy/OmniSQL-datasets) + + +The dataset download is 22.2GB and include BIRD, Spider, ScienceBenchmark, EHRSQL, Spider2-SQLite, Spider-DK, Spider-Realistic, Spider-Syn, and SynSQL-2.5M. In our training pipeline, we only need to access databases from SynSQL-2.5M and Spider. + +To download, run: + +```bash +hf download seeklhy/OmniSQL-datasets data.zip --repo-type dataset --local-dir $HOME/data/sql/db_files/ + +unzip $HOME/data/sql/db_files/data.zip -d $HOME/data/sql/db_files/ +``` + +If you modify the db_files path, update `DB_PATH` in `run_sql_fsdp.sh` accordingly. + +## Training + +See the bash scripts in this repo to launch training. Take a look at the file of interest, and modify any training configuration parameters as needed. + +To start training, run: + +```bash +bash examples/text_to_sql/.sh +``` \ No newline at end of file diff --git a/skyrl-train/examples/tis_correction/main_tis_dapo.py b/skyrl-train/examples/tis_correction/main_tis_dapo.py new file mode 100644 index 0000000000..0337609d26 --- /dev/null +++ b/skyrl-train/examples/tis_correction/main_tis_dapo.py @@ -0,0 +1,98 @@ +""" +uv run --isolated --extra vllm -m examples.tis_correction.main_tis_dapo +""" + +import ray +import hydra +import torch +from typing import List +from omegaconf import DictConfig +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.utils import initialize_ray +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg + +from skyrl_train.generators.base import GeneratorOutput + + +class DAPOTrainer(RayPPOTrainer): + """ + Custom trainer for DAPO. + + Overrides the postprocess_generator_output method to additionally apply soft overlong punishment to rewards. + """ + + @torch.no_grad() + def postprocess_generator_output(self, generator_output: GeneratorOutput, uids: List[str]) -> GeneratorOutput: + """ + Overrides the postprocess_generator_output method to additionally apply DAPO specific soft overlong punishment to rewards. + + Args: + generator_output: GeneratorOutput + uids: List[str] + + Returns: + GeneratorOutput + """ + overlong_buffer_len = self.cfg.trainer.algorithm.overlong_buffer.len + overlong_buffer_penalty_factor = self.cfg.trainer.algorithm.overlong_buffer.penalty_factor + # modify rewards here + prompt_token_ids = generator_output["prompt_token_ids"] + response_ids = generator_output["response_ids"] + rewards = generator_output["rewards"] + + assert not isinstance(rewards[0], list), "we assume verifiable sequence level rewards here" + + # get the prompt length + prompt_lengths = [len(prompt) for prompt in prompt_token_ids] + + # get the response length + response_lengths = [len(response) for response in response_ids] + + # get the max context length + max_context_length = ( + self.cfg.generator.max_input_length + self.cfg.generator.sampling_params.max_generate_length + ) + + # apply soft overlong punishment + for i, (prompt_length, response_length) in enumerate(zip(prompt_lengths, response_lengths)): + # max_exceed_length is the beginning of the overlong buffer + max_exceed_length = max_context_length - overlong_buffer_len - prompt_length + # if the response is within the overlong buffer, apply the penalty + if response_length > max_exceed_length and response_length <= max_context_length - prompt_length: + exceed_length = response_length - max_exceed_length + penalty = exceed_length / overlong_buffer_len * overlong_buffer_penalty_factor + + rewards[i] -= penalty + # if the response is outside the overlong buffer, set the reward to 0 + elif response_length > max_context_length - prompt_length: + # if self.cfg.generator.apply_overlong_filtering is true, loss masks are already set to 0 for these responses + rewards[i] = 0.0 + + generator_output["rewards"] = rewards + + # use base class impl for metrics and per-token reward conversion + return super().postprocess_generator_output(generator_output, uids) + + +class DAPOExp(BasePPOExp): + def get_trainer(self, *args, **kwargs): + return DAPOTrainer(*args, **kwargs) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = DAPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/examples/tis_correction/run_dapo_tis.sh b/skyrl-train/examples/tis_correction/run_dapo_tis.sh new file mode 100644 index 0000000000..a0995dd3c8 --- /dev/null +++ b/skyrl-train/examples/tis_correction/run_dapo_tis.sh @@ -0,0 +1,95 @@ +set -x + +# Colocated DAPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K. + +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/algorithms/dapo/run_dapo_gsm8k.sh + +DATA_DIR="$HOME/data/gsm8k" +NUM_GPUS=4 +LOGGER="wandb" # change to "console" to print to stdout + +# TIS parameters +TIS_IMP_RATIO_CAP=2.0 +USE_TIS=true +# returns rollout logprobs for the generated tokens; required for TIS +LOGPROBS=0 + +# main DAPO parameters +EPS_CLIP_LOW=0.2 +EPS_CLIP_HIGH=0.28 +DYNAMIC_SAMPLING_TYPE=filter +DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES=30 +LOSS_REDUCTION="token_mean" +# applies overlong filtering (but not soft overlong punishment) +APPLY_OVERLONG_FILTERING=true +# apply soft overlong punishment with custom trainer impl in main_dapo.py +OVERLONG_BUFFER_LEN=512 +OVERLONG_BUFFER_PENALTY_FACTOR=1.0 + +# other DAPO parameters +USE_KL_LOSS=false +TEMPERATURE=1.0 +TOP_P=1.0 +EVAL_TOP_P=0.7 +CLIP_RATIO_C=10.0 +MAX_RESPONSE_LENGTH=1024 + +uv run --isolated --extra vllm -m examples.tis_correction.main_tis_dapo \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.algorithm.policy_loss_type="dual_clip" \ + +trainer.algorithm.overlong_buffer.len=$OVERLONG_BUFFER_LEN \ + +trainer.algorithm.overlong_buffer.penalty_factor=$OVERLONG_BUFFER_PENALTY_FACTOR \ + trainer.algorithm.eps_clip_low=$EPS_CLIP_LOW \ + trainer.algorithm.eps_clip_high=$EPS_CLIP_HIGH \ + trainer.algorithm.dynamic_sampling.type=$DYNAMIC_SAMPLING_TYPE \ + trainer.algorithm.dynamic_sampling.max_sample_batches=$DYNAMIC_SAMPLING_MAX_SAMPLE_BATCHES \ + trainer.algorithm.loss_reduction=$LOSS_REDUCTION \ + generator.apply_overlong_filtering=$APPLY_OVERLONG_FILTERING \ + generator.sampling_params.temperature=$TEMPERATURE \ + generator.sampling_params.top_p=$TOP_P \ + generator.sampling_params.logprobs=$LOGPROBS \ + generator.eval_sampling_params.top_p=$EVAL_TOP_P \ + trainer.algorithm.use_kl_loss=$USE_KL_LOSS \ + trainer.algorithm.clip_ratio_c=$CLIP_RATIO_C \ + trainer.algorithm.use_tis=$USE_TIS \ + trainer.algorithm.tis_imp_ratio_cap=$TIS_IMP_RATIO_CAP \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy.optimizer_config.weight_decay=0.1 \ + trainer.policy.optimizer_config.max_grad_norm=1.0 \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_dapo_tis" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt" \ + $@ diff --git a/skyrl-train/examples/training_backends/fsdp/run_fsdp.sh b/skyrl-train/examples/training_backends/fsdp/run_fsdp.sh new file mode 100644 index 0000000000..adac46d97f --- /dev/null +++ b/skyrl-train/examples/training_backends/fsdp/run_fsdp.sh @@ -0,0 +1,43 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K with FSDP backend. +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/training_backends/fsdp/run_fsdp.sh + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$HOME/data/gsm8k/train.parquet']" \ + data.val_data="['$HOME/data/gsm8k/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.train_batch_size=1024 \ + trainer.micro_train_batch_size_per_gpu=40 \ + trainer.micro_forward_batch_size_per_gpu=40 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=256 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt_fsdp" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ diff --git a/skyrl-train/examples/training_backends/fsdp/run_fsdp2.sh b/skyrl-train/examples/training_backends/fsdp/run_fsdp2.sh new file mode 100644 index 0000000000..343ebdff95 --- /dev/null +++ b/skyrl-train/examples/training_backends/fsdp/run_fsdp2.sh @@ -0,0 +1,43 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K with FSDP2 backend. +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/training_backends/fsdp/run_fsdp2.sh + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$HOME/data/gsm8k/train.parquet']" \ + data.val_data="['$HOME/data/gsm8k/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.train_batch_size=1024 \ + trainer.micro_train_batch_size_per_gpu=40 \ + trainer.micro_forward_batch_size_per_gpu=40 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=256 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt_fsdp2" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ diff --git a/skyrl-train/examples/training_backends/run_no_seq_pack.sh b/skyrl-train/examples/training_backends/run_no_seq_pack.sh new file mode 100644 index 0000000000..b17fc2bc11 --- /dev/null +++ b/skyrl-train/examples/training_backends/run_no_seq_pack.sh @@ -0,0 +1,44 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K without sequence packing. +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# export WANDB_API_KEY= +# bash examples/training_backends/run_no_seq_pack.sh + +uv run --isolated --extra vllm -m skyrl_train.entrypoints.main_base \ + trainer.algorithm.advantage_estimator="grpo" \ + data.train_data="['$HOME/data/gsm8k/train.parquet']" \ + data.val_data="['$HOME/data/gsm8k/validation.parquet']" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.train_batch_size=16 \ + trainer.max_prompt_length=512 \ + trainer.use_sample_packing=false \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.micro_forward_batch_size_per_gpu=32 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.policy_mini_batch_size=4 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k" \ + trainer.run_name="gsm8k_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_1.5B_ckpt_no_seq_pack" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + $@ diff --git a/skyrl-train/examples/turn_level_rewards/README.md b/skyrl-train/examples/turn_level_rewards/README.md new file mode 100644 index 0000000000..a675290a3a --- /dev/null +++ b/skyrl-train/examples/turn_level_rewards/README.md @@ -0,0 +1,46 @@ +### Turn-level rewards (GSM8K multi-turn) + +This example shows how to train with turn-level rewards in a multi-turn environment using GSM8K. + +### How the GSM8K multi-turn environment works + +We give the model up to `max_turns` attempts to answer the question. The episode ends early if a correct final numeric answer is produced. + +We assign the following per-turn rewards: + +- **Correct strict match**: 1.0 and the episode terminates. +- **Well-formatted but incorrect** (answer includes `#### ANSWER`): `0.2 / max_turns`. +- **Otherwise**: 0.0. + +**Per-token credit assignment**: Turn-level rewards are converted to token level by assigning the entire turn reward to the final token of the assistant's response for that turn (all other tokens receive 0). This is how rewards are passed to the trainer. + +**Custom advantage estimators**: You can plug in custom advantage estimators to operate on the per-turn rewards via `AdvantageEstimatorRegistry` (e.g., using `@register_advantage_estimator`). See `examples/algorithms/custom_advantage_estimator` for an example. + +## How to run the example + +### 1) Generate the dataset + +This pulls `openai/gsm8k` from Hugging Face, adds a short instruction and the ground-truth answer: + +```bash +# Adjust output_dir and max_turns as needed +uv run examples/turn_level_rewards/gsm8k_multi_turn_dataset.py \ + --output_dir "$HOME/data/gsm8k_multi_turn" \ + --max_turns 5 +``` + +Outputs: +- `$HOME/data/gsm8k_multi_turn/train.parquet` +- `$HOME/data/gsm8k_multi_turn/validation.parquet` + +If you change `--output_dir`, update the `DATA_DIR` variable in the run script below accordingly. + +### 2) Launch training + +Modify training config parameters in the training script as needed. Commonly modified parameters are: `NUM_GPUS`, `LOGGER`, and `INFERENCE_BACKEND`. + +Then run the training script: + +```bash +bash examples/turn_level_rewards/run_gsm8k_multi_turn.sh +``` \ No newline at end of file diff --git a/skyrl-train/examples/turn_level_rewards/gsm8k_multi_turn_dataset.py b/skyrl-train/examples/turn_level_rewards/gsm8k_multi_turn_dataset.py new file mode 100644 index 0000000000..da7146dada --- /dev/null +++ b/skyrl-train/examples/turn_level_rewards/gsm8k_multi_turn_dataset.py @@ -0,0 +1,78 @@ +""" +Preprocess the Multi-turn GSM8k dataset to parquet format +""" + +import argparse +import re +import os + +import datasets + + +def extract_solution(solution_str): + solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) + assert solution is not None + final_solution = solution.group(0) + final_solution = final_solution.split("#### ")[1].replace(",", "") + return final_solution + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", default="~/data/gsm8k_multi_turn") + parser.add_argument("--max_turns", type=int, default=5) + + args = parser.parse_args() + + args.output_dir = os.path.expanduser(args.output_dir) + + data_source = "openai/gsm8k" + + dataset = datasets.load_dataset(data_source, "main") + + train_dataset = dataset["train"] + val_dataset = dataset["test"] + + instruction_following = 'Let\'s think step by step and output a tentative numeric answer after "####".' + + # add a row to each data item that represents a unique id + def make_map_fn(split): + def process_fn(example, idx): + question_raw = example.pop("question") + + question = question_raw + " " + instruction_following + + answer_raw = example.pop("answer") + solution = extract_solution(answer_raw) + data = { + "data_source": data_source, + "prompt": [ + { + "role": "user", + "content": question, + } + ], + "env_class": "gsm8k_multi_turn", + "reward_spec": { + "method": "rule", + "ground_truth": solution, + }, + "extra_info": { + "max_turns": args.max_turns, + "split": split, + "index": idx, + "answer": answer_raw, + "question": question_raw, + }, + } + return data + + return process_fn + + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + val_dataset = val_dataset.map(function=make_map_fn("test"), with_indices=True) + + output_dir = args.output_dir + os.makedirs(output_dir, exist_ok=True) + train_dataset.to_parquet(os.path.join(output_dir, "train.parquet")) + val_dataset.to_parquet(os.path.join(output_dir, "validation.parquet")) diff --git a/skyrl-train/examples/turn_level_rewards/run_gsm8k_multi_turn.sh b/skyrl-train/examples/turn_level_rewards/run_gsm8k_multi_turn.sh new file mode 100644 index 0000000000..576b068c55 --- /dev/null +++ b/skyrl-train/examples/turn_level_rewards/run_gsm8k_multi_turn.sh @@ -0,0 +1,56 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-1.5B-Instruct on GSM8K with turn-level rewards. + +# uv run examples/turn_level_rewards/gsm8k_multi_turn_dataset.py --output_dir $HOME/data/gsm8k_multi_turn +# export WANDB_API_KEY= +# bash examples/turn_level_rewards/run_gsm8k_multi_turn.sh + +# NOTE (sumanthrh): `micro_train_batch_size_per_gpu` and `micro_forward_batch_size_per_gpu` can be tuned + +DATA_DIR="$HOME/data/gsm8k_multi_turn" +ENV_CLASS="gsm8k_multi_turn" +NUM_GPUS=1 +LOGGER="wandb" # or "console" to print to stdout + +INFERENCE_BACKEND="vllm" # or "sglang" + +uv run --isolated --extra $INFERENCE_BACKEND -m skyrl_train.entrypoints.main_base \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-0.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=false \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=256 \ + trainer.policy_mini_batch_size=128 \ + trainer.micro_forward_batch_size_per_gpu=32 \ + trainer.micro_train_batch_size_per_gpu=32 \ + trainer.ckpt_interval=-1 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=$ENV_CLASS \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="$ENV_CLASS" \ + trainer.run_name="$ENV_CLASS_test" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$HOME/ckpts/gsm8k_multi_turn_ckpt" \ + $@ \ No newline at end of file diff --git a/skyrl-train/integrations/fleet/README.md b/skyrl-train/integrations/fleet/README.md new file mode 100644 index 0000000000..33d22d5d07 --- /dev/null +++ b/skyrl-train/integrations/fleet/README.md @@ -0,0 +1,129 @@ +# Fleet Integration for SkyRL + +This module provides a SkyRL-compatible environment wrapper for Fleet-hosted tasks using OpenEnv as the abstraction layer. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ SkyRL FleetTaskEnv (integrations/fleet/env.py) │ +│ │ +│ __init__(env_config, extras) │ +│ └── Stores config, task_key, api_key │ +│ └── self.openenv_task_env = None (not created yet) │ +│ │ +│ init(prompt) ←── Called by SkyRL trainer to start episode │ +│ │ │ +│ ├── OpenEnvFleetTaskEnv(task_config, ...) │ +│ │ ├── fleet.make() ←── Creates Fleet env (provisions VM) │ +│ │ └── list_tools() ←── Fetches & caches tools │ +│ │ │ +│ ├── reset_async() │ +│ │ ├── _orch.reset() ←── Resets episode state │ +│ │ └── Returns cached tools │ +│ │ │ +│ └── Builds system prompt with tools │ +│ │ +│ step(action) │ +│ ├── Parse tool call from LLM response │ +│ ├── Execute via OpenEnv step_async() │ +│ └── Return observation, reward, done │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ OpenEnv FleetTaskEnv (envs.fleet_env.task_env) │ +│ │ +│ __init__(task_config, api_key, ...) │ +│ ├── FleetEnvClient.from_fleet() ←── HTTP: Creates Fleet env instance │ +│ │ └── fleet.make() ←── Provisions cloud VM/container │ +│ └── list_tools() ←── MCP: Fetches available tools │ +│ └── Cached in _tools_cache │ +│ │ +│ reset_async() │ +│ ├── _orch.reset() ←── HTTP: Resets episode (logs warning if fails) │ +│ └── Returns obs with cached _tools_cache │ +│ │ +│ step_async(action) │ +│ └── _tools.call_tool() ←── MCP: Executes tool │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Fleet SDK │ +│ │ +│ fleet.make(env_key, ...) │ +│ └── HTTP call to Fleet API │ +│ └── Provisions environment instance (VM/container) │ +│ └── Returns env handle with URLs for HTTP + MCP endpoints │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +## Key Design Decisions + +1. **Environment created in `__init__()`**: Fleet environment is provisioned and tools are fetched when `OpenEnvFleetTaskEnv` is instantiated, not during `reset()`. This ensures tools are available immediately. + +2. **Tools cached**: Tools are fetched once during initialization and cached in `_tools_cache`. Every `reset_async()` and `step_async()` returns the cached tools. + +3. **Reset failure handling**: If `_orch.reset()` fails (timeout), a warning is logged but the episode continues with empty observation. Tools remain available from cache. + +4. **OpenEnv as abstraction layer**: SkyRL does not call Fleet SDK directly. All Fleet interactions go through OpenEnv's `FleetTaskEnv`. + +## Tool Flow + +``` +init() + └── Tools fetched via MCP list_tools() + └── Tools injected into system prompt (JSON format) + └── Tools cached for episode + +step() + └── LLM generates: {"name": "...", "arguments": {...}} + └── SkyRL parses tool call + └── OpenEnv executes via MCP call_tool() + └── Result returned to LLM +``` + +## Configuration + +### Environment Config +```yaml +environment: + env_class: fleet_task + skyrl_gym: + fleet_task: + tasks_file: /path/to/tasks.json # Exported from Fleet + api_key: ${FLEET_API_KEY} # Or set via environment + ttl_seconds: null # Auto: CUA=1800s, tool_use=600s (or override) +``` + +### Task JSON Format +```json +{ + "tasks": [ + { + "key": "task-001", + "prompt": "Search for flights from NYC to LA", + "env_id": "booking-com", + "version": "v1.2.3", + "task_modality": "tool_use", + "verifier_code": "async def verify(env): ..." + } + ] +} +``` + +## Error Handling + +| Failure Point | Behavior | +|--------------|----------| +| `fleet.make()` fails | RuntimeError raised, episode fails | +| `list_tools()` fails | RuntimeError raised, episode fails | +| `_orch.reset()` fails | Warning logged, continues with empty observation | +| `call_tool()` fails | Error returned in observation, episode continues | + +## Dependencies + +- **OpenEnv**: `pip install openenv[fleet]` or add to PYTHONPATH +- **Fleet SDK**: Installed as OpenEnv dependency +- **FLEET_API_KEY**: Must be set in environment or config diff --git a/skyrl-train/integrations/fleet/__init__.py b/skyrl-train/integrations/fleet/__init__.py new file mode 100644 index 0000000000..b6161df621 --- /dev/null +++ b/skyrl-train/integrations/fleet/__init__.py @@ -0,0 +1,15 @@ +# Fleet Task Environment Integration for SkyRL +# +# This module provides a SkyRL-compatible environment wrapper for Fleet-hosted tasks. +# It uses the Fleet SDK directly (no OpenEnv dependency). + +__all__ = ["FleetTaskEnv"] + + +def __getattr__(name: str): + """Lazy import to avoid import errors when dependencies are not installed.""" + if name == "FleetTaskEnv": + from integrations.fleet.env import FleetTaskEnv + + return FleetTaskEnv + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/skyrl-train/integrations/fleet/entrypoints/__init__.py b/skyrl-train/integrations/fleet/entrypoints/__init__.py new file mode 100644 index 0000000000..3ba7648c80 --- /dev/null +++ b/skyrl-train/integrations/fleet/entrypoints/__init__.py @@ -0,0 +1 @@ +# Fleet entrypoints diff --git a/skyrl-train/integrations/fleet/entrypoints/main_fleet.py b/skyrl-train/integrations/fleet/entrypoints/main_fleet.py new file mode 100644 index 0000000000..9ebcb9f4e4 --- /dev/null +++ b/skyrl-train/integrations/fleet/entrypoints/main_fleet.py @@ -0,0 +1,133 @@ +""" +Fleet Task Training Entrypoint for SkyRL. + +This entrypoint registers the FleetTaskEnv and runs GRPO training +on Fleet-hosted environments with checkpoint management. + +Features: +- S3 checkpoint upload (if AWS credentials set) +- S3 checkpoint download for cross-VM resume +- Local cleanup to prevent disk exhaustion (keeps latest checkpoint for resume) + +Usage: + python -m integrations.fleet.entrypoints.main_fleet \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file=/path/to/tasks.json \ + data.train_data=./data/fleet/train.parquet \ + data.val_data=./data/fleet/validation.parquet + +Environment Variables for S3 Checkpoint Management: + AWS_ACCESS_KEY_ID: AWS access key + AWS_SECRET_ACCESS_KEY: AWS secret key + AWS_REGION: AWS region (default: us-east-1) + S3_CHECKPOINT_BUCKET: S3 bucket name (default: skyrl-checkpoints) + RESUME_RUN_NAME: Run name to resume from (downloads checkpoint from S3) +""" + +import asyncio +import logging +import os +from pathlib import Path + +import hydra +import ray +from omegaconf import DictConfig +from skyrl_gym.envs import register +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils import initialize_ray + +logger = logging.getLogger(__name__) + + +class FleetPPOExp(BasePPOExp): + """ + Fleet-specific PPO experiment with checkpoint management. + + Always wraps trainer to: + - Download checkpoint from S3 if RESUME_RUN_NAME is set (cross-VM resume) + - Clean up old checkpoints before saving (prevents disk exhaustion) + - Upload to S3 if AWS credentials are set + - Keep local checkpoints for same-VM resume + """ + + def run(self): + trainer = self._setup_trainer() + + # Download checkpoint from S3 if RESUME_RUN_NAME is set (for cross-VM resume) + resume_run_name = os.environ.get("RESUME_RUN_NAME", "") + if resume_run_name: + try: + from integrations.fleet.s3_checkpoints import download_checkpoint_from_s3 + + ckpt_path = trainer.cfg.trainer.ckpt_path + model_path = getattr(trainer.cfg.trainer.policy.model, "path", "unknown-model") + model_name = Path(model_path).name + project_name = getattr(trainer.cfg.trainer, "project_name", "skyrl") + download_checkpoint_from_s3( + ckpt_path=ckpt_path, + run_name=resume_run_name, + project_name=project_name, + model_name=model_name, + ) + except Exception as e: + logger.warning(f"Failed to download checkpoint from S3: {e}") + + # Always wrap trainer for checkpoint management + # keep_local=True so checkpoints persist for resume after crash/preemption + try: + from integrations.fleet.s3_checkpoints import wrap_trainer_with_s3_upload + + trainer = wrap_trainer_with_s3_upload(trainer) + except Exception as e: + logger.warning(f"Failed to setup checkpoint management: {e}") + + # Start the training loop + asyncio.run(trainer.train()) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + """ + Ray remote function that registers the Fleet environment and runs training. + + This must be a Ray remote function because environment registration needs + to happen in the worker processes, not the driver. + """ + # Register the Fleet task environment + register( + id="fleet_task", + entry_point="integrations.fleet.env:FleetTaskEnv", + ) + + # Run training with checkpoint management + exp = FleetPPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + """ + Main entry point for Fleet task training. + + Required configuration: + environment.env_class: "fleet_task" + environment.skyrl_gym.fleet_task.tasks_file: Path to tasks JSON file + data.train_data: Path to training parquet file + data.val_data: Path to validation parquet file + + Optional configuration: + environment.skyrl_gym.fleet_task.api_key: Fleet API key (or use FLEET_API_KEY env var) + generator.max_turns: Maximum turns per episode (default: 50) + """ + # Validate config args + validate_cfg(cfg) + + # Initialize Ray cluster + initialize_ray(cfg) + + # Run training in Ray + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/fleet/entrypoints/main_fleet_tinker.py b/skyrl-train/integrations/fleet/entrypoints/main_fleet_tinker.py new file mode 100644 index 0000000000..704ca67571 --- /dev/null +++ b/skyrl-train/integrations/fleet/entrypoints/main_fleet_tinker.py @@ -0,0 +1,939 @@ +""" +Fleet Task Training with Tinker Backend. + +This entrypoint uses Tinker (hosted) for training and inference, +combined with Fleet environments via OpenEnv for rollout collection. + +Usage: + python -m integrations.fleet.entrypoints.main_fleet_tinker \ + --model-name Qwen/Qwen3-VL-30B-A3B-Instruct \ + --tasks-file /path/to/tasks.json \ + --dataset-file /path/to/train.parquet \ + --eval-dataset-file /path/to/validation.parquet + +Environment Variables: + TINKER_API_KEY: Tinker API key for authentication (required) + TINKER_API_URL: Tinker service URL (optional, SDK uses default if not set) + FLEET_API_KEY: Fleet API key for environment access + WANDB_API_KEY: Weights & Biases API key for logging + +Architecture: + 1. Load tasks from JSON file (same format as SkyRL Fleet integration) + 2. For each training step: + a. Save current model weights for sampling + b. Create SamplingClient from Tinker + c. Collect rollouts using FleetTaskEnv (OpenEnv) + Tinker inference + d. Compute GRPO advantages + e. Train using Tinker's forward_backward + optim_step + 3. Checkpoints saved via Tinker API + +Metrics (matching SkyRL): + - reward/avg_pass_at_{n}: Pass@k across all prompts + - reward/variance_per_prompt: Mean within-prompt reward variance (GRPO learning signal) + - reward/{env_key}/pass_at_{n}: Per-environment pass@k + - reward/{env_key}/variance_per_prompt: Per-environment variance (learning signal) + - eval/all/pass_at_{n}: Evaluation pass@k + - eval/{env_key}/pass_at_{n}: Per-environment eval pass@k +""" + +import asyncio +import logging +import os +import random +import time +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime +from typing import Any, Dict, List, Optional + +import numpy as np +from pydantic import BaseModel +from tqdm import tqdm +import tinker +import torch +import wandb +from tinker import types +from tinker.types.tensor_data import TensorData +from transformers import AutoTokenizer +from datasets import load_dataset +from torch.utils.data import DataLoader + +# Use SkyRL's FleetTaskEnv wrapper (now supports async via init_async/step_async) +from omegaconf import OmegaConf +from integrations.fleet.env import FleetTaskEnv + +# Import SkyRL's overlong filtering for parity +from skyrl_train.generators.utils import apply_overlong_filtering + +# Import shared metrics module for consistent metric calculation with SkyRL trainer +from skyrl_train.metrics.reward_metrics import ( + compute_pass_at_n as _compute_pass_at_n, + compute_reward_metrics, + compute_per_group_metrics, + sanitize_metric_key, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + datefmt="%H:%M:%S", +) +logger = logging.getLogger(__name__) +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("mcp").setLevel(logging.WARNING) + +# Thread pool for env operations - isolates MCP connections per thread (like SkyRL) +_env_executor: ThreadPoolExecutor = None + + +def _get_env_executor(max_workers: int = 16) -> ThreadPoolExecutor: + global _env_executor + if _env_executor is None: + _env_executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="fleet-env-") + return _env_executor + + +async def _run_in_executor(func, *args): + """Run sync function in thread pool - each thread gets isolated event loop/connections.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_get_env_executor(), func, *args) + + +class RolloutOutput(BaseModel): + """Output from a single rollout collection.""" + + prompt_ids: List[int] + response_ids: List[int] + logprobs: List[float] + loss_mask: List[int] + reward: float + task_key: str + env_key: str + turns: int + tool_calls: int + tool_errors: int = 0 # Count of tool call errors in this rollout + stop_reason: str + duration: float + # Timing breakdown for WandB + total_gen_time: float = 0.0 # Total Tinker generation time + total_step_time: float = 0.0 # Total MCP/Fleet step time + total_tokens: int = 0 # Total tokens generated + error: Optional[str] = None + + class Config: + frozen = True + + +def set_seed(seed: int): + """Set random seed for reproducibility.""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +def normalize_advantages(advantages: List[float]) -> List[float]: + """Normalize advantages to have mean 0 and std 1.""" + if not advantages or len(advantages) == 1: + return advantages + mean = np.mean(advantages) + std = np.std(advantages) + if std < 1e-8: + return [0.0] * len(advantages) + return [(a - mean) / (std + 1e-8) for a in advantages] + + +def compute_advantages_grpo( + rewards: List[float], + group_size: int = None, + normalize: bool = True, +) -> List[float]: + """ + GRPO (Group Relative Policy Optimization) advantage estimation. + + For each group of trajectories from the same prompt, compute advantages + as deviations from the group mean. + """ + rewards = np.array(rewards) + + if group_size is None: + group_size = len(rewards) + + n_groups = len(rewards) // group_size + advantages = [] + + for i in range(n_groups): + start_idx = i * group_size + end_idx = start_idx + group_size + group_rewards = rewards[start_idx:end_idx] + group_mean = group_rewards.mean() + group_advantages = group_rewards - group_mean + advantages.extend(group_advantages.tolist()) + + remaining = len(rewards) % group_size + if remaining > 0: + remaining_rewards = rewards[-remaining:] + remaining_mean = remaining_rewards.mean() + advantages.extend((remaining_rewards - remaining_mean).tolist()) + + if normalize: + advantages = normalize_advantages(advantages) + + return advantages + + +def compute_pass_at_n(rollouts: List[Dict[str, Any]], n_samples_per_prompt: int) -> float: + """ + Compute pass@n metric using the shared metrics module. + + For each unique prompt (task_key), if ANY of the n trajectories has reward > 0, + that counts as a "pass". + + This function is a thin wrapper around the shared compute_pass_at_n for backward + compatibility with the rollout dict format. + """ + rewards = [r.get("reward", 0.0) for r in rollouts] + uids = [r.get("task_key", "unknown") for r in rollouts] + return _compute_pass_at_n(rewards, uids) + + +def compute_per_env_metrics(rollouts: List[Dict[str, Any]], n_samples_per_prompt: int) -> Dict[str, float]: + """ + Compute per-environment metrics using the shared metrics module. + + This function is a thin wrapper around the shared compute_per_group_metrics for + backward compatibility with the rollout dict format. + """ + rewards = [r.get("reward", 0.0) for r in rollouts] + uids = [r.get("task_key", "unknown") for r in rollouts] + env_keys = [r.get("env_key", "unknown") for r in rollouts] + + return compute_per_group_metrics( + rewards=rewards, + uids=uids, + groups=env_keys, + n_samples_per_prompt=n_samples_per_prompt, + prefix="reward", + ) + + +def compute_rollout_metrics( + rollouts: List[Dict[str, Any]], + valid_rollouts: List[Dict[str, Any]], + rewards: List[float], + advantages: List[float], + n_samples_per_prompt: int, +) -> Dict[str, Any]: + """ + Compute all rollout metrics using the shared metrics module. + + Args: + rollouts: All rollouts (including failed ones) + valid_rollouts: Only valid rollouts + rewards: Rewards for valid rollouts + advantages: GRPO advantages for valid rollouts + n_samples_per_prompt: Number of samples per prompt + + Returns: + Dict of metrics for wandb logging + """ + metrics = {} + + # Extract data for shared module + uids = [r.get("task_key", "unknown") for r in valid_rollouts] + env_keys = [r.get("env_key", "unknown") for r in valid_rollouts] + + # Core reward metrics using shared module + core_metrics = compute_reward_metrics(rewards, uids, n_samples_per_prompt) + metrics[f"reward/avg_pass_at_{n_samples_per_prompt}"] = core_metrics[f"pass_at_{n_samples_per_prompt}"] + metrics["reward/variance_per_prompt"] = core_metrics["variance_per_prompt"] + metrics["reward/mean_positive_reward"] = core_metrics["mean_positive_reward"] + + # Advantage metrics (Tinker-specific) + metrics["advantage/mean"] = np.mean(advantages) + metrics["advantage/std"] = np.std(advantages) + metrics["rollouts/valid"] = len(valid_rollouts) + metrics["rollouts/total"] = len(rollouts) + + # Per-environment reward metrics using shared module + per_env_metrics = compute_per_group_metrics( + rewards=rewards, + uids=uids, + groups=env_keys, + n_samples_per_prompt=n_samples_per_prompt, + prefix="reward", + ) + metrics.update(per_env_metrics) + + # Per-environment rollout stats (turns, tool_calls, tool_errors, duration) - Tinker-specific + rollout_stats = defaultdict(list) + for r in valid_rollouts: + env_key = sanitize_metric_key(r.get("env_key", "unknown")) + rollout_stats[f"rollout/{env_key}/turns"].append(r.get("turns", 0)) + rollout_stats[f"rollout/{env_key}/tool_calls"].append(r.get("tool_calls", 0)) + rollout_stats[f"rollout/{env_key}/tool_errors"].append(r.get("tool_errors", 0)) + rollout_stats[f"rollout/{env_key}/duration"].append(r.get("duration", 0.0)) + + for key, values in rollout_stats.items(): + metrics[key] = np.mean(values) + + # Compute tool error rate per environment + env_keys_seen = set() + for r in valid_rollouts: + env_keys_seen.add(sanitize_metric_key(r.get("env_key", "unknown"))) + for env_key in env_keys_seen: + total_calls = sum(rollout_stats[f"rollout/{env_key}/tool_calls"]) + total_errors = sum(rollout_stats[f"rollout/{env_key}/tool_errors"]) + if total_calls > 0: + metrics[f"rollout/{env_key}/tool_error_rate"] = total_errors / total_calls + else: + metrics[f"rollout/{env_key}/tool_error_rate"] = 0.0 + + # Overall rollout duration stats + durations = [r.get("duration", 0.0) for r in valid_rollouts] + metrics["rollout/avg_duration"] = np.mean(durations) + metrics["rollout/max_duration"] = np.max(durations) + metrics["rollout/min_duration"] = np.min(durations) + + return metrics + + +def prepare_training_data( + rollouts: List[Dict[str, Any]], + advantages: List[float], + tokenizer: AutoTokenizer, + max_sequence_length: int, +) -> tuple: + """ + Prepare training data from rollouts (matching SkyRL's generate_batched pattern). + + Applies: + 1. DAPO overlong filtering (zero loss mask if response doesn't end with EOS) + 2. Sequence truncation for max_sequence_length + 3. Builds Tinker Datum objects for training + + Args: + rollouts: List of rollout dicts with prompt_ids, response_ids, logprobs, loss_mask + advantages: GRPO advantages for each rollout + tokenizer: Tokenizer for EOS token ID + max_sequence_length: Maximum sequence length for training + + Returns: + Tuple of (training_datums, truncated_count) + """ + # Apply DAPO overlong filtering (zero out loss mask if response doesn't end with EOS) + all_response_ids = [r.response_ids for r in rollouts] + all_loss_masks = [r.loss_mask for r in rollouts] + filtered_loss_masks = apply_overlong_filtering(all_loss_masks, all_response_ids, tokenizer.eos_token_id) + + training_datums = [] + truncated_count = 0 + + for idx, rollout in enumerate(rollouts): + prompt_ids = rollout.prompt_ids + response_ids = rollout.response_ids + logprobs = rollout.logprobs + loss_mask_data = filtered_loss_masks[idx] + + full_sequence = prompt_ids + response_ids + prompt_len = len(prompt_ids) + + # Truncate sequences exceeding model's max length for Tinker API + if len(full_sequence) > max_sequence_length: + truncated_count += 1 + full_sequence = full_sequence[:max_sequence_length] + response_len = len(full_sequence) - prompt_len + response_ids = response_ids[:response_len] + logprobs = logprobs[:response_len] if logprobs else [] + loss_mask_data = loss_mask_data[:response_len] + + # Target tokens (shifted by 1) + target_tokens = full_sequence[1:] + + # Logprobs (0 for prompt, actual for response) + full_logprobs = [0.0] * prompt_len + logprobs + full_logprobs = full_logprobs[1:] + + # Loss mask (0 for prompt, actual for response) + full_mask = [0] * prompt_len + loss_mask_data + full_mask = full_mask[1:] + + # Advantages (apply only where loss mask is 1) + advantage_value = advantages[idx] + full_advantages = torch.zeros(len(full_sequence)) + for i in range(prompt_len, len(full_sequence)): + if i - 1 < len(full_mask) and full_mask[i - 1] > 0: + full_advantages[i] = advantage_value + full_advantages = full_advantages[1:] + + datum = types.Datum( + model_input=types.ModelInput.from_ints(tokens=full_sequence[:-1]), + loss_fn_inputs={ + "target_tokens": TensorData.from_torch(torch.tensor(target_tokens)), + "logprobs": TensorData.from_torch(torch.tensor(full_logprobs)), + "advantages": TensorData.from_torch(full_advantages), + }, + ) + training_datums.append(datum) + + return training_datums, truncated_count + + +def tokenize_chat(tokenizer: AutoTokenizer, chat_history: List[Dict], add_generation_prompt: bool = True) -> List[int]: + """ + Tokenize chat history and ensure we get a plain list of token IDs. + + apply_chat_template can return different types depending on the tokenizer: + - List[int] for some tokenizers + - BatchEncoding dict with 'input_ids' key for others + + Tinker's ModelInput.from_ints() requires a plain list of integers. + """ + result = tokenizer.apply_chat_template(chat_history, add_generation_prompt=add_generation_prompt, tokenize=True) + # Handle BatchEncoding (dict-like) vs plain list + if hasattr(result, "input_ids"): + return list(result.input_ids) + elif isinstance(result, dict) and "input_ids" in result: + return list(result["input_ids"]) + else: + return list(result) + + +async def collect_fleet_rollout( + task_config: Dict[str, Any], + tasks_file: str, + sampling_client: tinker.SamplingClient, + tokenizer: AutoTokenizer, + max_turns: int = 50, + max_generate_length: int = 2048, + max_input_length: int = 30720, + temperature: float = 1.0, +) -> Dict[str, Any]: + """ + Collect a single trajectory using Fleet environment and Tinker inference. + + Uses SkyRL's FleetTaskEnv wrapper with async methods for environment interaction. + + Args: + max_generate_length: Max tokens per generation step. + max_input_length: Max context length before ending rollout (matching SkyRL). + """ + rollout_start = time.time() + + task_key = task_config.get("task_key") or task_config.get("key") + + # Create SkyRL FleetTaskEnv wrapper + # TTL of 2 hours - some rollouts with many turns can take 30+ minutes + env_config = OmegaConf.create({"tasks_file": tasks_file, "ttl_seconds": 7200}) + extras = {"task_key": task_key, "max_turns": max_turns} + + env = FleetTaskEnv(env_config=env_config, extras=extras) + + try: + # Initialize environment in thread pool - isolates MCP connections + chat_history, metadata = await _run_in_executor(env.init, []) + env_key = metadata.get("env_key", "unknown") + + # Tokenize initial prompt + prompt_ids = tokenize_chat(tokenizer, chat_history, add_generation_prompt=True) + + all_response_ids = [] + all_logprobs = [] + loss_mask = [] + done = False + total_reward = 0.0 + stop_reason = "stop" + # Timing accumulators for WandB + total_gen_time = 0.0 + total_step_time = 0.0 + total_tokens = 0 + + while not done and env.turns < max_turns: + turn_num = env.turns + 1 # 1-indexed for logging + + # Prepare input for Tinker (use env's chat_history) + input_ids = tokenize_chat(tokenizer, env.chat_history, add_generation_prompt=True) + + # Check context length limit (matching SkyRL's skyrl_gym_generator.py:274) + if len(input_ids) > max_input_length: + logger.info( + f"[{task_key}] Turn {turn_num}: context length ({len(input_ids)}) exceeds max ({max_input_length}), ending" + ) + stop_reason = "length" + break + + # Generate with Tinker + gen_start = time.time() + sampling_params = types.SamplingParams( + max_tokens=max_generate_length, + temperature=temperature, + top_p=1.0, + ) + + # Use async sampling to avoid blocking the event loop + result = await sampling_client.sample_async( + prompt=types.ModelInput.from_ints(tokens=input_ids), + num_samples=1, + sampling_params=sampling_params, + ) + gen_time = time.time() - gen_start + total_gen_time += gen_time + + if not result.sequences or len(result.sequences) == 0: + logger.warning(f"[{task_key}] Turn {turn_num}: no sequences returned from Tinker") + break + + sequence = result.sequences[0] + output_ids = sequence.tokens + output_logprobs = sequence.logprobs if sequence.logprobs else [] + + # Decode output + output_text = tokenizer.decode(output_ids, skip_special_tokens=True) + + # Collect trajectory data (assistant response tokens - trainable) + all_response_ids.extend(output_ids) + if output_logprobs: + all_logprobs.extend(output_logprobs) + else: + all_logprobs.extend([0.0] * len(output_ids)) + loss_mask.extend([1] * len(output_ids)) + + # Step environment in thread pool - isolates MCP connections + step_start = time.time() + step_output = await _run_in_executor(env.step, output_text) + step_time = time.time() - step_start + total_step_time += step_time + total_tokens += len(output_ids) + + # Get observation content for tokenization (masked out for loss) + # Note: BaseTextEnvStepOutput is a TypedDict, use dict access + if step_output["observations"]: + obs_content = step_output["observations"][0].get("content", "") + obs_ids = tokenizer.encode(obs_content, add_special_tokens=False) + all_response_ids.extend(obs_ids) + all_logprobs.extend([0.0] * len(obs_ids)) + loss_mask.extend([0] * len(obs_ids)) + + total_reward = step_output["reward"] + done = step_output["done"] + + return RolloutOutput( + prompt_ids=prompt_ids, + response_ids=all_response_ids, + logprobs=all_logprobs, + loss_mask=loss_mask, + reward=total_reward, + task_key=task_key, + env_key=env_key, + turns=env.turns, + tool_calls=env.tool_calls, + tool_errors=env.tool_errors, + stop_reason=stop_reason, + duration=time.time() - rollout_start, + total_gen_time=total_gen_time, + total_step_time=total_step_time, + total_tokens=total_tokens, + ) + + finally: + env.close() + + +async def collect_batch_rollouts( + batch: List[Dict[str, Any]], + tasks_file: str, + sampling_client: tinker.SamplingClient, + tokenizer: AutoTokenizer, + max_turns: int = 50, + max_generate_length: int = 2048, + max_input_length: int = 30720, + n_samples_per_prompt: int = 1, + max_concurrent: int = 8, +) -> List[Dict[str, Any]]: + """Collect rollouts for a batch of tasks with limited concurrency. + + Args: + max_concurrent: Maximum number of concurrent Fleet environment connections. + Now safe to increase since ThreadPoolExecutor isolates connections. + """ + # Semaphore to limit concurrent Fleet environment connections + semaphore = asyncio.Semaphore(max_concurrent) + + async def collect_single_rollout(task_config: Dict[str, Any], index: int) -> tuple: + """Wrapper to collect a single rollout with error handling and concurrency limit.""" + async with semaphore: + rollout_start = time.time() + try: + rollout = await collect_fleet_rollout( + task_config=task_config, + tasks_file=tasks_file, + sampling_client=sampling_client, + tokenizer=tokenizer, + max_turns=max_turns, + max_generate_length=max_generate_length, + max_input_length=max_input_length, + ) + return index, rollout + except Exception as e: + logger.error(f"Failed to collect rollout for {task_config.get('task_key')}: {e}") + return index, RolloutOutput( + prompt_ids=[], + response_ids=[], + logprobs=[], + loss_mask=[], + reward=0.0, + task_key=task_config.get("task_key", "unknown"), + env_key=task_config.get("env_key", "unknown"), + turns=0, + tool_calls=0, + tool_errors=0, + stop_reason="error", + error=str(e), + duration=time.time() - rollout_start, + ) + + # Create all rollout tasks (batch_size * n_samples_per_prompt) + tasks = [] + index = 0 + for task_config in batch: + for _ in range(n_samples_per_prompt): + tasks.append(collect_single_rollout(task_config, index)) + index += 1 + + total = len(tasks) + logger.info(f" Collecting {total} rollouts (max {max_concurrent} concurrent)...") + rollouts = [None] * total + completed = 0 + last_logged = 0 + log_interval = max(1, total // 4) # Log at ~25%, 50%, 75%, 100% + + # Run rollouts with limited concurrency via semaphore + for coro in asyncio.as_completed(tasks): + idx, rollout = await coro + rollouts[idx] = rollout + completed += 1 + + # Log progress at intervals + if completed - last_logged >= log_interval or completed == total: + logger.info(f" Progress: {completed}/{total} rollouts completed") + last_logged = completed + + return rollouts + + +def collate_fn(batch): + """Return batch as-is without tensor collation.""" + return batch + + +async def main( + model_name: str = "Qwen/Qwen3-VL-30B-A3B-Instruct", + tasks_file: str = None, + dataset_file: str = None, + eval_dataset_file: str = None, + batch_size: int = 8, + eval_batch_size: int = 32, + learning_rate: float = 4e-5, + lora_rank: int = 16, + max_steps: int = 200, + max_turns: int = 50, + max_generate_length: int = 2048, + max_input_length: int = 30720, + max_sequence_length: int = 32768, + n_samples_per_prompt: int = 4, + eval_every: int = 20, + seed: int = 42, + wandb_project: str = "fleet-tinker-grpo", + wandb_name: str = None, +): + """ + Main training loop using Tinker for training/inference and Fleet for environments. + """ + set_seed(seed) + + # Setup WandB run name + if wandb_name is None: + wandb_name = f"{model_name.split('/')[-1]}_{datetime.now().strftime('%m%d_%H%M')}" + + # Initialize WandB + wandb.init( + project=wandb_project, + name=wandb_name, + config={ + "model_name": model_name, + "batch_size": batch_size, + "learning_rate": learning_rate, + "lora_rank": lora_rank, + "max_turns": max_turns, + "max_generate_length": max_generate_length, + "max_input_length": max_input_length, + "max_sequence_length": max_sequence_length, + "n_samples_per_prompt": n_samples_per_prompt, + }, + ) + + # Load datasets + train_dataset = load_dataset("parquet", data_files=dataset_file)["train"] + eval_dataset = load_dataset("parquet", data_files=eval_dataset_file)["train"] if eval_dataset_file else None + + logger.info(f"Loaded {len(train_dataset)} training samples") + if eval_dataset: + logger.info(f"Loaded {len(eval_dataset)} eval samples") + + # Setup Tinker + tinker_url = os.environ.get("TINKER_API_URL") + tinker_api_key = os.environ.get("TINKER_API_KEY") + + service_client_kwargs = {} + if tinker_url: + service_client_kwargs["base_url"] = tinker_url + if tinker_api_key: + service_client_kwargs["api_key"] = tinker_api_key + + service_client = tinker.ServiceClient(**service_client_kwargs) + training_client = await service_client.create_lora_training_client_async(base_model=model_name, rank=lora_rank) + + adam_params = types.AdamParams(learning_rate=learning_rate, beta1=0.9, beta2=0.95, eps=1e-8) + tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Create dataloader + def create_dataloader(epoch: int): + return DataLoader( + train_dataset, + batch_size=batch_size, + shuffle=True, + collate_fn=collate_fn, + generator=torch.Generator().manual_seed(seed + epoch), + ) + + steps_per_epoch = (len(train_dataset) + batch_size - 1) // batch_size + current_epoch = 0 + train_dataloader = create_dataloader(current_epoch) + train_iterator = iter(train_dataloader) + + # Training loop + pbar = tqdm(range(max_steps), desc="Training", unit="step") + for step in pbar: + step_start = time.time() + metrics = {"step": step, "epoch": step // steps_per_epoch} + + # Get sampler weights for rollout inference + sampling_path = training_client.save_weights_for_sampler(name=f"step_{step:06d}").result().path + sampling_client = service_client.create_sampling_client(model_path=sampling_path) + + # Get batch + try: + batch = next(train_iterator) + except StopIteration: + current_epoch += 1 + train_dataloader = create_dataloader(current_epoch) + train_iterator = iter(train_dataloader) + batch = next(train_iterator) + + # Collect rollouts + logger.info(f"Step {step}: Collecting rollouts for {len(batch)} tasks...") + rollout_start = time.time() + + rollouts = await collect_batch_rollouts( + batch=batch, + tasks_file=tasks_file, + sampling_client=sampling_client, + tokenizer=tokenizer, + max_turns=max_turns, + max_generate_length=max_generate_length, + max_input_length=max_input_length, + n_samples_per_prompt=n_samples_per_prompt, + ) + + metrics["time/rollout"] = time.time() - rollout_start + + # Filter valid rollouts and log invalid ones + # Note: rollouts are RolloutOutput Pydantic objects - use attribute access + valid_rollouts = [] + invalid_rollouts = [] + for r in rollouts: + if r.response_ids and not r.error: + valid_rollouts.append(r) + else: + invalid_rollouts.append(r) + + if invalid_rollouts: + for r in invalid_rollouts: + task_key = r.task_key + error = r.error or "no response_ids" + stop_reason = r.stop_reason + logger.warning(f"Step {step}: Invalid rollout for {task_key}: {error} (stop_reason={stop_reason})") + metrics["rollouts/invalid"] = len(invalid_rollouts) + + if not valid_rollouts: + logger.warning(f"Step {step}: No valid rollouts, skipping") + continue + + # Compute GRPO advantages + rewards = [r.reward for r in valid_rollouts] + advantages = compute_advantages_grpo(rewards, group_size=n_samples_per_prompt, normalize=True) + + # Compute all rollout metrics (convert to dicts for metrics functions) + rollout_metrics = compute_rollout_metrics( + rollouts=[r.model_dump() for r in rollouts], + valid_rollouts=[r.model_dump() for r in valid_rollouts], + rewards=rewards, + advantages=advantages, + n_samples_per_prompt=n_samples_per_prompt, + ) + metrics.update(rollout_metrics) + + # Compute timing metrics from valid rollouts + gen_times = [r.total_gen_time for r in valid_rollouts] + step_times = [r.total_step_time for r in valid_rollouts] + tokens = [r.total_tokens for r in valid_rollouts] + durations = [r.duration for r in valid_rollouts] + + metrics["time/gen_total"] = sum(gen_times) + metrics["time/gen_mean"] = np.mean(gen_times) + metrics["time/step_total"] = sum(step_times) + metrics["time/step_mean"] = np.mean(step_times) + metrics["time/gen_pct"] = 100 * sum(gen_times) / sum(durations) if sum(durations) > 0 else 0 + metrics["time/step_pct"] = 100 * sum(step_times) / sum(durations) if sum(durations) > 0 else 0 + metrics["throughput/tokens_total"] = sum(tokens) + metrics["throughput/tokens_per_sec_gen"] = sum(tokens) / sum(gen_times) if sum(gen_times) > 0 else 0 + metrics["throughput/tokens_per_sec_effective"] = sum(tokens) / sum(durations) if sum(durations) > 0 else 0 + + # Prepare training data (DAPO filtering + truncation + datum creation) + training_datums, truncated_count = prepare_training_data( + rollouts=valid_rollouts, + advantages=advantages, + tokenizer=tokenizer, + max_sequence_length=max_sequence_length, + ) + + metrics["rollouts/truncated_overlong"] = truncated_count + if truncated_count > 0: + logger.info(f"Step {step}: Truncated {truncated_count} overlong sequences") + + if not training_datums: + logger.warning(f"Step {step}: No valid training sequences after filtering, skipping") + continue + + # Training step + logger.info(f"Step {step}: Training on {len(training_datums)} sequences...") + train_start = time.time() + + fwd_bwd_future = training_client.forward_backward(training_datums, loss_fn="ppo") + optim_step_future = training_client.optim_step(adam_params) + + fwd_bwd_future.result() + optim_step_future.result() + + metrics["time/train"] = time.time() - train_start + metrics["time/total"] = time.time() - step_start + + # Log metrics (commit=True forces immediate sync) + wandb.log(metrics, step=step, commit=True) + pbar.set_postfix( + { + f"pass@{n_samples_per_prompt}": f"{metrics[f'reward/avg_pass_at_{n_samples_per_prompt}']:.3f}", + "reward": f"{metrics['reward/avg_raw_reward']:.3f}", + "time": f"{metrics['time/total']:.1f}s", + } + ) + + # Evaluation + if eval_every > 0 and eval_dataset and step % eval_every == 0: + logger.info(f"Step {step}: Running evaluation...") + eval_dataloader = DataLoader(eval_dataset, batch_size=eval_batch_size, shuffle=False, collate_fn=collate_fn) + + all_eval_rollouts = [] + for eval_batch in eval_dataloader: + eval_rollouts = await collect_batch_rollouts( + batch=eval_batch, + tasks_file=tasks_file, + sampling_client=sampling_client, + tokenizer=tokenizer, + max_turns=max_turns, + max_generate_length=max_generate_length, + max_input_length=max_input_length, + n_samples_per_prompt=1, + ) + all_eval_rollouts.extend([r for r in eval_rollouts if not r.error]) + + if all_eval_rollouts: + eval_rewards = [r.reward for r in all_eval_rollouts] + # Convert to dicts for metrics functions + eval_rollouts_dicts = [r.model_dump() for r in all_eval_rollouts] + eval_pass_at_1 = compute_pass_at_n(eval_rollouts_dicts, 1) + eval_per_env = compute_per_env_metrics(eval_rollouts_dicts, 1) + + eval_metrics = { + "eval/all/pass_at_1": eval_pass_at_1, + "eval/all/mean_positive_reward": ( + np.mean([r for r in eval_rewards if r > 0]) if any(r > 0 for r in eval_rewards) else 0.0 + ), + "eval/num_samples": len(all_eval_rollouts), + } + # Add per-env eval metrics (rename from reward/ to eval/) + for key, value in eval_per_env.items(): + eval_key = key.replace("reward/", "eval/") + eval_metrics[eval_key] = value + + wandb.log(eval_metrics, step=step, commit=True) + logger.info(f"Step {step}: eval pass@1={eval_pass_at_1:.3f}, num_samples={len(all_eval_rollouts)}") + + wandb.finish() + logger.info("Training completed!") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Fleet Task Training with Tinker") + parser.add_argument("--model-name", type=str, default="Qwen/Qwen3-VL-30B-A3B-Instruct") + parser.add_argument("--tasks-file", type=str, required=True, help="Path to tasks JSON file") + parser.add_argument("--dataset-file", type=str, required=True, help="Path to training parquet") + parser.add_argument("--eval-dataset-file", type=str, default=None, help="Path to eval parquet") + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--eval-batch-size", type=int, default=32) + parser.add_argument("--learning-rate", type=float, default=4e-5) + parser.add_argument("--lora-rank", type=int, default=16) + parser.add_argument("--max-steps", type=int, default=200) + parser.add_argument("--max-turns", type=int, default=50) + parser.add_argument("--max-generate-length", type=int, default=2048, help="Max tokens per generation") + parser.add_argument("--max-input-length", type=int, default=30720, help="Max context length before ending rollout") + parser.add_argument("--max-sequence-length", type=int, default=32768, help="Max sequence length for training") + parser.add_argument("--n-samples-per-prompt", type=int, default=4) + parser.add_argument("--eval-every", type=int, default=20) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--wandb-project", type=str, default="fleet-tinker-grpo") + parser.add_argument("--wandb-name", type=str, default=None) + parser.add_argument( + "--track-extra-gradient-metrics", + type=bool, + default=False, + help="Track additional gradient metrics (for parity with SkyRL config)", + ) + + args = parser.parse_args() + + asyncio.run( + main( + model_name=args.model_name, + tasks_file=args.tasks_file, + dataset_file=args.dataset_file, + eval_dataset_file=args.eval_dataset_file, + batch_size=args.batch_size, + eval_batch_size=args.eval_batch_size, + learning_rate=args.learning_rate, + lora_rank=args.lora_rank, + max_steps=args.max_steps, + max_turns=args.max_turns, + max_generate_length=args.max_generate_length, + max_input_length=args.max_input_length, + max_sequence_length=args.max_sequence_length, + n_samples_per_prompt=args.n_samples_per_prompt, + eval_every=args.eval_every, + seed=args.seed, + wandb_project=args.wandb_project, + wandb_name=args.wandb_name, + ) + ) diff --git a/skyrl-train/integrations/fleet/env.py b/skyrl-train/integrations/fleet/env.py new file mode 100644 index 0000000000..aebda50c7b --- /dev/null +++ b/skyrl-train/integrations/fleet/env.py @@ -0,0 +1,742 @@ +""" +Fleet Task Environment for SkyRL. + +This module provides a SkyRL-compatible environment wrapper for Fleet-hosted tasks. +It uses OpenEnv's FleetTaskEnv as the abstraction layer for Fleet environments. +""" + +import ast +import asyncio +import json +import logging +import os +import re +import time +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +from omegaconf import DictConfig + +from skyrl_gym.envs.base_text_env import ( + BaseTextEnv, + BaseTextEnvStepOutput, + ConversationType, +) +from envs.fleet_env import FleetTaskEnv as OpenEnvFleetTaskEnv +from envs.fleet_env import ContextManager, configure_fleet_telemetry + +# Reduce MCP client log noise +# - loguru: some MCP libs use loguru +# - standard logging: mcp.client.streamable_http uses standard logging +try: + from loguru import logger as loguru_logger + + loguru_logger.disable("mcp") +except ImportError: + pass +logging.getLogger("mcp").setLevel(logging.WARNING) + +logger = logging.getLogger(__name__) + +# Global task cache to avoid reloading JSON for each env instance +_TASK_CACHE: Dict[str, Dict[str, Any]] = {} + + +def load_tasks_from_json(tasks_file: str) -> Dict[str, Any]: + """Load tasks from JSON file with caching. + + Returns a dict mapping task_key -> task_config dict. + """ + if tasks_file not in _TASK_CACHE: + expanded_path = os.path.expanduser(tasks_file) + if not os.path.exists(expanded_path): + raise FileNotFoundError(f"Tasks file not found: {expanded_path}") + + with open(expanded_path, "r") as f: + data = json.load(f) + + # Handle both formats: array or {"tasks": [...]} + if isinstance(data, list): + tasks = data + elif isinstance(data, dict) and "tasks" in data: + tasks = data["tasks"] + else: + raise ValueError(f"Invalid JSON format in {tasks_file}: expected array or object with 'tasks' key") + + if not tasks: + raise ValueError(f"No tasks found in {tasks_file}") + + # Index by task_key (support both 'key' and 'task_key' fields) + _TASK_CACHE[tasks_file] = {t.get("key") or t.get("task_key"): t for t in tasks} + + return _TASK_CACHE[tasks_file] + + +def _try_parse_json(raw: str) -> Optional[Dict[str, Any]]: + """Try to parse JSON, repairing missing trailing braces if needed.""" + raw = raw.strip() + try: + parsed = json.loads(raw) + if isinstance(parsed, dict): + return parsed + except (json.JSONDecodeError, ValueError): + pass + + # Repair: models often drop trailing closing braces on nested JSON. + # Try appending up to 3 closing braces. + for extra in range(1, 4): + try: + parsed = json.loads(raw + "}" * extra) + if isinstance(parsed, dict): + return parsed + except (json.JSONDecodeError, ValueError): + continue + + return None + + +def parse_tool_call(action: str) -> Optional[Dict[str, Any]]: + """ + Parse tool call from LLM response. + + Supports tag-based formats: + - {"name": "...", "arguments": {...}} + - {"name": "...", "arguments": {...}} + + Also handles cases where the closing tag is missing (e.g., when + is used as the stop string and not included in the output). + + Returns dict with "name" and "arguments" keys, or None if not found. + """ + # Try common tag formats + for tag in ["tool_call", "function_call"]: + # First try with closing tag + match = re.search(rf"<{tag}>(.*?)", action, re.DOTALL) + if not match: + # Try without closing tag (for when is the stop string) + # Match from opening tag to end of string or next special token + match = re.search(rf"<{tag}>(.*?)(?:<\||\Z)", action, re.DOTALL) + if match: + parsed = _try_parse_json(match.group(1)) + if parsed is None: + continue + # Normalize keys + name = parsed.get("name") or parsed.get("tool") + args = parsed.get("arguments") or parsed.get("params", {}) + if name: + return {"name": name, "arguments": args} + + return None + + +class FleetTaskEnv(BaseTextEnv): + """ + SkyRL environment for Fleet-hosted tasks. + + Uses OpenEnv's FleetTaskEnv as the abstraction layer for Fleet environments. + This provides a clean separation between SkyRL's training interface and + Fleet's environment management. + """ + + _trace_config: Optional[Dict[str, str]] = None + + @classmethod + def set_trace_config(cls, job_id: str, model: str): + """Set trace config for uploading eval traces to Fleet.""" + cls._trace_config = {"job_id": job_id, "model": model} + + @classmethod + def clear_trace_config(cls): + """Clear trace config after eval is done.""" + cls._trace_config = None + + def __init__( + self, + env_config: DictConfig, + extras: Dict[str, Any] = {}, + ): + super().__init__() + + self.extras = extras + self.max_turns = extras.get("max_turns", 50) + + # Task configuration from extras (set by dataset) + self.task_key = extras.get("task_key") + self.tasks_file = env_config.get("tasks_file") or extras.get("tasks_file") + + if not self.task_key: + raise ValueError("task_key must be provided in extras (from dataset)") + if not self.tasks_file: + raise ValueError("tasks_file must be provided in env_config or extras") + + # Expand path + self.tasks_file = os.path.expanduser(self.tasks_file) + + # Load task config from JSON + tasks = load_tasks_from_json(self.tasks_file) + self.task_config = tasks.get(self.task_key) + if not self.task_config: + available_keys = list(tasks.keys())[:5] + raise ValueError( + f"Task '{self.task_key}' not found in {self.tasks_file}. " f"Available keys (first 5): {available_keys}" + ) + + # API key + self.api_key = env_config.get("api_key") or os.environ.get("FLEET_API_KEY") + if not self.api_key: + raise ValueError("FLEET_API_KEY must be set in env_config or environment") + + # Logfire telemetry (no-op if LOGFIRE_TOKEN is not set) + logfire_token = os.environ.get("LOGFIRE_TOKEN") + if logfire_token: + configure_fleet_telemetry(token=logfire_token) + + # TTL for Fleet environment instances + self.ttl_seconds = env_config.get("ttl_seconds", None) # None = auto (CUA: 1800s, tool_use: 600s) + + # Partial reward: use verifier accumulator counts instead of binary 0/1 + self.partial_reward = env_config.get("partial_reward", False) + + # Hint config + self.enable_hints = env_config.get("enable_hints", False) + + # Environment state (initialized on init()) + self.openenv_task_env: Optional[OpenEnvFleetTaskEnv] = None + self.chat_history: ConversationType = [] + self.turns = 0 + self.tool_calls = 0 + self.tool_errors = 0 + self.last_reward: Optional[float] = None + self.tools: List[Dict[str, Any]] = [] + + # Verifier feedback (captured at close time for hint generation) + self._verifier_stdout: Optional[str] = None + self._verifier_error: Optional[str] = None + self._tool_error_messages: List[str] = [] + + # Context management (uses OpenEnv's ContextManager) + # Read from env_config (Hydra config) not extras (per-sample dataset config) + self.enable_context_tools = env_config.get("enable_context_tools", False) + self.context_manager: Optional[ContextManager] = None + if self.enable_context_tools: + logger.info( + f"Enabling context management tools with max_output_chars={extras.get('max_output_chars', 10000)}" + ) + self.context_manager = ContextManager(max_output_chars=extras.get("max_output_chars", 10000)) + + def _normalize_task_config(self) -> Dict[str, Any]: + """Normalize task config to OpenEnv's expected format.""" + config = self.task_config.copy() + + # Map field names if needed + if "key" in config and "task_key" not in config: + config["task_key"] = config["key"] + if "env_id" in config and "env_key" not in config: + config["env_key"] = config["env_id"] + if "version" in config and "env_version" not in config: + config["env_version"] = config["version"] + + return config + + async def init_async(self, prompt: ConversationType) -> Tuple[ConversationType, Dict[str, Any]]: + """ + Initialize the Fleet environment and return initial observation (async version). + + Creates Fleet environment via OpenEnv's FleetTaskEnv and returns the task prompt. + OpenEnv's FleetTaskEnv.__init__() creates the Fleet env and fetches tools. + """ + # Close any existing environment + self.close() + + # Create OpenEnv's FleetTaskEnv with normalized config + # This creates the Fleet env (fleet.make) and fetches tools (list_tools) + task_config = self._normalize_task_config() + + try: + self.openenv_task_env = OpenEnvFleetTaskEnv( + task_config=task_config, + api_key=self.api_key, + ttl_seconds=self.ttl_seconds, + max_steps=self.max_turns, + partial_reward=self.partial_reward, + ) + except Exception as e: + raise RuntimeError(f"Failed to create OpenEnv FleetTaskEnv: {e}") from e + + # Reset episode state (tools are already cached from __init__) + obs = await self.openenv_task_env.reset_async() + + # Reset state + self.turns = 0 + self.tool_calls = 0 + self.tool_errors = 0 + self.last_reward = None + + # Reset context manager if enabled + if self.context_manager: + self.context_manager.reset() + + # Get tools from observation (cached from __init__) + self.tools = obs.get("tools", []) + + # Add context management tools if enabled + if self.context_manager: + self.tools = self.tools + self.context_manager.get_tools() + if not self.tools: + raise RuntimeError(f"Task {self.task_key}: no tools found in observation. Fleet env requires tools.") + + # Build initial prompt with task instruction + task_prompt = self.task_config.get("prompt", "") + + # Inject hint from previous failed attempt if provided + hint = self.extras.get("hint") + if hint: + task_prompt = f"{task_prompt}\n\nHere is feedback from a previous attempt to help you:\n{hint}" + + # Build system prompt with tool definitions + tools_json = json.dumps(self.tools, indent=2) + # Include current date so model uses correct year for date-related tasks + current_date = datetime.now().strftime("%Y-%m-%d") + + # Build environment context section from env_variables + env_context = "" + env_vars = self.task_config.get("env_variables", {}) + if env_vars: + env_lines = [] + if "LOGGED_IN_USER" in env_vars: + env_lines.append(f"- Logged in user ID: {env_vars['LOGGED_IN_USER']}") + if "LOGGED_IN_NAME" in env_vars: + env_lines.append(f"- Logged in as: {env_vars['LOGGED_IN_NAME']}") + # Include other env variables (skip CURRENT_DATE as it's handled separately) + for key, value in env_vars.items(): + if key not in ("LOGGED_IN_USER", "LOGGED_IN_NAME", "CURRENT_DATE"): + env_lines.append(f"- {key}: {value}") + if env_lines: + env_context = "\n## Environment Context\n" + "\n".join(env_lines) + "\n" + + # Add environment-specific hints + env_key = self.task_config.get("env_key") or self.task_config.get("env_id") + env_hints = "" + if env_key == "fostgres": + env_hints = """ +## Database Exploration +Before writing SQL queries, first explore the database schema: +- List tables: SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' +- List columns: SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'your_table' +""" + + system_content = f"""You are a helpful agent. Complete the task by calling tools. + +## Current Date +Today's date is {current_date}. When dates are mentioned without a year, assume the current year ({datetime.now().year}) or a future date. +{env_context}{env_hints} +## Available Tools +{tools_json} + +## Tool Call Format +{{"name": "tool_name", "arguments": {{"param": "value"}}}} + +## Error Handling +If a tool call returns an error: +- Read the error message carefully +- Do NOT repeat the same call with identical arguments +- Change your approach: use different parameters, try a different tool, or break the task into smaller steps + +## Response Format +EVERY response MUST end with exactly ONE of: +1. A tool call: ... - to perform an action +2. Done signal: - ONLY when the task is fully complete + +IMPORTANT: When the task is complete, first output your final answer with the requested information, THEN say . Do not just say without providing the answer. + +NEVER respond with just a message. NEVER say "feel free to ask" or offer further help. +If the task is complete, provide your answer then say . Otherwise, make a tool call.""" + + # Build conversation with system prompt + system_message = {"role": "system", "content": system_content} + user_message = {"role": "user", "content": task_prompt} + self.chat_history = [system_message, user_message] + + metadata = { + "task_key": self.task_key, + "env_key": self.task_config.get("env_key") or self.task_config.get("env_id"), + "tools": self.tools, + "modality": self.task_config.get("task_modality", "tool_use"), + } + + return self.chat_history.copy(), metadata + + def init(self, prompt: ConversationType) -> Tuple[ConversationType, Dict[str, Any]]: + """ + Initialize the Fleet environment and return initial observation (sync wrapper). + + For async contexts, use init_async() instead. + """ + return asyncio.run(self.init_async(prompt)) + + async def step_async(self, action: str) -> BaseTextEnvStepOutput: + """ + Execute one step in the Fleet environment (async version). + + Parses the action for tool calls, executes via OpenEnv's FleetTaskEnv, + and returns observation. Reward is computed by the verifier on completion. + """ + step_start = time.time() + self.turns += 1 + assistant_msg = {"role": "assistant", "content": action} + self.chat_history.append(assistant_msg) + if self.context_manager: + self.context_manager.track_message(assistant_msg) + + max_turns_reached = self.turns >= self.max_turns + + # Check if agent signals completion + agent_done = "" in action.lower() or "[done]" in action.lower() + + # Parse tool call from LLM response + tool_call = parse_tool_call(action) + + tool_result = None + error = None + reward = 0.0 + mcp_time = 0.0 + + # Handle context management tools locally (no MCP call) + if tool_call and self.context_manager and self.context_manager.is_context_tool(tool_call["name"]): + tool_result, self.chat_history = self.context_manager.execute_tool( + tool_call["name"], tool_call.get("arguments", {}), self.chat_history + ) + # Execute tool call if present via OpenEnv + elif tool_call and self.openenv_task_env: + self.tool_calls += 1 + # Build action dict for OpenEnv + openenv_action = { + "tool": tool_call["name"], + "params": tool_call.get("arguments", {}), + "done": agent_done, + } + + try: + # Use async step method + mcp_start = time.time() + obs, reward, done, info = await self.openenv_task_env.step_async(openenv_action) + mcp_time = time.time() - mcp_start + tool_result = obs.get("observation") + if "tool_error" in info: + error = info["tool_error"] + + # Truncate long tool outputs and store full version for retrieval if context management is enabled + if tool_result and isinstance(tool_result, str) and self.context_manager: + tool_result = self.context_manager.truncate_output(tool_result) + except Exception as e: + mcp_time = time.time() - mcp_start + error = str(e) + elif agent_done and self.openenv_task_env: + # Agent signaled done without tool call + openenv_action = {"done": True} + try: + mcp_start = time.time() + obs, reward, done, info = await self.openenv_task_env.step_async(openenv_action) + mcp_time = time.time() - mcp_start + except Exception as e: + mcp_time = time.time() - mcp_start + error = str(e) + + # Detect error patterns in tool_result (tool returned error as successful response) + if not error and tool_result: + result_str = str(tool_result) if not isinstance(tool_result, str) else tool_result + # Check for common error patterns in tool response + if result_str.strip().startswith("Error:") or result_str.strip().startswith("error:"): + error = result_str + tool_result = None + elif isinstance(tool_result, dict) and tool_result.get("error"): + error = tool_result["error"] + tool_result = None + + # Check if episode is done + episode_done = agent_done or max_turns_reached + + # Upload trace at episode end if trace config is set + if episode_done and FleetTaskEnv._trace_config: + try: + from envs.fleet_env.trace import upload_trace + + inst_id = None + orch = getattr(self.openenv_task_env, "_orch", None) + if orch: + fleet_env = getattr(orch, "_fleet_env", None) + if fleet_env: + inst_id = getattr(fleet_env, "instance_id", None) + await upload_trace( + api_key=self.api_key, + job_id=FleetTaskEnv._trace_config["job_id"], + task_key=self.task_key, + model=FleetTaskEnv._trace_config["model"], + chat_history=self.chat_history, + reward=reward, + instance_id=inst_id, + metadata={"env_key": self.task_config.get("env_key"), "turns": self.turns}, + ) + except Exception as e: + logger.warning(f"Failed to upload trace for {self.task_key}: {e}") + + # Build observation message + if max_turns_reached: + return BaseTextEnvStepOutput( + observations=[], + reward=reward, + done=True, + metadata={"done_reason": "max_turns", "task_key": self.task_key}, + ) + + # Build response observation + if error: + self.tool_errors += 1 + obs_content = f"Error: {error}" + elif tool_result: + if isinstance(tool_result, dict): + obs_content = f"Tool result:\n{json.dumps(tool_result, indent=2)}" + else: + obs_content = f"Tool result:\n{tool_result}" + elif agent_done: + obs_content = "Task marked as complete." + elif not tool_call: + obs_content = 'No tool call found. Use {"name": "...", "arguments": {...}} format.' + else: + obs_content = "Action executed." + + new_obs = {"role": "user", "content": obs_content} + self.chat_history.append(new_obs) + if self.context_manager: + self.context_manager.track_message(new_obs) + + step_time = time.time() - step_start + metadata = { + "task_key": self.task_key, + "turn": self.turns, + "tool_call": tool_call, + "tool_result": tool_result, + "error": error, + "done_reason": "agent_done" if agent_done else None, + "step_time": step_time, + "mcp_time": mcp_time, + } + + # If context was modified (e.g., manage_context dropped turns), return full chat_history + # so the generator can replace its copy. This is required for stepwise training to work. + if tool_call and self.context_manager and self.context_manager.is_context_tool(tool_call["name"]): + if tool_call["name"] == "manage_context": + metadata["modified_chat_history"] = self.chat_history.copy() + + return BaseTextEnvStepOutput( + observations=[new_obs], + reward=reward, + done=episode_done, + metadata=metadata, + ) + + def step(self, action: str) -> BaseTextEnvStepOutput: + """ + Execute one step in the Fleet environment (sync wrapper). + + For async contexts, use step_async() instead. + """ + return asyncio.run(self.step_async(action)) + + def _capture_verifier_feedback(self): + """Capture verifier feedback from OpenEnv before nulling the env instance.""" + if self.openenv_task_env: + self._verifier_stdout = getattr(self.openenv_task_env, "verifier_stdout", None) + self._verifier_error = getattr(self.openenv_task_env, "verifier_error", None) + self._tool_error_messages = getattr(self.openenv_task_env, "tool_errors_list", []) + + def close(self): + """Close the Fleet environment and cleanup resources.""" + if self.openenv_task_env: + try: + self.openenv_task_env.close() + if self.openenv_task_env.final_reward is not None: + self.last_reward = self.openenv_task_env.final_reward + self._capture_verifier_feedback() + except Exception as e: + print(f"Warning: Failed to close Fleet environment: {e}") + self.openenv_task_env = None + + async def close_async(self): + """Close the Fleet environment and cleanup resources (async version). + + Runs verifier via OpenEnv's close_async() to get actual reward for + orphaned rollouts (context overflow, early termination by SkyRL). + """ + if self.openenv_task_env: + try: + await self.openenv_task_env.close_async() + if self.openenv_task_env.final_reward is not None: + self.last_reward = self.openenv_task_env.final_reward + self._capture_verifier_feedback() + except Exception as e: + print(f"Warning: Failed to close Fleet environment: {e}") + self.openenv_task_env = None + + def get_metrics(self) -> Dict[str, Any]: + """Return environment metrics for this episode.""" + metrics = { + "task_key": self.task_key, + "env_key": self.task_config.get("env_key") or self.task_config.get("env_id"), + "turns": self.turns, + "tool_calls": self.tool_calls, + "tool_errors": self.tool_errors, + "is_hinted": bool(self.extras.get("hint")), + } + if self.last_reward is not None: + metrics["final_reward"] = self.last_reward + # Include verifier feedback for hint generation + if self._verifier_stdout is not None: + metrics["verifier_stdout"] = self._verifier_stdout + if self._verifier_error is not None: + metrics["verifier_error"] = self._verifier_error + if self._tool_error_messages: + metrics["tool_error_messages"] = self._tool_error_messages + return metrics + + @staticmethod + def build_hint_text( + verifier_stdout: Optional[str], + verifier_error: Optional[str], + tool_error_messages: Optional[List[str]], + ) -> str: + """Build hint text from verifier feedback. No LLM call. + + Parses ERROR_ACCUMULATOR / SUCCESS_ACCUMULATOR from verifier stdout + and formats tool errors into structured feedback for the next attempt. + """ + parts = [] + + if verifier_stdout: + err_match = re.search( + r">>> ERROR_ACCUMULATOR >>>\n(.+?)\n<<< ERROR_ACCUMULATOR <<<", + verifier_stdout, + re.DOTALL, + ) + suc_match = re.search( + r">>> SUCCESS_ACCUMULATOR >>>\n(.+?)\n<<< SUCCESS_ACCUMULATOR <<<", + verifier_stdout, + re.DOTALL, + ) + if err_match or suc_match: + try: + errors = ast.literal_eval(err_match.group(1)) if err_match else [] + successes = ast.literal_eval(suc_match.group(1)) if suc_match else [] + except Exception: + errors, successes = [], [] + if successes: + parts.append(f"Checks passed ({len(successes)}): " + ", ".join(str(s)[:100] for s in successes[:5])) + if errors: + parts.append(f"Checks failed ({len(errors)}): " + ", ".join(str(e)[:100] for e in errors[:5])) + + if verifier_error: + parts.append(f"Verifier: {verifier_error}") + + if tool_error_messages: + unique = list(dict.fromkeys(tool_error_messages))[:5] + parts.append("Tool errors: " + "; ".join(e[:200] for e in unique)) + + return "\n".join(parts) if parts else "The previous attempt failed. Try a different approach." + + @staticmethod + def aggregate_metrics(metrics: List[Dict[str, Any]]) -> Dict[str, Any]: + """Aggregate metrics across episodes with per-env breakdown. + + Handles two types of metrics: + 1. Normal episode metrics: {env_key, turns, tool_calls, tool_errors} + 2. Init failure metrics: {env_init_failed, env_init_failed/{env_key}} + """ + if not metrics: + return {} + + # Track init failures per env (from env_init_failed/{env_key} pattern) + env_init_failures: Dict[str, int] = {} + total_init_failures = 0 + + # Group normal metrics by env_key + env_data: Dict[str, Dict[str, List[int]]] = {} + for m in metrics: + # Check for init failure metrics (env_init_failed/{env_key} pattern) + for key, value in m.items(): + if key.startswith("env_init_failed/"): + env_key = key.split("/", 1)[1] + env_init_failures[env_key] = env_init_failures.get(env_key, 0) + int(value) + total_init_failures += int(value) + elif key == "env_init_failed": + # Already counted via per-env key + pass + + # Normal episode metrics (has env_key field) + env_key = m.get("env_key") + if env_key: + if env_key not in env_data: + env_data[env_key] = {"turns": [], "tool_calls": [], "tool_errors": []} + env_data[env_key]["turns"].append(m.get("turns", 0)) + env_data[env_key]["tool_calls"].append(m.get("tool_calls", 0)) + env_data[env_key]["tool_errors"].append(m.get("tool_errors", 0)) + + result: Dict[str, Any] = {} + total_turns = 0 + total_tool_calls = 0 + total_tool_errors = 0 + total_episodes = 0 + + # Per-env_key metrics + for env_key, data in env_data.items(): + turns_list = data["turns"] + tool_calls_list = data["tool_calls"] + tool_errors_list = data["tool_errors"] + + avg_turns = sum(turns_list) / len(turns_list) + avg_tool_calls = sum(tool_calls_list) / len(tool_calls_list) + avg_tool_errors = sum(tool_errors_list) / len(tool_errors_list) + # Tool calls per turn (avoid div by zero) + total_env_turns = sum(turns_list) + total_env_tool_calls = sum(tool_calls_list) + total_env_tool_errors = sum(tool_errors_list) + tool_calls_per_turn = total_env_tool_calls / total_env_turns if total_env_turns > 0 else 0 + # Tool error rate (errors per tool call) + tool_error_rate = total_env_tool_errors / total_env_tool_calls if total_env_tool_calls > 0 else 0 + + result[f"{env_key}/avg_turns"] = avg_turns + result[f"{env_key}/min_turns"] = min(turns_list) + result[f"{env_key}/max_turns"] = max(turns_list) + result[f"{env_key}/avg_tool_calls"] = avg_tool_calls + result[f"{env_key}/tool_calls_per_turn"] = tool_calls_per_turn + result[f"{env_key}/avg_tool_errors"] = avg_tool_errors + result[f"{env_key}/total_tool_errors"] = total_env_tool_errors + result[f"{env_key}/tool_error_rate"] = tool_error_rate + result[f"{env_key}/num_episodes"] = len(turns_list) + + total_turns += total_env_turns + total_tool_calls += total_env_tool_calls + total_tool_errors += total_env_tool_errors + total_episodes += len(turns_list) + + # Overall metrics + result["avg_turns"] = total_turns / total_episodes if total_episodes > 0 else 0 + result["avg_tool_calls"] = total_tool_calls / total_episodes if total_episodes > 0 else 0 + result["tool_calls_per_turn"] = total_tool_calls / total_turns if total_turns > 0 else 0 + result["avg_tool_errors"] = total_tool_errors / total_episodes if total_episodes > 0 else 0 + result["total_tool_errors"] = total_tool_errors + result["tool_error_rate"] = total_tool_errors / total_tool_calls if total_tool_calls > 0 else 0 + result["total_episodes"] = total_episodes + + # Add init failure metrics (per-env and total) + for env_key, failures in env_init_failures.items(): + result[f"{env_key}/env_init_failed"] = failures + if total_init_failures > 0: + result["total_env_init_failed"] = total_init_failures + + return result + + +def clear_caches(): + """Clear global caches. Useful for testing.""" + global _TASK_CACHE + _TASK_CACHE = {} diff --git a/skyrl-train/integrations/fleet/export_tasks.py b/skyrl-train/integrations/fleet/export_tasks.py new file mode 100644 index 0000000000..7b1ed956fc --- /dev/null +++ b/skyrl-train/integrations/fleet/export_tasks.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Export tasks from Fleet API to JSON file. + +Usage: + python -m integrations.fleet.export_tasks --output ~/data/fleet/tasks.json + python -m integrations.fleet.export_tasks --output ~/data/fleet/tasks.json --env-key github +""" + +import argparse +import json +import os +import sys + + +def export_tasks(output_file: str, env_key: str | None = None, modality: str = "tool_use"): + """Export tasks from Fleet API to JSON file.""" + try: + from fleet import Fleet + except ImportError: + print("Fleet SDK not available. Install with: pip install fleet-python") + sys.exit(1) + + api_key = os.environ.get("FLEET_API_KEY") + if not api_key: + print("ERROR: FLEET_API_KEY environment variable not set") + sys.exit(1) + + fleet = Fleet(api_key=api_key) + + print(f"Loading tasks from Fleet API (env_key={env_key})...") + tasks = fleet.load_tasks(env_key=env_key) + print(f"Loaded {len(tasks)} tasks") + + # Convert to JSON format + task_dicts = [] + for task in tasks: + task_dicts.append( + { + "key": task.key, + "prompt": task.prompt, + "env_id": task.env_id, + "version": task.version, + "data_id": task.data_id, + "data_version": task.data_version, + "verifier_func": task.verifier_func, + "task_modality": modality, + } + ) + + # Ensure output directory exists + os.makedirs(os.path.dirname(os.path.expanduser(output_file)), exist_ok=True) + + output_path = os.path.expanduser(output_file) + with open(output_path, "w") as f: + json.dump(task_dicts, f, indent=2) + + print(f"Exported {len(task_dicts)} tasks to {output_path}") + + +def main(): + parser = argparse.ArgumentParser(description="Export Fleet tasks to JSON") + parser.add_argument("--output", "-o", required=True, help="Output JSON file path") + parser.add_argument("--env-key", default=None, help="Filter by environment key") + parser.add_argument("--modality", default="tool_use", help="Task modality") + args = parser.parse_args() + + export_tasks(args.output, args.env_key, args.modality) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/fleet/prepare_dataset.py b/skyrl-train/integrations/fleet/prepare_dataset.py new file mode 100644 index 0000000000..cdc465b8d7 --- /dev/null +++ b/skyrl-train/integrations/fleet/prepare_dataset.py @@ -0,0 +1,564 @@ +""" +Prepare Fleet tasks for SkyRL training. + +Converts Fleet task JSON files to SkyRL parquet dataset format. + +Usage: + python -m integrations.fleet.prepare_dataset \ + --tasks-json /path/to/all_tool_use.json \ + --output-dir ./data/fleet \ + --modality tool_use + +Split Strategy: + - Stratified by environment (each env maintains train/eval ratio) + - Hash-based deterministic assignment (same task always goes to same split) + - 20% eval ratio, capped at 20 samples per env (MAX_EVAL_SAMPLES) + - Minimum 5 eval samples per env (otherwise all go to train) + - Held-out eval envs: instacart (computer_use only) + +v0.3.2 Changes: + - Increased eval_ratio from 10% to 20% to include carlisle/outlook in eval + - Result: 11 envs in eval (was 9), ~183 eval samples (was ~146) + +v0.3.1 Changes: + - Added MAX_ENV_TRAIN_RATIO=0.20 to prevent any single env from dominating + - Hash-based deterministic sampling for reproducibility + +v0.3.0 Changes: + - Increased eval_ratio from 2% to 10% + - Added MAX_EVAL_SAMPLES=30 cap per environment + - MIN_EVAL_SAMPLES stays at 5 + - Result: ticketmaster now gets ~22 eval samples for trace analysis +""" + +import argparse +import hashlib +import json +import os +from collections import defaultdict +from typing import Any, Dict, List, Optional + +from datasets import Dataset + +# Held-out environments for eval only (not used in train) +HELD_OUT_ENVS = { + "tool_use": [], # v0.3: all envs split normally (outlook now included in train) + "computer_use": [], +} + +# Excluded environments (removed from both train and eval) +# v0.3.6: google-maps excluded due to broken MCP server (502 errors, "database is locked") +# v0.4.0: dropbox excluded due to broken env (instance creation timeouts) +EXCLUDED_ENVS = { + "tool_use": ["dropbox"], + "computer_use": ["dropbox"], +} + +# Tasks excluded due to missing CURRENT_DATE in env_variables (v0.4.0) +# These tasks have partial dates (e.g., "January 30th" without year) but their +# tool calls require mm/dd/yy format. Without CURRENT_DATE, the model cannot +# compute the correct year, causing date validation failures. +# See: https://github.com/fleet-ai/SkyRL/pull/246 +TASKS_MISSING_CURRENT_DATE = { + "task_a44hx6crecg4_1769052238469_i7dxxtjvq", # zillow - February 1st + "task_a7rlslof7gdy_1768337837679_8be6pguu3", # zillow - March 11th + "task_axtmgwocana_1768544478249_k2ozcylyf", # zillow - January 21st + "task_b1fxgn0k3yms_1768542773490_ddbhj5bai", # zillow - January 30th + "task_b4v77hb3owof_1768546181946_efsedxv9g", # zillow - February 14th + "task_b5zt6ipf0nbl_1768346335430_i23gknp4t", # zillow - January 15th + "task_bafrpi5qgyzh_1768546181946_2cebmq91r", # zillow - February 14th + "task_bdmnfipwxlqv_1769052238469_4nglwjqfm", # zillow - February 1st + "task_bxqzfjc2dbte_1768337837679_2qvnm9rq7", # zillow - March 11th + "task_c3jwlxmfvbop_1768544478249_efo6hxylr", # zillow - January 21st + "task_c7o0c7ehhv9t_1768542773490_2t9w2l1z5", # zillow - January 30th + "task_ceqj4h9t0ygi_1768346335430_8j1w8w5xp", # zillow - January 15th + "task_cgpxfxp78bvp_1768346335430_6v4n8wlt8", # zillow - January 15th + "task_cgsz56tqjlv6_1768346335430_hqgsjy4wt", # zillow - January 15th + "task_dpv4bpdpz6db_1768542773490_f3g6w8e8g", # zillow - January 30th + "task_f7lgb6fxfwln_1768337837679_d1dxk6ahv", # zillow - March 11th + "task_fl1rq3d2wbj9_1768337837679_d2x4k8p93", # zillow - March 11th + "task_fn1k5mvjx6r1_1768544478249_1nfmnp6r2", # zillow - January 21st + "task_fnh5f0x7hv6w_1768544478249_8wptm6zqp", # zillow - January 21st + "task_g2dwb1rfx69c_1769052238469_bc1y9h9d7", # zillow - February 1st + "task_g3wpj1mcl0lf_1768546181946_59vtqn9fw", # zillow - February 14th +} + +# Minimum number of samples required to create an eval split for an env +MIN_EVAL_SAMPLES = 5 + +# Maximum number of eval samples per environment (v0.3.1: reduced from 30 to 20) +# Ensures small envs get eval traces without blowing up eval set size +MAX_EVAL_SAMPLES = 20 + +# Maximum fraction of training data any single environment can have (v0.3.1) +# Prevents dominant environments from skewing training +MAX_ENV_TRAIN_RATIO = 0.20 + +# Maximum total eval prompts across all environments (v0.3.2) +# With eval_n_samples_per_prompt=3 and 30s per trajectory: +# 96 prompts × 3 samples = 288 trajectories (~8 tasks/env × 12 envs) +MAX_EVAL_PROMPTS = 96 + + +def load_tasks_from_json(json_path: str) -> List[Dict[str, Any]]: + """Load tasks from JSON file (Fleet export format).""" + with open(json_path, "r") as f: + data = json.load(f) + + # Handle both formats: array or {"tasks": [...]} + if isinstance(data, list): + return data + elif isinstance(data, dict) and "tasks" in data: + return data["tasks"] + else: + raise ValueError("Invalid JSON format: expected array or object with 'tasks' key") + + +def hash_to_split(task_key: str, eval_ratio: float = 0.10) -> str: + """Deterministically assign task to train or eval based on hash. + + Uses MD5 hash of task_key to get a deterministic float in [0, 1). + This ensures the same task always goes to the same split. + """ + hash_bytes = hashlib.md5(task_key.encode()).digest() + hash_int = int.from_bytes(hash_bytes[:8], byteorder="big") + hash_float = hash_int / (2**64) + return "eval" if hash_float < eval_ratio else "train" + + +def hash_to_float(task_key: str) -> float: + """Convert task_key to deterministic float in [0, 1) for sampling.""" + hash_bytes = hashlib.md5(task_key.encode()).digest() + hash_int = int.from_bytes(hash_bytes[:8], byteorder="big") + return hash_int / (2**64) + + +def cap_training_distribution( + train_records: List[Dict[str, Any]], + max_env_ratio: float, +) -> tuple[List[Dict[str, Any]], Dict[str, Dict[str, int]]]: + """Cap each environment's contribution to training data. + + Uses hash-based deterministic sampling so the same tasks are always selected. + + Args: + train_records: List of training records with 'data_source' (env_key) and 'task_key' + max_env_ratio: Maximum fraction any single env can contribute (e.g., 0.20 = 20%) + + Returns: + Tuple of (capped_records, cap_stats) where cap_stats shows per-env before/after counts + """ + if max_env_ratio >= 1.0: + return train_records, {} + + total_train = len(train_records) + max_per_env = int(total_train * max_env_ratio) + + # Group by environment + records_by_env: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for record in train_records: + env_key = record.get("data_source", "unknown") + records_by_env[env_key].append(record) + + # Cap each environment + capped_records = [] + cap_stats: Dict[str, Dict[str, int]] = {} + + for env_key, records in records_by_env.items(): + before_count = len(records) + + if before_count <= max_per_env: + # No capping needed + capped_records.extend(records) + cap_stats[env_key] = {"before": before_count, "after": before_count, "capped": False} + else: + # Sort by hash for deterministic selection + records_sorted = sorted(records, key=lambda r: hash_to_float(r.get("task_key", ""))) + selected = records_sorted[:max_per_env] + capped_records.extend(selected) + cap_stats[env_key] = {"before": before_count, "after": max_per_env, "capped": True} + + return capped_records, cap_stats + + +def prepare_fleet_dataset( + tasks_json: str, + output_dir: str, + modality: Optional[str] = "tool_use", + eval_ratio: float = 0.20, # v0.3.2: increased to 20% to include carlisle/outlook in eval + env_filter: Optional[str] = None, + difficulty_filter: Optional[str] = None, # v0.4.0: filter by difficulty (1=easy, 2=medium, 3=hard) + max_tasks: Optional[int] = None, + max_env_ratio: float = MAX_ENV_TRAIN_RATIO, # v0.3.1: cap dominant environments + max_eval_prompts: Optional[int] = MAX_EVAL_PROMPTS, # v0.3.2: cap total eval prompts +): + """ + Convert Fleet tasks JSON to SkyRL parquet dataset. + + Args: + tasks_json: Path to Fleet tasks JSON file + output_dir: Output directory for parquet files + modality: Task modality filter ("tool_use" or "computer_use"), None for all + eval_ratio: Fraction of data for evaluation (default: 0.02) + env_filter: Optional env_key filter (e.g., "github", "booking") + max_tasks: Optional maximum number of tasks to include + max_env_ratio: Maximum fraction any single env can contribute to training (default: 0.20) + """ + # Log applied filters at the start + print("\n=== Dataset Filters ===") + print(f" Source: {tasks_json}") + print(f" Modality: {modality or 'all'}") + print(f" Env filter: {env_filter or 'none'}") + print(f" Difficulty filter: {difficulty_filter or 'all (1,2,3)'}") + print(f" Max tasks: {max_tasks or 'unlimited'}") + print(f" Max env ratio: {max_env_ratio:.0%}") + print(f" Max eval prompts: {max_eval_prompts or 'unlimited'}") + print() + + print(f"Loading tasks from {tasks_json}...") + tasks = load_tasks_from_json(tasks_json) + print(f"Loaded {len(tasks)} tasks") + + # Filter by modality if specified + if modality: + tasks = [t for t in tasks if t.get("task_modality") == modality] + print(f"After modality filter ({modality}): {len(tasks)} tasks") + + # Filter by env_key(s) if specified - supports comma-separated list + if env_filter: + env_list = [e.strip() for e in env_filter.split(",") if e.strip()] + tasks = [t for t in tasks if t.get("env_key") in env_list or t.get("env_id") in env_list] + print(f"After env filter ({env_list}): {len(tasks)} tasks") + + # Filter by difficulty if specified - supports comma-separated list (e.g., "1,2" for easy+medium) + if difficulty_filter: + diff_list = [int(d.strip()) for d in difficulty_filter.split(",") if d.strip()] + tasks = [t for t in tasks if t.get("difficulty") in diff_list] + print(f"After difficulty filter ({diff_list}): {len(tasks)} tasks") + + # Limit tasks if specified + if max_tasks and len(tasks) > max_tasks: + tasks = tasks[:max_tasks] + print(f"Limited to {max_tasks} tasks") + + if not tasks: + print("No tasks remaining after filtering. Exiting.") + return + + # Deduplicate by task_key (keep first occurrence) + seen_task_keys: set = set() + unique_tasks = [] + duplicate_count = 0 + env_duplicate_counts: Dict[str, int] = defaultdict(int) + + for task in tasks: + task_key = task.get("key") or task.get("task_key") + if not task_key: + continue + if task_key in seen_task_keys: + duplicate_count += 1 + env_key = task.get("env_key") or task.get("env_id") or "unknown" + env_duplicate_counts[env_key] += 1 + else: + seen_task_keys.add(task_key) + unique_tasks.append(task) + + if duplicate_count > 0: + print(f"\n⚠️ WARNING: Removed {duplicate_count} duplicate task_keys") + print(" By environment:") + for env, count in sorted(env_duplicate_counts.items(), key=lambda x: -x[1]): + print(f" {env}: {count} duplicates removed") + print() + + tasks = unique_tasks + print(f"After deduplication: {len(tasks)} unique tasks") + + # Get excluded envs for this modality (removed entirely) + excluded_envs = set(EXCLUDED_ENVS.get(modality, [])) + if excluded_envs: + before_count = len(tasks) + tasks = [t for t in tasks if t.get("env_key") not in excluded_envs] + print(f"Excluded environments: {excluded_envs}") + print(f"After excluding envs: {len(tasks)} tasks (removed {before_count - len(tasks)})") + + # Exclude specific tasks missing CURRENT_DATE + if TASKS_MISSING_CURRENT_DATE: + before_count = len(tasks) + tasks = [t for t in tasks if (t.get("key") or t.get("task_key")) not in TASKS_MISSING_CURRENT_DATE] + removed = before_count - len(tasks) + if removed > 0: + print(f"Excluded tasks missing CURRENT_DATE: {removed} tasks") + + # Get held-out envs for this modality + held_out_envs = set(HELD_OUT_ENVS.get(modality, [])) + if held_out_envs: + print(f"Held-out test environments: {held_out_envs}") + + # Group tasks by environment + tasks_by_env: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for task in tasks: + env_key = task.get("env_key") or task.get("env_id") or "unknown" + tasks_by_env[env_key].append(task) + + # Prepare records with stratified split + train_records = [] + eval_records = [] + + # Track per-env counts for summary table + env_split_counts: Dict[str, Dict[str, int]] = {} + + print("\n=== Per-Environment Split ===") + for env_key in sorted(tasks_by_env.keys()): + env_tasks = tasks_by_env[env_key] + + # Check if this env is held out for eval only + if env_key in held_out_envs: + env_eval_count = 0 + for task in env_tasks: + record = _task_to_record(task, env_key) + if record: + eval_records.append(record) + env_eval_count += 1 + env_split_counts[env_key] = {"train": 0, "eval": env_eval_count} + print(f" {env_key}: {len(env_tasks)} -> EVAL only (held-out)") + continue + + # Calculate target eval size: use ratio but cap at MAX_EVAL_SAMPLES + target_eval_size = min(int(len(env_tasks) * eval_ratio), MAX_EVAL_SAMPLES) + + # If not enough samples for eval, put all in train + if target_eval_size < MIN_EVAL_SAMPLES: + env_train_count = 0 + for task in env_tasks: + record = _task_to_record(task, env_key) + if record: + train_records.append(record) + env_train_count += 1 + env_split_counts[env_key] = {"train": env_train_count, "eval": 0} + print(f" {env_key}: {len(env_tasks)} -> all TRAIN (< {MIN_EVAL_SAMPLES} eval samples)") + continue + + # Compute effective eval ratio to achieve target_eval_size (capped at MAX_EVAL_SAMPLES) + effective_eval_ratio = target_eval_size / len(env_tasks) + + # Stratified split using hash with effective ratio + env_train = 0 + env_eval = 0 + for task in env_tasks: + task_key = task.get("key") or task.get("task_key") + record = _task_to_record(task, env_key) + if not record: + continue + + split = hash_to_split(task_key, effective_eval_ratio) + if split == "eval": + eval_records.append(record) + env_eval += 1 + else: + train_records.append(record) + env_train += 1 + + env_split_counts[env_key] = {"train": env_train, "eval": env_eval} + print(f" {env_key}: {len(env_tasks)} -> {env_train} train, {env_eval} eval") + + print(f"\nTotal: {len(train_records)} train, {len(eval_records)} eval") + + # Apply total eval cap (v0.3.2) - stratified sampling across environments + if max_eval_prompts and len(eval_records) > max_eval_prompts: + print(f"\n=== Capping Eval Prompts ({max_eval_prompts} max total) ===") + + # Group by environment + eval_by_env: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for record in eval_records: + eval_by_env[record.get("data_source", "unknown")].append(record) + + # Take min(8, available) from each env, then distribute remaining quota proportionally + min_per_env = 8 + capped_eval_records = [] + + for env_key, records in eval_by_env.items(): + # Sort by hash for deterministic selection + records.sort(key=lambda r: hash_to_float(r.get("task_key", ""))) + # Take at least min_per_env (or all if fewer available) + take = min(min_per_env, len(records)) + capped_eval_records.extend(records[:take]) + + # If we have budget remaining, distribute round-robin across envs + remaining_budget = max_eval_prompts - len(capped_eval_records) + if remaining_budget > 0: + # Records not yet selected (sorted by hash for determinism) + remaining_by_env = { + env: records[min_per_env:] for env, records in eval_by_env.items() if len(records) > min_per_env + } + + # Round-robin until budget exhausted + env_keys = sorted(remaining_by_env.keys()) + idx = 0 + while remaining_budget > 0 and any(remaining_by_env.values()): + env = env_keys[idx % len(env_keys)] + if remaining_by_env[env]: + capped_eval_records.append(remaining_by_env[env].pop(0)) + remaining_budget -= 1 + idx += 1 + + # Update env_split_counts + for env_key in eval_by_env: + count = sum(1 for r in capped_eval_records if r.get("data_source") == env_key) + if env_key in env_split_counts: + env_split_counts[env_key]["eval"] = count + print(f" {env_key}: {len(eval_by_env[env_key])} -> {count}") + + eval_records = capped_eval_records + print(f"\nAfter capping: {len(eval_records)} eval prompts") + + # Apply per-environment cap to training data (v0.3.1) + if max_env_ratio < 1.0 and train_records: + train_records, cap_stats = cap_training_distribution(train_records, max_env_ratio) + + # Print capping summary + capped_envs = [env for env, stats in cap_stats.items() if stats["capped"]] + if capped_envs: + print(f"\n=== Training Distribution Cap ({max_env_ratio:.0%} max per env) ===") + for env in sorted(capped_envs): + stats = cap_stats[env] + print(f" {env}: {stats['before']} -> {stats['after']} ({stats['before'] - stats['after']} removed)") + print(f"\nAfter capping: {len(train_records)} train") + + # Update env_split_counts with capped values + for env, stats in cap_stats.items(): + if env in env_split_counts: + env_split_counts[env]["train"] = stats["after"] + + # Create datasets + train_dataset = Dataset.from_list(train_records) if train_records else None + eval_dataset = Dataset.from_list(eval_records) if eval_records else None + + # Save to parquet + os.makedirs(output_dir, exist_ok=True) + + if train_dataset: + train_path = os.path.join(output_dir, "train.parquet") + train_dataset.to_parquet(train_path) + print(f"Saved train dataset to {train_path}") + + if eval_dataset: + eval_path = os.path.join(output_dir, "validation.parquet") + eval_dataset.to_parquet(eval_path) + print(f"Saved validation dataset to {eval_path}") + + # Print summary statistics + print("\n=== Dataset Summary ===") + print(f"Train: {len(train_records)}") + print(f"Eval: {len(eval_records)} (includes held-out: {held_out_envs or 'none'})") + + # Print per-environment breakdown table + print("\n=== Per-Environment Breakdown ===") + print(f"{'Environment':<20} {'Train':>8} {'Eval':>8} {'Total':>8}") + print("-" * 48) + for env_key in sorted(env_split_counts.keys()): + counts = env_split_counts[env_key] + total = counts["train"] + counts["eval"] + print(f"{env_key:<20} {counts['train']:>8} {counts['eval']:>8} {total:>8}") + print("-" * 48) + print( + f"{'TOTAL':<20} {len(train_records):>8} {len(eval_records):>8} " f"{len(train_records) + len(eval_records):>8}" + ) + + +def _task_to_record(task: Dict[str, Any], env_key: str) -> Optional[Dict[str, Any]]: + """Convert a task dict to a dataset record.""" + task_key = task.get("key") or task.get("task_key") + prompt = task.get("prompt", "") + + if not task_key or not prompt: + return None + + return { + # Required fields for SkyRL + "prompt": [{"role": "user", "content": prompt}], + "env_class": "fleet_task", # This tells SkyRL to use FleetTaskEnv + # Task identification (passed as env_extras) + "task_key": task_key, + # Data source for per-environment metrics in WandB + "data_source": env_key, + } + + +def main(): + parser = argparse.ArgumentParser(description="Prepare Fleet tasks for SkyRL training") + parser.add_argument( + "--tasks-json", + type=str, + required=True, + help="Path to Fleet tasks JSON file", + ) + parser.add_argument( + "--output-dir", + type=str, + default="./data/fleet", + help="Output directory for parquet files", + ) + parser.add_argument( + "--modality", + type=str, + default="tool_use", + choices=["tool_use", "computer_use", "all"], + help="Task modality filter ('all' for no filter)", + ) + parser.add_argument( + "--eval-ratio", + type=float, + default=0.20, + help="Fraction of data for evaluation (default: 0.20)", + ) + parser.add_argument( + "--env-filter", + type=str, + default=None, + help="Optional env_key filter (e.g., 'github', 'booking')", + ) + parser.add_argument( + "--difficulty-filter", + type=str, + default=None, + help="Optional difficulty filter: 1=easy, 2=medium, 3=hard (e.g., '1,2' for easy+medium)", + ) + parser.add_argument( + "--max-tasks", + type=int, + default=None, + help="Maximum number of tasks to include", + ) + parser.add_argument( + "--max-env-ratio", + type=float, + default=MAX_ENV_TRAIN_RATIO, + help=f"Maximum fraction of training data per environment (default: {MAX_ENV_TRAIN_RATIO})", + ) + parser.add_argument( + "--max-eval-prompts", + type=int, + default=MAX_EVAL_PROMPTS, + help=f"Maximum total eval prompts across all environments (default: {MAX_EVAL_PROMPTS})", + ) + + args = parser.parse_args() + + # Handle 'all' modality + modality = None if args.modality == "all" else args.modality + + prepare_fleet_dataset( + tasks_json=args.tasks_json, + output_dir=args.output_dir, + modality=modality, + eval_ratio=args.eval_ratio, + env_filter=args.env_filter, + difficulty_filter=args.difficulty_filter, + max_tasks=args.max_tasks, + max_env_ratio=args.max_env_ratio, + max_eval_prompts=args.max_eval_prompts, + ) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/fleet/s3_checkpoints.py b/skyrl-train/integrations/fleet/s3_checkpoints.py new file mode 100644 index 0000000000..5db5aebb40 --- /dev/null +++ b/skyrl-train/integrations/fleet/s3_checkpoints.py @@ -0,0 +1,439 @@ +""" +S3 Checkpoint Management for SkyRL Training. + +Provides checkpoint upload to S3, download from S3 for resume, and local cleanup. + +Key behavior: +- Cleans up old local checkpoints BEFORE saving new one (prevents disk full) +- Uploads to S3 asynchronously (non-blocking, training continues) +- Downloads checkpoint from S3 before training for cross-VM resume +- Uploads eval results to S3 for persistence + +Usage: + from integrations.fleet.s3_checkpoints import ( + wrap_trainer_with_s3_upload, + download_checkpoint_from_s3, + upload_eval_results_to_s3, + ) + + # Download checkpoint before training (for resume on new VM) + download_checkpoint_from_s3(ckpt_path, run_name) + + trainer = wrap_trainer_with_s3_upload(trainer, bucket="skyrl-checkpoints") + upload_eval_results_to_s3(local_dir, run_name, global_step) + +Environment Variables: + AWS_ACCESS_KEY_ID: AWS access key + AWS_SECRET_ACCESS_KEY: AWS secret key + AWS_REGION: AWS region (default: us-east-1) + S3_CHECKPOINT_BUCKET: S3 bucket for checkpoints (default: skyrl-checkpoints) + S3_TRAJECTORY_BUCKET: S3 bucket for eval trajectories (default: skyrl-trajectories) +""" + +import os +import shutil +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Optional +import logging + +logger = logging.getLogger(__name__) + + +class S3CheckpointUploader: + """ + Uploads checkpoint directories to S3 asynchronously. + + Uses a background thread pool to avoid blocking training. + Deletes local checkpoints after successful upload. + """ + + def __init__( + self, + bucket: str, + prefix: str, + region: str = "us-east-1", + max_workers: int = 2, + ): + self.bucket = bucket + self.prefix = prefix + self.region = region + self._executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="s3-upload") + self._pending: set = set() + self._lock = threading.Lock() + + def _upload_sync(self, local_dir: str) -> bool: + """Synchronous upload that runs in thread pool.""" + try: + import boto3 + from botocore.config import Config + from boto3.s3.transfer import TransferConfig + + config = Config( + retries={"max_attempts": 3, "mode": "adaptive"}, + connect_timeout=30, + read_timeout=120, + ) + + s3 = boto3.client("s3", region_name=self.region, config=config) + + local_path = Path(local_dir) + if not local_path.exists(): + logger.warning(f"Checkpoint directory does not exist: {local_dir}") + return False + + checkpoint_name = local_path.name + s3_prefix = f"{self.prefix}/{checkpoint_name}" + + transfer_config = TransferConfig( + multipart_threshold=64 * 1024 * 1024, + multipart_chunksize=64 * 1024 * 1024, + max_concurrency=4, + use_threads=True, + ) + + uploaded_files = 0 + total_size = 0 + + for file_path in local_path.rglob("*"): + if file_path.is_file(): + relative_path = file_path.relative_to(local_path) + s3_key = f"{s3_prefix}/{relative_path}" + file_size = file_path.stat().st_size + total_size += file_size + + logger.info(f"Uploading {file_path.name} ({file_size / 1e6:.1f} MB)") + + s3.upload_file(str(file_path), self.bucket, s3_key, Config=transfer_config) + uploaded_files += 1 + + logger.info( + f"Uploaded {checkpoint_name}: {uploaded_files} files, {total_size / 1e9:.2f} GB to s3://{self.bucket}/{s3_prefix}/" + ) + + # Delete local after successful upload to free disk space + logger.info(f"Deleting local checkpoint after S3 upload: {local_dir}") + shutil.rmtree(local_dir) + + return True + + except Exception as e: + logger.error(f"S3 upload failed for {local_dir}: {e}") + return False + finally: + with self._lock: + self._pending.discard(local_dir) + + def upload_async(self, local_dir: str) -> None: + """Queue checkpoint for async upload. Non-blocking.""" + with self._lock: + if local_dir in self._pending: + return + self._pending.add(local_dir) + + logger.info(f"Queuing checkpoint for S3 upload: {local_dir}") + self._executor.submit(self._upload_sync, local_dir) + + def wait_for_uploads(self, timeout: Optional[float] = None) -> None: + """Wait for all pending uploads to complete.""" + self._executor.shutdown(wait=True) + self._executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="s3-upload") + + +def cleanup_old_local_checkpoints(ckpt_path: str, keep_n: int = 2) -> None: + """ + Delete old local checkpoints, keeping only the most recent N. + + Args: + ckpt_path: Base checkpoint directory + keep_n: Number of recent checkpoints to keep (default: 2 for safety) + """ + ckpt_dir = Path(ckpt_path) + if not ckpt_dir.exists(): + return + + checkpoint_dirs = sorted( + [d for d in ckpt_dir.iterdir() if d.is_dir() and d.name.startswith("global_step_")], + key=lambda x: int(x.name.split("_")[-1]), + reverse=True, + ) + + for old_dir in checkpoint_dirs[keep_n:]: + logger.info(f"Cleaning up old local checkpoint: {old_dir}") + try: + shutil.rmtree(old_dir) + except Exception as e: + logger.warning(f"Failed to delete {old_dir}: {e}") + + +def wrap_trainer_with_s3_upload( + trainer, + bucket: Optional[str] = None, + prefix: Optional[str] = None, + region: Optional[str] = None, +): + """ + Wrap a SkyRL trainer to: + 1. Clean up old checkpoints BEFORE saving (prevents disk full) + 2. Upload to S3 asynchronously AFTER saving (if credentials set) + 3. Delete local checkpoint after successful S3 upload (frees disk) + + Args: + trainer: SkyRL trainer instance + bucket: S3 bucket (default: from S3_CHECKPOINT_BUCKET env var) + prefix: S3 prefix (default: from trainer config) + region: AWS region (default: from AWS_REGION env var) + + Returns: + The trainer (modified in place) + """ + bucket = bucket or os.environ.get("S3_CHECKPOINT_BUCKET", "skyrl-checkpoints") + region = region or os.environ.get("AWS_REGION", "us-east-1") + + # Build prefix from trainer config + if prefix is None: + run_name = getattr(trainer.cfg.trainer, "run_name", None) + project_name = getattr(trainer.cfg.trainer, "project_name", "skyrl") + model_path = getattr(trainer.cfg.trainer.policy.model, "path", "unknown-model") + model_name = Path(model_path).name + prefix = f"{project_name}/{model_name}/{run_name}" if run_name else f"{project_name}/{model_name}" + + # Check AWS credentials + aws_key = os.environ.get("AWS_ACCESS_KEY_ID") + aws_secret = os.environ.get("AWS_SECRET_ACCESS_KEY") + s3_enabled = bool(aws_key and aws_secret) + + if s3_enabled: + logger.info(f"S3 checkpoint upload ENABLED: s3://{bucket}/{prefix}/") + uploader = S3CheckpointUploader(bucket=bucket, prefix=prefix, region=region) + else: + logger.warning( + "AWS credentials not found. S3 upload DISABLED. " + "Using aggressive local cleanup (keeping only 2 checkpoints). " + "Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to enable S3." + ) + uploader = None + + original_save_checkpoints = trainer.save_checkpoints + ckpt_path = trainer.cfg.trainer.ckpt_path + + def save_checkpoints_with_cleanup(): + """Wrapped save_checkpoints with pre-save cleanup and async S3 upload.""" + # CRITICAL: Clean up old checkpoints BEFORE saving to free disk space + # With S3: keep only 1 (we have S3 backup), allows room for new checkpoint + # Without S3: keep 2 for safety + keep_n = 1 if s3_enabled else 2 + cleanup_old_local_checkpoints(ckpt_path, keep_n=keep_n) + + # Now save the new checkpoint (disk has space) + original_save_checkpoints() + + # Queue async S3 upload (non-blocking) + if s3_enabled and uploader: + global_step = trainer.global_step + checkpoint_dir = os.path.join(ckpt_path, f"global_step_{global_step}") + if os.path.exists(checkpoint_dir): + uploader.upload_async(checkpoint_dir) + + trainer.save_checkpoints = save_checkpoints_with_cleanup + trainer._s3_uploader = uploader + + return trainer + + +def download_checkpoint_from_s3( + ckpt_path: str, + run_name: str, + bucket: Optional[str] = None, + region: Optional[str] = None, + project_name: str = "fleet-task-grpo", + model_name: str = "Qwen3-32B", +) -> bool: + """ + Download the latest checkpoint from S3 for resume on a fresh VM. + + Looks for checkpoint directories under the S3 prefix matching the run_name, + downloads the latest one, and writes latest_ckpt_global_step.txt. + + Args: + ckpt_path: Local checkpoint directory (e.g., ~/ckpts/fleet_tool_use_32b) + run_name: W&B run name used as S3 prefix (e.g., fleet_tool_use_32b_d7167c1c) + bucket: S3 bucket (default: from S3_CHECKPOINT_BUCKET env var) + region: AWS region (default: from AWS_REGION env var) + project_name: Project name used in S3 prefix + model_name: Model name used in S3 prefix + + Returns: + True if checkpoint was downloaded, False otherwise + """ + bucket = bucket or os.environ.get("S3_CHECKPOINT_BUCKET", "skyrl-checkpoints") + region = region or os.environ.get("AWS_REGION", "us-east-1") + + aws_key = os.environ.get("AWS_ACCESS_KEY_ID") + aws_secret = os.environ.get("AWS_SECRET_ACCESS_KEY") + if not (aws_key and aws_secret): + logger.info("No AWS credentials, skipping S3 checkpoint download") + return False + + # Check if local checkpoint already exists + latest_file = os.path.join(ckpt_path, "latest_ckpt_global_step.txt") + if os.path.exists(latest_file): + with open(latest_file, "r") as f: + step = f.read().strip() + local_ckpt = os.path.join(ckpt_path, f"global_step_{step}") + if os.path.exists(local_ckpt): + logger.info(f"Local checkpoint already exists at step {step}, skipping S3 download") + return False + + try: + import boto3 + from botocore.config import Config + + config = Config( + retries={"max_attempts": 3, "mode": "adaptive"}, + connect_timeout=30, + read_timeout=120, + ) + s3 = boto3.client("s3", region_name=region, config=config) + + # S3 prefix matches what wrap_trainer_with_s3_upload builds + s3_prefix = f"{project_name}/{model_name}/{run_name}/" + + # List all checkpoint directories in S3 + paginator = s3.get_paginator("list_objects_v2") + checkpoint_steps = set() + for page in paginator.paginate(Bucket=bucket, Prefix=s3_prefix, Delimiter="/"): + for prefix_obj in page.get("CommonPrefixes", []): + dir_name = prefix_obj["Prefix"].rstrip("/").split("/")[-1] + if dir_name.startswith("global_step_"): + try: + step = int(dir_name.split("_")[-1]) + checkpoint_steps.add(step) + except ValueError: + pass + + if not checkpoint_steps: + logger.info(f"No checkpoints found in s3://{bucket}/{s3_prefix}") + return False + + latest_step = max(checkpoint_steps) + s3_ckpt_prefix = f"{s3_prefix}global_step_{latest_step}/" + local_ckpt_dir = os.path.join(ckpt_path, f"global_step_{latest_step}") + + logger.info(f"Downloading checkpoint step {latest_step} from s3://{bucket}/{s3_ckpt_prefix}") + + os.makedirs(local_ckpt_dir, exist_ok=True) + + downloaded_files = 0 + total_size = 0 + for page in paginator.paginate(Bucket=bucket, Prefix=s3_ckpt_prefix): + for obj in page.get("Contents", []): + s3_key = obj["Key"] + relative_path = s3_key[len(s3_ckpt_prefix) :] + if not relative_path: + continue + local_file = os.path.join(local_ckpt_dir, relative_path) + os.makedirs(os.path.dirname(local_file), exist_ok=True) + file_size = obj["Size"] + total_size += file_size + logger.info(f"Downloading {relative_path} ({file_size / 1e6:.1f} MB)") + s3.download_file(bucket, s3_key, local_file) + downloaded_files += 1 + + # Write latest_ckpt_global_step.txt so SkyRL's resume_mode=latest can find it + os.makedirs(ckpt_path, exist_ok=True) + with open(latest_file, "w") as f: + f.write(str(latest_step)) + + logger.info( + f"Downloaded checkpoint: {downloaded_files} files, {total_size / 1e9:.2f} GB " + f"from s3://{bucket}/{s3_ckpt_prefix} to {local_ckpt_dir}" + ) + return True + + except Exception as e: + logger.error(f"Failed to download checkpoint from S3: {e}") + return False + + +def upload_eval_results_to_s3( + local_dir: str, + run_name: str, + global_step: Optional[int] = None, + bucket: Optional[str] = None, + region: Optional[str] = None, + delete_local: bool = False, +) -> bool: + """ + Upload eval results directory to S3. + + Args: + local_dir: Local directory containing eval JSONL files + run_name: Run name for S3 prefix (e.g., "fleet_tool_use_abc123") + global_step: Global step number (for organizing in S3) + bucket: S3 bucket (default: from S3_TRAJECTORY_BUCKET env var) + region: AWS region (default: from AWS_REGION env var) + delete_local: If True, delete local files after upload + + Returns: + True if upload succeeded, False otherwise + """ + bucket = bucket or os.environ.get("S3_TRAJECTORY_BUCKET", "skyrl-trajectories") + region = region or os.environ.get("AWS_REGION", "us-east-1") + + # Check AWS credentials + aws_key = os.environ.get("AWS_ACCESS_KEY_ID") + aws_secret = os.environ.get("AWS_SECRET_ACCESS_KEY") + if not (aws_key and aws_secret): + logger.warning("AWS credentials not found. Skipping S3 upload for eval results.") + return False + + local_path = Path(local_dir) + if not local_path.exists(): + logger.warning(f"Eval directory does not exist: {local_dir}") + return False + + try: + import boto3 + from botocore.config import Config + + config = Config( + retries={"max_attempts": 3, "mode": "adaptive"}, + connect_timeout=30, + read_timeout=60, + ) + + s3 = boto3.client("s3", region_name=region, config=config) + + # Build S3 prefix: evals/{run_name}/global_step_{N}/ + step_suffix = f"global_step_{global_step}" if global_step is not None else "eval_only" + s3_prefix = f"evals/{run_name}/{step_suffix}" + + uploaded_files = 0 + total_size = 0 + + for file_path in local_path.rglob("*"): + if file_path.is_file(): + relative_path = file_path.relative_to(local_path) + s3_key = f"{s3_prefix}/{relative_path}" + file_size = file_path.stat().st_size + total_size += file_size + + s3.upload_file(str(file_path), bucket, s3_key) + uploaded_files += 1 + + logger.info( + f"Uploaded eval results: {uploaded_files} files, {total_size / 1e6:.2f} MB " + f"to s3://{bucket}/{s3_prefix}/" + ) + + if delete_local: + shutil.rmtree(local_dir) + logger.info(f"Deleted local eval directory: {local_dir}") + + return True + + except Exception as e: + logger.error(f"S3 upload failed for eval results {local_dir}: {e}") + return False diff --git a/skyrl-train/integrations/fleet/scripts/create_flash_attn_stub.py b/skyrl-train/integrations/fleet/scripts/create_flash_attn_stub.py new file mode 100644 index 0000000000..b1417845e3 --- /dev/null +++ b/skyrl-train/integrations/fleet/scripts/create_flash_attn_stub.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +""" +Create a minimal flash_attn stub package. + +SkyRL training code needs flash_attn.bert_padding.pad_input/unpad_input. +vLLM needs flash_attn.ops.triton.rotary.apply_rotary for rotary embeddings. + +These are pure Python/PyTorch implementations - no CUDA compilation needed. +This avoids all CUDA import issues (no flash_attn_2_cuda or nvcc needed). + +Sources: +- bert_padding: https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/bert_padding.py +- rotary: https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/ops/triton/rotary.py +""" + +import os +import site + + +def create_stub(): + site_packages = site.getsitepackages()[0] + + # Create flash_attn package in site-packages + pkg_dir = os.path.join(site_packages, "flash_attn") + os.makedirs(pkg_dir, exist_ok=True) + + # Create dist-info directory for importlib.metadata compatibility + # This is required because transformers calls: + # importlib.metadata.version("flash_attn") + # which looks for flash_attn-*.dist-info/METADATA, not __version__ + dist_info_dir = os.path.join(site_packages, "flash_attn-2.7.0+stub.dist-info") + os.makedirs(dist_info_dir, exist_ok=True) + + # Create METADATA file (minimal PEP 566 format) + metadata_content = """Metadata-Version: 2.1 +Name: flash-attn +Version: 2.7.0+stub +Summary: Stub package for flash_attn (no CUDA required) +""" + with open(os.path.join(dist_info_dir, "METADATA"), "w") as f: + f.write(metadata_content) + + # Create INSTALLER file + with open(os.path.join(dist_info_dir, "INSTALLER"), "w") as f: + f.write("stub\n") + + # Create top_level.txt + with open(os.path.join(dist_info_dir, "top_level.txt"), "w") as f: + f.write("flash_attn\n") + + # Create RECORD file (empty is fine for our purposes) + with open(os.path.join(dist_info_dir, "RECORD"), "w") as f: + f.write("") + + print(f"Created flash_attn-2.7.0+stub.dist-info at {dist_info_dir}") + + # Create ops/triton directory structure for rotary embedding + ops_dir = os.path.join(pkg_dir, "ops") + triton_dir = os.path.join(ops_dir, "triton") + os.makedirs(triton_dir, exist_ok=True) + + # Create __init__.py files for package structure + with open(os.path.join(pkg_dir, "__init__.py"), "w") as f: + f.write("# Minimal flash_attn stub - provides bert_padding and rotary ops\n") + f.write("__version__ = '2.7.0+stub'\n") + + with open(os.path.join(ops_dir, "__init__.py"), "w") as f: + f.write("# flash_attn.ops stub\n") + + with open(os.path.join(triton_dir, "__init__.py"), "w") as f: + f.write("# flash_attn.ops.triton stub\n") + + # Create rotary.py with pure PyTorch implementation (no Triton needed) + rotary_code = '''""" +Pure PyTorch implementation of rotary embeddings. +Replaces the Triton kernel version for environments without flash_attn CUDA build. + +This is a fallback implementation that matches the flash_attn API but uses +standard PyTorch operations instead of Triton kernels. +""" +import torch +from typing import Optional, Union + + +def _rotate_half(x: torch.Tensor) -> torch.Tensor: + """Rotate half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def _apply_rotary_emb_torch( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + interleaved: bool = False, +) -> torch.Tensor: + """ + Apply rotary embedding to input tensor using PyTorch operations. + + Args: + x: Input tensor of shape (batch, seqlen, nheads, headdim) or (seqlen, nheads, headdim) + cos: Cosine values of shape (seqlen, rotary_dim // 2) or (seqlen, rotary_dim) + sin: Sine values matching cos shape + interleaved: If True, use interleaved layout + + Returns: + Tensor with rotary embedding applied + """ + ro_dim = cos.shape[-1] * 2 if cos.shape[-1] * 2 <= x.shape[-1] else cos.shape[-1] + + # Handle different cos/sin shapes + if cos.dim() == 2: + # cos is (seqlen, dim) - need to broadcast to x's shape + seqlen = x.shape[-3] if x.dim() == 4 else x.shape[-3] + cos = cos[:seqlen] + sin = sin[:seqlen] + + # Reshape for broadcasting: (seqlen, 1, dim) + cos = cos.unsqueeze(-2) + sin = sin.unsqueeze(-2) + + # Split x into rotary and pass-through parts + x_ro = x[..., :ro_dim] + x_pass = x[..., ro_dim:] if ro_dim < x.shape[-1] else None + + if interleaved: + # Interleaved layout: [x0, x1, x2, x3, ...] -> rotate pairs + x1 = x_ro[..., ::2] + x2 = x_ro[..., 1::2] + + cos_half = cos[..., : cos.shape[-1] // 2] if cos.shape[-1] == ro_dim else cos + sin_half = sin[..., : sin.shape[-1] // 2] if sin.shape[-1] == ro_dim else sin + + o1 = x1 * cos_half - x2 * sin_half + o2 = x1 * sin_half + x2 * cos_half + + x_ro_out = torch.stack([o1, o2], dim=-1).flatten(-2) + else: + # Non-interleaved: first half and second half are separate + cos_expanded = cos + sin_expanded = sin + + # Handle rotary_dim being half of headdim + if cos.shape[-1] * 2 == ro_dim: + cos_expanded = torch.cat([cos, cos], dim=-1) + sin_expanded = torch.cat([sin, sin], dim=-1) + + x_ro_out = x_ro * cos_expanded + _rotate_half(x_ro) * sin_expanded + + if x_pass is not None: + return torch.cat([x_ro_out, x_pass], dim=-1) + return x_ro_out + + +def apply_rotary( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + seqlen_offsets: Union[int, torch.Tensor] = 0, + cu_seqlens: Optional[torch.Tensor] = None, + max_seqlen: Optional[int] = None, + interleaved: bool = False, + inplace: bool = False, + conjugate: bool = False, +) -> torch.Tensor: + """ + Apply rotary positional embedding to input tensor. + + Pure PyTorch implementation matching flash_attn.ops.triton.rotary API. + + Args: + x: Input tensor (batch, seqlen, nheads, headdim) or (total_seqlen, nheads, headdim) + cos: Precomputed cosine values (seqlen_ro, rotary_dim / 2) + sin: Precomputed sine values matching cos shape + seqlen_offsets: Integer or tensor of size (batch,) for offset adjustments + cu_seqlens: Optional cumulative sequence lengths for variable-length sequences + max_seqlen: Maximum sequence length (required when cu_seqlens is provided) + interleaved: If True, use interleaved data layout + inplace: If True, modify x in-place (ignored in PyTorch fallback) + conjugate: If True, conjugate sine values (negate sin) + + Returns: + Tensor with rotary embedding applied, same shape as input + """ + if conjugate: + sin = -sin + + # Handle sequence offsets by slicing cos/sin appropriately + if isinstance(seqlen_offsets, int) and seqlen_offsets != 0: + seqlen = x.shape[-3] if x.dim() == 4 else x.shape[0] + cos = cos[seqlen_offsets : seqlen_offsets + seqlen] + sin = sin[seqlen_offsets : seqlen_offsets + seqlen] + elif isinstance(seqlen_offsets, torch.Tensor): + # Per-batch offsets - more complex handling needed + # For now, use a simple loop (can be optimized if needed) + if x.dim() == 4: + batch_size = x.shape[0] + seqlen = x.shape[1] + outputs = [] + for b in range(batch_size): + offset = int(seqlen_offsets[b].item()) + cos_b = cos[offset : offset + seqlen] + sin_b = sin[offset : offset + seqlen] + outputs.append(_apply_rotary_emb_torch(x[b:b+1], cos_b, sin_b, interleaved)) + return torch.cat(outputs, dim=0) + + # Handle cu_seqlens (variable length sequences packed together) + if cu_seqlens is not None: + # x is (total_seqlen, nheads, headdim) + # Need to apply rotary per-sequence + outputs = [] + for i in range(len(cu_seqlens) - 1): + start = int(cu_seqlens[i].item()) + end = int(cu_seqlens[i + 1].item()) + seq_len = end - start + x_seq = x[start:end] + cos_seq = cos[:seq_len] + sin_seq = sin[:seq_len] + outputs.append(_apply_rotary_emb_torch(x_seq.unsqueeze(0), cos_seq, sin_seq, interleaved).squeeze(0)) + return torch.cat(outputs, dim=0) + + return _apply_rotary_emb_torch(x, cos, sin, interleaved) +''' + + with open(os.path.join(triton_dir, "rotary.py"), "w") as f: + f.write(rotary_code) + + print(f"Created flash_attn.ops.triton.rotary stub at {triton_dir}") + + # Create bert_padding.py with the functions SkyRL needs + bert_padding_code = '''""" +Minimal bert_padding functions from flash-attention. +Only includes pad_input and unpad_input used by SkyRL training. +""" +import torch +from torch import Tensor +from einops import rearrange + + +class IndexFirstAxis(torch.autograd.Function): + @staticmethod + def forward(ctx, input, indices): + ctx.save_for_backward(indices) + assert input.ndim >= 2 + ctx.first_axis_dim, other_shape = input.shape[0], input.shape[1:] + second_dim = other_shape.numel() + return torch.gather( + rearrange(input, "b ... -> b (...)"), 0, + indices.unsqueeze(-1).expand(-1, second_dim) + ).reshape(-1, *other_shape) + + @staticmethod + def backward(ctx, grad_output): + (indices,) = ctx.saved_tensors + assert grad_output.ndim >= 2 + other_shape = grad_output.shape[1:] + grad_output = rearrange(grad_output, "b ... -> b (...)") + grad_input = torch.zeros( + [ctx.first_axis_dim, grad_output.shape[1]], + device=grad_output.device, + dtype=grad_output.dtype, + ) + grad_input.scatter_( + 0, indices.unsqueeze(-1).expand(-1, grad_output.shape[1]), grad_output + ) + return grad_input.reshape(ctx.first_axis_dim, *other_shape), None + + +index_first_axis = IndexFirstAxis.apply + + +class IndexPutFirstAxis(torch.autograd.Function): + @staticmethod + def forward(ctx, values, indices, first_axis_dim): + ctx.save_for_backward(indices) + assert indices.ndim == 1 + assert values.ndim >= 2 + output = torch.zeros( + first_axis_dim, *values.shape[1:], device=values.device, dtype=values.dtype + ) + output[indices] = values + return output + + @staticmethod + def backward(ctx, grad_output): + (indices,) = ctx.saved_tensors + grad_values = grad_output[indices] + return grad_values, None, None + + +index_put_first_axis = IndexPutFirstAxis.apply + + +def unpad_input(hidden_states, attention_mask, unused_mask=None): + """ + Remove padding from hidden states based on attention mask. + + Arguments: + hidden_states: (batch, seqlen, ...) + attention_mask: (batch, seqlen), 1 for valid tokens, 0 for padding + unused_mask: unused, kept for API compatibility + + Returns: + hidden_states_unpad: (total_valid_tokens, ...) + indices: (total_valid_tokens,), indices of valid tokens in flattened batch + cu_seqlens: (batch + 1,), cumulative sequence lengths + max_seqlen_in_batch: int, maximum sequence length + seqlens: (batch,), sequence lengths + """ + seqlens = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = int(seqlens.max().item()) + cu_seqlens = torch.nn.functional.pad( + torch.cumsum(seqlens, dim=0, dtype=torch.int32), (1, 0) + ) + return ( + index_first_axis(rearrange(hidden_states, "b s ... -> (b s) ..."), indices), + indices, + cu_seqlens, + max_seqlen_in_batch, + seqlens, + ) + + +def pad_input(hidden_states_unpad, indices, batch, seqlen): + """ + Pad hidden states back to original shape. + + Arguments: + hidden_states_unpad: (total_valid_tokens, ...) + indices: (total_valid_tokens,) + batch: int, batch size + seqlen: int, sequence length + + Returns: + hidden_states: (batch, seqlen, ...) + """ + output = index_put_first_axis(hidden_states_unpad, indices, batch * seqlen) + return rearrange(output, "(b s) ... -> b s ...", b=batch) +''' + + with open(os.path.join(pkg_dir, "bert_padding.py"), "w") as f: + f.write(bert_padding_code) + + print(f"Created flash_attn stub package at {pkg_dir}") + print(" - bert_padding: pad_input, unpad_input") + print(" - ops.triton.rotary: apply_rotary (pure PyTorch)") + + +if __name__ == "__main__": + create_stub() diff --git a/skyrl-train/integrations/fleet/tests/__init__.py b/skyrl-train/integrations/fleet/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/integrations/fleet/tests/test_prepare_dataset.py b/skyrl-train/integrations/fleet/tests/test_prepare_dataset.py new file mode 100644 index 0000000000..d0470171c2 --- /dev/null +++ b/skyrl-train/integrations/fleet/tests/test_prepare_dataset.py @@ -0,0 +1,365 @@ +"""Tests for prepare_dataset.py - stratified eval split functionality.""" + +import json +import os +import tempfile + +import pytest + +from integrations.fleet.prepare_dataset import ( + HELD_OUT_ENVS, + MIN_EVAL_SAMPLES, + _task_to_record, + hash_to_split, + load_tasks_from_json, + prepare_fleet_dataset, +) + + +class TestHashToSplit: + """Tests for deterministic hash-based split assignment.""" + + def test_deterministic_same_key(self): + """Same task_key always produces same split.""" + key = "github_task_123" + results = [hash_to_split(key) for _ in range(100)] + assert len(set(results)) == 1, "Same key should always produce same split" + + def test_deterministic_across_calls(self): + """Split assignment is reproducible.""" + keys = ["task_a", "task_b", "task_c", "github_456", "booking_789"] + first_run = [hash_to_split(k) for k in keys] + second_run = [hash_to_split(k) for k in keys] + assert first_run == second_run + + def test_respects_eval_ratio(self): + """Approximately eval_ratio of tasks go to eval split.""" + # Use many keys to get statistical significance + keys = [f"task_{i}" for i in range(10000)] + splits = [hash_to_split(k, eval_ratio=0.1) for k in keys] + eval_count = splits.count("eval") + # Should be roughly 10% (allow some variance) + assert 800 < eval_count < 1200, f"Expected ~1000 eval, got {eval_count}" + + def test_different_eval_ratios(self): + """Different eval_ratio produces different distribution.""" + keys = [f"task_{i}" for i in range(1000)] + splits_10 = [hash_to_split(k, eval_ratio=0.1) for k in keys] + splits_20 = [hash_to_split(k, eval_ratio=0.2) for k in keys] + eval_10 = splits_10.count("eval") + eval_20 = splits_20.count("eval") + assert eval_20 > eval_10, "Higher eval_ratio should produce more eval samples" + + def test_returns_valid_split(self): + """Only returns 'train' or 'eval'.""" + keys = [f"key_{i}" for i in range(100)] + splits = [hash_to_split(k) for k in keys] + assert all(s in ("train", "eval") for s in splits) + + +class TestTaskToRecord: + """Tests for _task_to_record conversion.""" + + def test_valid_task(self): + """Converts valid task to record format.""" + task = { + "key": "github_123", + "prompt": "Create a new repository", + "env_key": "github", + } + record = _task_to_record(task, "github") + assert record is not None + assert record["prompt"] == [{"role": "user", "content": "Create a new repository"}] + assert record["env_class"] == "fleet_task" + assert record["task_key"] == "github_123" + assert record["data_source"] == "github" + + def test_task_key_field(self): + """Handles 'task_key' field name.""" + task = { + "task_key": "booking_456", + "prompt": "Search for hotels", + } + record = _task_to_record(task, "booking") + assert record["task_key"] == "booking_456" + + def test_missing_key_returns_none(self): + """Returns None if task_key is missing.""" + task = {"prompt": "Some prompt"} + record = _task_to_record(task, "test") + assert record is None + + def test_missing_prompt_returns_none(self): + """Returns None if prompt is missing.""" + task = {"key": "task_123"} + record = _task_to_record(task, "test") + assert record is None + + def test_empty_prompt_returns_none(self): + """Returns None if prompt is empty.""" + task = {"key": "task_123", "prompt": ""} + record = _task_to_record(task, "test") + assert record is None + + +class TestLoadTasksFromJson: + """Tests for JSON loading with different formats.""" + + def test_array_format(self): + """Loads tasks from array format JSON.""" + tasks_data = [ + {"key": "task_1", "prompt": "Do something"}, + {"key": "task_2", "prompt": "Do another thing"}, + ] + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(tasks_data, f) + f.flush() + tasks = load_tasks_from_json(f.name) + os.unlink(f.name) + assert len(tasks) == 2 + assert tasks[0]["key"] == "task_1" + + def test_object_with_tasks_key(self): + """Loads tasks from object format with 'tasks' key.""" + tasks_data = { + "tasks": [ + {"key": "task_1", "prompt": "Do something"}, + {"key": "task_2", "prompt": "Do another thing"}, + ], + "metadata": {"version": "1.0"}, + } + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(tasks_data, f) + f.flush() + tasks = load_tasks_from_json(f.name) + os.unlink(f.name) + assert len(tasks) == 2 + + def test_invalid_format_raises(self): + """Raises ValueError for invalid JSON format.""" + tasks_data = {"invalid": "format"} + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(tasks_data, f) + f.flush() + with pytest.raises(ValueError, match="Invalid JSON format"): + load_tasks_from_json(f.name) + os.unlink(f.name) + + +class TestHeldOutEnvs: + """Tests for held-out eval environment configuration.""" + + def test_tool_use_held_out(self): + """Outlook is held out for tool_use.""" + assert "outlook" in HELD_OUT_ENVS["tool_use"] + + def test_computer_use_held_out(self): + """Instacart is held out for computer_use.""" + assert "instacart" in HELD_OUT_ENVS["computer_use"] + + def test_min_eval_samples_threshold(self): + """MIN_EVAL_SAMPLES is set to 5.""" + assert MIN_EVAL_SAMPLES == 5 + + +class TestPrepareFleetDataset: + """Integration tests for prepare_fleet_dataset.""" + + def _create_test_tasks(self, env_counts: dict, modality: str = "tool_use"): + """Create test tasks for multiple environments.""" + tasks = [] + for env_key, count in env_counts.items(): + for i in range(count): + tasks.append( + { + "key": f"{env_key}_task_{i}", + "prompt": f"Task {i} for {env_key}", + "env_key": env_key, + "task_modality": modality, + } + ) + return tasks + + def test_held_out_env_goes_to_eval(self): + """Held-out environments go entirely to eval split.""" + tasks = self._create_test_tasks( + {"github": 500, "outlook": 24}, # outlook is held out for tool_use + modality="tool_use", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + json_path = os.path.join(tmpdir, "tasks.json") + with open(json_path, "w") as f: + json.dump(tasks, f) + + prepare_fleet_dataset( + tasks_json=json_path, + output_dir=tmpdir, + modality="tool_use", + ) + + # Check validation.parquet contains outlook tasks (held-out go to eval) + val_path = os.path.join(tmpdir, "validation.parquet") + assert os.path.exists(val_path) + + import pyarrow.parquet as pq + + val_df = pq.read_table(val_path).to_pandas() + # Should have outlook (24) + github eval (~10 with 2%) + outlook_tasks = [k for k in val_df["task_key"] if "outlook" in k] + assert len(outlook_tasks) == 24 + + def test_small_env_all_to_train(self): + """Environments with < MIN_EVAL_SAMPLES go entirely to train.""" + # 50 tasks with 2% eval = 1 expected, less than MIN_EVAL_SAMPLES (5) + tasks = self._create_test_tasks({"small_env": 50}, modality="tool_use") + + with tempfile.TemporaryDirectory() as tmpdir: + json_path = os.path.join(tmpdir, "tasks.json") + with open(json_path, "w") as f: + json.dump(tasks, f) + + prepare_fleet_dataset( + tasks_json=json_path, + output_dir=tmpdir, + modality="tool_use", + ) + + train_path = os.path.join(tmpdir, "train.parquet") + val_path = os.path.join(tmpdir, "validation.parquet") + + import pyarrow.parquet as pq + + train_df = pq.read_table(train_path).to_pandas() + assert len(train_df) == 50 + + # validation.parquet should not exist or be empty + assert not os.path.exists(val_path) + + def test_stratified_split(self): + """Large environments get stratified train/eval split.""" + # 500 tasks with 2% eval = ~10 expected in eval (above MIN_EVAL_SAMPLES=5) + tasks = self._create_test_tasks({"github": 500}, modality="tool_use") + + with tempfile.TemporaryDirectory() as tmpdir: + json_path = os.path.join(tmpdir, "tasks.json") + with open(json_path, "w") as f: + json.dump(tasks, f) + + prepare_fleet_dataset( + tasks_json=json_path, + output_dir=tmpdir, + modality="tool_use", + eval_ratio=0.1, + ) + + import pyarrow.parquet as pq + + train_df = pq.read_table(os.path.join(tmpdir, "train.parquet")).to_pandas() + val_df = pq.read_table(os.path.join(tmpdir, "validation.parquet")).to_pandas() + + total = len(train_df) + len(val_df) + assert total == 500 + + # Check roughly 10% in eval (allow some variance due to hash) + eval_ratio = len(val_df) / total + assert 0.05 < eval_ratio < 0.15 + + def test_modality_filter(self): + """Only tasks matching modality are included.""" + tasks = self._create_test_tasks({"github": 50}, modality="tool_use") + self._create_test_tasks( + {"amazon": 50}, modality="computer_use" + ) + + with tempfile.TemporaryDirectory() as tmpdir: + json_path = os.path.join(tmpdir, "tasks.json") + with open(json_path, "w") as f: + json.dump(tasks, f) + + prepare_fleet_dataset( + tasks_json=json_path, + output_dir=tmpdir, + modality="tool_use", + ) + + import pyarrow.parquet as pq + + train_df = pq.read_table(os.path.join(tmpdir, "train.parquet")).to_pandas() + # Only github (tool_use) tasks, not amazon (computer_use) + assert all("github" in k for k in train_df["task_key"]) + + def test_env_filter(self): + """env_filter limits to specific environment.""" + tasks = self._create_test_tasks( + {"github": 500, "booking": 500}, + modality="tool_use", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + json_path = os.path.join(tmpdir, "tasks.json") + with open(json_path, "w") as f: + json.dump(tasks, f) + + prepare_fleet_dataset( + tasks_json=json_path, + output_dir=tmpdir, + modality="tool_use", + env_filter="github", + ) + + import pyarrow.parquet as pq + + train_df = pq.read_table(os.path.join(tmpdir, "train.parquet")).to_pandas() + # Only github tasks + assert all("github" in k for k in train_df["task_key"]) + + def test_deterministic_split_across_runs(self): + """Same tasks produce same split across multiple runs.""" + tasks = self._create_test_tasks({"github": 500}, modality="tool_use") + + with tempfile.TemporaryDirectory() as tmpdir: + json_path = os.path.join(tmpdir, "tasks.json") + with open(json_path, "w") as f: + json.dump(tasks, f) + + # Run twice with different output dirs + out1 = os.path.join(tmpdir, "run1") + out2 = os.path.join(tmpdir, "run2") + + prepare_fleet_dataset(tasks_json=json_path, output_dir=out1, modality="tool_use") + prepare_fleet_dataset(tasks_json=json_path, output_dir=out2, modality="tool_use") + + import pyarrow.parquet as pq + + train1 = set(pq.read_table(os.path.join(out1, "train.parquet")).to_pandas()["task_key"]) + train2 = set(pq.read_table(os.path.join(out2, "train.parquet")).to_pandas()["task_key"]) + val1 = set(pq.read_table(os.path.join(out1, "validation.parquet")).to_pandas()["task_key"]) + val2 = set(pq.read_table(os.path.join(out2, "validation.parquet")).to_pandas()["task_key"]) + + assert train1 == train2, "Train splits should be identical" + assert val1 == val2, "Validation splits should be identical" + + def test_computer_use_held_out(self): + """Instacart is held out for computer_use modality.""" + tasks = self._create_test_tasks( + {"amazon": 100, "instacart": 32}, + modality="computer_use", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + json_path = os.path.join(tmpdir, "tasks.json") + with open(json_path, "w") as f: + json.dump(tasks, f) + + prepare_fleet_dataset( + tasks_json=json_path, + output_dir=tmpdir, + modality="computer_use", + ) + + import pyarrow.parquet as pq + + val_df = pq.read_table(os.path.join(tmpdir, "validation.parquet")).to_pandas() + # Should have instacart (32) - amazon (100 tasks at 2% = 2 expected) goes all to train + instacart_tasks = [k for k in val_df["task_key"] if "instacart" in k] + assert len(instacart_tasks) == 32 diff --git a/skyrl-train/integrations/fleet/tests/test_tinker_training.py b/skyrl-train/integrations/fleet/tests/test_tinker_training.py new file mode 100644 index 0000000000..f33a33a07f --- /dev/null +++ b/skyrl-train/integrations/fleet/tests/test_tinker_training.py @@ -0,0 +1,372 @@ +""" +Tests for Tinker training utilities. + +Run with: + python -m pytest skyrl-train/integrations/fleet/tests/test_tinker_training.py -v +""" + +import sys +import os + +# Add parent to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from utils import ( + truncate_sequence, + truncate_auxiliary_data, + apply_overlong_filtering_simple, + prepare_training_sequence, +) + + +class TestTruncateSequence: + """Tests for sequence truncation logic.""" + + def test_no_truncation_within_limit(self): + """Sequences within max_sequence_length should not be modified.""" + prompt_ids = [1, 2, 3] + response_ids = [4, 5] + max_sequence_length = 10 + + full_seq, truncated_resp, resp_len = truncate_sequence(prompt_ids, response_ids, max_sequence_length) + + assert full_seq == [1, 2, 3, 4, 5] + assert truncated_resp == [4, 5] + assert resp_len == 2 + + def test_truncate_overlong_sequence(self): + """Sequences exceeding max_sequence_length should be truncated.""" + prompt_ids = [1, 2, 3] + response_ids = [4, 5, 6, 7, 8, 9, 10] + max_sequence_length = 8 + + full_seq, truncated_resp, resp_len = truncate_sequence(prompt_ids, response_ids, max_sequence_length) + + assert len(full_seq) == max_sequence_length + assert full_seq == [1, 2, 3, 4, 5, 6, 7, 8] + assert truncated_resp == [4, 5, 6, 7, 8] + assert resp_len == 5 + + def test_truncation_preserves_prompt(self): + """Truncation should preserve prompt and truncate response.""" + prompt_ids = [1, 2, 3, 4, 5] # 5 tokens + response_ids = [6, 7, 8, 9, 10] # 5 tokens + max_sequence_length = 7 + + full_seq, truncated_resp, resp_len = truncate_sequence(prompt_ids, response_ids, max_sequence_length) + + # Prompt should be fully preserved + assert full_seq[:5] == prompt_ids + # Response should be truncated + assert resp_len == 2 # Only 2 response tokens fit + assert truncated_resp == [6, 7] + + def test_exact_limit(self): + """Sequences exactly at limit should not be modified.""" + prompt_ids = [1, 2, 3] + response_ids = [4, 5, 6, 7] + max_sequence_length = 7 + + full_seq, truncated_resp, resp_len = truncate_sequence(prompt_ids, response_ids, max_sequence_length) + + assert len(full_seq) == 7 + assert full_seq == [1, 2, 3, 4, 5, 6, 7] + assert truncated_resp == response_ids + assert resp_len == 4 + + def test_empty_response(self): + """Empty response should work correctly.""" + prompt_ids = [1, 2, 3] + response_ids = [] + max_sequence_length = 10 + + full_seq, truncated_resp, resp_len = truncate_sequence(prompt_ids, response_ids, max_sequence_length) + + assert full_seq == [1, 2, 3] + assert truncated_resp == [] + assert resp_len == 0 + + +class TestTruncateAuxiliaryData: + """Tests for auxiliary data truncation.""" + + def test_truncate_to_shorter_length(self): + """Data should be truncated to target length.""" + data = [1, 2, 3, 4, 5] + result = truncate_auxiliary_data(data, 3) + assert result == [1, 2, 3] + + def test_no_truncation_needed(self): + """Data within limit should not be modified.""" + data = [1, 2, 3] + result = truncate_auxiliary_data(data, 5) + assert result == [1, 2, 3] + + def test_exact_length(self): + """Data exactly at limit should not be modified.""" + data = [1, 2, 3] + result = truncate_auxiliary_data(data, 3) + assert result == [1, 2, 3] + + def test_truncate_floats(self): + """Float data (like logprobs) should work.""" + data = [-0.1, -0.2, -0.3, -0.4, -0.5] + result = truncate_auxiliary_data(data, 3) + assert result == [-0.1, -0.2, -0.3] + + def test_empty_data(self): + """Empty data should work correctly.""" + data = [] + result = truncate_auxiliary_data(data, 3) + assert result == [] + + +class TestOverlongFiltering: + """Tests for DAPO overlong filtering.""" + + def test_zeros_truncated_responses(self): + """Responses not ending with EOS should have loss mask zeroed.""" + loss_masks = [[1, 1, 1], [1, 1, 1]] + response_ids = [[1, 2, 3], [4, 5, 99]] # First doesn't end with EOS, second does + eos_token_id = 99 + + result = apply_overlong_filtering_simple(loss_masks, response_ids, eos_token_id) + + assert result[0] == [0, 0, 0], "Truncated response should have zeroed mask" + assert result[1] == [1, 1, 1], "Complete response should keep original mask" + + def test_preserves_length(self): + """Output masks should have same length as input masks.""" + loss_masks = [[1, 0, 1, 1], [1, 1]] + response_ids = [[1, 2, 3, 4], [5, 6]] + eos_token_id = 99 + + result = apply_overlong_filtering_simple(loss_masks, response_ids, eos_token_id) + + assert len(result[0]) == len(loss_masks[0]) + assert len(result[1]) == len(loss_masks[1]) + + def test_empty_response(self): + """Empty responses should have loss mask zeroed.""" + loss_masks = [[1, 1, 1]] + response_ids = [[]] + eos_token_id = 99 + + result = apply_overlong_filtering_simple(loss_masks, response_ids, eos_token_id) + + assert result[0] == [0, 0, 0], "Empty response should have zeroed mask" + + def test_all_complete(self): + """All responses ending with EOS should preserve masks.""" + loss_masks = [[1, 1], [1, 0, 1]] + response_ids = [[1, 99], [2, 3, 99]] + eos_token_id = 99 + + result = apply_overlong_filtering_simple(loss_masks, response_ids, eos_token_id) + + assert result[0] == [1, 1] + assert result[1] == [1, 0, 1] + + def test_all_truncated(self): + """All responses not ending with EOS should have zeroed masks.""" + loss_masks = [[1, 1], [1, 0, 1]] + response_ids = [[1, 2], [3, 4, 5]] + eos_token_id = 99 + + result = apply_overlong_filtering_simple(loss_masks, response_ids, eos_token_id) + + assert result[0] == [0, 0] + assert result[1] == [0, 0, 0] + + +class TestPrepareTrainingSequence: + """Tests for combined preparation function.""" + + def test_no_truncation(self): + """Sequences within limit should not be truncated.""" + prompt_ids = [1, 2, 3] + response_ids = [4, 5] + logprobs = [-0.1, -0.2] + loss_mask = [1, 1] + max_sequence_length = 10 + + full_seq, trunc_logprobs, trunc_mask, was_truncated = prepare_training_sequence( + prompt_ids, response_ids, logprobs, loss_mask, max_sequence_length + ) + + assert full_seq == [1, 2, 3, 4, 5] + assert trunc_logprobs == [-0.1, -0.2] + assert trunc_mask == [1, 1] + assert was_truncated is False + + def test_with_truncation(self): + """Overlong sequences should be truncated with flag set.""" + prompt_ids = [1, 2, 3] + response_ids = [4, 5, 6, 7, 8] + logprobs = [-0.1, -0.2, -0.3, -0.4, -0.5] + loss_mask = [1, 1, 1, 1, 1] + max_sequence_length = 6 + + full_seq, trunc_logprobs, trunc_mask, was_truncated = prepare_training_sequence( + prompt_ids, response_ids, logprobs, loss_mask, max_sequence_length + ) + + assert len(full_seq) == 6 + assert full_seq == [1, 2, 3, 4, 5, 6] + assert trunc_logprobs == [-0.1, -0.2, -0.3] + assert trunc_mask == [1, 1, 1] + assert was_truncated is True + + def test_preserves_mask_values(self): + """Truncation should preserve original mask values.""" + prompt_ids = [1, 2] + response_ids = [3, 4, 5, 6] + logprobs = [-0.1, -0.2, -0.3, -0.4] + loss_mask = [1, 0, 1, 0] # Mixed mask + max_sequence_length = 4 + + full_seq, trunc_logprobs, trunc_mask, was_truncated = prepare_training_sequence( + prompt_ids, response_ids, logprobs, loss_mask, max_sequence_length + ) + + assert trunc_mask == [1, 0] # First 2 values preserved + assert was_truncated is True + + +class TestCombinedFlow: + """ + Tests for combined DAPO filtering + truncation flow. + + These tests validate the pattern used by prepare_training_data() in + main_fleet_tinker.py, which: + 1. Applies DAPO overlong filtering (zeros mask if no EOS) + 2. Truncates sequences to max_sequence_length + 3. Builds training datums (Tinker-specific, not tested here) + """ + + def test_dapo_then_truncate(self): + """DAPO filtering should be applied, then truncation.""" + # Simulate the flow in prepare_training_data() + response_ids = [1, 2, 3, 4, 5] # Doesn't end with EOS + loss_mask = [1, 1, 1, 1, 1] + eos_token_id = 99 + + # Step 1: Apply DAPO overlong filtering + filtered_masks = apply_overlong_filtering_simple([loss_mask], [response_ids], eos_token_id) + filtered_mask = filtered_masks[0] + + # Should be zeroed because response doesn't end with EOS + assert filtered_mask == [0, 0, 0, 0, 0] + + # Step 2: Prepare training sequence with truncation + prompt_ids = [10, 11, 12] + max_sequence_length = 6 + + full_seq, _, trunc_mask, _ = prepare_training_sequence( + prompt_ids, response_ids, [0.0] * 5, filtered_mask, max_sequence_length + ) + + # Both should be truncated + assert len(full_seq) == max_sequence_length + assert len(trunc_mask) == 3 # 6 - 3 = 3 response tokens + assert trunc_mask == [0, 0, 0] # Still zeroed + + def test_complete_response_then_truncate(self): + """Complete response (ends with EOS) but still needs truncation.""" + response_ids = [1, 2, 3, 4, 99] # Ends with EOS + loss_mask = [1, 1, 1, 1, 1] + eos_token_id = 99 + + # Step 1: Apply DAPO overlong filtering + filtered_masks = apply_overlong_filtering_simple([loss_mask], [response_ids], eos_token_id) + filtered_mask = filtered_masks[0] + + # Should NOT be zeroed because response ends with EOS + assert filtered_mask == [1, 1, 1, 1, 1] + + # Step 2: Prepare training sequence with truncation + prompt_ids = [10, 11, 12] + max_sequence_length = 6 + + full_seq, _, trunc_mask, was_truncated = prepare_training_sequence( + prompt_ids, response_ids, [0.0] * 5, filtered_mask, max_sequence_length + ) + + # Truncated but mask values preserved + assert len(full_seq) == max_sequence_length + assert len(trunc_mask) == 3 + assert trunc_mask == [1, 1, 1] # Original values preserved + assert was_truncated is True + + def test_batch_processing(self): + """Test processing multiple rollouts (as prepare_training_data does).""" + eos_token_id = 99 + + # Simulate batch of rollouts with mixed completion status + rollouts = [ + { + "prompt_ids": [1, 2, 3], + "response_ids": [4, 5, 6], # No EOS - should be zeroed + "logprobs": [-0.1, -0.2, -0.3], + "loss_mask": [1, 1, 1], + }, + { + "prompt_ids": [10, 11], + "response_ids": [12, 13, 99], # Ends with EOS - should be preserved + "logprobs": [-0.4, -0.5, -0.6], + "loss_mask": [1, 1, 1], + }, + { + "prompt_ids": [20, 21, 22], + "response_ids": [23, 24, 25, 26, 27], # No EOS, needs truncation + "logprobs": [-0.1, -0.2, -0.3, -0.4, -0.5], + "loss_mask": [1, 1, 1, 1, 1], + }, + ] + max_sequence_length = 6 + + # Step 1: Apply DAPO filtering to all rollouts + all_response_ids = [r["response_ids"] for r in rollouts] + all_loss_masks = [r["loss_mask"] for r in rollouts] + filtered_masks = apply_overlong_filtering_simple(all_loss_masks, all_response_ids, eos_token_id) + + # Rollout 0: No EOS -> zeroed + assert filtered_masks[0] == [0, 0, 0] + # Rollout 1: Has EOS -> preserved + assert filtered_masks[1] == [1, 1, 1] + # Rollout 2: No EOS -> zeroed + assert filtered_masks[2] == [0, 0, 0, 0, 0] + + # Step 2: Prepare each sequence with truncation + results = [] + for idx, rollout in enumerate(rollouts): + full_seq, trunc_logprobs, trunc_mask, was_truncated = prepare_training_sequence( + rollout["prompt_ids"], + rollout["response_ids"], + rollout["logprobs"], + filtered_masks[idx], + max_sequence_length, + ) + results.append( + { + "full_seq": full_seq, + "trunc_mask": trunc_mask, + "was_truncated": was_truncated, + } + ) + + # Rollout 0: 3+3=6, no truncation needed, mask zeroed + assert len(results[0]["full_seq"]) == 6 + assert results[0]["trunc_mask"] == [0, 0, 0] + assert results[0]["was_truncated"] is False + + # Rollout 1: 2+3=5, no truncation needed, mask preserved + assert len(results[1]["full_seq"]) == 5 + assert results[1]["trunc_mask"] == [1, 1, 1] + assert results[1]["was_truncated"] is False + + # Rollout 2: 3+5=8 > 6, truncated to 6, mask zeroed + assert len(results[2]["full_seq"]) == 6 + assert len(results[2]["trunc_mask"]) == 3 # 6 - 3 = 3 response tokens + assert results[2]["trunc_mask"] == [0, 0, 0] + assert results[2]["was_truncated"] is True diff --git a/skyrl-train/integrations/fleet/utils.py b/skyrl-train/integrations/fleet/utils.py new file mode 100644 index 0000000000..caa64eaab0 --- /dev/null +++ b/skyrl-train/integrations/fleet/utils.py @@ -0,0 +1,118 @@ +""" +Utility functions for Fleet task training with Tinker. + +These functions handle sequence truncation and loss mask filtering, +matching SkyRL's skyrl_gym_generator patterns. +""" + +from typing import List, Tuple + + +def truncate_sequence( + prompt_ids: List[int], + response_ids: List[int], + max_sequence_length: int, +) -> Tuple[List[int], List[int], int]: + """ + Truncate a sequence to fit within max_sequence_length. + + The prompt is preserved fully; only the response is truncated. + + Args: + prompt_ids: Token IDs for the prompt. + response_ids: Token IDs for the response. + max_sequence_length: Maximum total sequence length. + + Returns: + Tuple of (full_sequence, truncated_response_ids, response_len). + """ + full_sequence = prompt_ids + response_ids + prompt_len = len(prompt_ids) + + if len(full_sequence) > max_sequence_length: + full_sequence = full_sequence[:max_sequence_length] + response_len = len(full_sequence) - prompt_len + truncated_response_ids = response_ids[:response_len] + else: + response_len = len(response_ids) + truncated_response_ids = response_ids + + return full_sequence, truncated_response_ids, response_len + + +def truncate_auxiliary_data( + data: List, + response_len: int, +) -> List: + """ + Truncate auxiliary data (logprobs, loss_mask) to match truncated response length. + + Args: + data: List of values corresponding to response tokens. + response_len: Target length after truncation. + + Returns: + Truncated list. + """ + if len(data) > response_len: + return data[:response_len] + return data + + +def apply_overlong_filtering_simple( + loss_masks: List[List[int]], + response_ids: List[List[int]], + eos_token_id: int, +) -> List[List[int]]: + """ + Apply DAPO overlong filtering: zero out loss mask for responses not ending with EOS. + + This is a simplified version for testing - the actual SkyRL function is in + skyrl_train.generators.utils.apply_overlong_filtering. + + Args: + loss_masks: List of loss masks for each response. + response_ids: List of response token IDs for each response. + eos_token_id: The EOS token ID. + + Returns: + Filtered loss masks (zeroed if response doesn't end with EOS). + """ + filtered = [] + for mask, response in zip(loss_masks, response_ids): + # Empty response or doesn't end with EOS -> zero out mask + if not response or response[-1] != eos_token_id: + filtered.append([0] * len(mask)) + else: + filtered.append(list(mask)) + return filtered + + +def prepare_training_sequence( + prompt_ids: List[int], + response_ids: List[int], + logprobs: List[float], + loss_mask: List[int], + max_sequence_length: int, +) -> Tuple[List[int], List[float], List[int], bool]: + """ + Prepare a training sequence with truncation if needed. + + Args: + prompt_ids: Token IDs for the prompt. + response_ids: Token IDs for the response. + logprobs: Log probabilities for response tokens. + loss_mask: Loss mask for response tokens. + max_sequence_length: Maximum total sequence length. + + Returns: + Tuple of (full_sequence, truncated_logprobs, truncated_loss_mask, was_truncated). + """ + full_sequence, truncated_response, response_len = truncate_sequence(prompt_ids, response_ids, max_sequence_length) + + was_truncated = len(prompt_ids) + len(response_ids) > max_sequence_length + + truncated_logprobs = truncate_auxiliary_data(logprobs, response_len) + truncated_loss_mask = truncate_auxiliary_data(loss_mask, response_len) + + return full_sequence, truncated_logprobs, truncated_loss_mask, was_truncated diff --git a/skyrl-train/integrations/modal/README.md b/skyrl-train/integrations/modal/README.md new file mode 100644 index 0000000000..675d32a726 --- /dev/null +++ b/skyrl-train/integrations/modal/README.md @@ -0,0 +1,132 @@ +# SkyRL Modal Integration + +This integration allows you to run SkyRL commands inside a containerized environment hosted by [Modal](https://modal.com/), providing seamless access to GPU resources without managing infrastructure. + +## Overview + +The Modal integration: +- **Runs commands** from your local SkyRL repo inside Modal's cloud infrastructure +- **Sets up a container** based on the SkyRL base image (`novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8`) +- **Mounts your local SkyRL repository** to `/root/SkyRL` in the container +- **Attaches persistent volumes** for data storage at `/root/data` +- **Initializes Ray** automatically for distributed computing +- **Streams live output** from your commands back to your local terminal + +## Prerequisites + +1. Install Modal: `pip install modal` +2. Set up Modal authentication: `modal setup` +3. Ensure you have the SkyRL repository cloned locally + +## Usage + +### Basic Command + +```bash +modal run main.py --command "your-command-here" +``` + +### Examples + +#### 1. Test GPU availability +```bash +modal run main.py --command "nvidia-smi" +``` + +#### 2. Generate GSM8K dataset +```bash +modal run main.py --command "uv run examples/gsm8k/gsm8k_dataset.py --output_dir /root/data/gsm8k" +``` + +#### 3. Run training script +```bash +modal run main.py --command "bash examples/gsm8k/run_gsm8k_modal.sh" +``` + +#### 4. Other scripts +Be sure to override config variables such as `data.train_data`, `data.val_data`, `trainer.ckpt_path`, +and `trainer.export_path` so they point to the correct persistence location inside the container, which +defaults to `/root/data` + +## Configuration + +### Command Parameters + +- `--command`: The command to execute in the container (default: `"nvidia-smi"`) + +### Environment Variables + +Configure the integration using environment variables: + +| Variable | Description | Default | +|----------|-------------|---------| +| `MODAL_APP_NAME` | Name of your Modal app (useful for team collaboration) | `"my_skyrl_app"` | +| `MODAL_GPU` | GPU configuration. See [Modal docs](https://modal.com/docs/guide/gpu). Defaults to L4:1 if not specified | `"A100:4"` | + +### Example with Environment Variables + +```bash +MODAL_APP_NAME=benji_skyrl_app \ +MODAL_GPU=A100:4 \ +modal run main.py --command "bash examples/gsm8k/run_gsm8k_modal.sh" +``` + +## How It Works + +### 1. Image Creation +The `create_modal_image()` function: +- Pulls the SkyRL base Docker image +- Sets environment variables (`SKYRL_REPO_ROOT`) +- Mounts your local SkyRL repository to `/root/SkyRL` in the container +- Excludes unnecessary files (`.venv`, `.git`, `__pycache__`, etc.) + +### 2. Volume Management +The `create_modal_volume()` function: +- Creates or attaches a persistent volume named `"skyrl-data"` +- Mounts it at `/root/data` for data persistence across runs + +### 3. Command Execution +The `run_script()` function: +- Changes to the specified directory within the SkyRL repo +- Ensures `skyrl-gym` is available for dependencies +- Starts a Ray cluster with `ray start --head` +- Executes your command with live output streaming +- Returns exit codes for error handling + +## Resource Configuration + +By default, the integration uses: +- **GPU**: 1x NVIDIA L4 GPU +- **Timeout**: 3600 seconds (1 hour) + +To modify resources, edit the `@app.function()` decorator in `main.py`: + +```python +@app.function( + image=image, + gpu="L4:1", # use env var to configure this + volumes=volume, + timeout=7200, # Change timeout (in seconds) +) +``` + +## Data Persistence + +Data stored in `/root/data` persists across Modal runs. This is useful for: +- Storing generated datasets +- Saving model checkpoints +- Caching intermediate results + +## Notes + +- The local SkyRL repository is automatically detected by traversing parent directories +- Files in `.gitignore` are excluded from the container mount +- Ray is automatically configured to use `127.0.0.1:6379` +- All output is streamed in real-time to your terminal + +## Support + +For issues with: +- **Modal platform**: See [Modal documentation](https://modal.com/docs) +- **SkyRL integration**: Check SkyRL repository issues or documentation + diff --git a/skyrl-train/integrations/modal/main.py b/skyrl-train/integrations/modal/main.py new file mode 100644 index 0000000000..6ebdb6ba73 --- /dev/null +++ b/skyrl-train/integrations/modal/main.py @@ -0,0 +1,170 @@ +import modal +import os +from pathlib import Path + + +def _find_local_repo_root() -> Path: + """Computes full path of local SkyRL repo robustly + + Raises: + Exception: if cannot find local SkyRL repo + + Returns: + Path: path object describing full path of local SkyRL repo + """ + # If running inside Modal container, use the environment variable + if "SKYRL_REPO_ROOT" in os.environ: + return Path(os.environ["SKYRL_REPO_ROOT"]) + + candidates = [Path(__file__).resolve(), Path.cwd()] + for start in candidates: + for base in [start] + list(start.parents): + if (base / "skyrl-train").exists() and (base / "skyrl-gym").exists(): + return base + raise Exception("SkyRL root repo path not found") + + +def create_modal_image() -> modal.Image: + """Creates a Modal image for Modal container. This uses the SkyRL container as + a base image. It also mounts the local SkyRL repo to the container + + Returns: + modal.Image: container image + """ + + local_repo_path = _find_local_repo_root() + print(f"Root path: {local_repo_path}") + + envs = { + "SKYRL_REPO_ROOT": "/root/SkyRL", # where to put SkyRL in container + } + + return ( + modal.Image.from_registry("novaskyai/skyrl-train-ray-2.51.1-py3.12-cu12.8") + .env(envs) + .add_local_dir( + local_path=str(local_repo_path), + remote_path="/root/SkyRL", + ignore=[ + ".venv", + "*.pyc", + "__pycache__", + ".git", + "*.egg-info", + ".pytest_cache", + "node_modules", + ".DS_Store", + ], + ) + ) + + +def create_modal_volume(volume_name: str = "skyrl-data") -> dict[str, modal.Volume]: + """Creates volume to attach to container. + + Args: + volume_name (str, optional): Name of volume. Creates a new + volume if given name does not exist. Defaults to "skyrl-data". + + Returns: + dict[str, modal.Volume]: location in container to attach the volume & volume itself + """ + data_volume = modal.Volume.from_name(volume_name, create_if_missing=True) + return {"/root/data": data_volume} # mounts volume at /root/data inside container + + +app = modal.App(os.getenv("MODAL_APP_NAME", "my_skyrl_app")) +image = create_modal_image() +volume = create_modal_volume() + + +@app.function( + image=image, + gpu=os.environ.get("MODAL_GPU", "L4:1"), + volumes=volume, + timeout=3600, # 1 hour +) +def run_script(command: str): + """ + Runs COMMAND inside SkyRL/skyrl-train + """ + import subprocess + import os + + # The repo root is already set in the image environment + repo_root = os.environ.get("SKYRL_REPO_ROOT", "/root/SkyRL") + + # Print current environment for debugging + print(f"Container repo root: {repo_root}") + print(f"Initial working directory: {os.getcwd()}") + + # Change to the skyrl-train directory + run_command_dir = os.path.join(repo_root, "skyrl-train") + os.chdir(run_command_dir) + print(f"Changed to directory: {os.getcwd()}") + + # Ensure skyrl-gym exists inside working_dir so uv can resolve editable path + gym_src = os.path.join("..", "skyrl-gym") + gym_dst = os.path.join(".", "skyrl-gym") + if not os.path.exists(gym_dst): + if os.path.exists(gym_src): + print("Copying ../skyrl-gym into working_dir for uv packaging") + subprocess.run( + f"cp -r {gym_src} {gym_dst}", + shell=True, + check=True, + ) + else: + raise Exception("Cannot find skyrl-gym source") + + print("Initializing ray cluster in command line") + subprocess.run( + "ray start --head", + shell=True, + check=True, + ) + # Use 'auto' to automatically detect the Ray cluster instead of hardcoded IP + os.environ["RAY_ADDRESS"] = "auto" + + print(f"Running command: {command}") + print(f"Working directory: {os.getcwd()}") + print("=" * 60) + + # Run the command with live output streaming + process = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Merge stderr into stdout + text=True, + bufsize=1, # Line buffered + universal_newlines=True, + ) + + # Stream output line by line + for line in process.stdout: + print(line, end="") # Print each line as it comes + + # Wait for process to complete + returncode = process.wait() + + print("=" * 60) + if returncode != 0: + raise Exception(f"Command failed with exit code {returncode}") + + +@app.local_entrypoint() +def main(command: str = "nvidia-smi"): + """Main entry-point for running a command in Modal-integrated SkyRL environmenmt. + The given command will be run inside SkyRL/skyrl-train/ + + Args: + command (str, optional): Command to run. Defaults to "nvidia-smi". + + Examples: + modal run main.py --command "uv run examples/gsm8k/gsm8k_dataset.py --output_dir /root/data/gsm8k" + MODAL_GPU=A100:4 MODAL_APP_NAME=benji_skyrl_app modal run main.py --command "bash examples/gsm8k/run_gsm8k_modal.sh" + """ + print(f"{'=' * 5} Submitting command to Modal: {command} {'=' * 5}") + run_script.remote(command) + print(f"\n{'=' * 5} Command completed successfully {'=' * 5}") diff --git a/skyrl-train/integrations/openenv/README.md b/skyrl-train/integrations/openenv/README.md new file mode 100644 index 0000000000..1cb93ff4cd --- /dev/null +++ b/skyrl-train/integrations/openenv/README.md @@ -0,0 +1,102 @@ +## Guide: OpenEnv + SkyRL + +This directory holds the workflow to train on PyTorch OpenEnv environments with SkyRL. + +In this guide, we walk through how to train a reinforcement learning agent using SkyRL with [PyTorch OpenEnv](https://github.com/meta-pytorch/OpenEnv) environments. OpenEnv provides isolated execution environments for agentic RL training with Gymnasium-style APIs. + + +Start by following the SkyRL [installation instructions](https://skyrl.readthedocs.io/en/latest/getting-started/installation.html), then enter the `skyrl-train` directory: +```bash +cd SkyRL/skyrl-train +``` + +### 1) Environment Setup + +Prerequisites: Ensure that you have Docker installed and the required OpenEnv environment images pulled locally. + +First, install the OpenEnv environments (i.e., download the images for each environment): + +```bash +# Execute from skyrl-train directory +uv run integrations/openenv/install_environment.py echo-env +# Or install all environments: +# uv run integrations/openenv/install_environment.py +``` + +This will pull the necessary Docker images for the OpenEnv environments. + +Available environments: ``echo-env``, ``coding-env``, ``openspiel-env``, ``atari-env``, ``sumo-rl-env``, ``finrl-env``. + + +### 2) Dataset Preparation + +For training, we use simple example datasets generated by the ``prepare_dummy_dataset.py`` script: + +```bash +# Execute from skyrl-train directory +uv run integrations/openenv/prepare_dummy_dataset.py --output_dir ~/data/openenv --env_name echo_env +# Or generate datasets for all environments: +# uv run integrations/openenv/prepare_dummy_dataset.py --output_dir ~/data/openenv +``` + +This creates training and validation datasets with example prompts for the specified environment (we provide two examples in ``echo_env`` and ``coding_env``) + +`prepare_dummy_dataset.py` has additional optional parameters: + - `--output_dir`: directory to place datasets (default: `~/data/openenv`) + - `--env_name`: specific environment to prepare dataset for (default: all environments) + +Notes on dataset generation: +- This script will generate the following Parquet files under `output_dir`: + - `train.parquet` + - `validation.parquet` +- For issues in loading the dataset, see the Troubleshooting section below. + +### 3) Training + +We provide an example training script for Qwen2.5-0.5B-Instruct on OpenEnv environments: + +```bash +# Execute from skyrl-train directory +bash integrations/openenv/run_openenv.sh +``` + +Currently, the supporting environments are: ``echo_env``, ``coding_env``, ``openspiel-env``, ``atari-env``, ``sumo-rl-env``, ``finrl-env``. +You can customize the training by setting environment variables: + +```bash +ENV_NAME=coding_env NUM_GPUS=2 bash integrations/openenv/run_openenv.sh +``` + +Or modify the commonly-edited training settings in `run_openenv.sh` as needed: +```bash +ENV_NAME="coding_env" +DATA_DIR="$HOME/data/openenv/$ENV_NAME" +NUM_GPUS=4 +LOGGER="wandb" +``` + +All training parameters can be modified in `run_openenv.sh`, such as the model choice (`trainer.policy.model.path`), GRPO group size (`generator.n_samples_per_prompt`), or training batch size (`trainer.train_batch_size`). + +See all available training configuration parameters in `ppo_base_config.yaml`. + + + +## Tips + +- **Docker Resources**: Ensure sufficient Docker resources are available, especially for computationally intensive environments like Atari or OpenSpiel. +- **Generation Format**: The generation format right now is expected to be a single action wrapped in ``...`` tags for dummy testing. Change `_get_openenv_action` in the OpenEnv environment wrapper (`integrations/openenv/env.py`) for custom parsing logic. +- **Environment Variables**: You can override default values with environment variables like `NUM_GPUS=1`, `ENV_NAME=coding_env`, `MAX_TURNS=1` etc. +- **Logging**: Set `LOGGER=console` to print logs to stdout instead of using wandb. + +## Troubleshooting + +For issues with SkyRL or the integration with OpenEnv, please [open an Issue](https://github.com/NovaSky-AI/SkyRL/issues/new). + +### Datasets + +We use dummy datasets for all the environment integration now. Please modify `prepare_dummy_dataset.py` as needed to extract and prepare the correct datasets. + +## TODOs and Limitations +We welcome any contributions to help resolve the remaining tasks! +* Make it easier to specify different OpenEnv environments used for training and validation. +* Make it smoother to specify which dataset splits to use diff --git a/skyrl-train/integrations/openenv/entrypoints/main_openenv.py b/skyrl-train/integrations/openenv/entrypoints/main_openenv.py new file mode 100644 index 0000000000..5777e41a52 --- /dev/null +++ b/skyrl-train/integrations/openenv/entrypoints/main_openenv.py @@ -0,0 +1,30 @@ +import hydra +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils import initialize_ray +import ray +from skyrl_gym.envs import register + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # Register the multiply environment inside the entrypoint task (no need to modify the skyrl-gym package). + register( + id="openenv", + entry_point="integrations.openenv.env:OpenEnv", + ) + exp = BasePPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # Validate config args. + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/openenv/env.py b/skyrl-train/integrations/openenv/env.py new file mode 100644 index 0000000000..5b34b6da57 --- /dev/null +++ b/skyrl-train/integrations/openenv/env.py @@ -0,0 +1,189 @@ +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput, ConversationType +from typing import Dict, Type, Any +from omegaconf import DictConfig + +# Demonstrate five different environments for now +from envs.echo_env import EchoEnv, EchoAction +from envs.coding_env import CodingEnv, CodeAction +from envs.openspiel_env import OpenSpielEnv, OpenSpielAction +from envs.atari_env import AtariEnv, AtariAction +from envs.sumo_rl_env import SumoRLEnv, SumoAction +from envs.finrl_env import FinRLEnv, FinRLAction +import re +import ast + +import dataclasses + + +def serialize_observation(obs: Any, max_list_len: int = 20) -> str: + """ + Convert any Observation (dataclass or dict) into a flat string. + Long lists are truncated for readability. + """ + if obs is None: + return "Observation: None" + + # Convert to dict representation + if dataclasses.is_dataclass(obs): + obs_dict = dataclasses.asdict(obs) + elif hasattr(obs, "__dict__"): + obs_dict = vars(obs) + elif isinstance(obs, dict): + obs_dict = obs + else: + # Fallback for unexpected types + return f"Observation: {str(obs)}" + + flat_lines = [] + for k, v in obs_dict.items(): + if isinstance(v, list): + display = v[:max_list_len] + suffix = " ..." if len(v) > max_list_len else "" + flat_lines.append(f"{k}: {display}{suffix} (len={len(v)})") + else: + flat_lines.append(f"{k}: {v}") + + return "\n".join(flat_lines) + "\n given this information, try again." + + +class OpenEnv(BaseTextEnv): + """ + Environment for LiveCodeBench execution environment. + """ + + def __init__( + self, + env_config: DictConfig, + extras: Dict[str, Any] = {}, + ): + super().__init__() + + self.extras = extras + self.max_turns = extras["max_turns"] if "max_turns" in extras else 1 + + # NOTE: find a way to get the env class + self.env_name = extras["env_name"] + self.env_type = self._get_env_class(self.env_name) + self.env = self.env_type.from_docker_image(self.env_name + ":latest") + + # Reset before start + self.initial_step_result = self.env.reset() + + # Look at the state of the environment + # self.env.state() + + # Chat history + # role (user, assistant), content (tool observation or LLM response) + self.chat_history: ConversationType = [] + + def _get_env_class(self, env_name: str) -> Type: + if env_name == "echo_env": + return EchoEnv + elif env_name == "coding_env": + return CodingEnv + elif env_name == "openspiel-env": + return OpenSpielEnv + elif env_name == "atari-env": + return AtariEnv + elif env_name == "sumo-rl-env": + return SumoRLEnv + elif env_name == "finrl-env": + return FinRLEnv + else: + raise ValueError(f"Unknown environment '{env_name}'") + + def _get_openenv_action(self, env_name: str, action: str): + """ + Parse the action string to detect things to pass into the OpenEnv environment. + Assume a simple fixed format: ... + + Returns: + Action object to pass into the OpenEnv environment. + """ + matches = [] + if "" in action and "" in action: + matches = re.findall(r"(.*?)", action) + + action = matches[-1] if len(matches) > 0 else None + + if not action: + raise ValueError(f"No action found in action string: {action}") + + if env_name == "echo_env": + return EchoAction(message=action) + elif env_name == "coding_env": + return CodeAction(code=action) + elif env_name == "openspiel-env": + # NOTE: optionally pass in game names + if "game_name" in self.extras: + return OpenSpielAction(action_id=int(action), game_name=self.extras["game_name"]) + else: + return OpenSpielAction(action_id=int(action)) + elif env_name == "atari-env": + if not action.isdigit(): + raise ValueError(f"Atari action must be numeric, got: {action}") + return AtariAction(action_id=int(action)) + elif env_name == "sumo-rl-env": + return SumoAction(phase_id=int(action)) + elif env_name == "finrl-env": + try: + actions_list = ast.literal_eval(action) + except Exception as e: + raise ValueError(f"Invalid FinRL action format '{action}', needs to be a list of floats: {e}") + + return FinRLAction(actions=list(actions_list)) + else: + raise ValueError(f"Unknown environment '{env_name}'") + + def _is_done(self) -> bool: + return self.turns >= self.max_turns + + def step(self, action: str) -> BaseTextEnvStepOutput: + self.turns += 1 + self.chat_history.append({"role": "assistant", "content": action}) + + max_turns_reached = self._is_done() + + error = None + try: + action = self._get_openenv_action(self.env_name, action) + result = self.env.step(action) + observation = serialize_observation(result.observation) + reward = 0.0 if not result.reward else result.reward + done = result.done + except Exception as e: + error = str(e) + observation = None + reward = -1 + + if max_turns_reached: + # If reached max turns, just return the reward and done + return BaseTextEnvStepOutput(observations=[], reward=reward, done=max_turns_reached, metadata={}) + else: + # Return observation for multi-turn interaction + if observation: + new_obs = {"role": "user", "content": observation} + elif error: + new_obs = {"role": "user", "content": error} + else: + new_obs = None + + if new_obs: + self.chat_history.append(new_obs) + + info = { + "env_class": self.env_name, + "action": action, + "observation": observation, + } + # print("chat history: ", self.chat_history) + + return BaseTextEnvStepOutput( + observations=[new_obs] if new_obs else [], + reward=reward, + done=done, + metadata=info, + ) + + def close(self): + self.env.close() diff --git a/skyrl-train/integrations/openenv/install_environment.py b/skyrl-train/integrations/openenv/install_environment.py new file mode 100644 index 0000000000..b428116004 --- /dev/null +++ b/skyrl-train/integrations/openenv/install_environment.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# +# Helper that pulls prebuilt OpenEnv environment Docker images and tags them locally. +# +# Examples: +# uv run skyrl_gym/envs/openenv/install_environment.py echo-env +# uv run skyrl_gym/envs/openenv/install_environment.py coding-env +# uv run skyrl_gym/envs/openenv/install_environment.py atari-env +# uv run skyrl_gym/envs/openenv/install_environment.py openspiel-env +# uv run skyrl_gym/envs/openenv/install_environment.py sumo-rl-env +# pulls all images +# + +import argparse +import sys +import subprocess + +# Image mapping: from https://github.com/meta-pytorch/OpenEnv/pkgs/container/ +ENV_IMAGES = { + "base": "ghcr.io/meta-pytorch/openenv-base:sha-64d4b10", + "atari-env": "ghcr.io/meta-pytorch/openenv-atari-env:sha-64d4b10", + "coding-env": "ghcr.io/meta-pytorch/openenv-coding-env:sha-64d4b10", + "echo-env": "ghcr.io/meta-pytorch/openenv-echo-env:sha-64d4b10", + "openspiel-env": "ghcr.io/meta-pytorch/openenv-openspiel-base:sha-e622c7e", + "sumo-rl-env": "ghcr.io/meta-pytorch/openenv-sumo-rl-env:sha-c25298c", + "finrl-env": "", +} + + +def run_command(cmd, check=True): + print(f"Running: {' '.join(cmd)}") + subprocess.run(cmd, check=check) + + +def pull_image(image_url, local_tag): + print(f"Pulling image from URL:{image_url}") + run_command(["docker", "pull", image_url]) + print(f"Tagging as {local_tag}:latest") + run_command(["docker", "tag", image_url, f"{local_tag}:latest"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Pull prebuilt OpenEnv Docker images.") + parser.add_argument( + "env_name", + nargs="?", + help="Environment name (e.g., echo-env, coding-env, atari-env, openspiel-env, sumo-rl-env). Leave blank to pull all.", + ) + args = parser.parse_args() + + try: + print("Pulling OpenEnv base image...") + pull_image(ENV_IMAGES["base"], "openenv-base") + + if args.env_name: + env_name = args.env_name + if env_name not in ENV_IMAGES: + print(f"Error: Unknown environment '{env_name}'.") + print("Available environments:", ", ".join(ENV_IMAGES.keys())) + sys.exit(1) + + print(f"Pulling {env_name} image...") + + if env_name == "finrl-env": + # NOTE(shu): finrl-env is not available remotely; need to build from source + print("Building finrl-env from source (no remote package available)...") + + repo_url = "https://github.com/meta-pytorch/OpenEnv.git" + repo_dir = "OpenEnv" + + try: + # clone and build base + run_command(["git", "clone", "--depth", "1", repo_url, repo_dir]) + + # NOTE(shu): original docker file does not work; need to replace finrl==0.3.6 with 0.3.7 + finrl_dockerfile = f"{repo_dir}/src/envs/finrl_env/server/Dockerfile" + run_command(["sed", "-i", "s/finrl==0.3.6/finrl==0.3.7/g", finrl_dockerfile]) + + base_dockerfile = f"{repo_dir}/src/core/containers/images/Dockerfile" + run_command(["docker", "build", "-t", "envtorch-base:latest", "-f", base_dockerfile, repo_dir]) + + finrl_dockerfile = f"{repo_dir}/src/envs/finrl_env/server/Dockerfile" + run_command(["docker", "build", "-t", "finrl-env:latest", "-f", finrl_dockerfile, repo_dir]) + + # Now start the docker with default sample data + run_command(["docker", "run", "-p", "8000:8000", "finrl-env:latest"]) + + finally: + run_command(["rm", "-rf", repo_dir]) + else: + pull_image(ENV_IMAGES[env_name], env_name) + + else: + print("No environment specified. Pulling all environments...") + for name, url in ENV_IMAGES.items(): + if name == "base": + continue + pull_image(url, name) + + print("All images pulled successfully.") + + except subprocess.CalledProcessError as e: + print(f"Command failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/openenv/prepare_dummy_dataset.py b/skyrl-train/integrations/openenv/prepare_dummy_dataset.py new file mode 100644 index 0000000000..046255da24 --- /dev/null +++ b/skyrl-train/integrations/openenv/prepare_dummy_dataset.py @@ -0,0 +1,112 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocess the OpenEnv dataset to parquet format +""" + +import argparse +import os +from typing import List +from datasets import Dataset +import math + + +def example_question(env_name: str) -> List[str]: + """ + Return simple example prompts for each environment. + Each prompt instructs the LLM to generate an ... response. + """ + + # Shared instruction appended to each example + instruction = "Wrap the action between and tags.\n" "For example: ACTION_HERE." + + if env_name == "echo_env": + # Echo environment simply echoes the text back. + examples = [ + "Send a short test message to the echo environment.", + ] + elif env_name == "coding_env": + # Coding environment executes code and returns stdout. + examples = [ + "Print 'I love RL!' using Python.", + "Provide a function fib(n) that can compute the n-th Fibonacci number", + "Using a for loop, print numbers 1 to 5 and their squares.", + ] + instruction = ( + "Write the python code inside ... tags. Example: print('Hello, World!')" + ) + else: + # NOTE: we only provide dummy train set examples for echo_env and coding_env + # For other environments, you need to provide your own train / validation set examples. + raise ValueError(f"Unknown environment name: {env_name}") + + return [f"{ex}\n\n{instruction}" for ex in examples] + + +def build_dataset(env_name: str, output_dir: str, data_source="openenv", target_size=32): + """Generate and save train/validation datasets for one environment.""" + base_questions = example_question(env_name) + repeat_factor = math.ceil(target_size / len(base_questions)) + questions = (base_questions * repeat_factor)[:target_size] + + # Split the questions into train and validation sets + split_index = target_size // 2 + train_questions = questions[:split_index] + val_questions = questions[split_index:] + + def make_map_fn(split): + def process_fn(example, idx): + return { + "data_source": data_source, + "prompt": [{"role": "user", "content": example["question"]}], + "env_class": "openenv", + "env_name": env_name, + } + + return process_fn + + train_dataset = Dataset.from_dict({"question": train_questions}) + val_dataset = Dataset.from_dict({"question": val_questions}) + + train_dataset = train_dataset.map(make_map_fn("train"), with_indices=True) + val_dataset = val_dataset.map(make_map_fn("validation"), with_indices=True) + + os.makedirs(output_dir, exist_ok=True) + train_dataset.to_parquet(os.path.join(output_dir, "train.parquet")) + val_dataset.to_parquet(os.path.join(output_dir, "validation.parquet")) + + print(f"Saved {len(train_dataset)} train and {len(val_dataset)} validation examples to {output_dir}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", default="~/data/openenv") + parser.add_argument("--env_name", default=None) + args = parser.parse_args() + + args.output_dir = os.path.expanduser(args.output_dir) + all_envs = ["echo_env", "coding_env"] + + if args.env_name is None: + print("No --env_name specified. Preparing datasets for all available environments:") + for env in all_envs: + print(f" - {env}") + print() + for env in all_envs: + build_dataset(env, os.path.join(args.output_dir, env)) + else: + if args.env_name not in all_envs: + print(f"Error: Unknown environment '{args.env_name}'. Available environments are: {', '.join(all_envs)}") + exit(1) + build_dataset(args.env_name, os.path.join(args.output_dir, args.env_name)) diff --git a/skyrl-train/integrations/openenv/run_openenv.sh b/skyrl-train/integrations/openenv/run_openenv.sh new file mode 100644 index 0000000000..22121ad15f --- /dev/null +++ b/skyrl-train/integrations/openenv/run_openenv.sh @@ -0,0 +1,73 @@ +set -x + +# Colocated GRPO training+generation for Qwen2.5-0.5B-Instruct on OpenEnv. + +# uv run examples/openenv/dummy_openenv_dataset.py --output_dir $HOME/data/openenv --env_name echo_env +# Env name: echo_env, coding_env, openspiel-env, atari-env, sumo-rl-env, finrl-env + +# Prestart the docker environment with +# cd /skyrl-gym +# uv run envs/openenv/install_environment.py + +# export WANDB_API_KEY= +# bash examples/openenv/run_dummy_openenv.sh + +# You can override the default values with e.g.: `NUM_GPUS=1 bash examples/openenv/run_dummy_openenv.sh`. + +: "${ENV_NAME:="echo_env"}" +: "${DATA_DIR:="$HOME/data/openenv/$ENV_NAME"}" +: "${CKPT_PATH:="$HOME/ckpts/openenv_${ENV_NAME}_1.5B"}" +: "${NUM_GPUS:=4}" +: "${LOGGER:=wandb}" # change to "console" to print to stdout + +: "${INFERENCE_BACKEND:=vllm}" +# : "${INFERENCE_BACKEND:=sglang}" +: "${MAX_TURNS:=1}" + +uv run --isolated --extra $INFERENCE_BACKEND --with "openenv@git+https://github.com/meta-pytorch/OpenEnv.git" --with "litellm>=1.75.5" -m integrations.openenv.entrypoints.main_openenv \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.critic_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.eval_batch_size=4 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=4 \ + trainer.policy_mini_batch_size=4 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + environment.env_class=openenv \ + generator.use_conversation_multi_turn=false \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="openenv" \ + trainer.run_name="openenv_${ENV_NAME}_latest" \ + trainer.resume_mode=latest \ + trainer.ckpt_path=$CKPT_PATH \ + trainer.dump_data_batch=true \ + $@ \ No newline at end of file diff --git a/skyrl-train/integrations/swe_tasks/README.md b/skyrl-train/integrations/swe_tasks/README.md new file mode 100644 index 0000000000..fa1548a129 --- /dev/null +++ b/skyrl-train/integrations/swe_tasks/README.md @@ -0,0 +1,74 @@ +# SWE Task Generator Integration + +Train a coding agent on tasks generated by [swe-task-generator](https://github.com/fleet-ai/swe-task-generator). + +## How It Works + +1. **Task instances** are generated from GitHub PRs (see swe-task-generator Part 1) +2. Each task has a Docker image on Docker Hub + an eval_script.sh +3. During training, the agent interacts with the Docker container via bash commands +4. The agent tries to fix the bug described in the problem statement +5. Reward = 1.0 if eval_script passes, 0.0 if it fails + +## Quick Start + +### 1. Prepare Dataset + +```bash +cd SkyRL/skyrl-train + +uv run --isolated integrations/swe_tasks/prepare_dataset.py \ + --tasks-dir /path/to/swe-task-generator/tasks \ + --output-dir ~/data/swe_tasks +``` + +### 2. Launch Training (GPU cluster) + +```bash +# Local GPU cluster (8xH100) +bash integrations/swe_tasks/run_swe_tasks_8B.sh + +# Via SkyPilot (cloud GPU) +sky launch tasks/swe-tasks-grpo-qwen3-8b.yaml --env WANDB_API_KEY= +``` + +### 3. Verify Reward Signal (no GPU needed) + +```bash +cd /path/to/swe-task-generator +python show_reward.py +``` + +## Architecture + +``` +SkyRL Training Loop + │ + ├── Qwen3-8B generates bash commands + │ (explores repo, reads code, writes fixes) + │ + ├── mini-swe-agent executes in Docker container + │ (image pulled from Docker Hub) + │ + ├── Agent submits: git diff --cached + │ + └── Evaluation: + ├── git apply + ├── bash eval_script.sh + └── reward = 1.0 if exit_code == 0 else 0.0 +``` + +## Configuration + +- `swe_tasks.yaml` — Agent system prompt, step limit, Docker settings +- `run_swe_tasks_8B.sh` — Training hyperparameters (batch size, lr, etc.) +- `tasks/swe-tasks-grpo-qwen3-8b.yaml` — SkyPilot cloud deployment config + +## Task Format + +Each task instance in the dataset has: +- `instance_id`: e.g., "marshmallow-code-marshmallow-2894" +- `problem_statement`: Description of the bug to fix +- `eval_script`: Shell script that returns 0 on success +- `image_name`: Docker Hub image with the repo at the buggy commit +- `gold_patch`: Reference fix (used for evaluation, not shown to agent) diff --git a/skyrl-train/integrations/swe_tasks/__init__.py b/skyrl-train/integrations/swe_tasks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/integrations/swe_tasks/entrypoints/__init__.py b/skyrl-train/integrations/swe_tasks/entrypoints/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/integrations/swe_tasks/entrypoints/main_swe_tasks.py b/skyrl-train/integrations/swe_tasks/entrypoints/main_swe_tasks.py new file mode 100644 index 0000000000..3285bc6e0c --- /dev/null +++ b/skyrl-train/integrations/swe_tasks/entrypoints/main_swe_tasks.py @@ -0,0 +1,44 @@ +""" +SkyRL training entrypoint for swe-task-generator tasks. + +Usage: + cd SkyRL/skyrl-train + bash integrations/swe_tasks/run_swe_tasks_8B.sh +""" + +import hydra +from omegaconf import DictConfig, OmegaConf +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils import initialize_ray +import ray + +from integrations.swe_tasks.swe_tasks_generator import SWETasksGenerator + + +class SWETasksPPOExp(BasePPOExp): + def get_generator(self, cfg, tokenizer, inference_engine_client): + generator = SWETasksGenerator( + generator_cfg=cfg.generator, + skyrl_gym_cfg=OmegaConf.create({"max_env_workers": 0}), + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + model_name=self.cfg.trainer.policy.model.path, + ) + return generator + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = SWETasksPPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + validate_cfg(cfg) + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/swe_tasks/env.py b/skyrl-train/integrations/swe_tasks/env.py new file mode 100644 index 0000000000..2840af5a8b --- /dev/null +++ b/skyrl-train/integrations/swe_tasks/env.py @@ -0,0 +1,106 @@ +""" +SWE Task environment utilities for SkyRL integration. + +Mirrors the pattern from examples/mini_swe_agent/mini_swe_utils.py but +adapted for our swe-task-generator task format. Key differences: + - image_name comes directly from task.json (already on Docker Hub) + - eval_script is stored in the instance metadata + - No SWE-Bench specific image naming conventions needed +""" + +import uuid +import traceback +from typing import TypedDict, Optional, Dict, Any + +from loguru import logger +from jinja2 import Template +from minisweagent.environments import Environment, get_environment + + +class SWETaskEvaluationResult(TypedDict): + instance_id: str + resolved: bool + eval_error: Optional[str] + + +def get_swe_task_environment(config: dict, instance: dict) -> Environment: + """ + Create a Docker environment for a SWE task instance. + + Args: + config: Environment configuration (from swe_tasks.yaml) + instance: Task instance dict with 'image_name' field + + Returns: + Environment: A minisweagent Docker environment ready for agent interaction + """ + env_config = config.setdefault("environment", {}) + env_config["environment_class"] = env_config.get("environment_class", "docker") + + image_name = instance["image_name"] + + if env_config["environment_class"] == "docker": + env_config["image"] = image_name + elif env_config["environment_class"] == "singularity": + env_config["image"] = f"docker://{image_name}" + + env = get_environment(env_config) + + if startup_command := config.get("run", {}).get("env_startup_command"): + startup_command = Template(startup_command).render(**instance) + out = env.execute(startup_command) + if out["returncode"] != 0: + raise RuntimeError(f"Error executing startup command: {out}") + + return env + + +def evaluate_trajectory(instance: Dict[str, Any], model_patch: str, config: dict) -> SWETaskEvaluationResult: + """ + Evaluate a model-generated patch against the task's eval_script. + + This is called after the agent finishes its trajectory. It: + 1. Creates a fresh Docker environment + 2. Applies the model's git patch + 3. Runs the eval_script from the task instance + 4. Returns whether the task was resolved + + Args: + instance: Task instance dict with 'instance_id', 'eval_script', 'image_name' + model_patch: The git diff patch generated by the model + config: Environment configuration + + Returns: + SWETaskEvaluationResult with resolved=True/False + """ + ret = SWETaskEvaluationResult( + instance_id=instance["instance_id"], + resolved=False, + eval_error=None, + ) + + env = None + try: + env = get_swe_task_environment(config, instance) + except Exception as e: + ret["eval_error"] = f"Env creation failed: {e}" + logger.info(f"Environment creation failed: {e}\n{traceback.format_exc()}") + return ret + + # Apply model's git patch using heredoc to avoid ARG_MAX issues + delimiter = f"PATCH_{uuid.uuid4().hex}" + command = f"cd /workspace && git apply <<'{delimiter}'\n{model_patch}\n{delimiter}" + + obs = env.execute(command) + + if obs["returncode"] != 0: + ret["eval_error"] = obs["output"] + else: + # Run eval script from the task instance + eval_script = instance["eval_script"] + eval_cmd = f"bash <<'EOF'\n{eval_script}\nEOF" + obs = env.execute(eval_cmd, timeout=3600) + ret["resolved"] = obs["returncode"] == 0 + ret["eval_error"] = f"(truncated to last 1000 chars)\n{obs['output'][-1000:]}" if not ret["resolved"] else None + + return ret diff --git a/skyrl-train/integrations/swe_tasks/prepare_dataset.py b/skyrl-train/integrations/swe_tasks/prepare_dataset.py new file mode 100644 index 0000000000..b230c1e189 --- /dev/null +++ b/skyrl-train/integrations/swe_tasks/prepare_dataset.py @@ -0,0 +1,103 @@ +""" +Prepare SkyRL-compatible dataset from swe-task-generator task instances. + +Reads task.json files from a tasks/ directory and outputs train.parquet + validation.parquet +in the format expected by SkyRL's PromptDataset and our MiniSweAgentGenerator. + +Dataset columns (matching mini_swe_agent format): + - data_source: str + - prompt: List[Dict[str, str]] (chat messages) + - env_class: str ("null" — custom Docker env) + - instance: Dict (full task metadata: instance_id, eval_script, image_name, gold_patch, etc.) + +Usage: + uv run --isolated integrations/swe_tasks/prepare_dataset.py \\ + --tasks-dir /path/to/swe-task-generator/tasks \\ + --output-dir ~/data/swe_tasks +""" + +import argparse +import json +import os +import glob + +import datasets + + +def load_tasks(tasks_dir: str) -> list: + """Load all task.json files from the tasks directory.""" + tasks = [] + task_dirs = sorted(glob.glob(os.path.join(tasks_dir, "task_*"))) + for task_dir in task_dirs: + task_json = os.path.join(task_dir, "task.json") + if os.path.exists(task_json): + with open(task_json) as f: + task = json.load(f) + tasks.append(task) + print(f" Loaded: {task['instance_id']}") + return tasks + + +def make_dataset(tasks: list, data_source: str) -> datasets.Dataset: + """Convert task list to a HuggingFace Dataset in SkyRL format.""" + records = [] + for task in tasks: + records.append( + { + "data_source": data_source, + "prompt": [{"role": "user", "content": task["problem_statement"]}], + "env_class": "null", + # Store full instance dict — pyarrow will serialize nested dicts + "instance_id": task["instance_id"], + "problem_statement": task["problem_statement"], + "eval_script": task["eval_script"], + "image_name": task["image_name"], + "gold_patch": task.get("gold_patch", ""), + "test_patch": task.get("test_patch", ""), + "fix_patch": task.get("fix_patch", ""), + "repo": task.get("repo", ""), + "base_commit": task.get("base_commit", ""), + } + ) + + return datasets.Dataset.from_list(records) + + +def main(): + parser = argparse.ArgumentParser(description="Prepare SkyRL dataset from swe-task-generator tasks") + parser.add_argument("--tasks-dir", required=True, help="Path to tasks/ directory") + parser.add_argument("--output-dir", default="~/data/swe_tasks", help="Output directory for parquet files") + parser.add_argument("--data-source", default="swe-task-generator", help="Dataset identifier") + args = parser.parse_args() + + tasks_dir = os.path.abspath(args.tasks_dir) + output_dir = os.path.expanduser(args.output_dir) + os.makedirs(output_dir, exist_ok=True) + + print(f"Loading tasks from {tasks_dir}...") + tasks = load_tasks(tasks_dir) + + if not tasks: + print("ERROR: No task instances found!") + return + + print(f"\nFound {len(tasks)} task instances") + + ds = make_dataset(tasks, args.data_source) + + # For our small dataset, use same set for train and validation + # NOTE: In production with more tasks, split into train/val + train_path = os.path.join(output_dir, "train.parquet") + val_path = os.path.join(output_dir, "validation.parquet") + + ds.to_parquet(train_path) + print(f" Wrote train: {train_path} ({len(ds)} instances)") + + ds.to_parquet(val_path) + print(f" Wrote val: {val_path} ({len(ds)} instances)") + + print(f"\nDataset ready at {output_dir}/") + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/swe_tasks/run_swe_tasks_8B.sh b/skyrl-train/integrations/swe_tasks/run_swe_tasks_8B.sh new file mode 100644 index 0000000000..d8e65a0964 --- /dev/null +++ b/skyrl-train/integrations/swe_tasks/run_swe_tasks_8B.sh @@ -0,0 +1,79 @@ +set -x + +# GRPO training for Qwen3-8B on swe-task-generator tasks. +# Uses 1 node with 8 GPUs (8xH100 recommended). +# +# Prerequisites: +# 1. Prepare dataset: +# uv run --isolated integrations/swe_tasks/prepare_dataset.py \ +# --tasks-dir /path/to/swe-task-generator/tasks \ +# --output-dir ~/data/swe_tasks +# +# 2. Ensure Docker images are accessible: +# docker pull erranli/swe-task-marshmallow-code-marshmallow-2894:latest +# docker pull erranli/swe-task-marshmallow-code-marshmallow-2892:latest +# +# 3. Run: +# cd SkyRL/skyrl-train +# bash integrations/swe_tasks/run_swe_tasks_8B.sh + +DATA_DIR="${DATA_DIR:-$HOME/data/swe_tasks}" +CKPT_PATH="${CKPT_PATH:-$HOME/ckpts/swe_tasks}" +SWE_TASKS_TRAJ_DIR="${SWE_TASKS_TRAJ_DIR:-$HOME/swe_tasks_trajs}" + +NUM_GPUS=8 +NNODES=1 +NUM_INFERENCE_ENGINES=4 +TP_SIZE=2 +LOGGER=wandb + +uv run --isolated --extra vllm --extra miniswe \ + -m integrations.swe_tasks.entrypoints.main_swe_tasks \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-8B" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.policy_num_nodes=$NNODES \ + trainer.placement.ref_num_nodes=$NNODES \ + trainer.policy.sequence_parallel_size=2 \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.epochs=20 \ + trainer.eval_batch_size=2 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=2 \ + trainer.policy_mini_batch_size=2 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.dump_data_batch=true \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=4096 \ + generator.sampling_params.max_generate_length=4096 \ + generator.max_input_length=30720 \ + generator.max_turns=20 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=True \ + generator.enable_http_endpoint=True \ + generator.http_endpoint_host='127.0.0.1' \ + generator.http_endpoint_port=8001 \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="swe_tasks" \ + trainer.run_name="swe_tasks_8B_marshmallow" \ + trainer.resume_mode=null \ + trainer.ckpt_path="$CKPT_PATH" \ + +generator.swe_tasks_config_path="integrations/swe_tasks/swe_tasks.yaml" \ + +generator.swe_tasks_traj_dir=$SWE_TASKS_TRAJ_DIR \ + $@ diff --git a/skyrl-train/integrations/swe_tasks/swe_tasks.yaml b/skyrl-train/integrations/swe_tasks/swe_tasks.yaml new file mode 100644 index 0000000000..488415e745 --- /dev/null +++ b/skyrl-train/integrations/swe_tasks/swe_tasks.yaml @@ -0,0 +1,138 @@ +# Agent and environment configuration for swe-task-generator tasks. +# Adapted from examples/mini_swe_agent/swebench.yaml +# +# Key differences from mini_swe_agent: +# - cwd is /workspace (our Dockerfiles clone repos to /workspace) +# - Docker images come from Docker Hub (our pushed images) +# - No need for SWE-Bench specific image naming + +agent: + system_template: | + You are a helpful assistant that can interact multiple times with a computer shell to solve programming tasks. + Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||). + + Include a THOUGHT section before your command where you explain your reasoning process. + Format your response as shown in . + + + THOUGHT: Your reasoning and analysis here + + ```bash + your_command_here + ``` + + + Failure to follow these rules will cause your response to be rejected. + instance_template: | + + Consider the following PR description: + {{task}} + + + + # Task Instructions + + ## Overview + You're a software engineer interacting continuously with a computer by submitting commands. + You'll be helping implement necessary changes to meet requirements in the PR description. + Your task is specifically to make changes to non-test files in the current directory in order to fix the issue described in the PR description in a way that is general and consistent with the codebase. + + IMPORTANT: This is an interactive process where you will think and issue ONE command, see its result, then think and issue your next command. + + For each response: + 1. Include a THOUGHT section explaining your reasoning and what you're trying to accomplish + 2. Provide exactly ONE bash command to execute + + ## Important Boundaries + - MODIFY: Regular source code files in /workspace (this is the working directory for all your subsequent commands) + - DO NOT MODIFY: Tests, configuration files (pyproject.toml, setup.cfg, etc.) + + ## Recommended Workflow + 1. Analyze the codebase by finding and reading relevant files + 2. Create a script to reproduce the issue + 3. Edit the source code to resolve the issue + 4. Verify your fix works by running your script again + 5. Test edge cases to ensure your fix is robust + + ## Command Execution Rules + You are operating in an environment where + 1. You write a single command + 2. The system executes that command in a subshell + 3. You see the result + 4. You write your next command + + Each response should include: + 1. A **THOUGHT** section where you explain your reasoning and plan + 2. A single bash code block with your command + + Commands must be specified in a single bash code block: + + ```bash + your_command_here + ``` + + **CRITICAL REQUIREMENTS:** + - Your response SHOULD include a THOUGHT section explaining your reasoning + - Your response MUST include EXACTLY ONE bash code block + - This bash block MUST contain EXACTLY ONE command (or a set of commands connected with && or ||) + - If you include zero or multiple bash blocks, or no command at all, YOUR RESPONSE WILL FAIL + - Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + - However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files + + ## Submission + When you've completed your work (reading, editing, testing), and cannot make further progress + issue exactly the following command: + + ```bash + echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && git add -A && git diff --cached + ``` + + This command will submit your work. + You cannot continue working (reading, editing, testing) in any way on this task after submitting. + + action_observation_template: | + {{output.returncode}} + {% if output.output | length < 10000 -%} + + {{ output.output -}} + + {%- else -%} + + The output of your last command was too long. + Please try a different command that produces less output. + + {%- set elided_chars = output.output | length - 10000 -%} + + {{ output.output[:5000] }} + + + {{ elided_chars }} characters elided + + + {{ output.output[-5000:] }} + + {%- endif -%} + format_error_template: | + Please always provide EXACTLY ONE action in triple backticks, found {{actions|length}} actions. + step_limit: 20 + cost_limit: 3. + +environment: + cwd: "/workspace" + timeout: 180 + pull_timeout: 1200 + env: + PAGER: cat + MANPAGER: cat + LESS: -R + PIP_PROGRESS_BAR: 'off' + TQDM_DISABLE: '1' + environment_class: docker + # Use 'docker' on most systems; change to 'podman' if needed + executable: docker + +model: + model_name: "" + model_kwargs: + drop_params: true + cost_tracking: ignore_errors \ No newline at end of file diff --git a/skyrl-train/integrations/swe_tasks/swe_tasks_generator.py b/skyrl-train/integrations/swe_tasks/swe_tasks_generator.py new file mode 100644 index 0000000000..6a0f2ff167 --- /dev/null +++ b/skyrl-train/integrations/swe_tasks/swe_tasks_generator.py @@ -0,0 +1,292 @@ +""" +SWE Tasks Generator for SkyRL. + +Adapts the MiniSweAgentGenerator pattern for our swe-task-generator task format. +Uses mini-swe-agent for multi-step Docker execution, with our custom Docker images +from Docker Hub and eval_script from task.json. +""" + +import asyncio +from typing import Dict, List, Optional, Any, Tuple +from omegaconf import DictConfig +import yaml +import traceback +import ray +from pathlib import Path + +from minisweagent.models import get_model +from minisweagent.agents.default import DefaultAgent +from minisweagent.run.utils.save import save_traj +from minisweagent.config import get_config_path +from .env import evaluate_trajectory, get_swe_task_environment + +from skyrl_train.generators.skyrl_gym_generator import SkyRLGymGenerator, GeneratorOutput, GeneratorInput +from skyrl_train.generators.base import TrajectoryID, TrainingPhase, BatchMetadata +from skyrl_train.inference_engines.base import ConversationType +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.generators.utils import ( + get_rollout_metrics, + get_response_ids_and_loss_mask_from_messages, +) + + +class DefaultAgentWithReminder(DefaultAgent): + """Agent that reminds the model of remaining turns.""" + + def get_observation(self, response: dict) -> dict: + output = self.execute_action(self.parse_action(response)) + observation = self.render_template(self.config.action_observation_template, output=output) + remaining = self.config.step_limit - self.model.n_calls + + if remaining == 1: + observation = f"{observation}\nREMINDER: You only have 1 turn left. Please provide the final answer" + elif remaining > 1: + observation = f"{observation}\nREMINDER: You have {remaining} turns left to arrive at the solution." + + self.add_message("user", observation) + return output + + +@ray.remote(num_cpus=0.01) +def init_and_run( + instance: dict, + litellm_model_name: str, + sweagent_config: dict, + generator_cfg: DictConfig, + sampling_params: dict, + trajectory_id: TrajectoryID, + global_step: int, + training_phase: TrainingPhase, + vllm_base_url: str = "http://127.0.0.1:8000", +): + """ + Ray remote task: initialize environment and run agent trajectory. + + Adapted from mini_swe_agent but uses our task instance format: + - image_name from task.json (Docker Hub) + - eval_script from task.json + """ + import os + from loguru import logger + + # Disable minisweagent cost tracking for local models not in litellm pricing DB + os.environ["MSWEA_COST_TRACKING"] = "ignore_errors" + + model_config = sweagent_config.get("model", {}) + model_config.setdefault("model_kwargs", {}).update(sampling_params) + # Point litellm to the local vLLM inference server + model_config["model_kwargs"]["api_base"] = f"{vllm_base_url}/v1" + model_config["model_kwargs"]["api_key"] = "dummy" + model = get_model(litellm_model_name, model_config) + + agent = None + env = None + extra_info = None + result = None + reward = 0 + error = None + + try: + env = get_swe_task_environment(sweagent_config, instance) + agent = DefaultAgentWithReminder(model, env, **sweagent_config.get("agent", {})) + exit_status, result = agent.run(instance["problem_statement"]) + except Exception as e: + logger.error(f"Error processing instance {instance['instance_id']}: {e}", exc_info=True) + exit_status, result = type(e).__name__, str(e) + error = str(e) + extra_info = {"traceback": traceback.format_exc()} + finally: + # Save trajectory for debugging + path = Path(generator_cfg.swe_tasks_traj_dir) / f"step_{global_step}" / training_phase + path.mkdir(parents=True, exist_ok=True) + instance_id = instance["instance_id"] + filename = f"{instance_id}_{trajectory_id.repetition_id}.json" + path = path / filename + + if agent is not None: + eval_error = None + try: + result = evaluate_trajectory(instance, result, sweagent_config) + reward = int(result["resolved"]) + eval_error = result["eval_error"] + if eval_error: + error = eval_error + logger.debug(f"Eval error: {eval_error}") + except Exception as e: + logger.debug(f"Eval error: {e}") + logger.debug(f"Traceback: {traceback.format_exc()}") + eval_error = str(e) + error = str(e) + + save_traj( + agent, + path, + exit_status=exit_status, + result=result, + extra_info=extra_info, + reward=reward, + eval_error=eval_error, + ) + + return (agent.messages if agent is not None else [], reward, error) + + +class SWETasksGenerator(SkyRLGymGenerator): + """ + Custom generator for swe-task-generator tasks. + + Uses mini-swe-agent for multi-step Docker execution with our custom + Docker images and eval scripts. The reward is binary: + 1.0 = eval_script passes (agent fixed the bug) + 0.0 = eval_script fails (agent did not fix the bug) + """ + + def __init__( + self, + generator_cfg: DictConfig, + skyrl_gym_cfg: DictConfig, + inference_engine_client: InferenceEngineClient, + tokenizer, + model_name: str, + ): + super().__init__(generator_cfg, skyrl_gym_cfg, inference_engine_client, tokenizer, model_name) + + self.http_endpoint_host = generator_cfg.get("http_endpoint_host", "127.0.0.1") + self.http_endpoint_port = generator_cfg.get("http_endpoint_port", 8001) + self.base_url = f"http://{self.http_endpoint_host}:{self.http_endpoint_port}" + self.generator_cfg = generator_cfg + self.tokenizer = tokenizer + self.model_name = model_name + self.litellm_model_name = "openai/" + self.model_name + + if self.generator_cfg.chat_template.name_or_path is not None: + raise NotImplementedError("SWETasksGenerator doesn't support custom chat template") + + async def swe_tasks_agent_loop( + self, + prompt: ConversationType, + env_extras: Dict[str, Any], + max_tokens: int, + max_input_length: int, + sampling_params: Dict[str, Any], + trajectory_id: TrajectoryID, + batch_metadata: BatchMetadata, + ) -> Tuple[List[int], float, str, List[int], List[int], Optional[List[int]]]: + """Run a single agent trajectory on a SWE task instance.""" + + sweagent_config = yaml.safe_load(get_config_path(self.generator_cfg.swe_tasks_config_path).read_text()) + + # Build instance dict from env_extras (which come from the dataset row) + instance = { + "instance_id": env_extras["instance_id"], + "problem_statement": env_extras["problem_statement"], + "eval_script": env_extras["eval_script"], + "image_name": env_extras["image_name"], + "gold_patch": env_extras.get("gold_patch", ""), + } + + messages, reward, error = await init_and_run.remote( + instance, + self.litellm_model_name, + sweagent_config, + self.generator_cfg, + sampling_params, + trajectory_id, + batch_metadata.global_step, + batch_metadata.training_phase, + vllm_base_url=self.base_url, + ) + + if not len(messages): + return None, None, None, None, None, None + + # First two messages are system + user (same as mini_swe_agent) + response_messages = messages[2:] + + for message in messages[:2]: + assert message["role"] in ("system", "user") + + initial_input_ids = self.tokenizer.apply_chat_template(messages[:2], add_generation_prompt=False, tokenize=True) + initial_prompt_length = len(initial_input_ids) + + # Remove trailing user messages (final git diff capture) + if not response_messages: + # No response beyond system+user — agent produced nothing + return None, None, None, None, None, None + last_idx = len(response_messages) - 1 + while last_idx >= 0 and response_messages[last_idx]["role"] == "user": + last_idx -= 1 + if last_idx < 0: + # Only user messages, no assistant messages at all + return None, None, None, None, None, None + response_messages = response_messages[: last_idx + 1] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages( + response_messages, + self.tokenizer, + assistant_logprobs=None, + ) + + prompt_ids = initial_input_ids + max_response_tokens = max_tokens + max_input_length - initial_prompt_length + + stop_reason = "complete" + if len(response_ids) > max_response_tokens: + stop_reason = "length" + + response_ids = response_ids[:max_response_tokens] + loss_mask = loss_mask[:max_response_tokens] + + return (response_ids, reward, stop_reason, loss_mask, prompt_ids, None) + + async def generate(self, input_batch: GeneratorInput) -> GeneratorOutput: + """Generate trajectories for the input batch.""" + prompts = input_batch["prompts"] + env_extras = input_batch["env_extras"] + trajectory_ids = input_batch["trajectory_ids"] + batch_metadata = input_batch["batch_metadata"] + max_tokens = self.generator_cfg.sampling_params.max_generate_length + max_input_length = self.generator_cfg.max_input_length + sampling_params = get_sampling_params_for_backend( + self.generator_cfg.backend, self.generator_cfg.sampling_params + ) + + tasks = [] + for i in range(len(prompts)): + tasks.append( + self.swe_tasks_agent_loop( + prompts[i], + env_extras[i], + max_tokens=max_tokens, + max_input_length=max_input_length, + sampling_params=sampling_params, + trajectory_id=trajectory_ids[i], + batch_metadata=batch_metadata, + ) + ) + + all_outputs = await asyncio.gather(*tasks) + + responses = [output[0] for output in all_outputs if output[0] is not None] + rewards = [output[1] for output in all_outputs if output[0] is not None] + stop_reasons = [output[2] for output in all_outputs if output[0] is not None] + loss_masks = [output[3] for output in all_outputs if output[0] is not None] + prompt_token_ids = [output[4] for output in all_outputs if output[0] is not None] + + if not len(responses): + raise ValueError("No valid responses. All trajectories failed — check environment setup.") + + rollout_metrics = get_rollout_metrics(responses, rewards) + + generator_output: GeneratorOutput = { + "prompt_token_ids": prompt_token_ids, + "response_ids": responses, + "rewards": rewards, + "loss_masks": loss_masks, + "stop_reasons": stop_reasons, + "rollout_metrics": rollout_metrics, + "rollout_logprobs": None, + } + + return generator_output diff --git a/skyrl-train/integrations/verifiers/README.md b/skyrl-train/integrations/verifiers/README.md new file mode 100644 index 0000000000..43ea9f66f0 --- /dev/null +++ b/skyrl-train/integrations/verifiers/README.md @@ -0,0 +1,82 @@ +## Guide: Verifiers (Environments Hub) + SkyRL + +This directory holds the workflow to train on Environments Hub environments with SkyRL. + +To start training, follow three simple steps: +1) Install the environment from Environments Hub. +2) Prepare the environment's training and validation datasets. +3) Launch training! + +Start by following the SkyRL [installation instructions](https://skyrl.readthedocs.io/en/latest/getting-started/installation.html), then enter the `skyrl-train` directory: +```bash +cd SkyRL/skyrl-train +``` + +### 1) Install the environment +First, specify your desired environment from the [Environments Hub](https://app.primeintellect.ai/dashboard/environments): +```bash +export ENV_ID="will/wordle" +``` + +Then, install the environment, which adds it to the `uv` project: +```bash +uv run integrations/verifiers/install_environment.py $ENV_ID +``` + +### 2) Prepare the dataset +Next, load the environment's dataset and convert to SkyRL format: +```bash +uv run --isolated --with verifiers \ + python integrations/verifiers/prepare_dataset.py \ + --env_id $ENV_ID +``` + +`prepare_dataset.py` has additional optional parameters: + - `--output_dir`: directory to place datasets (default: `~/data/{env_id}`) + - `--num_train`: number of training samples to generate (-1 for no limit) + - `--num_eval`: number of validation samples to generate (-1 for no limit) + +Notes on dataset generation: +- This script will generate the following Parquet files under `output_dir`: + - `train.parquet` + - `validation.parquet`, if included in the environment +- For issues in loading the dataset, see the Troubleshooting section below. + +### 3) Launch training +Open `run_verifiers.sh`, which specifies the training configuration parameters and is the primary interface for launching training runs. + +Modify the commonly-edited training settings as needed: +```bash +ENV_ID="will/wordle" +DATA_DIR="$HOME/data/$ENV_ID" +NUM_GPUS=1 +LOGGER="wandb" +``` + +Finally, launch your training run: + +```bash +bash integrations/verifiers/run_verifiers.sh +``` + +All training parameters can be modified in `run_verifiers.sh`, such as the model choice (`trainer.policy.model.path`), GRPO group size (`generator.n_samples_per_prompt`), or training batch size (`trainer.train_batch_size`). + +See all available training configuration parameters in `ppo_base_config.yaml`. + + +## Troubleshooting + +For issues with SkyRL or the integration with Verifiers, please [open an Issue](https://github.com/NovaSky-AI/SkyRL/issues/new). + + +### Datasets +Verifiers environments can handle dataset splits in different ways. Some environments require passing a `dataset_split` argument to `load_environment()` (e.g., to specify `train` vs `test`), others implement both `vf_env.load_dataset()` and `vf_env.load_eval_dataset()`. + +The implementation in `prepare_dataset.py` assumes the latter approach to get datasets, which may be incorrect for some environments. Please modify `prepare_dataset.py` as needed to extract and prepare the correct datasets. + + +## TODOs and Limitations +We welcome any contributions to help resolve the remaining tasks! +* Make it easier to specify different Verifiers environments used for training and validation. +* Make it smoother to specify which dataset splits to use (ie, resolve the challenge specified in the `Troubleshooting` section.) +* Consider plumbing Verifiers-specific configuration to the VerifiersGenerator for easy override. For example: `zero_truncated_completions` and `mask_truncated_completions`. diff --git a/skyrl-train/integrations/verifiers/entrypoints/main_verifiers.py b/skyrl-train/integrations/verifiers/entrypoints/main_verifiers.py new file mode 100644 index 0000000000..8f702044e1 --- /dev/null +++ b/skyrl-train/integrations/verifiers/entrypoints/main_verifiers.py @@ -0,0 +1,38 @@ +import hydra +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils import initialize_ray +import ray +from integrations.verifiers.verifiers_generator import VerifiersGenerator +from transformers import PreTrainedTokenizer +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient + + +class VerifiersEntrypoint(BasePPOExp): + def get_generator( + self, cfg: DictConfig, tokenizer: PreTrainedTokenizer, inference_engine_client: InferenceEngineClient + ): + return VerifiersGenerator( + generator_cfg=cfg.generator, + tokenizer=tokenizer, + model_name=cfg.trainer.policy.model.path, + ) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + exp = VerifiersEntrypoint(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # Validate config args. + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/verifiers/install_environment.py b/skyrl-train/integrations/verifiers/install_environment.py new file mode 100644 index 0000000000..6ee20a86ed --- /dev/null +++ b/skyrl-train/integrations/verifiers/install_environment.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# +# Helper that installs the Verifiers environment and adds it to the uv project. +# +# Example: +# uv run integrations/verifiers/install_environment.py will/wordle@0.1.4 +# + +import argparse +import sys +import subprocess + + +def verifiers_env_to_uv_args(spec: str): + org, sep, rest = spec.partition("/") + if not sep: + rest = spec + name, _, version = rest.partition("@") + index_url = f"https://hub.primeintellect.ai/{org}/simple/" + with_req = name if not version else f"{name}=={version}" + return [with_req, "--index", index_url] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Install a Verifiers environment into the current uv project.") + parser.add_argument( + "env_id", + help="Environment ID in the form 'org/name@version' (e.g., will/wordle@0.1.4)", + ) + args = parser.parse_args() + + env_id = args.env_id + uv_args = verifiers_env_to_uv_args(env_id) + + result = subprocess.run(["uv", "add", "--active", *uv_args]) + sys.exit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/integrations/verifiers/prepare_dataset.py b/skyrl-train/integrations/verifiers/prepare_dataset.py new file mode 100644 index 0000000000..42cf6ef4b0 --- /dev/null +++ b/skyrl-train/integrations/verifiers/prepare_dataset.py @@ -0,0 +1,90 @@ +import argparse +import os +from functools import partial +from typing import Any, Dict + +from verifiers import load_environment + + +def extract_env_name(env_id: str) -> str: + """Return only the environment name from strings like 'org/name@version' or 'name@version'.""" + base = env_id.split("/")[-1] + return base.split("@")[0] + + +def build_row(sample: Dict[str, Any], idx: int, data_source: str, env_name: str) -> Dict[str, Any]: + if "prompt" not in sample: + raise ValueError("Example must contain a 'prompt' field") + prompt = sample["prompt"] # Already formatted by the environment as chat messages + + answer = sample.get("answer", "") + info = sample.get("info", None) + task = sample.get("task", "default") + + full_sample = { + "data_source": data_source, + "prompt": prompt, + "example_id": sample.get("example_id", idx), + "verifiers": { + "answer": answer, + "task": task, + "environment": env_name, + }, + } + + if info: + full_sample["verifiers"]["info"] = info + + return full_sample + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate Parquet dataset from a verifiers environment.") + parser.add_argument( + "--env_id", default="primeintellect/wordle", help="Environment identifier to load (e.g., 'wordle')." + ) + parser.add_argument( + "--output_dir", default=None, help="Output directory for Parquet files. Defaults to ~/data/{env_id} in-code." + ) + parser.add_argument( + "--num_train", type=int, default=-1, help="Number of training examples to generate. -1 for no limit." + ) + parser.add_argument( + "--num_eval", type=int, default=-1, help="Number of evaluation examples to generate. -1 for no limit." + ) + + args = parser.parse_args() + + # Resolve output directory + output_dir_name = args.output_dir if args.output_dir else f"~/data/{args.env_id}" + output_dir = os.path.expanduser(output_dir_name) + os.makedirs(output_dir, exist_ok=True) + + # Load verifiers environment + env_name = extract_env_name(args.env_id) + vf_env = load_environment(env_id=env_name) + data_source = f"verifiers/{env_name}" + map_fn = partial(build_row, data_source=data_source, env_name=env_name) + + # Load train dataset + try: + train_ds = vf_env.get_dataset(args.num_train) + except ValueError: + train_ds = None + print(f"WARNING: Environment {args.env_id} does not have a training dataset. Loading the eval dataset only.") + if train_ds: + train_ds = train_ds.map(map_fn, with_indices=True, num_proc=16) + # Drop top-level 'info' column, which often defaults to empty dict and cannot be serialized to parquet. + if "info" in train_ds.column_names: + train_ds = train_ds.remove_columns("info") + train_path = os.path.join(output_dir, "train.parquet") + train_ds.to_parquet(train_path) + + # Load eval dataset + eval_ds = vf_env.get_eval_dataset(args.num_eval) + eval_ds = eval_ds.map(map_fn, with_indices=True, num_proc=16) + # Drop top-level 'info' column, which often defaults to empty dict and cannot be serialized to parquet. + if "info" in eval_ds.column_names: + eval_ds = eval_ds.remove_columns("info") + val_path = os.path.join(output_dir, "validation.parquet") + eval_ds.to_parquet(val_path) diff --git a/skyrl-train/integrations/verifiers/run_verifiers.sh b/skyrl-train/integrations/verifiers/run_verifiers.sh new file mode 100644 index 0000000000..00e67ef94d --- /dev/null +++ b/skyrl-train/integrations/verifiers/run_verifiers.sh @@ -0,0 +1,42 @@ +# Launches SkyRL training on the Verifiers environment. +# +# Example: +# bash integrations/verifiers/run_verifiers.sh +# +set -x + +# Specify environment ID from Environments Hub in form "org/name@version" (e.g., will/wordle@0.1.4) +ENV_ID="will/wordle" +DATA_DIR="$HOME/data/$ENV_ID" +NUM_GPUS=1 +LOGGER="wandb" # change to "console" to print to stdout + +uv run --isolated --with verifiers --extra vllm -m integrations.verifiers.entrypoints.main_verifiers \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS \ + generator.num_inference_engines=$NUM_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + generator.n_samples_per_prompt=5 \ + trainer.epochs=20 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.train_batch_size=128 \ + trainer.policy_mini_batch_size=128 \ + trainer.micro_forward_batch_size_per_gpu=32 \ + trainer.micro_train_batch_size_per_gpu=32 \ + trainer.max_prompt_length=8192 \ + generator.max_input_length=8192 \ + generator.sampling_params.max_generate_length=1024 \ + generator.enable_http_endpoint=true \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + environment.env_class="$ENV_ID" \ + trainer.project_name="verifiers" \ + trainer.run_name="verifiers_test" \ + trainer.ckpt_interval=-1 \ + trainer.ckpt_path="$HOME/ckpts/verifiers_ckpt" + $@ \ No newline at end of file diff --git a/skyrl-train/integrations/verifiers/verifiers_generator.py b/skyrl-train/integrations/verifiers/verifiers_generator.py new file mode 100644 index 0000000000..4515c6d402 --- /dev/null +++ b/skyrl-train/integrations/verifiers/verifiers_generator.py @@ -0,0 +1,115 @@ +from typing import Optional +from skyrl_train.generators.base import GeneratorInterface, GeneratorInput, GeneratorOutput +from omegaconf import DictConfig +from openai import AsyncOpenAI +import httpx +from verifiers import load_environment +from verifiers.types import GenerateOutputs, ProcessedOutputs, RolloutInput +from skyrl_train.generators.utils import get_rollout_metrics + + +class VerifiersGenerator(GeneratorInterface): + def __init__( + self, + generator_cfg: DictConfig, + tokenizer, + model_name: str, + ): + """ + Args: + generator_cfg: DictConfig object containing the generator configuration + tokenizer: tokenizer object for encoding and decoding text + """ + self.generator_cfg = generator_cfg + self.tokenizer = tokenizer + self.model_name = model_name + + assert generator_cfg.enable_http_endpoint, "HTTP endpoint must be enabled for VerifiersGenerator" + self.base_url = f"http://{generator_cfg.http_endpoint_host}:{generator_cfg.http_endpoint_port}/v1" + self.client = self._setup_client(connection_limit=None) # None means unlimited connections + + def _setup_client(self, connection_limit: Optional[int]) -> AsyncOpenAI: + timeout = httpx.Timeout(timeout=600, connect=5.0) + limits = httpx.Limits( + max_connections=connection_limit, # OAI default: 1000 + max_keepalive_connections=connection_limit, # OAI default: 100 + ) + http_client = httpx.AsyncClient(limits=limits, timeout=timeout) + return AsyncOpenAI( + base_url=self.base_url, + api_key="dummy", # Make OAI client happy. + max_retries=10, # OAI default: 2 + http_client=http_client, + ) + + async def generate(self, input_batch: GeneratorInput) -> GeneratorOutput: + assert "env_extras" in input_batch, "Verifiers dataset fields are passed through env_extras" + + # Defaults are based on Verifiers' defaults. + verifiers_dicts = [sample["verifiers"] for sample in input_batch["env_extras"]] + rollout_inputs = [] + for i, item in enumerate(verifiers_dicts): + rollout_inputs.append( + RolloutInput( + prompt=input_batch["prompts"][i], + answer=item.get("answer", ""), + example_id=item["example_id"], + info=item.get("info", {}), + task=item.get("task", "default"), + ) + ) + + # Assumes all training samples correspond to the same Verifiers environment. + # For now, if multiple environments are needed, use Verifiers' EnvGroup abstraction. + environment_id = verifiers_dicts[0]["environment"] + vf_env = load_environment(environment_id) + + # Verifiers requires logprobs from vLLM for post-processing. + sampling_params = input_batch.get("sampling_params", {}).copy() + sampling_params["logprobs"] = True + sampling_params["top_logprobs"] = 1 + sampling_params["extra_body"] = { + "return_tokens_as_token_ids": True, + } + + # Clean the sampling params for Verifiers' generate. + extra_body_keys = [ + "min_tokens", + "skip_special_tokens", + "include_stop_str_in_output", + "top_k", + "min_p", + "repetition_penalty", + ] + for key in extra_body_keys: + if key in sampling_params: + sampling_params["extra_body"][key] = sampling_params[key] + del sampling_params[key] + + # Generate the trajectories. + generate_outputs: GenerateOutputs = await vf_env.generate( + inputs=rollout_inputs, + client=self.client, + model=self.model_name, + sampling_args=sampling_params, + ) + + processed_outputs: ProcessedOutputs = vf_env.process_env_results_vllm( + prompts=generate_outputs.prompt, + completions=generate_outputs.completion, + states=generate_outputs.state, + rewards=generate_outputs.reward, + processing_class=self.tokenizer, + max_seq_len=self.generator_cfg.max_input_length + self.generator_cfg.sampling_params.max_generate_length, + mask_env_responses=True, + ) + + # Convert output to SkyRL format. + return GeneratorOutput( + prompt_token_ids=processed_outputs.prompt_ids, + response_ids=processed_outputs.completion_ids, + rewards=processed_outputs.rewards, + loss_masks=processed_outputs.completion_mask, + rollout_logprobs=processed_outputs.completion_logprobs, + rollout_metrics=get_rollout_metrics(processed_outputs.completion_ids, processed_outputs.rewards), + ) diff --git a/skyrl-train/pyproject.toml b/skyrl-train/pyproject.toml new file mode 100644 index 0000000000..5495bdd102 --- /dev/null +++ b/skyrl-train/pyproject.toml @@ -0,0 +1,264 @@ +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "skyrl-train" +version = "0.3.1" +description = "skyrl-train" +authors = [ + {name = "NovaSkyAI", email = "novasky.berkeley@gmail.com"} +] +license = {text = "MIT"} +readme = "README.md" +requires-python = "==3.12.*" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +dependencies = [ + "loguru", + "tqdm", + "ninja", + "tensorboard", + "func_timeout", + "transformers>=4.51.0", + "hydra-core==1.3.2", + "accelerate", + "torchdata", + "omegaconf", + "ray==2.51.1", + "peft", + "debugpy==1.8.0", + "hf_transfer", + "wandb", + "datasets==4.0.0", + "tensordict", + "jaxtyping", + "skyrl-gym", + "flash-attn", + "causal-conv1d>=1.6.0", + "flash-linear-attention", + "polars", + "s3fs", + "fastapi", + "uvicorn", + "pybind11", + "setuptools", +] + +[tool.uv] +required-version = ">=0.8.10" +conflicts = [ + [ + { extra = "vllm" }, + { extra = "sglang" }, + ], + [ + { extra = "vllm" }, + { extra = "flashrl" }, + { extra = "sglang" }, + ], + [ + { extra = "flashrl" }, + { extra = "miniswe" }, + ], + [ + { extra = "mcore" }, + { extra = "vllm" }, + { extra = "sglang" }, + { extra = "flashrl" }, + ] +] +no-build-isolation-package = [ + "transformer-engine-torch", + "transformer-engine", + "nv-grouped-gemm", +] +override-dependencies = [ + "nvidia-resiliency-ext; sys_platform == 'never'", + "mamba-ssm; sys_platform == 'never'", + "transformer-engine[pytorch]==2.10.0", + "megatron-core==0.15.0" +] +[tool.uv.extra-build-dependencies] +flash-attn = [{requirement = "torch", match-runtime = true}] +transformer-engine = [{requirement = "torch", match-runtime = true}, "build_tools", "ninja"] +transformer-engine-torch = [{requirement = "torch", match-runtime = true}, "build_tools", "ninja"] + +[tool.uv.extra-build-variables] +flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE"} + +[tool.uv.sources] +skyrl-gym = { path = "./skyrl-gym" , editable = true } +torch = { index = "pytorch-cu128" } +torchvision = { index = "pytorch-cu128" } +# We use `flashinfer-jit-cache` to avoid slow JIT compilation on first run. +# Different inference engines may pin different compatible flashinfer versions, so we provide the option to pin different versions for vllm/sglang +flashinfer-jit-cache = { index = "flashinfer-cu128", marker = "extra == 'mcore'" } +flashinfer-python = [ + { url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl", marker = "extra == 'sglang' and extra != 'mcore' and extra != 'vllm'" } +] +megatron-bridge = {git = "https://github.com/NVIDIA-NeMo/Megatron-Bridge", rev = "04e370eedf8cc44a812189a19f2171d90555c07a"} + + +[project.optional-dependencies] +dev = [ + "ruff==0.11.9", + "black==24.10.0", + "pytest>=6.2.5", + "pytest-asyncio", + "pre-commit", + "litellm", +] +docs = [ + "sphinx>=7.0.0", + "sphinx-rtd-theme>=2.0.0", + "sphinx-autodoc-typehints>=1.25.0", + "myst-parser>=2.0.0", + "sphinx-copybutton>=0.5.0", + "sphinx-autobuild>=2021.3.14" +] +# TODO(tgriggs): Add `sandboxes` here once available as a package. +sandboxes = [ + "litellm[proxy]>=1.67.5", +] +vllm = [ + "vllm==0.17.0", + "flash-attn==2.8.3", + "torch==2.10.0", + "flashinfer-python", + "torchvision" +] +sglang = [ + "sglang[srt,openai,torch_memory_saver]==0.4.8.post1", # 0.4.9.post1 causes non-colocate weight broadcast to hang + "flashinfer-python", + "flash-attn==2.8.3", + "torch==2.7.1", + "torchvision", +] +mcore = [ + "transformer-engine[pytorch]==2.10.0", + "flash-attn==2.8.1", + "vllm==0.13.0", + "torch==2.9.0", + "flashinfer-python==0.5.3", + "torchvision", + "megatron-bridge", + "megatron-core==0.15.0", + "flashinfer-jit-cache==0.5.3", + "nvidia-modelopt", +] +flashrl = [ + # NOTE: Custom vLLM wheel must be installed separately. + # See examples/flash_rl/README.md for installation instructions. + "flash-attn==2.8.3", + "torch==2.7.0", + "flashinfer-python", + "torchvision", +] +miniswe = [ + # NOTE (sumanthrh): Needs to be a commit after https://github.com/SWE-agent/mini-swe-agent/commit/4f5d445e99d13b5482478c23508bf2fbf7c0670c + "mini-swe-agent>=1.12.0", + "litellm", +] + + +[[tool.uv.index]] +name = "pytorch-cu128" +url = "https://download.pytorch.org/whl/cu128" +explicit = true + +[[tool.uv.index]] +name = "flashinfer-cu128" +url = "https://flashinfer.ai/whl/cu128" +explicit = true + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["skyrl_train*"] + +[tool.setuptools.dynamic] +version = {attr = "skyrl_train.__version__"} +readme = {file = ["README.md"]} + +# Ship default config files in the wheel +[tool.setuptools.package-data] +"skyrl_train" = ["config/**"] + +[tool.pytest.ini_options] +addopts = "-v -s" +testpaths = [ + "tests", +] + + + +[tool.isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 120 +known_third_party = "wandb" + +[tool.black] +line-length = 120 +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.flake8] +max-line-length = 120 +max-doc-length = 120 +extend-ignore = [ + # Default ignored errors by flake8 + "E121", "E123", "E126", "E226", "E24", "E704", + # F401 module imported but unused + "F401", + # E203 whitespace before ':' (conflict with black) + "E203", + # E231 missing whitespace after ',' (conflict with black) + "E231", + # E501 line too long (conflict with black) + "E501", + # E741 do not use variables named 'l', 'O', or 'I' + "E741", + # W503 line break before binary operator (conflict with black) + "W503", + # W504 line break after binary operator (conflict with black) + "W504", + # W505 doc line too long (conflict with black) + "W505", + # W605 invalid escape sequence 'x' (conflict with latex within docs) + "W605", +] + +[tool.ruff.lint] +ignore = [ + "F722" # Syntax error in annotation - ignored because this doesn't play well with jaxtyping +] + +[pytest] +markers = [ + "vllm", + "sglang", + "integrations", + "megatron", +] diff --git a/skyrl-train/scripts/README.md b/skyrl-train/scripts/README.md new file mode 100644 index 0000000000..2114d966b2 --- /dev/null +++ b/skyrl-train/scripts/README.md @@ -0,0 +1,106 @@ +# Fleet Training Scripts + +Shared scripts for Fleet GRPO training via SkyPilot. These replace the inline shell in each task YAML with reusable, multi-node-aware scripts. + +## Scripts + +| Script | Purpose | +|--------|---------| +| `fleet-common-setup.sh` | Env validation, Python venv, pip deps, OpenEnv, S3 dataset download, prepare_dataset | +| `fleet-common-run.sh` | Ray cluster setup (multi-node), wandb login, training launch with common hydra overrides | +| `fleet-qwen35-extra-setup.sh` | Qwen3.5-specific: transformers nightly, flash-attn wheel, CUDA toolkit, writes `$HOME/.cuda_env` | + +## Usage + +### Single-node (e.g., 9B) + +```yaml +setup: | + bash skyrl-train/scripts/fleet-common-setup.sh \ + --openenv-branch deniz/fleet_client \ + --extra-setup skyrl-train/scripts/fleet-qwen35-extra-setup.sh + +run: | + bash skyrl-train/scripts/fleet-common-run.sh \ + --use-python-direct --cuda-env "$HOME/.cuda_env" \ + --set-ulimit --no-pytorch-alloc-conf -- \ + trainer.policy.model.path="Qwen/Qwen3.5-9B" \ + trainer.epochs=20 ... +``` + +Data root is auto-detected: `/workspace` if writable (RunPod), otherwise `$HOME` (GCP, Lambda, etc.). Override with `--data-root DIR`. + +### Multi-node (e.g., 35B on 2 nodes) + +Set `num_nodes: 2` in the YAML. The run script auto-detects `SKYPILOT_NODE_RANK`: +- **Rank 0 (head):** starts Ray head, calculates `TOTAL_GPUS = gpus_per_node * num_nodes`, launches training +- **Rank > 0 (workers):** joins Ray cluster, sleeps + +No changes needed in the run block — multi-node works automatically. + +## Setup Script Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--openenv-branch BRANCH` | `deniz/fleet_client` | OpenEnv git ref to install | +| `--extra-setup SCRIPT` | *(none)* | Script to source after `uv sync` (model-specific deps) | +| `--data-root DIR` | auto-detect | Root for dataset download (`DIR/data/fleet/`). Uses `/workspace` if writable, else `$HOME` | +| `--skip-uv-isolated` | `false` | Flag for configs that use `python` directly | + +## Run Script Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--data-root DIR` | auto-detect | Root for dataset files. Uses `/workspace` if writable, else `$HOME` | +| `--ckpt-root DIR` | same as data-root | Root for checkpoints, exports, and tmp dir | +| `--use-python-direct` | `false` | Use `python -m` instead of `uv run --isolated` | +| `--cuda-env FILE` | *(none)* | Source this file for CUDA_HOME (e.g., `$HOME/.cuda_env`) | +| `--set-ulimit` | `false` | Set `ulimit -n 65536` (needed for Ray+vLLM) | +| `--no-pytorch-alloc-conf` | `false` | Skip setting `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` | +| `--nccl-heartbeat SEC` | *(none)* | Set `TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC` (useful for multi-node) | +| `--` | | Everything after this is passed as hydra overrides | + +## Common Hydra Overrides (auto-injected by run script) + +These are always set by `fleet-common-run.sh` — don't repeat them in the YAML: + +- `data.train_data`, `data.val_data` (from `--data-root`) +- `environment.env_class=fleet_task` +- `environment.skyrl_gym.fleet_task.tasks_file` +- `trainer.placement.colocate_all=true`, `trainer.strategy=fsdp2` +- `trainer.placement.{policy,ref}_num_gpus_per_node`, `{policy,ref}_num_nodes` +- `generator.num_inference_engines=$TOTAL_GPUS` + +## Environment Variables + +Required in SkyPilot YAML `envs:`: + +| Variable | Description | +|----------|-------------| +| `FLEET_API_KEY` | Fleet API key | +| `WANDB_API_KEY` | Weights & Biases key | +| `AWS_ACCESS_KEY_ID` | S3 dataset download | +| `AWS_SECRET_ACCESS_KEY` | S3 dataset download | +| `MODALITY` | `tool_use` or `computer_use` | +| `DATA_VERSION` | S3 dataset version (e.g., `v54`) | +| `S3_DATASET_BUCKET` | Dataset bucket name | +| `INFERENCE_BACKEND` | `vllm` or `sglang` | + +## Adding a New Model Config + +1. Copy an existing YAML (e.g., `openenv-fleet-grpo-qwen3_5-9b.yaml`) +2. If the model needs special deps, create a new `fleet--extra-setup.sh` +3. Update `setup:` to reference your extra-setup script +4. Update `run:` with model-specific hydra overrides after `--` +5. Add the task name to `.github/workflows/openenv-fleet-train.yaml` + +## GCP Support + +GCP requires the correct GPU image for H200/B200 (default SkyPilot image has driver 535, needs 550+): + +```yaml +- accelerators: H200:8 + cloud: gcp + use_spot: true + image_id: projects/deeplearning-platform-release/global/images/common-cu128-ubuntu-2204-nvidia-570-v20260305 +``` diff --git a/skyrl-train/scripts/fill_results_from_wandb.py b/skyrl-train/scripts/fill_results_from_wandb.py new file mode 100755 index 0000000000..c4a90d4088 --- /dev/null +++ b/skyrl-train/scripts/fill_results_from_wandb.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +""" +Fill results tables in project.md from WandB run data. + +Usage: + python fill_results_from_wandb.py --run-id [--entity thefleet] [--project fleet-task-grpo] + +Environment: + WANDB_API_KEY: WandB API key + +Example: + WANDB_API_KEY= python fill_results_from_wandb.py --run-id mk6nr5ij +""" + +import argparse +import os +import wandb + + +def fetch_run_metrics(entity: str, project: str, run_id: str) -> dict: + """Fetch metrics from a WandB run.""" + api = wandb.Api() + run = api.run(f"{entity}/{project}/{run_id}") + + metrics = { + "summary": dict(run.summary), + "config": dict(run.config), + "name": run.name, + "state": run.state, + } + + # Get history for time series data + history = run.history() + metrics["history"] = history + + return metrics + + +def extract_env_metrics(metrics: dict) -> dict: + """Extract per-environment metrics from WandB data.""" + summary = metrics["summary"] + results = {} + + # Look for environment-specific metrics + envs = [ + "outlook", + "github", + "booking", + "reddit", + "ticketmaster", + "fira", + "zillow", + "hubspot", + "google-maps", + "dropbox", + ] + + for env in envs: + env_data = {} + + # Look for pass@k metrics + for key, value in summary.items(): + if env in key.lower(): + env_data[key] = value + + # Look for avg_turns + turns_key = f"environment/fleet_task/{env}/avg_turns" + if turns_key in summary: + env_data["avg_turns"] = summary[turns_key] + + # Look for reward/score + for pattern in [f"{env}/reward", f"{env}/score", f"{env}/pass"]: + for key, value in summary.items(): + if pattern in key.lower(): + env_data[key] = value + + if env_data: + results[env] = env_data + + return results + + +def print_results_table(metrics: dict): + """Print results in a format suitable for markdown tables.""" + summary = metrics["summary"] + + print("\n=== Run Summary ===") + print(f"Run Name: {metrics['name']}") + print(f"State: {metrics['state']}") + + print("\n=== All Metrics ===") + for key in sorted(summary.keys()): + if not key.startswith("_"): + print(f"{key}: {summary[key]}") + + print("\n=== Per-Environment Metrics ===") + env_metrics = extract_env_metrics(metrics) + for env, data in env_metrics.items(): + print(f"\n{env}:") + for key, value in data.items(): + print(f" {key}: {value}") + + +def main(): + parser = argparse.ArgumentParser(description="Fill results tables from WandB") + parser.add_argument("--run-id", required=True, help="WandB run ID") + parser.add_argument("--entity", default="thefleet", help="WandB entity") + parser.add_argument("--project", default="fleet-task-grpo", help="WandB project") + args = parser.parse_args() + + if not os.environ.get("WANDB_API_KEY"): + print("Error: WANDB_API_KEY environment variable required") + return 1 + + print(f"Fetching metrics from {args.entity}/{args.project}/{args.run_id}...") + metrics = fetch_run_metrics(args.entity, args.project, args.run_id) + print_results_table(metrics) + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/skyrl-train/scripts/fleet-common-run.sh b/skyrl-train/scripts/fleet-common-run.sh new file mode 100755 index 0000000000..ce297ddc50 --- /dev/null +++ b/skyrl-train/scripts/fleet-common-run.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# Fleet shared run: Ray cluster setup (multi-node aware) + training launch +# +# Usage (from SkyPilot YAML run block): +# bash skyrl-train/scripts/fleet-common-run.sh \ +# --use-python-direct --cuda-env "$HOME/.cuda_env" \ +# --set-ulimit --no-pytorch-alloc-conf -- \ +# trainer.policy.model.path="Qwen/Qwen3.5-9B" \ +# trainer.epochs=20 ... +# +# Multi-node: +# Rank 0 (head): starts Ray head, launches training +# Rank >0 (workers): joins Ray cluster, sleeps +# +# Required env vars: WANDB_API_KEY, MODALITY, INFERENCE_BACKEND, +# SKYPILOT_NUM_GPUS_PER_NODE, SKYPILOT_NODE_IPS +# Optional env vars: SKYPILOT_NUM_NODES, SKYPILOT_NODE_RANK +set -euo pipefail + +# Defaults +DATA_ROOT="" +CKPT_ROOT="" +USE_PYTHON_DIRECT=false +CUDA_ENV="" +SET_ULIMIT=false +NO_PYTORCH_ALLOC_CONF=false +NCCL_HEARTBEAT="" +HYDRA_OVERRIDES=() + +# Parse args +while [[ $# -gt 0 ]]; do + case "$1" in + --data-root) DATA_ROOT="$2"; shift 2 ;; + --ckpt-root) CKPT_ROOT="$2"; shift 2 ;; + --use-python-direct) USE_PYTHON_DIRECT=true; shift ;; + --cuda-env) CUDA_ENV="$2"; shift 2 ;; + --set-ulimit) SET_ULIMIT=true; shift ;; + --no-pytorch-alloc-conf) NO_PYTORCH_ALLOC_CONF=true; shift ;; + --nccl-heartbeat) NCCL_HEARTBEAT="$2"; shift 2 ;; + --) shift; HYDRA_OVERRIDES=("$@"); break ;; + *) echo "ERROR: Unknown arg: $1"; exit 1 ;; + esac +done + +# Auto-detect data/ckpt root: /workspace if writable (RunPod), else $HOME (GCP, Lambda, etc.) +if [ -z "$DATA_ROOT" ]; then + if [ -d "/workspace" ] && [ -w "/workspace" ]; then + DATA_ROOT="/workspace" + else + DATA_ROOT="$HOME" + fi +fi +if [ -z "$CKPT_ROOT" ]; then + CKPT_ROOT="$DATA_ROOT" +fi + +echo "=== Fleet Common Run ===" +echo "Data root: $DATA_ROOT" +echo "Ckpt root: $CKPT_ROOT" + +cd skyrl-train +source .venv/bin/activate + +# --- Optional settings --- +if [ "$SET_ULIMIT" = true ]; then + ulimit -n 65536 +fi + +if [ -n "$CUDA_ENV" ]; then + source "$CUDA_ENV" 2>/dev/null || true +fi + +if [ "$NO_PYTORCH_ALLOC_CONF" = false ]; then + export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +fi + +if [ -n "$NCCL_HEARTBEAT" ]; then + export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC="$NCCL_HEARTBEAT" +fi + +TMP_DIR="${CKPT_ROOT}/skyrl-tmp" +mkdir -p "$TMP_DIR" +export TMPDIR="$TMP_DIR" + +TASKS_FILE="${DATA_ROOT}/data/fleet/tasks_${MODALITY}.json" +DATA_DIR="${DATA_ROOT}/data/fleet/${MODALITY}" + +# --- System diagnostics --- +echo "=== System Diagnostics ===" +free -h +nvidia-smi --query-gpu=name,driver_version,memory.total,memory.free --format=csv 2>/dev/null || true +echo "--- /dev/shm ---" +df -h /dev/shm 2>/dev/null || echo "/dev/shm not mounted" +ls -la /dev/shm/ 2>/dev/null | head -5 || true +echo "--- GPU Topology ---" +nvidia-smi topo -m 2>/dev/null || true +echo "--- cgroup memory limits ---" +cat /sys/fs/cgroup/memory.max 2>/dev/null || cat /sys/fs/cgroup/memory/memory.limit_in_bytes 2>/dev/null || echo "No cgroup memory limit found" +echo "--- ulimits ---" +ulimit -a 2>/dev/null || true +echo "--- NCCL env vars ---" +env | grep -i NCCL || echo "No NCCL env vars set" +echo "--- kernel overcommit ---" +cat /proc/sys/vm/overcommit_memory 2>/dev/null || true +echo "=== End Diagnostics ===" + +# --- wandb login --- +python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + +# --- Fabric Manager check (NVSwitch GPUs: B200, H200 SXM) --- +# On non-GCP clouds (RunPod, Lambda, etc.), Fabric Manager is required for NVLink +# P2P on NVSwitch systems. Without it, dist.broadcast() in FSDP causes SIGKILL. +# +# On GCP, NVSwitch is managed at the HOST level — the guest VM does not have +# NVSwitch devices, so FM reports "NV_WARN_NOTHING_TO_DO" and cannot start. +# This is EXPECTED. NVLink P2P works through GCP's host-managed fabric without FM. +# GCP also provides a custom NCCL shim (gIB) that manages all NCCL configuration. +# Do NOT set NCCL_P2P_DISABLE, NCCL_NVLS_ENABLE, or NCCL_CUMEM_ENABLE on GCP — +# the shim's "Guest Config Checker" expects these to be unset. +ON_GCP=false +if [ -d "/usr/local/gib" ]; then + ON_GCP=true +elif [ -f "/sys/class/dmi/id/product_name" ] && grep -qi "google" /sys/class/dmi/id/product_name 2>/dev/null; then + ON_GCP=true +fi + +FM_STATUS=$(systemctl is-active nvidia-fabricmanager 2>/dev/null || echo "unknown") +echo "Fabric Manager status: $FM_STATUS" +echo "On GCP: $ON_GCP" + +if [ "$ON_GCP" = true ]; then + echo "GCP detected — skipping Fabric Manager restart (host manages NVSwitch)" + + # GCP's deep learning images install /etc/profile.d/nccl_env.sh which auto-sources + # /usr/local/gib/scripts/set_nccl_env.sh and adds /usr/local/gib/lib64 to LD_LIBRARY_PATH. + # This sets NCCL_NET=gIB, forcing the gIB network plugin for RDMA/InfiniBand. + # + # Problem: gIB requires RDMA hardware (ConnectX NICs + multiple GPUDirect VPC networks). + # SkyPilot provisions VMs with a single management NIC — no RDMA networking. + # When NCCL_NET=gIB is forced but gIB can't init, NCCL fails with + # "Failed to initialize any NET plugin" → SIGKILL during dist.broadcast(). + # + # Fix: check for RDMA devices. If absent, strip gIB so NCCL falls back to + # NVLink P2P for intra-node communication. Multi-node uses GKE with RDMA. + if [ -d "/sys/class/infiniband" ] && [ "$(ls /sys/class/infiniband/ 2>/dev/null)" ]; then + echo "RDMA devices found — keeping gIB for GPUDirect RDMA" + else + echo "No RDMA devices — disabling gIB" + # Remove gIB from LD_LIBRARY_PATH (set by /etc/profile.d/nccl_env.sh) + export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | sed 's|/usr/local/gib/lib64:||g; s|:/usr/local/gib/lib64||g; s|/usr/local/gib/lib64||g') + # Unset NCCL_NET=gIB so NCCL can fall back to NVLink P2P + unset NCCL_NET + # Clear gIB-specific vars set by set_nccl_env.sh + unset NCCL_CROSS_NIC NCCL_NET_GDR_LEVEL NCCL_P2P_NET_CHUNKSIZE NCCL_NVLS_CHUNKSIZE + unset NCCL_IB_ADAPTIVE_ROUTING NCCL_IB_QPS_PER_CONNECTION NCCL_IB_TC NCCL_IB_FIFO_TC + unset NCCL_TUNER_CONFIG_PATH + echo "Cleared gIB NCCL env vars. Using NVLink P2P (intra-node)." + fi + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + echo "NCCL vars:" + env | grep -i NCCL || echo " (none)" + + # Ensure /dev/shm is large enough for NCCL IPC (some GCP images have small default) + SHM_SIZE=$(df --output=size /dev/shm 2>/dev/null | tail -1 | tr -d ' ') + echo "Current /dev/shm size: ${SHM_SIZE}K" + if [ -n "$SHM_SIZE" ] && [ "$SHM_SIZE" -lt 16777216 ]; then + echo "WARNING: /dev/shm is only ${SHM_SIZE}K — remounting to 16G for NCCL" + sudo mount -o remount,size=16G /dev/shm 2>&1 || echo "Failed to remount /dev/shm" + df -h /dev/shm + fi +elif [ "$FM_STATUS" != "active" ]; then + echo "WARNING: Fabric Manager not active. Attempting restart..." + sudo nvidia-smi -pm 1 2>&1 || true + sudo systemctl stop nvidia-fabricmanager 2>&1 || true + sleep 1 + sudo systemctl start nvidia-fabricmanager 2>&1 || true + sleep 5 + FM_STATUS=$(systemctl is-active nvidia-fabricmanager 2>/dev/null || echo "unknown") + echo "Fabric Manager status after restart: $FM_STATUS" + if [ "$FM_STATUS" != "active" ]; then + echo "=== WARNING: Fabric Manager failed to start ===" + echo "Training may fail if this system has NVSwitch GPUs." + sudo journalctl -u nvidia-fabricmanager --no-pager -n 10 2>&1 || true + fi +fi + +# --- Ray cluster setup (multi-node aware) --- +export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook +export RAY_object_store_memory=10000000000 +# Disable Ray's memory monitor to prevent spurious worker kills +export RAY_DISABLE_MEMORY_MONITOR=1 +# NOTE: On GCP VMs without RDMA, gIB NCCL vars are stripped above. +# On GKE with RDMA, gIB is preserved for inter-node GPUDirect. + +read -r head_ip _ <<< "$SKYPILOT_NODE_IPS" + +wait_for_ray() { + local address=$1 + for _ in $(seq 1 24); do + if ray status --address "$address" >/dev/null 2>&1; then + return 0 + fi + sleep 5 + done + echo "ERROR: Ray cluster at $address failed to become ready" >&2 + return 1 +} + +if [ "${SKYPILOT_NODE_RANK:-0}" = "0" ]; then + # === Head node: start Ray head + launch training === + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + wait_for_ray 127.0.0.1:6479 + + TOTAL_GPUS=$((SKYPILOT_NUM_GPUS_PER_NODE * ${SKYPILOT_NUM_NODES:-1})) + export TOTAL_GPUS + echo "=== Head node: $TOTAL_GPUS GPUs across ${SKYPILOT_NUM_NODES:-1} node(s) ===" + + # Build training command + CMD_ARGS=() + if [ "$USE_PYTHON_DIRECT" = true ]; then + CMD_ARGS=(python -m integrations.fleet.entrypoints.main_fleet) + else + CMD_ARGS=(uv run --isolated --extra "$INFERENCE_BACKEND" -m integrations.fleet.entrypoints.main_fleet) + fi + + # Common hydra overrides (data paths, placement, strategy, checkpoints) + CMD_ARGS+=( + "data.train_data=['${DATA_DIR}/train.parquet']" + "data.val_data=['${DATA_DIR}/validation.parquet']" + environment.env_class=fleet_task + "environment.skyrl_gym.fleet_task.tasks_file=$TASKS_FILE" + trainer.placement.colocate_all=true + trainer.strategy=fsdp2 + "trainer.placement.policy_num_gpus_per_node=$SKYPILOT_NUM_GPUS_PER_NODE" + "trainer.placement.ref_num_gpus_per_node=$SKYPILOT_NUM_GPUS_PER_NODE" + "trainer.placement.policy_num_nodes=${SKYPILOT_NUM_NODES:-1}" + "trainer.placement.ref_num_nodes=${SKYPILOT_NUM_NODES:-1}" + "generator.num_inference_engines=$TOTAL_GPUS" + "trainer.ckpt_path=${CKPT_ROOT}/ckpts" + "trainer.export_path=${CKPT_ROOT}/exports" + ) + + # Append model-specific hydra overrides (passed after --) + if [ ${#HYDRA_OVERRIDES[@]} -gt 0 ]; then + CMD_ARGS+=("${HYDRA_OVERRIDES[@]}") + fi + + export HYDRA_FULL_ERROR=1 + echo "=== Launching Training ===" + set +e + "${CMD_ARGS[@]}" + EXIT_CODE=$? + set -e + + if [ $EXIT_CODE -ne 0 ]; then + echo "=== Training failed (exit code $EXIT_CODE) ===" + echo "--- dmesg (last 50 lines, unfiltered) ---" + sudo dmesg -T 2>/dev/null | tail -50 || true + echo "--- dmesg (OOM/kill/segfault) ---" + sudo dmesg -T 2>/dev/null | grep -iE "oom|kill|out of memory|segfault|sigsegv|general protection|cgroup" | tail -20 || true + echo "--- memory ---" + free -h + echo "--- GPU memory ---" + nvidia-smi --query-gpu=memory.used,memory.free --format=csv 2>/dev/null || true + echo "--- /dev/shm after crash ---" + df -h /dev/shm 2>/dev/null || true + echo "--- cgroup memory events ---" + cat /sys/fs/cgroup/memory.events 2>/dev/null || cat /sys/fs/cgroup/memory/memory.oom_control 2>/dev/null || true + echo "--- Ray worker logs (last errors) ---" + grep -r "SIGKILL\|SIGABRT\|SIGSEGV\|SYSTEM_ERROR\|RuntimeError\|NCCL" /tmp/ray/session_latest/logs/ 2>/dev/null | tail -30 || true + exit $EXIT_CODE + fi + +else + # === Worker node: join Ray cluster and wait === + echo "=== Worker node (rank ${SKYPILOT_NODE_RANK}), joining Ray cluster at $head_ip:6479 ===" + if ! ray status --address "$head_ip:6479" >/dev/null 2>&1; then + ray start --address "$head_ip:6479" --disable-usage-stats + fi + wait_for_ray "$head_ip:6479" + echo "Worker node joined. Sleeping..." + sleep infinity +fi diff --git a/skyrl-train/scripts/fleet-common-setup.sh b/skyrl-train/scripts/fleet-common-setup.sh new file mode 100755 index 0000000000..7e251bad34 --- /dev/null +++ b/skyrl-train/scripts/fleet-common-setup.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Fleet shared setup: env validation, venv, dependencies, OpenEnv, dataset download +# +# Usage (from SkyPilot YAML setup block): +# bash skyrl-train/scripts/fleet-common-setup.sh \ +# --openenv-branch deniz/fleet_client \ +# --extra-setup skyrl-train/scripts/fleet-qwen35-extra-setup.sh +# +# Required env vars: FLEET_API_KEY, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, +# MODALITY, DATA_VERSION, S3_DATASET_BUCKET +# Optional env vars: ENV_KEYS, DIFFICULTY +set -euo pipefail + +# Defaults +OPENENV_BRANCH="deniz/fleet_client" +EXTRA_SETUP="" +DATA_ROOT="" +SKIP_UV_ISOLATED=false + +# Parse args +while [[ $# -gt 0 ]]; do + case "$1" in + --openenv-branch) OPENENV_BRANCH="$2"; shift 2 ;; + --extra-setup) EXTRA_SETUP="$2"; shift 2 ;; + --data-root) DATA_ROOT="$2"; shift 2 ;; + --skip-uv-isolated) SKIP_UV_ISOLATED=true; shift ;; + *) echo "ERROR: Unknown arg: $1"; exit 1 ;; + esac +done + +# Auto-detect data root: /workspace if writable (RunPod), else $HOME (GCP, Lambda, etc.) +if [ -z "$DATA_ROOT" ]; then + if [ -d "/workspace" ] && [ -w "/workspace" ]; then + DATA_ROOT="/workspace" + else + DATA_ROOT="$HOME" + fi +fi + +# Resolve extra-setup path to absolute before cd (it's relative to repo root) +if [ -n "$EXTRA_SETUP" ]; then + EXTRA_SETUP="$(cd "$(dirname "$EXTRA_SETUP")" && pwd)/$(basename "$EXTRA_SETUP")" +fi + +cd skyrl-train + +echo "=== Fleet Common Setup ===" +echo "OpenEnv branch: $OPENENV_BRANCH" +echo "Data root: $DATA_ROOT" +echo "Extra setup: ${EXTRA_SETUP:-none}" + +# --- Environment validation --- +echo "Validating environment variables..." +if [ -z "${FLEET_API_KEY:-}" ]; then + echo "ERROR: FLEET_API_KEY is required"; exit 1 +fi +if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download"; exit 1 +fi +if [ "${MODALITY:-}" != "tool_use" ] && [ "${MODALITY:-}" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: ${MODALITY:-unset}"; exit 1 +fi +echo "Environment validation passed" + +# --- Fix Ray binary permissions (some cloud images strip +x) --- +for f in .venv/bin/ray .venv/lib/python*/site-packages/ray/core/src/ray/raylet/raylet; do + [ -f "$f" ] && chmod +x "$f" 2>/dev/null || true +done + +# --- System dependencies (GCP images may lack build tools) --- +if ! command -v c++ &>/dev/null; then + echo "Installing build-essential (c++ compiler required for causal-conv1d)..." + sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends build-essential +fi + +# --- Python environment --- +uv venv --python 3.12 --seed +source .venv/bin/activate +# vLLM 0.17.0 has native Qwen3.5 support (GDN via torch.ops.vllm.gdn_attention_core), +# FlashAttention 4, and PyTorch 2.10.0 +uv sync --extra vllm +uv pip install wandb boto3 awscli +uv pip install "litellm>=1.75.5" fleet-python logfire "mcp>=1.0.0" + +# --- Extra setup hook (model-specific dependencies) --- +if [ -n "$EXTRA_SETUP" ]; then + echo "Running extra setup: $EXTRA_SETUP" + source "$EXTRA_SETUP" +fi + +# --- OpenEnv (force reinstall for latest changes) --- +uv pip install --force-reinstall --no-cache-dir --no-deps "git+https://github.com/fleet-ai/OpenEnv.git@${OPENENV_BRANCH}" + +# --- Dataset download --- +mkdir -p "${DATA_ROOT}/data/fleet" +TASKS_FILE="${DATA_ROOT}/data/fleet/tasks_${MODALITY}.json" +S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" +echo "Downloading dataset from $S3_PATH..." +aws s3 cp "$S3_PATH" "$TASKS_FILE" +TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") +echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + +# --- Prepare dataset (parquet files) --- +DATA_DIR="${DATA_ROOT}/data/fleet/${MODALITY}" +PREPARE_CMD="python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" +[ -n "${ENV_KEYS:-}" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" +[ -n "${DIFFICULTY:-}" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" +eval "$PREPARE_CMD" + +echo "=== Fleet Common Setup Complete ===" diff --git a/skyrl-train/scripts/fleet-qwen35-extra-setup.sh b/skyrl-train/scripts/fleet-qwen35-extra-setup.sh new file mode 100755 index 0000000000..d1c80cfafe --- /dev/null +++ b/skyrl-train/scripts/fleet-qwen35-extra-setup.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Qwen3.5-specific dependencies (sourced by fleet-common-setup.sh via --extra-setup) +# +# Installs: transformers upgrade, flash-attn 2.8.3 wheel, CUDA toolkit (nvcc) +# Writes: $HOME/.cuda_env (sourced at run time for FlashInfer JIT) + +# Upgrade transformers to 5.3.0 for Qwen3.5-MoE (model_type=qwen3_5_moe). +# - Qwen3.5 launched Feb 2026; all 4.x releases predate it. +# - 5.1.0 doesn't register qwen3_5_moe in AUTO_CONFIG_MAPPING. +# - 5.3.0 is the first stable release with full qwen3_5_moe support. +# - Do NOT install from git main (renamed layer_type_validation, breaks vLLM 0.17). +uv pip install -U "transformers==5.3.0" + +# flash-attn 2.8.3 prebuilt wheel for torch 2.10 + CUDA 12 (training forward/backward) +uv pip install "https://github.com/lesj0610/flash-attention/releases/download/v2.8.3-cu12-torch2.10-cp312/flash_attn-2.8.3%2Bcu12torch2.10cxx11abiTRUE-cp312-cp312-linux_x86_64.whl" + +python -c "import torch; import torchvision; print(f'torch={torch.__version__}, torchvision={torchvision.__version__}')" + +# --- CUDA toolkit for FlashInfer JIT (GatedDeltaNet kernels) --- +# pip CUDA packages are incomplete (missing nv/target headers); use NVIDIA apt repo instead +CUDA_HOME="" +for d in /usr/local/cuda /usr/local/cuda-12.8 /usr/local/cuda-12.6 /usr/local/cuda-12.4; do + if [ -x "$d/bin/nvcc" ]; then + CUDA_HOME="$d" + break + fi +done +if [ -z "$CUDA_HOME" ] && command -v nvcc &>/dev/null; then + NVCC_PATH=$(command -v nvcc) + CUDA_HOME=$(dirname "$(dirname "$NVCC_PATH")") +fi +if [ -z "$CUDA_HOME" ]; then + echo "nvcc not found on system. Installing CUDA toolkit from NVIDIA apt repo..." + sudo apt-get update -qq + UBUNTU_VER=$(lsb_release -rs 2>/dev/null | tr -d '.' || echo "2204") + KEYRING_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VER}/x86_64/cuda-keyring_1.1-1_all.deb" + echo "Installing CUDA keyring from $KEYRING_URL" + wget -qO /tmp/cuda-keyring.deb "$KEYRING_URL" 2>&1 || curl -sLo /tmp/cuda-keyring.deb "$KEYRING_URL" + file /tmp/cuda-keyring.deb + sudo dpkg -i /tmp/cuda-keyring.deb + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends cuda-nvcc-12-8 libcublas-dev-12-8 cuda-nvrtc-dev-12-8 + CUDA_HOME="/usr/local/cuda-12.8" +fi +export CUDA_HOME +export PATH="$CUDA_HOME/bin:$PATH" +echo "CUDA_HOME=$CUDA_HOME" +"$CUDA_HOME/bin/nvcc" --version + +# Write cuda_env for run phase (fleet-common-run.sh sources this via --cuda-env) +echo "export CUDA_HOME=$CUDA_HOME" > "$HOME/.cuda_env" +echo "export PATH=$CUDA_HOME/bin:\$PATH" >> "$HOME/.cuda_env" diff --git a/skyrl-train/scripts/full_context/README.md b/skyrl-train/scripts/full_context/README.md new file mode 100644 index 0000000000..7e2cc0672b --- /dev/null +++ b/skyrl-train/scripts/full_context/README.md @@ -0,0 +1,11 @@ +# Full Context Training + +This folder contains scripts for full context training. The idea is to train for a few steps with the given parameters of `train_batch_size`, `policy_mini_batch_size`, `micro_forward_batch_size_per_gpu`, `micro_train_batch_size_per_gpu`, etc with the specified maximum sequence length catch OOMs early. + +The maximum sequence length is simply `generator.max_input_length` + `generator.sampling_params.max_generate_length`. Thus, we will perform `trainer.num_dummy_steps` number of iterations with batches of size `(trainer.train_batch_size, generator.max_input_length + generator.sampling_params.max_generate_length)` + +## Usage + +```bash +bash scripts/full_context/run.sh +``` \ No newline at end of file diff --git a/skyrl-train/scripts/full_context/__init__.py b/skyrl-train/scripts/full_context/__init__.py new file mode 100644 index 0000000000..9b47f0eac7 --- /dev/null +++ b/skyrl-train/scripts/full_context/__init__.py @@ -0,0 +1,3 @@ +from .trainer_full_ctx import FullCtxTrainer + +__all__ = ["FullCtxTrainer"] diff --git a/skyrl-train/scripts/full_context/main_full_ctx.py b/skyrl-train/scripts/full_context/main_full_ctx.py new file mode 100644 index 0000000000..29dc87716e --- /dev/null +++ b/skyrl-train/scripts/full_context/main_full_ctx.py @@ -0,0 +1,54 @@ +""" +uv run --isolated --extra vllm -m scripts.full_context.main_full_ctx +""" + +import hydra +from omegaconf import DictConfig +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir, validate_cfg +from skyrl_train.utils import initialize_ray +import ray +from .trainer_full_ctx import FullCtxTrainer + + +class FullCtxPPOExp(BasePPOExp): + def get_trainer( + self, + cfg, + tracker, + tokenizer, + train_dataset, + eval_dataset, + inference_engine_client, + generator, + colocate_pg, + ): + return FullCtxTrainer( + cfg=cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=colocate_pg, + ) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = FullCtxPPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/scripts/full_context/run_full_ctx.sh b/skyrl-train/scripts/full_context/run_full_ctx.sh new file mode 100644 index 0000000000..406f416913 --- /dev/null +++ b/skyrl-train/scripts/full_context/run_full_ctx.sh @@ -0,0 +1,46 @@ +set -x + +# Script to simulate full context training for GSM8K with Qwen2.5-1.5B-Instruct on 4 GPUs + +# NOTE: Make sure to tune the configurations for the setup you wish to test. + +DATA_DIR="$HOME/data/gsm8k" + +uv run --isolated --extra vllm -m scripts.full_context.main_full_ctx \ + data.train_data="['$DATA_DIR/train.parquet']" \ + data.val_data="['$DATA_DIR/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=4 \ + trainer.placement.ref_num_gpus_per_node=4 \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=1024 \ + trainer.policy_mini_batch_size=256 \ + trainer.critic_mini_batch_size=256 \ + trainer.micro_forward_batch_size_per_gpu=64 \ + trainer.micro_train_batch_size_per_gpu=64 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=512 \ + generator.sampling_params.max_generate_length=1024 \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k_full_ctx" \ + trainer.run_name="gsm8k_full_ctx_test" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + +trainer.num_dummy_steps=5 \ No newline at end of file diff --git a/skyrl-train/scripts/full_context/run_full_ctx_megatron.sh b/skyrl-train/scripts/full_context/run_full_ctx_megatron.sh new file mode 100644 index 0000000000..777bb3c9ed --- /dev/null +++ b/skyrl-train/scripts/full_context/run_full_ctx_megatron.sh @@ -0,0 +1,69 @@ +set -x + +# Script to simulate full context training for DAPO with Qwen3-4B on 8 GPUs with Megatron. +# uv run examples/gsm8k/gsm8k_dataset.py --output_dir $HOME/data/gsm8k +# bash scripts/full_context/run_full_ctx_megatron.sh + +# NOTE: Make sure to tune the configurations for the setup you wish to test. + +DATA_DIR="$HOME/data/gsm8k" +TRAIN_FILE="$DATA_DIR/train.parquet" +TEST_FILE="$DATA_DIR/validation.parquet" + +NUM_NODES=1 +NUM_GPUS_PER_NODE=8 +NUM_INFERENCE_ENGINES=4 +INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE=2 + +MEGATRON_TP=4 +MEGATRON_PP=2 +MEGATRON_CP=1 + +MAX_PROMPT_LENGTH=2048 +MAX_RESPONSE_LENGTH=8192 + +MODEL_NAME="Qwen/Qwen3-4B" + +uv run --isolated --extra mcore -m scripts.full_context.main_full_ctx \ + data.train_data="['$TRAIN_FILE']" \ + data.val_data="['$TEST_FILE']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path=$MODEL_NAME \ + trainer.placement.colocate_all=true \ + trainer.strategy=megatron \ + trainer.placement.policy_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + trainer.placement.ref_num_gpus_per_node=$NUM_GPUS_PER_NODE \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$INFERENCE_ENGINE_TENSOR_PARALLEL_SIZE \ + trainer.policy.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.policy.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.policy.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.tensor_model_parallel_size=$MEGATRON_TP \ + trainer.ref.megatron_config.context_parallel_size=$MEGATRON_CP \ + trainer.ref.megatron_config.pipeline_model_parallel_size=$MEGATRON_PP \ + trainer.epochs=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=64 \ + trainer.policy_mini_batch_size=32 \ + trainer.micro_forward_batch_size_per_gpu=8 \ + trainer.micro_train_batch_size_per_gpu=8 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=$MAX_PROMPT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_RESPONSE_LENGTH \ + trainer.policy.optimizer_config.lr=1.0e-5 \ + trainer.algorithm.use_kl_loss=false \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=false \ + generator.batched=true \ + environment.env_class=gsm8k \ + generator.n_samples_per_prompt=5 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="wandb" \ + trainer.project_name="gsm8k_full_ctx" \ + trainer.run_name="gsm8k_full_ctx_test_megatron_tp${MEGATRON_TP}_pp${MEGATRON_PP}_cp${MEGATRON_CP}_lora_rank${LORA_RANK}_alpha${LORA_ALPHA}" \ + trainer.eval_batch_size=1024 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + +trainer.num_dummy_steps=5 \ No newline at end of file diff --git a/skyrl-train/scripts/full_context/trainer_full_ctx.py b/skyrl-train/scripts/full_context/trainer_full_ctx.py new file mode 100644 index 0000000000..612b4bc340 --- /dev/null +++ b/skyrl-train/scripts/full_context/trainer_full_ctx.py @@ -0,0 +1,93 @@ +from skyrl_train.trainer import RayPPOTrainer +from loguru import logger +import random +from skyrl_train.utils.utils import Timer + + +class FullCtxTrainer(RayPPOTrainer): + """A dummy trainer that tests configurations with max sequence length. + + This trainer is meant to help users validate their configuration setup by: + 1. Creating max length sequences directly + 2. Running a few training steps + + This helps catch OOM issues early before running full training. + """ + + def train(self): + """Run a few training steps with max sequence length.""" + logger.info("Starting dummy training with max sequence length...") + + self.global_step = 0 + + # Initialize weight sync state + with Timer("init_weight_sync_state", self.all_timings): + self.init_weight_sync_state() + + # Run a few training steps + self.global_step += 1 # start from 1 + for step in range(self.cfg.trainer.num_dummy_steps): + logger.info(f"Running dummy training step {step + 1}/{self.cfg.trainer.num_dummy_steps}") + + # Run a single training step + with Timer("step", self.all_timings): + # Create training input directly with max length sequences + num_samples = self.cfg.trainer.train_batch_size * self.cfg.generator.n_samples_per_prompt + uids = [str(i) for i in range(self.cfg.trainer.train_batch_size)] + prompt_token_ids = [ + [random.randint(0, self.tokenizer.vocab_size - 1)] * self.cfg.generator.max_input_length + ] * self.cfg.trainer.train_batch_size + prompt_token_ids = sum( + [ + [prompt_token_id] * self.cfg.generator.n_samples_per_prompt + for prompt_token_id in prompt_token_ids + ], + [], + ) + response_ids = [ + [random.randint(0, self.tokenizer.vocab_size - 1)] + * self.cfg.generator.sampling_params.max_generate_length + ] * num_samples + uids = sum([[uid] * self.cfg.generator.n_samples_per_prompt for uid in uids], []) + + dummy_generator_output = { + "prompt_token_ids": prompt_token_ids, + "response_ids": response_ids, + "rewards": [ + [0] * (self.cfg.generator.sampling_params.max_generate_length - 1) + [random.randint(0, 1)] + ] + * num_samples, + "loss_masks": [[1] * self.cfg.generator.sampling_params.max_generate_length] * num_samples, + } + training_input = self.convert_to_training_input(dummy_generator_output, uids) + + with Timer("fwd_logprobs_values_reward", self.all_timings): + training_input = self.fwd_logprobs_values_reward(training_input) + + # 1.5 apply kl divergence penalty to rewards + if self.cfg.trainer.algorithm.use_kl_in_reward: + with Timer("apply_reward_kl_penalty", self.all_timings): + training_input = self.apply_reward_kl_penalty(training_input) + + # 3. calculate advantages and returns + with Timer("compute_advantages_and_returns", self.all_timings): + training_input = self.compute_advantages_and_returns(training_input) + # remove some unwanted keys + for key in ["rewards"]: + training_input.pop(key) + training_input.metadata.pop("uids") + + # 4. train policy/critic model + with Timer("train_critic_and_policy", self.all_timings): + status = self.train_critic_and_policy(training_input) + self.policy_model.offload_to_cpu(offload_optimizer=True, offload_model=False) + + self.tracker.log(self.all_metrics, step=self.global_step) + self.all_metrics = {} + self.tracker.log({"timing/" + k: v for k, v in self.all_timings.items()}, step=self.global_step) + self.all_timings = {} + self.global_step += 1 + + logger.info(f"Step {step + 1} completed. Status: {status}") + + logger.info("Dummy training completed successfully!") diff --git a/skyrl-train/scripts/launch_multiple_remote_servers.py b/skyrl-train/scripts/launch_multiple_remote_servers.py new file mode 100644 index 0000000000..2b945e5db4 --- /dev/null +++ b/skyrl-train/scripts/launch_multiple_remote_servers.py @@ -0,0 +1,251 @@ +""" +Helper script to launch multiple vLLM remote servers in a Ray cluster. + +The main purpose is to be able to test out the remote endpoint functionality in SkyRL easily. + +Example usage: +uv run --isolated --frozen --extra vllm scripts/launch_multiple_remote_servers.py --model-path Qwen/Qwen2.5-1.5B-Instruct --tp-size 2 --num-replicas 2 --gpu-memory-utilization 0.9 > my_server_logs.log 2>&1 +""" + +import argparse +import os +import ray +import subprocess +from ray.util.placement_group import placement_group, PlacementGroupSchedulingStrategy +from skyrl_train.utils import get_ray_pg_ready_with_timeout +from skyrl_train.utils.utils import initialize_ray +import socket +import signal +import time +import sys +import tempfile +from pathlib import Path +from omegaconf import OmegaConf +from concurrent.futures import ThreadPoolExecutor +from typing import List, Tuple +import psutil + + +def get_free_port(): + """Get a free port on the current node.""" + with socket.socket() as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + + +def wait_for_server(host: str, port: int, timeout_seconds: int = 180) -> bool: + """Wait for server to be ready by checking if we can connect.""" + start_time = time.time() + print(f"Waiting for server at {host}:{port}...") + + while True: + try: + with socket.socket() as sock: + sock.settimeout(1) + sock.connect((host, port)) + return True + except (socket.timeout, ConnectionRefusedError): + pass + + if time.time() - start_time >= timeout_seconds: + print(f"Timeout waiting for server at {host}:{port}") + return False + + time.sleep(0.5) + + +@ray.remote +class VLLMServer: + def __init__(self, model_path, tp_size, gpu_memory_utilization=0.7, distributed_executor_backend="mp", quiet=False): + self.model_path = model_path + self.tp_size = tp_size + self.port = get_free_port() + self.host = ray._private.services.get_node_ip_address().strip("[]") + self.gpu_memory_utilization = gpu_memory_utilization + self.distributed_executor_backend = distributed_executor_backend + self.pid = None + self.quiet = quiet + self.log_dir = Path(tempfile.gettempdir()) / "skyrl_vllm_server_logs" + self.log_dir.mkdir(exist_ok=True) + + def start(self): + """Start the VLLM server process.""" + cmd = [ + "python", + "-m", + "skyrl_train.inference_engines.vllm.vllm_server", + "--model", + self.model_path, + "--enforce-eager", + "--tensor-parallel-size", + str(self.tp_size), + "--seed", + "42", + "--distributed-executor-backend", + self.distributed_executor_backend, + "--enable-prefix-caching", + "--dtype", + "bfloat16", + "--trust-remote-code", + "--gpu-memory-utilization", + str(self.gpu_memory_utilization), + "--enable-sleep-mode", + "--host", + self.host, + "--port", + str(self.port), + "--worker-extension-cls", + "skyrl_train.inference_engines.vllm.vllm_engine.WorkerWrap", + ] + + # Set CUDA_VISIBLE_DEVICES based on Ray's GPU allocation + gpu_ids = ray.get_gpu_ids() + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpu_ids)) + + # Create log files for this server + timestamp = time.strftime("%Y%m%d_%H%M%S") + log_prefix = f"vllm_server_{self.host}_{self.port}_{timestamp}" + stdout_log = self.log_dir / f"{log_prefix}_stdout.log" if self.quiet else "sys.stdout" + stderr_log = self.log_dir / f"{log_prefix}_stderr.log" if self.quiet else "sys.stderr" + + # Configure subprocess output + if self.quiet: + stdout = open(stdout_log, "w") + stderr = open(stderr_log, "w") + else: + stdout = sys.stdout + stderr = sys.stderr + + # Start the server process + process = subprocess.Popen(cmd, stdout=stdout, stderr=stderr) + self.pid = process.pid + return process.pid, f"{self.host}:{self.port}", str(stdout_log), str(stderr_log) + + def kill(self): + if self.pid is not None: + try: + print(f"Killing server {self.pid}...") + # First kill child processes + print("Killing child processes...") + try: + parent = psutil.Process(self.pid) + children = parent.children(recursive=True) + for child in children: + print(f"Killing child process {child.pid}...") + try: + child.terminate() + except psutil.NoSuchProcess: + pass + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass # Process already terminated or not accessible + + # Then try SIGTERM on main process + os.kill(self.pid, signal.SIGTERM) + # Wait a bit for graceful shutdown + time.sleep(3) + + # If still running, force kill with SIGKILL + try: + os.kill(self.pid, signal.SIGKILL) + except ProcessLookupError: + pass # Process already terminated + except ProcessLookupError: + pass # Process already terminated + + +def wait_for_all_servers(server_addresses: List[Tuple[str, int]], timeout_seconds: int = 180) -> bool: + """Wait for all servers to be ready using thread pool.""" + with ThreadPoolExecutor(max_workers=len(server_addresses)) as executor: + futures = [executor.submit(wait_for_server, host, port, timeout_seconds) for host, port in server_addresses] + results = [f.result() for f in futures] + return all(results) + + +def print_server_info(server_infos: List[Tuple[int, str, str, str]], args): + for i, (pid, addr, stdout_log, stderr_log) in enumerate(server_infos): + print(f"Server {i+1}:") + print(f"TP: {args.tp_size}") + print(f" PID: {pid}") + print(f" Address: {addr}") + print(f" Logs: {stdout_log}, {stderr_log}") + + +def main(): + parser = argparse.ArgumentParser(description="Launch multiple VLLM servers in a Ray cluster") + parser.add_argument("--model-path", type=str, required=True, help="Path to the model") + parser.add_argument("--tp-size", type=int, required=True, help="Tensor parallel size for each replica") + parser.add_argument("--num-replicas", type=int, required=True, help="Number of server replicas to launch") + parser.add_argument("--gpu-memory-utilization", type=float, default=0.9, help="GPU memory utilization") + parser.add_argument("--quiet", action="store_true", help="Quiet mode") + parser.add_argument("--timeout", type=int, default=180, help="Timeout in seconds for server readiness") + args = parser.parse_args() + + cfg = OmegaConf.create() + cfg.generator = OmegaConf.create() + cfg.generator.backend = "vllm" + cfg.generator.weight_sync_backend = "nccl" + # Initialize Ray + initialize_ray(cfg) + + # Create placement group for the server + bundles = [[{"GPU": args.tp_size, "CPU": 1}] for _ in range(args.num_replicas)] + placement_groups = [placement_group(bundle) for bundle in bundles] + for pg in placement_groups: + print("Waiting for Ray placement group to be ready...") + get_ray_pg_ready_with_timeout(pg, timeout=180) + + # Launch servers + server_actors = [] + for i in range(args.num_replicas): + pg = placement_groups[i] + server = VLLMServer.options( + num_gpus=args.tp_size, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, + ), + ).remote( + model_path=args.model_path, + tp_size=args.tp_size, + gpu_memory_utilization=args.gpu_memory_utilization, + quiet=args.quiet, + ) + server_actors.append(server) + + # Start all servers + rets = ray.get([server.start.remote() for server in server_actors]) + + print(f"Launched {args.num_replicas} VLLM servers:") + server_addresses = [] + server_infos = [] + for i, ret in enumerate(rets): + pid, addr, stdout_log, stderr_log = ret + host, port = addr.split(":") + server_addresses.append((host, int(port))) + server_infos.append((pid, addr, stdout_log, stderr_log)) + + print_server_info(server_infos, args) + + try: + print("\nWaiting for all servers to be ready...") + if wait_for_all_servers(server_addresses, args.timeout): + print("\nAll servers are ready!") + print_server_info(server_infos, args) + else: + print("\nSome servers failed to start within timeout. Check logs for details.") + print("Shutting down...") + ray.get([server.kill.remote() for server in server_actors]) + ray.shutdown() + return + + print("\nPress Ctrl+C to shut down all the servers") + + while True: + time.sleep(1) + except KeyboardInterrupt: + print("\nShutting down servers. Please wait for a few seconds...") + ray.get([server.kill.remote() for server in server_actors]) + ray.shutdown() + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/scripts/multi_node_nccl_test.py b/skyrl-train/scripts/multi_node_nccl_test.py new file mode 100644 index 0000000000..53493b6baa --- /dev/null +++ b/skyrl-train/scripts/multi_node_nccl_test.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +"""Script to test multi-node NCCL communication with Ray + +This script is useful to debug if multi-node communication works and if the right network interfaces (eg: RDMA) is being used. +""" +import os +import sys +import ray +import torch +import torch.distributed as dist +from skyrl_train.utils.utils import initialize_ray, get_ray_pg_ready_with_timeout +from skyrl_train.env_vars import SKYRL_RAY_PG_TIMEOUT_IN_S +from ray.util.placement_group import placement_group +from omegaconf import OmegaConf +from loguru import logger +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("--num-nodes", type=int, default=2) +parser.add_argument("--master-port", type=int, default=12355) +args = parser.parse_args() + + +def log_versions(rank): + logger.info( + f"{rank} Python version: {sys.version} | " + f"PyTorch version: {torch.__version__} | " + f"CUDA available: {torch.cuda.is_available()} | " + f"CUDA version: {torch.version.cuda if torch.cuda.is_available() else 'N/A'} | " + f"Ray version: {ray.__version__}" + ) + + +@ray.remote(num_gpus=1) +class PyTorchDistActor: + def __init__(self, rank, world_size): + self.rank = rank + self.node_ip = self.get_node_ip() + self.world_size = world_size + + logger.info(f"Rank {self.rank} initialized with: node_ip={self.node_ip}, world_size={self.world_size}") + log_versions(rank) + + def get_node_ip(self): + return ray.util.get_node_ip_address() + + def set_master_node_addr(self, master_addr, master_port): + os.environ["MASTER_ADDR"] = str(master_addr) + os.environ["MASTER_PORT"] = str(master_port) + + def run(self): + # Initialize the process group + dist.init_process_group(backend="nccl", init_method="env://", world_size=self.world_size, rank=self.rank) + + # Create a dictionary to broadcast + if self.rank == 0: + data = {"message": "Hello from rank 0", "value": 123} + else: + data = None + + objects = [data] + + # Broadcast the object list + dist.broadcast_object_list(objects, src=0) + + if self.rank != 0: + logger.info(f"Rank {self.rank} received data: {objects[0]}") + assert objects[0] == { + "message": "Hello from rank 0", + "value": 123, + }, f"Data received at rank {self.rank} is not correct, got {objects[0]}" + else: + logger.info(f"Rank {self.rank} sent: {data}") + + dist.barrier() + # Clean up + dist.destroy_process_group() + return objects[0] + + +if __name__ == "__main__": + # Initialize Ray + cfg = OmegaConf.create() + cfg.generator = OmegaConf.create() + cfg.generator.backend = "vllm" + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer = OmegaConf.create() + cfg.trainer.strategy = "fsdp2" + initialize_ray(cfg) + + total_ranks = args.num_nodes + actors = [] + + # Create placement group for distributed training + pg = placement_group(bundles=[{"GPU": 1, "CPU": 1}] * total_ranks, strategy="STRICT_SPREAD") + get_ray_pg_ready_with_timeout(pg, timeout=SKYRL_RAY_PG_TIMEOUT_IN_S) + + # Create actors + for rank in range(total_ranks): + actor = PyTorchDistActor.options(placement_group=pg).remote(rank, total_ranks) + actors.append(actor) + + # set master node addr + master_addr = ray.get(actors[0].get_node_ip.remote()) + master_port = args.master_port + ray.get([actor.set_master_node_addr.remote(master_addr, master_port) for actor in actors]) + + # Run the distributed operation + results = ray.get([actor.run.remote() for actor in actors]) + print("All results:", results) diff --git a/skyrl-train/skyrl-gym b/skyrl-train/skyrl-gym new file mode 120000 index 0000000000..051b9f8c83 --- /dev/null +++ b/skyrl-train/skyrl-gym @@ -0,0 +1 @@ +../skyrl-gym/ \ No newline at end of file diff --git a/skyrl-train/skyrl_train/__init__.py b/skyrl-train/skyrl_train/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/skyrl_train/config/__init__.py b/skyrl-train/skyrl_train/config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/skyrl_train/config/megatron_config/policy.yaml b/skyrl-train/skyrl_train/config/megatron_config/policy.yaml new file mode 100644 index 0000000000..3ba0dc8f95 --- /dev/null +++ b/skyrl-train/skyrl_train/config/megatron_config/policy.yaml @@ -0,0 +1,54 @@ +# @package megatron_config.policy +tensor_model_parallel_size: 1 +pipeline_model_parallel_size: 1 +context_parallel_size: 1 +expert_model_parallel_size: 1 +expert_tensor_parallel_size: null + +# pass-through config to Megatron's `DistributedDataParallelConfig` object +# https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/distributed/distributed_data_parallel_config.py#L8 +ddp_config: + grad_reduce_in_fp32: true + overlap_grad_reduce: false + overlap_param_gather: false + average_in_collective: true + +# pass-through kwargs to the HuggingFace model config (i.e. for overriding vocab size, etc) +model_config_kwargs: {} + +torch_profiler_config: + enable: false + ranks: [] + save_path: null + +lora_config: + # see: https://docs.nvidia.com/nemo/megatron-bridge/0.2.0/apidocs/bridge/bridge.peft.lora.html for details - currently "lora" and "canonical_lora" are supported + lora_type: "lora" + +# pass-through kwargs to Megatron's `OptimizerConfig` object +# any overlapping arguments with those we attempt to resolve in trainer.policy.optimizer_config will be overridden by the values here +# https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/optimizer/optimizer_config.py#L12 +optimizer_config_kwargs: + # set these all to true (and optimizer_offload_fraction=1.0) for optimizer cpu offloading + overlap_cpu_optimizer_d2h_h2d: false + # NOTE: `use_precision_aware_optimizer=true` can cause checkpointing to fail. + # See: https://github.com/nvidia/megatron-lm/issues/1820. + # We recommend leaving this as `false` + use_precision_aware_optimizer: false + optimizer_cpu_offload: false + optimizer_offload_fraction: 0.0 + +# pass-through kwargs to the Megatron's `TransformerConfig` object +# https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/transformer/transformer_config.py#L33 +transformer_config_kwargs: + # Recompute config - used for gradient/activation checkpointing + # for details see: https://github.com/NVIDIA/Megatron-LM/blob/core_r0.13.0/megatron/core/transformer/transformer_config.py#L33 + # for the most aggresive memory savings, set recompute_granularity to "full", recompute_method to "uniform", and recompute_num_layers to 1 + recompute_granularity: full + recompute_modules: ["core_attn"] + recompute_method: uniform + recompute_num_layers: 1 + +# flag to manually empty torch's cuda cache between the forward/backward pass and the optimizer step +# this will free reserved but unallocated memory, and can help avoid OoMs in the optimizer +empty_cuda_cache: true diff --git a/skyrl-train/skyrl_train/config/megatron_config/ref.yaml b/skyrl-train/skyrl_train/config/megatron_config/ref.yaml new file mode 100644 index 0000000000..dd5447a6bb --- /dev/null +++ b/skyrl-train/skyrl_train/config/megatron_config/ref.yaml @@ -0,0 +1,12 @@ +# @package megatron_config.ref +tensor_model_parallel_size: 1 +pipeline_model_parallel_size: 1 +context_parallel_size: 1 +expert_model_parallel_size: 1 +expert_tensor_parallel_size: 1 + + +model_config_kwargs: {} + +# kwargs to override the Megatron TransformerConfig object +transformer_config_kwargs: {} \ No newline at end of file diff --git a/skyrl-train/skyrl_train/config/ppo_base_config.yaml b/skyrl-train/skyrl_train/config/ppo_base_config.yaml new file mode 100644 index 0000000000..f224d20982 --- /dev/null +++ b/skyrl-train/skyrl_train/config/ppo_base_config.yaml @@ -0,0 +1,333 @@ +defaults: + - _self_ + # @.: + - megatron_config@trainer.policy.megatron_config: policy + - megatron_config@trainer.ref.megatron_config: ref + - skyrl_gym_config: default + +data: + train_data: ["${oc.env:HOME}/data/gsm8k/train.parquet"] + val_data: ["${oc.env:HOME}/data/gsm8k/validation.parquet"] + +trainer: + placement: + colocate_all: true + colocate_policy_ref: true + policy_num_nodes: 1 + policy_num_gpus_per_node: 4 + critic_num_nodes: 1 + critic_num_gpus_per_node: 4 + ref_num_nodes: 1 + ref_num_gpus_per_node: 4 + sequence_parallel_backend: "ulysses" + strategy: fsdp2 + policy: + model: + path: "Qwen/Qwen2.5-1.5B-Instruct" + lora: + rank: 0 + alpha: 16 + dropout: 0 + lora_sync_path: "/tmp/skyrl_lora_sync" + target_modules: "all-linear" + exclude_modules: null + # For FSDP, this corresponds to `init_lora_weights` in PEFT. See: https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig.init_lora_weights + # For Megatron, this is used for `lora_A_init_method`, and "xavier", "normal", "kaiming", and "zero" are supported. + init_method: "kaiming" + optimizer_config: + lr: 1.0e-6 + adam_betas: [0.9, 0.999] + weight_decay: 1e-2 + max_grad_norm: 1.0 # gradient clipping + offload_after_step: true # offload optimizer state to cpu after each full training step. Applicable only when `colocate_all=true` + num_warmup_steps: 0 # number of mini-batch steps to warmup the optimizer for + scheduler: "constant_with_warmup" + fsdp_config: + cpu_offload: false # offload params + optimizer state to cpu during fwd pass + reshard_after_forward: true # fsdp2 only, [True, False, int between 1 and fsdp_size] + fsdp_size: -1 + sequence_parallel_size: 1 + # uses torch compile with logits calculation + use_torch_compile: false + # saves memory snapshots to os.path.join(ckpt_path, "memory_snapshots") - can visualize by dragging pickle files to https://docs.pytorch.org/memory_viz + record_memory: false + # whether to track additional gradient metrics. + track_extra_gradient_metrics: false + # pass through kwargs to the HuggingFace model config for FSDP training backends (i.e. for overriding vocab size, etc) - for megatron, use policy.megatron_config.transformer_config_kwargs instead + model_config_kwargs: {} + ref: + model: + path: ${trainer.policy.model.path} + sequence_parallel_size: 1 + fsdp_config: + cpu_offload: false + reshard_after_forward: true + fsdp_size: -1 + # pass through kwargs to the HuggingFace model config (i.e. for overriding vocab size, etc) + # pass through kwargs to the HuggingFace model config for FSDP training backends (i.e. for overriding vocab size, etc) - for megatron, use ref.megatron_config.transformer_config_kwargs instead + model_config_kwargs: {} + critic: + model: + path: null + lora: + rank: 0 + alpha: 16 + dropout: 0 + target_modules: "all-linear" + exclude_modules: null + init_method: "kaiming" + optimizer_config: + lr: 5.0e-6 + adam_betas: [0.9, 0.999] + weight_decay: 1e-2 + max_grad_norm: 1.0 # gradient clipping + offload_after_step: true # offload optimizer state to cpu after each full training step. Applicable only when `colocate_all=true` + num_warmup_steps: 0 # number of mini-batch steps to warmup the optimizer for + scheduler: "constant_with_warmup" + fsdp_config: + cpu_offload: false + reshard_after_forward: true + fsdp_size: -1 + sequence_parallel_size: 1 + # pass through kwargs to the HuggingFace model config (i.e. for overriding vocab size, etc) + model_config_kwargs: {} + algorithm: + advantage_estimator: "grpo" # "grpo", "gae", "rloo", "reinforce++", or customizable with AdvantageEstimatorRegistry + kl_ctrl: # only used if use_kl_in_reward is true (not applied in the case of use_kl_loss=true) - uses kl_loss_coef as the initial KL coefficient + type: "fixed" # "fixed" or "adaptive" + kl_target: 0.1 # target KL divergence for adaptive KL controller + horizon: 10000 # controls the update rate of the adaptive KL controller + kl_estimator_type: "k3" # "k1", "k2", "k3", "abs" - see http://joschu.net/blog/kl-approx.html for details + use_kl_estimator_k3: false # to be deprecated, use kl_estimator_type="k3" instead + use_abs_kl: false # to be deprecated, use kl_estimator_type="abs" instead + # note: use_kl_in_reward and use_kl_loss should be mutually exclusive + use_kl_in_reward: false # apply kl loss to rewards + use_kl_loss: true # used in policy model + kl_loss_coef: 0.001 + # entropy loss + use_entropy_loss: false + entropy_loss_coef: 0.01 + # this adds training batch level normalization to advantages + advantage_batch_normalize: false + value_head_prefix: "value_head" + policy_loss_type: "regular" # "regular", "dual_clip", "gspo", "clip_cov", "kl_cov", or customizable with PolicyLossRegistry + loss_reduction: "token_mean" # "token_mean", "sequence_mean", "seq_mean_token_sum_norm" + grpo_norm_by_std: true # set to false to disable normalization by std in GRPO + zero_variance_filter: false # set to true to loss mask out prompts with zero variance rewards. only applicable when rewards are response-level. + # GAE parameters + lambd: 1.0 + gamma: 1.0 + # PPO parameters + eps_clip_low: 0.2 + eps_clip_high: 0.2 + # dual clip parameters + clip_ratio_c: 3.0 + # Truncated Importance Sampling as proposed in https://fengyao.notion.site/off-policy-rl + tis_imp_ratio_cap: -1.0 + use_tis: false + # SAPO parameters (only used when policy_loss_type: "sapo") (https://arxiv.org/pdf/2511.20347) + sapo: + tau_pos: 1.0 + tau_neg: 1.05 # default values used in the paper with Qwen3-30B-A3B-Base + + # value loss parameters + value_clip: 0.2 + dynamic_sampling: + type: null # filter, replace, or null + max_sample_batches: 30 # sample at most this many batches before stopping, -1 to sample forever + min_replace_ratio: 0.3 # minimum proportion of good samples with which to replace bad samples (for replace strategy only) + + # clip-cov parameters (only used when policy_loss_type: "clip_cov" + clip_cov: + clip_ratio: 0.0002 # fraction of tokens to clip based on covariance + clip_cov_lb: 1.0 # lower bound for covariance clipping + clip_cov_ub: 5.0 # upper bound for covariance clipping + + # kl-cov parameters (only used when policy_loss_type: "kl_cov") + kl_cov: + kl_cov_frac: 0.2 # percentage of tokens to apply KL regularization to (20%) + ppo_kl_coef: 1.0 + + # cispo parameters (only used when policy_loss_type: "cispo") + cispo: + cispo_eps_clip_low: 0 # offset for lower bound of importance sampling ratio clipping (as opposed to PPO token update clipping) + cispo_eps_clip_high: 5 # offset for upper bound of importance sampling ratio clipping (as opposed to PPO token update clipping) + + # Fully async specific knobs. For more see http://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html#step-2-config-knobs-to-tune-for-fully-async-training + fully_async: + # The maximum number of off-policy steps allowed. If a group of trajectory is scheduled at step i, + # and it is used to train at step j, it is guaranteed that j - i <= max_staleness_steps. + # The larger the max_staleness_steps, the more off-policy the training is, and the more throughput we get. + max_staleness_steps: 4 + + # The number of generation workers to spawn, where each worker works on a group of trajcetories, + # being the same prompt repeated `generator.n_samples_per_prompt` times. + # It should be >= policy_mini_batch_size to avoid wasted throughput, and <= policy_mini_batch_size * (max_staleness_steps + 1) + # since it would be wasted due to capacity control. + # The larger the number, the more throughput, and likely more staleness (and hence off-policy-ness). + # Default value is: policy_mini_batch_size * (max_staleness_steps / 2 + 1) = 256 * (4 / 2 + 1) = 768 + num_parallel_generation_workers: 768 + + # Chunk size for lm_head logprobs computation. When > 0, avoids materializing the full + # (B, S, vocab_size) logits tensor by computing lm_head in chunks along the sequence dim. + # Reduces peak memory from O(S * V) to O(chunk_size * V). Set to 0 to disable. + loss_chunk_size: 0 + gradient_checkpointing: true + gradient_checkpointing_use_reentrant: false + seed: 42 + resume_mode: latest # null/"none", "latest", "from_path" + resume_path: null + ckpt_path: "${oc.env:HOME}/ckpts/" # Path for resumable training checkpoints (model state, optimizer state, etc.) + max_ckpts_to_keep: -1 # -1 to keep all checkpoints, N to keep the last N checkpoints + ckpt_interval: 10 # Save full training checkpoint every `ckpt_interval` steps. + hf_save_interval: -1 # Save HF format model(s)every `hf_save_interval` steps. + export_path: "${oc.env:HOME}/exports/" # Path for exported artifacts (HF models, debug dumps, etc.) + bf16: true + epochs: 1 # Number of passes over the full dataset + update_epochs_per_batch: 1 # Number of gradient update passes over each train batch + train_batch_size: 1024 # See `utils/utils.py::validate_batch_sizes` for train, mini, and micro batch size constraints. + # Hybrid environment sampling: ensures minimum representation from each environment per batch + # to prevent "unlucky" batches with only hard environments. Remaining slots filled proportionally. + use_hybrid_env_sampling: true + min_samples_per_env: 1 # Minimum samples from each environment per batch (only used when use_hybrid_env_sampling=true) + policy_mini_batch_size: 256 + critic_mini_batch_size: 256 + micro_train_batch_size_per_gpu: 1 + micro_forward_batch_size_per_gpu: 1 + update_ref_every_epoch: false + use_sample_packing: true + eval_batch_size: 1024 + eval_before_train: true + eval_interval: 5 # Set to -1 to disable evaluation. + # max prompt length in training dataset + max_prompt_length: 2048 + flash_attn: true + disable_fast_tokenizer: false + target_modules: null # NOTE: these are now deprecated, use trainer.policy.model.lora.target_modules or trainer.critic.model.lora.target_modules instead + exclude_modules: null # NOTE: these are now deprecated, use trainer.policy.model.lora.exclude_modules or trainer.critic.model.lora.exclude_modules instead + project_name: "skyrl" + run_name: "test_run" + logger: "wandb" + dump_data_batch: false + dump_eval_results: true + + # YaRN: + rope_scaling: null + rope_theta: null + # rope_scaling: + # rope_type: yarn + # factor: 1.0 + # original_max_position_embeddings: 32768 + + +generator: + model_name: ${trainer.policy.model.path} + model_dtype: "bfloat16" # should match dtype for inference engine + run_engines_locally: true + num_inference_engines: 1 + backend: "vllm" + weight_sync_backend: "nccl" + # if using cuda_ipc, we send in batches of this size in GB + weight_transfer_threshold_cuda_ipc_GB: 1.0 + inference_engine_tensor_parallel_size: 4 + inference_engine_pipeline_parallel_size: 1 + inference_engine_expert_parallel_size: 1 + inference_engine_data_parallel_size: 1 + n_samples_per_prompt: 5 + async_engine: true + batched: false + max_input_length: ${trainer.max_prompt_length} # max generator input length used for multi-turn conversations - for single turn set equal to max_prompt_length + # VLLM_ENABLE_V1_MULTIPROCESSING=0 for reproducibility + vllm_v1_disable_multiproc: true + enable_prefix_caching: true + enable_chunked_prefill: true + max_num_batched_tokens: 8192 + # Disable CUDA graphs by default for stability. Set to false for higher performance, but this may affect convergence for long-running and/or long context training jobs. + enforce_eager: true + fully_sharded_loras: false + gpu_memory_utilization: 0.8 + max_num_seqs: 1024 + remote_inference_engine_urls: ["127.0.0.1:8001"] + enable_http_endpoint: false + http_endpoint_host: "127.0.0.1" + http_endpoint_port: 8000 + max_turns: 1 + # Maximum time (seconds) for a single trajectory before timeout. 0 = no timeout. + trajectory_timeout_seconds: 900 + # Inject context status line into observations (e.g., "[Context: 24K/48K tokens (50%)]") + # Useful for training models to use context management tools + inject_context_status: false + + # chat template configuration for SkyRLGymGenerator. + # For `/chat/completions` with vllm via the HTTP endpoint for rollout, instead use say + # `+generator.engine_init_kwargs.chat_template=/path/to/templates/qwen3_acc_thinking.jinja2` + chat_template: + source: "name" # "name" or "file" + name_or_path: null # e.g., "qwen3_with_thinking" or "/path/to/template.j2" + + # Chat templating kwargs to pass to `tokenizer.apply_chat_template` + chat_template_kwargs: {} + + # Inference engine arguments. Arguments are passed directly to the vLLM or SGLang engine, so names must match + # the engine's args. To specify an engine arg in the CLI override, use the format: +generator.engine_init_kwargs.arg_name=value + engine_init_kwargs: {} + + override_existing_update_group: "auto" # "auto", "enable", "disable" + # sampling params for generation phase + sampling_params: + max_generate_length: 1024 + repetition_penalty: 1.0 + temperature: 1.0 + top_p: 1.0 + min_p: 0.0 + top_k: -1 + logprobs: 0 + stop: null + + # whether to use a conversation based format for multi-turn generations + # if false, append multi-turn model responses and env observations to the original assistant response + # if true, each multi-turn model response and env observations is stored in a separate assistant/user message respectively + use_conversation_multi_turn: true + + # Used when use_conversation_multi_turn is true, and sampling_params.stop is not null. + # If true, append tokenizer.eos_token_id to the end of the generation if the generation ends + # with stop_reason "stop" and matched a stop string in sampling_params.stop. + append_eos_token_after_stop_str_in_multi_turn: true + + # sampling params for evaluation + eval_sampling_params: + max_generate_length: ${generator.sampling_params.max_generate_length} + repetition_penalty: 1.0 + temperature: 0.0 + top_p: 1.0 + min_p: 0.0 + top_k: -1 + logprobs: 0 + stop: null + + # number of samples per prompt for evaluation + eval_n_samples_per_prompt: 1 + + # NOTE (sumanthrh): This flag sets the reward to 0 if the `stop_reason` is not `stop`. + # This is useful in cases where the LLM generation was truncated or aborted. + # Cases where this is useful: Often, we have format rewards for the LLM to follow, + # but in cases where the LLM didn't finish the response, we typically don't want to reward it. + # This is a general setting for all environments. + # TODO (erictang000): Show clear ablations for benefits of this on GSM8K or SQL. + zero_reward_on_non_stop: false + + # Whether to apply DAPO Overlong Filtering to the loss masks. + # For each trajectory that exceeds the max length (i.e., truncated and does not end with an + # EOS token), this masks out every token in the loss mask. + apply_overlong_filtering: false + + # rope parameters, can be optionally different from the trainer , useful in some cases like with thinking models. + rope_scaling: ${trainer.rope_scaling} + rope_theta: ${trainer.rope_theta} + + step_wise_trajectories: false + +environment: + env_class: "gsm8k" + # NOTE: environment specific defaults for environment.skyrl_gym are set at the following path: + # skyrl_gym: config/skyrl_gym_config/default.yaml diff --git a/skyrl-train/skyrl_train/config/skyrl_gym_config/default.yaml b/skyrl-train/skyrl_train/config/skyrl_gym_config/default.yaml new file mode 100644 index 0000000000..04b2363991 --- /dev/null +++ b/skyrl-train/skyrl_train/config/skyrl_gym_config/default.yaml @@ -0,0 +1,26 @@ +# @package environment.skyrl_gym +# number of background workers for env step calls. Set to 0 to disable background workers. +max_env_workers: 32 + +text2sql: + db_path: "/home/ray/default/sql_data" + +llm_as_a_judge: + model: "gpt-4o-mini" + base_url: null # or a local endpoint: http://localhost:8000/v1 + +search: + log_requests: false + search_url: "http://127.0.0.1:8000/retrieve" + topk: 3 + timeout: 30 + +fleet_task: + tasks_file: null # Path to exported Fleet tasks JSON file + api_key: null # Fleet API key (defaults to FLEET_API_KEY env var) + ttl_seconds: 900 # TTL for Fleet environment instances (15 min headroom for retries) + enable_context_tools: false # Enable context management tools (check_context, manage_context, etc.) + partial_reward: false # Compute partial scores from verifier accumulators instead of binary 0/1 + enable_hints: false # Run hinted rollouts when all raw samples fail (rescue GRPO signal) + hint_reward_threshold: 0.0 # Generate hint when max_reward < this (0.0 = only when ALL raw fail) + n_hint_samples: 2 # Number of hinted rollouts per prompt when triggered diff --git a/skyrl-train/skyrl_train/config/utils.py b/skyrl-train/skyrl_train/config/utils.py new file mode 100644 index 0000000000..4ec1432309 --- /dev/null +++ b/skyrl-train/skyrl_train/config/utils.py @@ -0,0 +1,15 @@ +from pathlib import Path +from hydra import compose, initialize_config_dir + +CONFIG_DIR = Path(__file__).parent # skyrl-train/config +DEFAULT_CONFIG_NAME = "ppo_base_config.yaml" + + +def get_default_config(): + with initialize_config_dir(config_dir=str(CONFIG_DIR)): + cfg = compose(config_name=DEFAULT_CONFIG_NAME) + return cfg + + +if __name__ == "__main__": + cfg = get_default_config() diff --git a/skyrl-train/skyrl_train/dataset/__init__.py b/skyrl-train/skyrl_train/dataset/__init__.py new file mode 100644 index 0000000000..b52d924404 --- /dev/null +++ b/skyrl-train/skyrl_train/dataset/__init__.py @@ -0,0 +1 @@ +from .dataset import PromptDataset as PromptDataset diff --git a/skyrl-train/skyrl_train/dataset/dataset.py b/skyrl-train/skyrl_train/dataset/dataset.py new file mode 100644 index 0000000000..12a81fa297 --- /dev/null +++ b/skyrl-train/skyrl_train/dataset/dataset.py @@ -0,0 +1,88 @@ +import datasets +from loguru import logger +import os +from typing import List +from transformers import PreTrainedTokenizerBase + + +class PromptDataset: + def __init__( + self, + datasets: str | List[str], + tokenizer: PreTrainedTokenizerBase, + max_prompt_length: int, + num_workers: int = 8, + prompt_key: str = "prompt", + env_class_key: str = "env_class", + ): + self.tokenizer = tokenizer + self.max_prompt_length = max_prompt_length + self.prompt_key = prompt_key + self.env_class_key = env_class_key + self.num_workers = num_workers + + self.datasets = datasets + if isinstance(self.datasets, str): + self.datasets = [self.datasets] + + self._read_files_and_tokenize() + + def _read_files_and_tokenize(self): + loaded_datasets = [] + for source in self.datasets: + ext = os.path.splitext(source)[-1].lower() + if ext == ".parquet": + ds = datasets.load_dataset("parquet", data_files=source, keep_in_memory=True)["train"] + elif ext in [".json", ".jsonl"]: + ds = datasets.load_dataset("json", data_files=source, keep_in_memory=True)["train"] + else: + # Treat as HF dataset spec: "name" or "name:split" + dataset_name, has_split, split = source.partition(":") + try: + ds_dict = datasets.load_dataset(path=dataset_name, keep_in_memory=True) + except ValueError: + raise ValueError(f"Dataset `{dataset_name}` not found on Hugging Face.") + split = split if has_split else "train" + if split not in ds_dict: + raise ValueError( + f"Split `{split}` not found in dataset `{dataset_name}`. Configured split was `{split}` and default is `train`" + ) + ds = ds_dict[split] + loaded_datasets.append(ds) + + self.dataframe: datasets.Dataset = datasets.concatenate_datasets(loaded_datasets) + + logger.info(f"Total dataset size: {len(self.dataframe)}") + + # filter out too long prompts + tokenizer = self.tokenizer + prompt_key = self.prompt_key + self.dataframe = self.dataframe.filter( + lambda doc: len( + tokenizer.apply_chat_template(doc[prompt_key], add_generation_prompt=True, return_dict=False) + ) + <= self.max_prompt_length, + num_proc=self.num_workers, + desc=f"Filtering prompts longer than {self.max_prompt_length} tokens", + ) + + logger.info(f"Filtered dataset size: {len(self.dataframe)}") + + def __getitem__(self, item): + row_dict: dict = self.dataframe[item] + messages = row_dict.pop(self.prompt_key) + env_class = row_dict.pop(self.env_class_key, None) + + extra = {key: value for key, value in row_dict.items() if key != self.prompt_key and key != self.env_class_key} + uid = str(item) + + return messages, env_class, extra, uid + + def collate_fn(self, item_list): + all_inputs = [] + for prompt, env_class, env_extras, item_uids in item_list: + all_inputs.append({"prompt": prompt, "env_class": env_class, "env_extras": env_extras, "uid": item_uids}) + return all_inputs + + def __len__(self): + return len(self.dataframe) diff --git a/skyrl-train/skyrl_train/dataset/preprocess.py b/skyrl-train/skyrl_train/dataset/preprocess.py new file mode 100644 index 0000000000..a17cea1b91 --- /dev/null +++ b/skyrl-train/skyrl_train/dataset/preprocess.py @@ -0,0 +1,132 @@ +from typing import List, Tuple, Optional +import torch +from transformers import AutoTokenizer +from jaxtyping import Float + + +def _verify_inputs( + prompts: List[List[int]], + responses: List[List[int]], + rewards: Optional[List[torch.Tensor]], + loss_masks: List[List[int]], +): + assert ( + len(prompts) == len(responses) and len(prompts) > 0 + ), "prompts and responses must have the same length and length must be greater than 0, got {} and {}".format( + len(prompts), len(responses) + ) + + if rewards is not None: + assert len(rewards) == len(prompts), "rewards must have the same length as prompts, got {} and {}".format( + len(rewards), len(prompts) + ) + assert len(loss_masks) == len(prompts), "loss_masks must have the same length as prompt, got {} and {}".format( + len(loss_masks), len(prompts) + ) + + +def convert_prompts_responses_to_batch_tensors( + tokenizer: AutoTokenizer, + prompts: List[List[int]], + responses: List[List[int]], + rewards: List[List[float]], + loss_masks: List[List[int]], + logprobs: Optional[List[List[float]]] = None, +) -> Tuple[ + Float[torch.Tensor, "batch seq_len"], + Float[torch.Tensor, "batch seq_len"], + Float[torch.Tensor, "batch response_len"], + Float[torch.Tensor, "batch response_len"], + Float[torch.Tensor, "batch response_len"], + Optional[Float[torch.Tensor, "batch response_len"]], +]: + """ + Convert prompts and responses to batch tensors for training. + + This function concatenates all prompts and responses to the following format: + + | [PAD] [PAD] token token token | token token [PAD] [PAD] | + | token token token token token | token token [PAD] [PAD] | + | [PAD] [PAD] [PAD] token token | token token token [PAD] | + |<---------- prompt ----------->|<-------- answer ------->| + + Assumes that the responses already contain an eos token at index -1. + + Args: + tokenizer: Model tokenizer + prompts: List of tokenized prompts + responses: List of tokenized responses + rewards: List of rewards for each response + loss_masks: List of loss masks for each response + logprobs: List of rollout log probs for each response + + Returns: + sequences: Full trajectories (padded and concatenated prompts and responses). Size: (batch, seq_len). + attention_mask: Attention mask for the model. Size: (batch, seq_len) + action_mask: Response mask for the model. Size: (batch, response_len) + rewards: Rewards for each output. Size: (batch, response_len) + loss_masks: Loss masks for each output. Size: (batch, response_len) + """ + _verify_inputs(prompts, responses, rewards, loss_masks) + + max_input_len, max_output_len = 0, 0 + prompt_token_lens, response_token_lens = [], [] + inputs_token_ids, outputs_token_ids = [], [] + for prompt, response in zip(prompts, responses): + + inputs_token_ids.append(prompt) + outputs_token_ids.append(response) + + prompt_token_len = len(prompt) + response_token_len = len(response) + prompt_token_lens.append(prompt_token_len) + response_token_lens.append(response_token_len) + + max_input_len = max(max_input_len, prompt_token_len) + max_output_len = max(max_output_len, response_token_len) + + pad_token_id = tokenizer.pad_token_id + sequences = [] + attention_masks = [] + action_masks = [] + for i, prompt in enumerate(prompts): + # left padding input + input_len = prompt_token_lens[i] + input_ids = [pad_token_id] * (max_input_len - input_len) + list(inputs_token_ids[i]) + input_attention_mask = [0] * (max_input_len - input_len) + [1] * input_len + + # right padding output + output_len = response_token_lens[i] + output_ids = list(outputs_token_ids[i]) + [pad_token_id] * (max_output_len - output_len) + output_attention_mask = [1] * output_len + [0] * (max_output_len - output_len) + + # concat input and output + sequences.append(input_ids + output_ids) + attention_masks.append(input_attention_mask + output_attention_mask) + action_masks.append(output_attention_mask) + + sequences = torch.tensor(sequences) + attention_mask = torch.tensor(attention_masks, dtype=torch.int64) + action_mask = torch.tensor(action_masks, dtype=torch.int64) + + # initialize ret loss masks to be the same as action mask + ret_loss_masks = torch.zeros_like(action_mask, dtype=torch.float) + for i, loss_mask in enumerate(loss_masks): + ret_loss_masks[i, : len(loss_mask)] = torch.tensor(loss_mask) + + # do the same for custom rewards + ret_rewards = torch.zeros_like(action_mask, dtype=torch.float) + for i, custom_reward in enumerate(rewards): + if isinstance(custom_reward, list): + custom_reward = torch.tensor(custom_reward) + ret_rewards[i, : len(custom_reward)] = custom_reward + + logprobs_tensor = None + if logprobs: + max_output_len = action_mask.size(1) + padded_logprobs = [ + sample_logprobs + [0.0] * (max_output_len - len(sample_logprobs)) for sample_logprobs in logprobs + ] + logprobs_tensor = torch.tensor(padded_logprobs, dtype=torch.float) + + return sequences, attention_mask, action_mask, ret_rewards, ret_loss_masks, logprobs_tensor diff --git a/skyrl-train/skyrl_train/dataset/replay_buffer.py b/skyrl-train/skyrl_train/dataset/replay_buffer.py new file mode 100644 index 0000000000..b4929919a9 --- /dev/null +++ b/skyrl-train/skyrl_train/dataset/replay_buffer.py @@ -0,0 +1,366 @@ +""" +Replay buffer utilities. +""" + +# NOTE (sumanthrh): These are replay buffer utilities from OpenRLHF. +# While currently unused, we may need them in the future for more involved off-policy algorithms. +import copy +import random +from abc import ABC +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union +from jaxtyping import Float, Integer + +import torch +import torch.nn.functional as F + + +BasicType = Union[int, float, str, bool] + + +def to(tensor: Union[torch.Tensor, List[torch.Tensor], BasicType], device): + if isinstance(tensor, list): + return [to(t, device) for t in tensor] + elif isinstance(tensor, torch.Tensor): + return tensor.to(device) + else: + return tensor + + +def pin_memory(tensor: Union[torch.Tensor, List[torch.Tensor], BasicType]): + if isinstance(tensor, list): + return [pin_memory(t) for t in tensor] + elif isinstance(tensor, torch.Tensor): + return tensor.pin_memory() + else: + return tensor + + +@dataclass +class Experience: + """Experience is a batch of data. + These data should have the the sequence length and number of actions. + Left padding for sequences is applied. + + Shapes of each tensor: + sequences: (B, S) + action_log_probs: (B, A) + base_action_log_probs: (B, A) + values: (B, A) + returns: (B, A) + advatanges: (B, A) + attention_mask: (B, S) + action_mask: (B, A) + kl: (B, A) + + "A" is the number of actions/ response length. + """ + + sequences: Integer[torch.Tensor, "batch seq_len"] + action_log_probs: Float[torch.Tensor, "batch response_len"] + base_action_log_probs: Optional[Float[torch.Tensor, "batch response_len"]] + values: Optional[Float[torch.Tensor, "batch response_len"]] + returns: Optional[Float[torch.Tensor, "batch response_len"]] + advantages: Optional[Float[torch.Tensor, "batch response_len"]] + attention_mask: Optional[Integer[torch.LongTensor, "batch seq_len"]] + loss_mask: Optional[Integer[torch.LongTensor, "batch response_len"]] + action_mask: Optional[Integer[torch.Tensor, "batch response_len"]] + rollout_logprobs: Optional[Float[torch.Tensor, "batch response_len"]] + num_actions: int + info: Optional[dict] + kl: Optional[Float[torch.Tensor, "batch response_len"]] = None + metadata: Optional[Dict[str, Any]] = None + + @torch.no_grad() + def to_device(self, device: torch.device) -> None: + self.sequences = to(self.sequences, device) + self.action_log_probs = to(self.action_log_probs, device) + if self.base_action_log_probs is not None: + self.base_action_log_probs = to(self.base_action_log_probs, device) + if self.values is not None: + self.values = to(self.values, device) + if self.returns is not None: + self.returns = to(self.returns, device) + if self.advantages is not None: + self.advantages = to(self.advantages, device) + if self.attention_mask is not None: + self.attention_mask = to(self.attention_mask, device) + if self.loss_mask is not None: + self.loss_mask = to(self.loss_mask, device) + if self.action_mask is not None: + self.action_mask = to(self.action_mask, device) + if self.rollout_logprobs is not None: + self.rollout_logprobs = to(self.rollout_logprobs, device) + + def pin_memory(self): + self.sequences = pin_memory(self.sequences) + self.action_log_probs = pin_memory(self.action_log_probs) + if self.base_action_log_probs is not None: + self.base_action_log_probs = pin_memory(self.base_action_log_probs) + if self.values is not None: + self.values = pin_memory(self.values) + if self.returns is not None: + self.returns = pin_memory(self.returns) + if self.advantages is not None: + self.advantages = pin_memory(self.advantages) + if self.attention_mask is not None: + self.attention_mask = self.attention_mask.pin_memory() + if self.loss_mask is not None: + self.loss_mask = self.loss_mask.pin_memory() + if self.action_mask is not None: + self.action_mask = self.action_mask.pin_memory() + if self.rollout_logprobs is not None: + self.rollout_logprobs = self.rollout_logprobs.pin_memory() + return self + + +@dataclass +class BufferItem: + """BufferItem is an item of experience data. + + Shapes of each tensor: + sequences: (S) + action_log_probs: (A) + base_action_log_probs: (A) + values: (1) + returns: (1) + advatanges: (1) + attention_mask: (S) + loss_mask: (A) + action_mask: (A) + + "A" is the number of actions. + """ + + sequences: Integer[torch.Tensor, "seq_len"] # noqa: F821 + action_log_probs: Float[torch.Tensor, "response_len"] # noqa: F821 + base_action_log_probs: Optional[Float[torch.Tensor, "response_len"]] # noqa: F821 + values: Optional[Float[torch.Tensor, "response_len"]] # noqa: F821 + returns: Optional[Float[torch.Tensor, "response_len"]] # noqa: F821 + advantages: Optional[Float[torch.Tensor, "response_len"]] # noqa: F821 + attention_mask: Optional[Integer[torch.LongTensor, "seq_len"]] # noqa: F821 + loss_mask: Optional[Integer[torch.LongTensor, "response_len"]] # noqa: F821 + action_mask: Optional[Integer[torch.Tensor, "response_len"]] # noqa: F821 + num_actions: int + info: Optional[dict] + + def to_json(self) -> dict: + def _to_json(obj): + if isinstance(obj, torch.Tensor): + return obj.cpu().tolist() + elif isinstance(obj, dict): + return {k: _to_json(v) for k, v in obj.items()} + elif isinstance(obj, (list, tuple)): + return [_to_json(x) for x in obj] + else: + return obj + + return {k: _to_json(v) for k, v in self.__dict__.items()} + + +def split_experience_batch(experience: Experience) -> List[BufferItem]: + keys = ( + "sequences", + "action_log_probs", + "base_action_log_probs", + "values", + "returns", + "advantages", + "attention_mask", + "loss_mask", + "action_mask", + "num_actions", + ) + if len(experience.sequences.shape) == 1: + # no padding + batch_size = 1 + for key in keys: + value = getattr(experience, key) + if value is not None: + if isinstance(value, torch.Tensor): + value = value.unsqueeze(0) + setattr(experience, key, value) + else: + batch_size = len(experience.sequences) + batch_kwargs = [{} for _ in range(batch_size)] + + for key in keys: + value = getattr(experience, key) + if value is None: + for i in range(batch_size): + batch_kwargs[i][key] = None + continue + vals = value + if isinstance(vals, torch.Tensor): + vals = torch.unbind(vals) + assert batch_size == len(vals) + for i, v in enumerate(vals): + if isinstance(v, torch.Tensor): + v = v.clone() + batch_kwargs[i][key] = v + + for i in range(batch_size): + batch_kwargs[i]["info"] = {} + for k, v in experience.info.items(): + vals = torch.unbind(v) + assert batch_size == len(vals) + for i, vv in enumerate(vals): + if isinstance(vv, torch.Tensor): + assert vv.numel() == 1, f"info[{k}] must be a scalar tensor, but got {vv.shape}" + vv = vv.item() + batch_kwargs[i]["info"][k] = vv + + items = [BufferItem(**kwargs) for kwargs in batch_kwargs] + return items + + +def zero_pad_sequences(sequences: List[torch.Tensor], side: str = "left") -> torch.Tensor: + assert side in ("left", "right") + max_len = max(seq.size(0) for seq in sequences) + padded_sequences = [] + for seq in sequences: + pad_len = max_len - seq.size(0) + padding = (pad_len, 0) if side == "left" else (0, pad_len) + padded_sequences.append(F.pad(seq, padding)) + return torch.stack(padded_sequences, dim=0) + + +def make_experience_batch(items: List[BufferItem]) -> Experience: + kwargs = {} + keys = ( + "sequences", + "action_log_probs", + "base_action_log_probs", + "values", + "returns", + "advantages", + "attention_mask", + "loss_mask", + "action_mask", + "num_actions", + ) + for key in keys: + vals = [getattr(item, key) for item in items] + # NOTE (sumanthrh): Assumes list of Tensors + batch_data = vals if vals[0] is not None else None + kwargs[key] = batch_data + + kwargs["info"] = {} + for key in items[0].info.keys(): + vals = torch.tensor([item.info[key] for item in items]) + kwargs["info"][key] = vals + return Experience(**kwargs) + + +def remove_padding_in_sequences(items): + for item in items: + seq, act_log_prob, base_act_log_prob, value, ret, adv, att_mask, act_mask = ( + item.sequences, + item.action_log_probs, + item.base_action_log_probs, + item.values, + item.returns, + item.advantages, + item.attention_mask, + item.action_mask, + ) + right_pad = (1 - act_mask.long()).sum() + right_pad = None if right_pad == 0 else -right_pad + + # left_pad for seq and att_mask + left_pad = att_mask.long().argmax() + ( + item.sequences, + item.action_log_probs, + item.base_action_log_probs, + item.values, + item.returns, + item.advantages, + item.attention_mask, + item.action_mask, + ) = ( + seq[left_pad:right_pad], + act_log_prob[:right_pad], + base_act_log_prob[:right_pad], + value[:right_pad] if value is not None else None, + ret[:right_pad] if ret is not None else None, + adv[:right_pad] if adv is not None else None, + att_mask[left_pad:right_pad] if att_mask is not None else None, + act_mask[:right_pad] if act_mask is not None else None, + ) + return items + + +class NaiveReplayBuffer(ABC): + """Naive replay buffer class. It stores experience. + + Args: + sample_batch_size (int): Batch size when sampling. + limit (int, optional): Limit of number of experience samples. A number <= 0 means unlimited. Defaults to 0. + cpu_offload (bool, optional): Whether to offload experience to cpu when sampling. Defaults to True. + """ + + def __init__( + self, + sample_batch_size: int, + limit: int = 0, + cpu_offload: bool = True, + ) -> None: + super().__init__() + self.sample_batch_size = sample_batch_size + # limit <= 0 means unlimited + self.limit = limit + self.cpu_offload = cpu_offload + self.target_device = ( + torch.device(f"cuda:{torch.cuda.current_device()}") if torch.cuda.is_available() else torch.device("cpu") + ) + self.items: List[BufferItem] = [] + + @torch.no_grad() + def split_to_n_batches(self, n_batches: int, drop_last: bool = False) -> List["NaiveReplayBuffer"]: + assert n_batches > 0 + if not drop_last: + batch_size = (len(self.items) + n_batches - 1) // n_batches + else: + batch_size = len(self.items) // n_batches + if len(self.items) < batch_size * n_batches: + # padding + padding_size = batch_size * n_batches - len(self.items) + padding_items = random.choices(self.items, k=padding_size) + self.items.extend(padding_items) + bfs = [] + items = copy.deepcopy(self.items) + random.shuffle(items) + for i in range(n_batches): + bf = NaiveReplayBuffer( + sample_batch_size=self.sample_batch_size, + limit=self.limit, + cpu_offload=self.cpu_offload, + ) + bf.items = items[i * batch_size : (i + 1) * batch_size] + bfs.append(bf) + return bfs + + @torch.no_grad() + def append(self, experience: Experience) -> None: + if self.cpu_offload: + experience.to_device(torch.device("cpu")) + items = split_experience_batch(experience) + self.items.extend(items) + if self.limit > 0: + samples_to_remove = len(self.items) - self.limit + if samples_to_remove > 0: + self.items = self.items[samples_to_remove:] + + def clear(self) -> None: + self.items.clear() + + def __len__(self) -> int: + return len(self.items) + + def __getitem__(self, idx: int) -> BufferItem: + return self.items[idx] + + def collate_fn(self, batch) -> Experience: + experience = make_experience_batch(batch) + return experience diff --git a/skyrl-train/skyrl_train/distributed/__init__.py b/skyrl-train/skyrl_train/distributed/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/skyrl_train/distributed/dispatch.py b/skyrl-train/skyrl_train/distributed/dispatch.py new file mode 100644 index 0000000000..5f53524dfa --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/dispatch.py @@ -0,0 +1,264 @@ +"""Defines dispatch and collect logic for distributed training""" + +from dataclasses import dataclass +from ray.actor import ActorHandle +from typing import List, Tuple, Optional, Dict, Type, Any +import asyncio +from abc import ABC, abstractmethod +import ray +from ray import ObjectRef +from skyrl_train.training_batch import TrainingInputBatch, TrainingOutputBatch +import inspect + + +@dataclass +class MeshRank: + """Represents a rank in the device mesh. + + This is a tuple of (DP, SP, TP, PP) ranks. + """ + + dp: int + sp: int + tp: int + pp: int + + world_size: int + dp_size: int + pp_size: int + + def is_collection_dp_rank(self) -> bool: + """Check if this rank is a DP rank to collect from + + This is the rank with (SP=0, TP=0, PP=pp_size-1) + + Note: double check this for ETP > 1 (but this is not a typically used case) + """ + return self.tp == 0 and self.pp == self.pp_size - 1 and self.sp == 0 + + def __str__(self) -> str: + return f"MeshRank(dp={self.dp}, sp={self.sp}, tp={self.tp}, pp={self.pp}, world_size={self.world_size}, dp_size={self.dp_size}, pp_size={self.pp_size})" + + def __repr__(self) -> str: + return self.__str__() + + +@dataclass +class ActorInfo: + """Actor information for distributed training. + + This includes the actor handle and the rank in the device mesh. + """ + + handle: ActorHandle + rank: MeshRank + + +class Dispatch(ABC): + """Base class for dispatch types + + Dispatch types are responsible for: + - dispatching method calls to actors handling data sharding if necessary + - collecting results from actors and concatenating results if necessary + - validating arguments for dispatch + """ + + @classmethod + @abstractmethod + def dispatch(cls, actor_infos: List[ActorInfo], method: str, *args, **kwargs) -> List[ObjectRef]: + """Dispatches method calls to the actors with data sharding if necessary.""" + pass + + @classmethod + @abstractmethod + async def async_collect( + cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef] + ) -> Optional[TrainingOutputBatch]: + """Collects results from the actors asynchronously in an asyncio-compatible way.""" + pass + + @classmethod + @abstractmethod + def sync_collect(cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef]) -> Optional[TrainingOutputBatch]: + """Collects results from the actors synchronously and returns a `TrainingOutputBatch`.""" + pass + + @classmethod + @abstractmethod + def validate_dispatch_args(cls, *args, **kwargs) -> Tuple[Tuple, Dict[str, Any]]: + """Validate and process arguments for dispatch. + + Returns: + Tuple of (args, kwargs) to be passed to dispatch + """ + pass + + +class MeshDispatch(Dispatch): + """Mesh dispatch type to dispatch data to a group of actors along the device mesh. + + Supports DP (Data Parallel), SP (Sequence Parallel), TP (Tensor Parallel) and PP (Pipeline Parallel) parallelism. + The actor method should accept a single argument - the data batch. + + For data dispatch: + + * The input data is chunked into `dp_size` equal chunks, where `dp_size` is the size of data parallelism. + * Each actor with the same DP rank processes the same data chunk in parallel. + + For data collection: + + * Data is collected only from the primary rank of each model/sequence parallel group. + * The primary rank is defined as the rank with (SP=0, TP=0, PP=0). + * The collected chunks are concatenated in order of DP rank to reconstruct the full data. + + Example: For a world size of 8, with DP size=2, SP size=2, TP size=2, PP size=1: + + * Data dispatch: The data is chunked into 2 chunks. All actors with DP rank 0 process the first chunk, + and all actors with DP rank 1 process the second chunk. + * Data collection: Only two actors contribute to the final output - the primary rank from each DP group: + (DP=0, SP=0, TP=0, PP=0) and (DP=1, SP=0, TP=0, PP=0). Their chunks are concatenated in order. + + """ + + @classmethod + def dispatch(cls, actor_infos: List[ActorInfo], method: str, data: TrainingInputBatch) -> List[ObjectRef]: + assert len(actor_infos) > 0, "actor_infos must be a non-empty list" + object_refs = [] + dp_size = actor_infos[0].rank.dp_size + assert len(data) % dp_size == 0, "data batch size must be divisible by dp_size, got {} and {}".format( + len(data), dp_size + ) + chunk_size = len(data) // dp_size + data_chunks: List[TrainingInputBatch] = data.chunk(chunk_size) + + for actor_info in actor_infos: + # index into tensordict to get the correct data to send + data_to_send = data_chunks[actor_info.rank.dp] + object_refs.append(getattr(actor_info.handle, method).remote(data_to_send)) + return object_refs + + @classmethod + async def async_collect( + cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef] + ) -> Optional[TrainingOutputBatch]: + assert len(actor_infos) == len(object_refs), "`actor_infos` and `object_refs` must have the same length" + all_objects = await asyncio.gather(*object_refs) + if len(all_objects) and all_objects[0] is not None: + return concatenate_outputs_after_mesh_dispatch(actor_infos, all_objects) + return + + @classmethod + def sync_collect(cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef]) -> Optional[TrainingOutputBatch]: + assert len(actor_infos) == len(object_refs), "`actor_infos` and `object_refs` must have the same length" + all_objects = ray.get(object_refs) + if len(all_objects) and all_objects[0] is not None: + return concatenate_outputs_after_mesh_dispatch(actor_infos, all_objects) + # all should be none + assert all(obj is None for obj in all_objects), "Got a mix of `None` and non-`None` objects" + return + + @classmethod + def validate_dispatch_args(cls, *args, **kwargs) -> Tuple[Tuple, Dict[str, Any]]: + sig = inspect.signature(cls.dispatch) + # pass dummy actor_infos and method_name + bound_args = sig.bind([], "dummy", *args, **kwargs) + bound_args.apply_defaults() + + # Check if there are any extra arguments + if len(bound_args.arguments) > 3: # data, actor_infos, method_name + # remove actor_infos and method_name - not added by user + bound_args.arguments.pop("actor_infos") + bound_args.arguments.pop("method") + raise ValueError(f"MeshDispatch only accepts 'data' as an argument, got extra args: {bound_args.arguments}") + + data = bound_args.arguments.get("data") + if not isinstance(data, TrainingInputBatch): + raise ValueError(f"For MeshDispatch, `data` entry should be a `TrainingInput`, got {data}") + args = (data,) + kwargs = {} + return args, kwargs + + +class PassThroughDispatch(Dispatch): + """PassThrough dispatch type to dispatch data to a group of actors without any sharding. + + This is useful for cases where we want to run the same method on all the actors. + Supports methods with any number of arguments. + """ + + @classmethod + def dispatch(cls, actor_infos: List[ActorInfo], method: str, *args, **kwargs) -> List[ObjectRef]: + return [getattr(actor_info.handle, method).remote(*args, **kwargs) for actor_info in actor_infos] + + @classmethod + async def async_collect( + cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef] + ) -> Optional[TrainingOutputBatch]: + all_objects = await asyncio.gather(*object_refs) + if len(all_objects) and all_objects[0] is not None: + return concatenate_outputs_after_mesh_dispatch(actor_infos, all_objects) + return + + @classmethod + def sync_collect(cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef]) -> Optional[TrainingOutputBatch]: + data_batches = ray.get(object_refs) + if len(data_batches) > 0 and data_batches[0] is not None: + assert isinstance( + data_batches[0], TrainingOutputBatch + ), "data_batches must be a list of `TrainingOutputBatch` objects" + return concatenate_outputs_after_mesh_dispatch(actor_infos, data_batches) + # all should be none + assert all(obj is None for obj in data_batches), "Got a mix of `None` and non-`None` objects" + return + + @classmethod + def validate_dispatch_args(cls, *args, **kwargs) -> Tuple[Tuple, Dict[str, Any]]: + # no validation needed just pass everything + return args, kwargs + + +class DispatchRegistry: + _registry: Dict[str, Type[Dispatch]] = {"mesh": MeshDispatch, "pass_through": PassThroughDispatch} + + @classmethod + def register(cls, name: str, dispatch_class: Type[Dispatch]) -> None: + """Register a new dispatch type.""" + assert issubclass(dispatch_class, Dispatch) + cls._registry[name] = dispatch_class + + @classmethod + def get(cls, name: str) -> Type[Dispatch]: + """Get a registered dispatch type.""" + if name not in cls._registry: + raise KeyError(f"Dispatch type '{name}' not registered") + return cls._registry[name] + + @classmethod + def list_registered(cls) -> Dict[str, Type[Dispatch]]: + """List all registered dispatch types.""" + return cls._registry + + +def register_dispatch_type(name: str, dispatch_class: Type) -> None: + DispatchRegistry.register(name, dispatch_class) + + +def concatenate_outputs_after_mesh_dispatch( + actor_infos: List[ActorInfo], data_batches: List[TrainingOutputBatch] +) -> TrainingOutputBatch: + """Concatenate data batches from different ranks after mesh dispatch. + + - Data is collected only from the primary DP rank. + - The collected chunks are concatenated in order of DP rank to reconstruct the full data. + """ + assert len(actor_infos) == len(data_batches), "`actor_infos` and `data_batches` must have the same length" + shards = [] + # collect in-order + dp_rank_to_shard = {} + for actor_info, data_batch in zip(actor_infos, data_batches): + if actor_info.rank.is_collection_dp_rank(): + dp_rank = actor_info.rank.dp + dp_rank_to_shard[dp_rank] = data_batch + for i in range(actor_infos[0].rank.dp_size): + shards.append(dp_rank_to_shard[i]) + return TrainingOutputBatch.cat(shards) diff --git a/skyrl-train/skyrl_train/distributed/fsdp_strategy.py b/skyrl-train/skyrl_train/distributed/fsdp_strategy.py new file mode 100644 index 0000000000..f2e12a3b9a --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/fsdp_strategy.py @@ -0,0 +1,654 @@ +import os +import copy +import random +from collections import defaultdict +from datetime import timedelta +from typing import List, Union, Optional +from jaxtyping import Float +import gc +import json +from loguru import logger +import numpy as np +import torch +import torch.nn as nn +from torch import optim +from torch import distributed as dist +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import CPUOffload, MixedPrecision + +from skyrl_train.distributed.strategy import DistributedStrategy +from skyrl_train.model_wrapper import HFModelWrapper +from skyrl_train.distributed.utils import ModelOrModelOptimPair +from skyrl_train.utils.io import io +from skyrl_train.distributed.fsdp_utils import ( + CPUOffloadPolicy, + MixedPrecisionPolicy, + init_fn, + get_fsdp_wrap_policy, + PrecisionType, + create_device_mesh, + fsdp2_clip_grad_norm_, + fsdp2_get_full_state_dict, + apply_fsdp2, + get_sharding_strategy, + offload_fsdp_model_to_cpu, + load_fsdp_model_to_gpu, + offload_fsdp_optimizer, + load_fsdp_optimizer, + get_fsdp_state_ctx, + fsdp_version, + fsdp2_load_full_state_dict, +) +from transformers.trainer import get_scheduler + +from packaging import version + +if version.parse(torch.__version__) >= version.parse("2.6"): + from torch.distributed.fsdp import CPUOffloadPolicy, FSDPModule, MixedPrecisionPolicy +elif version.parse(torch.__version__) >= version.parse("2.4"): + from torch.distributed._composable.fsdp import CPUOffloadPolicy, FSDPModule, MixedPrecisionPolicy +else: + CPUOffloadPolicy, FSDPModule, MixedPrecisionPolicy = None, None, None + + +class FSDPStrategy(DistributedStrategy): + """ + The strategy for training with FSDP. + """ + + def __init__( + self, + fsdp_config, + optimizer_config=None, + model_config=None, + fsdp_strategy: str = "fsdp", + seed: int = 42, + micro_train_batch_size_per_gpu=1, + num_training_steps: Optional[int] = None, + ) -> None: + super().__init__() + assert fsdp_strategy in ("fsdp", "fsdp2"), f"Unsupported FSDP strategy: {fsdp_strategy}" + self.fsdp_config = fsdp_config + self.optimizer_config = optimizer_config + self.model_config = model_config + self.fsdp_strategy = fsdp_strategy + self.max_norm = optimizer_config.max_grad_norm if optimizer_config is not None else 1.0 + self.micro_train_batch_size_per_gpu = micro_train_batch_size_per_gpu + self.seed = seed + self.device_mesh = None + self.total_training_steps: Optional[int] = num_training_steps + + # if we are using fsdp 1 or cpu offload is off for fsdp2, then we need to manually offload weights/optimizer to cpu + self.manual_offload = self.fsdp_strategy == "fsdp" or not self.fsdp_config.get("cpu_offload") + if self.optimizer_config is not None: + self.manual_offload_optimizer = ( + self.optimizer_config.get("offload_after_step", True) and self.manual_offload + ) + else: + self.manual_offload_optimizer = False + + # LoRA related configs + self.is_lora = self.model_config.lora.rank > 0 if self.model_config is not None else False + + self.time_steps = defaultdict(int) + + def set_seed(self, seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + def setup_distributed(self, timeout=timedelta(minutes=30)) -> None: + self.set_seed(self.seed) + + local_rank = int(os.environ.get("LOCAL_RANK", "-1")) + if local_rank != -1: + torch.cuda.set_device(local_rank) + + # Initializes the distributed backend which will take care of synchronizing nodes/GPUs + self.world_size = dist.get_world_size() + + self.device_mesh = create_device_mesh(world_size=self.world_size, fsdp_size=self.fsdp_config.fsdp_size) + + def offload_to_cpu( + self, model, optimizer, pin_memory=True, non_blocking=True, offload_optimizer=True, offload_model=True + ): + """ + Offload model weights and optimizer to CPU memory. + + For all cases except fsdp2 with cpu_offload=True, we need to manually offload weights/optimizer to cpu. + """ + if isinstance(model, HFModelWrapper): + model = model.model + else: + model = model + + if self.manual_offload: + if offload_model: + offload_fsdp_model_to_cpu(model, empty_cache=True) + + if optimizer is not None and self.manual_offload_optimizer and offload_optimizer: + offload_fsdp_optimizer(optimizer) + + torch.cuda.synchronize() + torch.cuda.empty_cache() + + def backload_to_gpu(self, model, optimizer, non_blocking=True, backload_optimizer=True, backload_model=True): + """Reload model weights back to GPU.""" + if isinstance(model, HFModelWrapper): + model = model.model + else: + model = model + + # if we are using fsdp 1 or cpu offload is off for fsdp2, then we need to manually backload weights/optimizer to gpu + if self.manual_offload: + if backload_model: + load_fsdp_model_to_gpu(model) + if optimizer is not None and self.manual_offload_optimizer and backload_optimizer: + load_fsdp_optimizer(optimizer, torch.cuda.current_device()) + + torch.cuda.synchronize() + + def backward(self, loss: torch.Tensor, model, optimizer: optim.Optimizer, **kwargs) -> None: + """Perform backward pass""" + loss.backward() + + def optimizer_step( + self, + optimizer: optim.Optimizer, + model, + scheduler, + name="model", + **kwargs, + ) -> Optional[Float[torch.Tensor, "1"]]: + """Perform optimizer step""" + grad_norm = None + if isinstance(model, HFModelWrapper): + model = model.model + + if self.max_norm > 0: + # NOTE (sumanthrh): All `grad_norm`s returned here are the original grad norms before clipping. + if isinstance(model, FSDP): + grad_norm = model.clip_grad_norm_(max_norm=self.max_norm) + elif isinstance(model, FSDPModule): + grad_norm = fsdp2_clip_grad_norm_(model.parameters(), max_norm=self.max_norm) + else: + grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=self.max_norm) + + # Skip update if gradient norm is not finite + if grad_norm is not None and not torch.isfinite(grad_norm): + if torch.distributed.is_initialized(): + rank = torch.distributed.get_rank() + logger.warning(f"rank {rank} grad_norm is not finite: {grad_norm}") + else: + logger.warning(f"grad_norm is not finite: {grad_norm}") + optimizer.zero_grad() + return grad_norm + + optimizer.step() + if scheduler is not None: + scheduler.step() + optimizer.zero_grad() + return grad_norm + + def prepare( + self, *models_or_model_optim_pairs: ModelOrModelOptimPair + ) -> Union[List[ModelOrModelOptimPair], ModelOrModelOptimPair]: + """Prepare models and optimizers with FSDP""" + ret = [] + for arg in models_or_model_optim_pairs: + if isinstance(arg, tuple): + assert len(arg) == 3, f'Expect (model, optimizer, scheduler) pair, got a tuple with size "{len(arg)}"' + ret.append(self._fsdp_init_train_model(*arg)) + else: + ret.append(self._fsdp_init_eval_model(arg)) + + return ret[0] if len(ret) == 1 else ret + + def _fsdp_init_model(self, model, is_train=True, is_wrapped=False): + # Initialize FSDP wrapping policy + wrap_policy = get_fsdp_wrap_policy( + module=model.model if is_wrapped else model, + config=self.fsdp_config.get("wrap_policy", None), + is_lora=self.is_lora, + ) + + # Setup mixed precision + mixed_precision_config = self.fsdp_config.get("mixed_precision", None) + if mixed_precision_config is not None: + param_dtype = PrecisionType.to_dtype(mixed_precision_config.get("param_dtype", "bf16")) + reduce_dtype = PrecisionType.to_dtype(mixed_precision_config.get("reduce_dtype", "fp32")) + buffer_dtype = PrecisionType.to_dtype(mixed_precision_config.get("buffer_dtype", "fp32")) + else: + param_dtype = torch.bfloat16 + reduce_dtype = torch.float32 + buffer_dtype = torch.float32 + + mixed_precision = MixedPrecision(param_dtype=param_dtype, reduce_dtype=reduce_dtype, buffer_dtype=buffer_dtype) + + cpu_offload = None + + # sharding strategy + fsdp_mesh = self.device_mesh + sharding_strategy = get_sharding_strategy(fsdp_mesh) + + # Wrap model with FSDP + if self.fsdp_strategy == "fsdp": + # cpu offloading will always be none for models that train with FSDP due to correctness issues with gradient accumulation - + # see https://docs.pytorch.org/docs/stable/fsdp.html + if not is_train and self.fsdp_config.get("cpu_offload", False): + cpu_offload = CPUOffload(offload_params=True) + fsdp_module = FSDP( + model.model if is_wrapped else model, + cpu_offload=cpu_offload, + param_init_fn=init_fn, + use_orig_params=False, + auto_wrap_policy=wrap_policy, + device_id=torch.cuda.current_device(), + sharding_strategy=sharding_strategy, + mixed_precision=mixed_precision, + sync_module_states=True, + device_mesh=self.device_mesh, + forward_prefetch=False, + ) + elif self.fsdp_strategy == "fsdp2": + assert CPUOffloadPolicy is not None, "PyTorch version >= 2.4 is required for using fully_shard API (FSDP2)" + mp_policy = MixedPrecisionPolicy( + param_dtype=param_dtype, reduce_dtype=reduce_dtype, cast_forward_inputs=True + ) + if self.fsdp_config.get("cpu_offload", False): + cpu_offload = CPUOffloadPolicy(pin_memory=True) + + fsdp_kwargs = { + "mesh": fsdp_mesh, + "mp_policy": mp_policy, + "offload_policy": cpu_offload, + "reshard_after_forward": self.fsdp_config.get("reshard_after_forward", True), + } + module = model.model if is_wrapped else model + full_state = module.state_dict() + apply_fsdp2(module, fsdp_kwargs, self.fsdp_config) + fsdp2_load_full_state_dict(module, full_state, cpu_offload) + fsdp_module = module + else: + raise NotImplementedError(f"{self.fsdp_strategy} not implemented") + + return fsdp_module + + def _fsdp_init_train_model(self, model, optimizer, scheduler): + """Initialize a model for training with FSDP""" + is_wrapped = isinstance(model, HFModelWrapper) + fsdp_module = self._fsdp_init_model(model, is_train=True, is_wrapped=is_wrapped) + + optim_config = self.optimizer_config + if optim_config is not None: + new_optimizer = optim.AdamW( + fsdp_module.parameters(), + lr=optim_config.lr, + betas=optim_config.adam_betas, + weight_decay=optim_config.weight_decay, + ) + + lr_scheduler = get_scheduler( + optim_config.scheduler, + new_optimizer, + num_warmup_steps=optim_config.num_warmup_steps, + num_training_steps=self.total_training_steps, + ) + else: + new_optimizer = None + lr_scheduler = None + + if is_wrapped: + model.model = fsdp_module + else: + model = fsdp_module + + return model, new_optimizer, lr_scheduler + + def _fsdp_init_eval_model(self, model): + """Initialize a model for evaluation with FSDP""" + is_wrapped = isinstance(model, HFModelWrapper) + fsdp_module = self._fsdp_init_model(model, is_train=False, is_wrapped=is_wrapped) + + if is_wrapped: + model.model = fsdp_module + else: + model = fsdp_module + + return model + + def _unwrap_model(self, model) -> nn.Module: + """Unwrap model from HFModelWrapper or FSDP""" + # Handle HFModelWrapper wrapper + if isinstance(model, HFModelWrapper): + return self._unwrap_model(model.model) + + # For FSDP2 models, check if the FSDP model itself has the necessary attributes + model_type = type(model).__name__ + if "FSDP" in model_type: + has_config = hasattr(model, "config") + has_lm_head = hasattr(model, "lm_head") + has_generate = hasattr(model, "generate") + if has_config and (has_lm_head or has_generate): + return model + + # Check for FSDP v1 unwrapping + if hasattr(model, "_fsdp_wrapped_module"): + return model._fsdp_wrapped_module + + # If no unwrapping needed, return the original model + return model + + def _fix_fsdp_config(self, config): + """Fix architecture names by removing FSDP prefix if present""" + # Determine which config to save + config_to_save = config + + # Fix architecture name by removing FSDP prefix if present + if hasattr(config_to_save, "architectures") and config_to_save.architectures: + # Create a copy of the config to avoid modifying the original + config_to_save = copy.deepcopy(config_to_save) + + # Fix architecture names to remove FSDP prefix + fixed_architectures = [] + for arch in config_to_save.architectures: + fixed_arch = arch + if arch.startswith("FSDP"): + # Remove "FSDP" prefix (for fsdp2) + fixed_arch = arch[len("FSDP") :] + self.print(f"[rank-0]: Fixed architecture name: {arch} -> {fixed_arch}") + fixed_architectures.append(fixed_arch) + + config_to_save.architectures = fixed_architectures + + return config_to_save + + def _save_lora_adapters(self, model, ckpt_dir): + """Save LoRA adapters in HuggingFace PEFT format""" + from dataclasses import asdict + from safetensors.torch import save_file + from skyrl_train.distributed.fsdp_utils import layered_summon_lora_params + + lora_save_path = os.path.join(ckpt_dir, "lora_adapter") + peft_config = {} + + if self.is_rank_0(): + io.makedirs(lora_save_path, exist_ok=True) + peft_config = asdict(model.peft_config.get("default", {})) + if peft_config: + peft_config["task_type"] = peft_config["task_type"].value + peft_config["peft_type"] = peft_config["peft_type"].value + peft_config["target_modules"] = list(peft_config["target_modules"]) + + lora_params = layered_summon_lora_params(model) + + if self.is_rank_0(): + save_file(lora_params, os.path.join(lora_save_path, "adapter_model.safetensors")) + with io.open_file(os.path.join(lora_save_path, "adapter_config.json"), "w") as f: + json.dump(peft_config, f, ensure_ascii=False, indent=4) + + self.print(f"[rank-0]: Saved LoRA adapter to: {lora_save_path}") + + dist.barrier() + + def save_checkpoint( + self, + model, + ckpt_dir, + node_local_rank, + optimizer=None, + scheduler=None, + client_state={}, + tag=None, + tokenizer=None, + ): + """Save model checkpoint for FSDP""" + import warnings + from torch.distributed.fsdp import ShardedStateDictConfig, ShardedOptimStateDictConfig, StateDictType + + if node_local_rank == 0: + io.makedirs(ckpt_dir, exist_ok=True) + + # Wait for checkpoint directory to be created. + dist.barrier() + + # Extract the actual model for saving + if isinstance(model, HFModelWrapper): + save_model = model.model + else: + save_model = model + + if self.fsdp_strategy not in ("fsdp", "fsdp2"): + raise ValueError(f"Unsupported FSDP strategy: {self.fsdp_strategy}") + + # Set up state dict configurations for sharded saving + state_dict_cfg = ShardedStateDictConfig(offload_to_cpu=True) + optim_cfg = ShardedOptimStateDictConfig(offload_to_cpu=True) + + # Define paths for saving individual rank files + rank = self.get_rank() + world_size = self.world_size + + with io.local_work_dir(ckpt_dir) as work_dir: + model_path = os.path.join(work_dir, f"model_world_size_{world_size}_rank_{rank}.pt") + optim_path = os.path.join(work_dir, f"optim_world_size_{world_size}_rank_{rank}.pt") + extra_path = os.path.join(work_dir, f"extra_state_world_size_{world_size}_rank_{rank}.pt") + + # Save using appropriate FSDP context + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with get_fsdp_state_ctx(save_model, StateDictType.SHARDED_STATE_DICT, state_dict_cfg, optim_cfg): + # Get and save model state dict + model_state_dict = save_model.state_dict() + self.print(f"[rank-{rank}]: Saving model to {model_path}") + with io.open_file(model_path, "wb") as f: + torch.save(model_state_dict, f) + + # Get and save optimizer state dict if optimizer is provided + optimizer_state_dict = {} + if optimizer is not None: + optimizer_state_dict = optimizer.state_dict() + self.print(f"[rank-{rank}]: Saving optim to {optim_path}") + with io.open_file(optim_path, "wb") as f: + torch.save(optimizer_state_dict, f) + + # Get scheduler state dict if scheduler is provided + lr_scheduler_state_dict = {} + if scheduler is not None: + lr_scheduler_state_dict = scheduler.state_dict() + + # Create extra state dict with client state and any additional info + extra_state_dict = { + "lr_scheduler": lr_scheduler_state_dict, + "client_state": client_state, + "tag": tag, + "fsdp_strategy": self.fsdp_strategy, + "world_size": world_size, + "rank": rank, + "rng": self.get_rng_state(), # Add RNG state for reproducibility + } + + # Save extra state + self.print(f"[rank-{rank}]: Saving extra_state to {extra_path}") + with io.open_file(extra_path, "wb") as f: + torch.save(extra_state_dict, f) + + # Garbage collect temporary buffers from materializing the state dicts + gc.collect() + + if self.is_rank_0(): + config_save_model = self._unwrap_model(model) + hf_dir = os.path.join(work_dir, "huggingface") + self.save_hf_configs(config_save_model.config, hf_dir, tokenizer) + + # Also save runtime FSDP config + fsdp_config_path = os.path.join(work_dir, "fsdp_config.json") + with io.open_file(fsdp_config_path, "w") as f: + json.dump({"fsdp_strategy": self.fsdp_strategy, "world_size": self.world_size}, f, indent=4) + + # Save LoRA adapters if using LoRA + if self.is_lora and hasattr(save_model, "peft_config"): + self._save_lora_adapters(save_model, ckpt_dir) + + # Final barrier to ensure all operations complete + dist.barrier() + torch.cuda.synchronize() + self.print(f"[rank-{rank}]: Checkpoint saved to {ckpt_dir}") + + def load_checkpoint( + self, + model, + ckpt_dir, + optimizer=None, + scheduler=None, + tag=None, + load_module_strict=True, + load_optimizer_states=True, + load_lr_scheduler_states=True, + ): + """Load model checkpoint for FSDP""" + import warnings + from torch.distributed.fsdp import ShardedStateDictConfig, ShardedOptimStateDictConfig, StateDictType + + if ckpt_dir is None: + raise ValueError("ckpt_dir cannot be None") + elif not io.exists(ckpt_dir): + raise FileNotFoundError(f"Checkpoint directory not found: {ckpt_dir}") + + # Extract the actual model for loading + load_model = model + if isinstance(model, HFModelWrapper): + load_model = model.model + + # Define paths for loading individual rank files + rank = self.get_rank() + world_size = self.world_size + + with io.local_read_dir(ckpt_dir) as read_dir: + model_path = os.path.join(read_dir, f"model_world_size_{world_size}_rank_{rank}.pt") + optim_path = os.path.join(read_dir, f"optim_world_size_{world_size}_rank_{rank}.pt") + extra_path = os.path.join(read_dir, f"extra_state_world_size_{world_size}_rank_{rank}.pt") + + # Check if checkpoint files exist + if not io.exists(model_path): + raise FileNotFoundError(f"Model checkpoint not found: {model_path}") + if not io.exists(extra_path): + raise FileNotFoundError(f"Extra state checkpoint not found: {extra_path}") + + # Optimizer path is optional since we may not save optimizer states initially + optim_exists = io.exists(optim_path) + + self.print(f"[rank-{rank}]: Loading model from {model_path}") + self.print(f"[rank-{rank}]: Loading extra_state from {extra_path}") + if optim_exists: + self.print(f"[rank-{rank}]: Loading optim from {optim_path}") + + # Load state dictionaries from disk + with io.open_file(model_path, "rb") as f: + model_state_dict = torch.load(f, map_location="cpu", weights_only=False) + with io.open_file(extra_path, "rb") as f: + extra_state_dict = torch.load(f, map_location="cpu", weights_only=False) + + optimizer_state_dict = {} + if optim_exists and load_optimizer_states: + with io.open_file(optim_path, "rb") as f: + optimizer_state_dict = torch.load(f, map_location="cpu", weights_only=False) + + # Extract scheduler state from extra state + lr_scheduler_state_dict = extra_state_dict.get("lr_scheduler", {}) + + # Set up state dict configurations for sharded loading + state_dict_cfg = ShardedStateDictConfig(offload_to_cpu=True) + optim_cfg = ShardedOptimStateDictConfig(offload_to_cpu=True) + + # Load using appropriate FSDP context + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with get_fsdp_state_ctx(load_model, StateDictType.SHARDED_STATE_DICT, state_dict_cfg, optim_cfg): + # Load model state dict + load_model.load_state_dict(model_state_dict, strict=load_module_strict) + self.print(f"[rank-{rank}]: Successfully loaded model state dict") + + # Load optimizer state dict if optimizer object is provided and loading is requested + if optimizer is not None and load_optimizer_states and optimizer_state_dict: + optimizer.load_state_dict(optimizer_state_dict) + self.print(f"[rank-{rank}]: Successfully loaded optimizer state") + + # Load scheduler state dict if scheduler object is provided and loading is requested + if scheduler is not None and load_lr_scheduler_states: + scheduler.load_state_dict(lr_scheduler_state_dict) + self.print(f"[rank-{rank}]: Successfully loaded scheduler state") + + # Load RNG state for reproducibility + if "rng" in extra_state_dict: + self.load_rng_state(extra_state_dict["rng"]) + + # Wait for all ranks to finish loading + dist.barrier() + + # Create states dict with extra information + client_state = extra_state_dict.get("client_state", {}) + states = { + "client_state": client_state, + "tag": extra_state_dict.get("tag", tag), + "fsdp_strategy": extra_state_dict.get("fsdp_strategy", self.fsdp_strategy), + "world_size": extra_state_dict.get("world_size", world_size), + "rank": extra_state_dict.get("rank", rank), + } + + self.print(f"[rank-{rank}]: Checkpoint loaded successfully from {ckpt_dir}") + + return ckpt_dir, states + + # TODO (erictang000): Test in multi-node setting + def save_hf_model(self, model: Union[HFModelWrapper, nn.Module], output_dir: str, tokenizer=None, **kwargs) -> None: + """Save model in HuggingFace safetensors format using FSDP's full state dict gathering""" + + # Step 1: Create output directory (rank 0 only) + if self.is_rank_0(): + io.makedirs(output_dir, exist_ok=True) + self.print(f"[rank-0]: Created output directory: {output_dir}") + + # Step 2: Extract models - get both the model for saving metadata and the FSDP model for state dict + model_to_save = self._unwrap_model(model) # For saving config/metadata + fsdp_model = model.model if isinstance(model, HFModelWrapper) else model # For state dict collection + + # Validate that we have a proper HuggingFace model + if not hasattr(model_to_save, "config") or not hasattr(model_to_save, "save_pretrained"): + raise ValueError("Model must be a HuggingFace model with config and save_pretrained method") + + # Step 3: Determine FSDP version and collect full state dict + fsdp_ver = fsdp_version(fsdp_model) + self.print(f"[rank-{self.get_rank()}]: Detected FSDP version: {fsdp_ver}") + + if fsdp_ver == 2: + # Use FSDP2 API - collects on rank 0 only + output_state_dict = fsdp2_get_full_state_dict(fsdp_model, cpu_offload=True, rank0_only=True) + elif fsdp_ver == 1: + from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict + + options = StateDictOptions(full_state_dict=True, cpu_offload=True, broadcast_from_rank0=False) + output_state_dict = get_model_state_dict(fsdp_model, options=options) + if not self.is_rank_0(): + output_state_dict.clear() + else: + raise ValueError(f"Unsupported FSDP version: {fsdp_ver}") + + # Step 4: Save on rank 0 only + if self.is_rank_0(): + with io.local_work_dir(output_dir) as work_dir: + # Save the model in HuggingFace format using safetensors + model_to_save.save_pretrained(work_dir, state_dict=output_state_dict, safe_serialization=True, **kwargs) + + # Fix and save the config + config_to_save = self._fix_fsdp_config(model_to_save.config) + config_to_save.save_pretrained(work_dir) + + # Save tokenizer if provided + if tokenizer is not None: + tokenizer.save_pretrained(work_dir) + + self.print(f"[rank-0]: Successfully saved model to {output_dir}") + + dist.barrier() diff --git a/skyrl-train/skyrl_train/distributed/fsdp_utils.py b/skyrl-train/skyrl_train/distributed/fsdp_utils.py new file mode 100644 index 0000000000..2f3f2d1495 --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/fsdp_utils.py @@ -0,0 +1,580 @@ +# This code is adapted from VERL +# https://github.com/volcengine/verl/blob/main/verl/utils/fsdp_utils.py +# The original copyright is reproduced below: +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +from contextlib import nullcontext +from typing import Union +from loguru import logger + +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.distributed import DeviceMesh +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp._runtime_utils import _lazy_init +from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy, transformer_auto_wrap_policy +from transformers.trainer_pt_utils import get_module_class_from_name +from torch.distributed.device_mesh import init_device_mesh +from collections import OrderedDict + +from packaging import version +from peft.utils.save_and_load import get_peft_model_state_dict + +if version.parse(torch.__version__) >= version.parse("2.6"): + from torch.distributed.fsdp import CPUOffloadPolicy, FSDPModule, MixedPrecisionPolicy, fully_shard +elif version.parse(torch.__version__) >= version.parse("2.4"): + from torch.distributed._composable.fsdp import CPUOffloadPolicy, FSDPModule, MixedPrecisionPolicy, fully_shard +else: + fully_shard, MixedPrecisionPolicy, FSDPModule, CPUOffloadPolicy = None, None, None, None + + +def init_fn(x: torch.nn.Module): + if torch.distributed.get_rank() != 0: + x = x.to_empty(device=torch.cuda.current_device(), recurse=False) + torch.cuda.empty_cache() + return x + + +def _patch_accelerate_for_torch_compat(): + """Patch Parameter.__new__ to handle _is_hf_initialized incompatibility with torch 2.10+. + + accelerate's register_empty_parameter grabs param.__dict__ (which includes + _is_hf_initialized set by transformers) and passes it as **kwargs to + Parameter(data, **kwargs). torch 2.10+ Parameter.__new__() rejects unknown kwargs. + + Fix: make Parameter.__new__ accept and ignore extra kwargs. + See: https://github.com/volcengine/verl/issues/4522 + """ + if getattr(torch.nn.Parameter, "_skyrl_patched", False): + return + + _original_param_new = torch.nn.Parameter.__new__ + + def _patched_param_new(cls, data=None, requires_grad=True, **_extra_kwargs): + return _original_param_new(cls, data, requires_grad) + + torch.nn.Parameter.__new__ = _patched_param_new + torch.nn.Parameter._skyrl_patched = True + + +def get_init_weight_context_manager(use_meta_tensor=True, mesh: DeviceMesh = None): + from accelerate import init_empty_weights + + _patch_accelerate_for_torch_compat() + + def cpu_init_weights(): + return torch.device("cpu") + + if use_meta_tensor: + if mesh is None: + init_context = init_empty_weights if torch.distributed.get_rank() != 0 else cpu_init_weights + else: + init_context = init_empty_weights if mesh.get_coordinate()[-1] != 0 else cpu_init_weights + else: + init_context = cpu_init_weights + return init_context + + +def get_fsdp_wrap_policy(module, config=None, is_lora=False): + """Get FSDP wrap policy for the module. + + Args: + module: The module to get wrap policy for + config: Configuration for wrap policy + is_lora: Whether to enable lambda policy for LoRA modules + """ + if config is None: + config = {} + + def _get_attr(attr_name, default_value=None): + if hasattr(config, "get"): + return config.get(attr_name, default_value) + else: + return getattr(config, attr_name, default_value) + + if _get_attr("disable", False): + return None + + default_transformer_cls_names_to_wrap = getattr(module, "_no_split_modules", None) + fsdp_transformer_layer_cls_to_wrap = _get_attr( + "transformer_layer_cls_to_wrap", default_transformer_cls_names_to_wrap + ) + min_num_params = _get_attr("min_num_params", 0) + auto_wrap_policy = None + + policies = [] + + from torch.distributed.fsdp.wrap import _or_policy, lambda_auto_wrap_policy + + # Add lambda policy for LoRA modules if is_lora is True + if is_lora: + + def lambda_policy_fn(module): + return bool( + len(list(module.named_children())) == 0 + and getattr(module, "weight", None) is not None + and module.weight.requires_grad + ) + + lambda_policy = functools.partial(lambda_auto_wrap_policy, lambda_fn=lambda_policy_fn) + policies.append(lambda_policy) + + if min_num_params > 0: + size_policy = functools.partial(size_based_auto_wrap_policy, min_num_params=min_num_params) + policies.append(size_policy) + elif fsdp_transformer_layer_cls_to_wrap is not None: + transformer_cls_to_wrap = set() + for layer_class in fsdp_transformer_layer_cls_to_wrap: + transformer_cls = get_module_class_from_name(module, layer_class) + if transformer_cls is None: + logger.warning(f"Could not find transformer layer class '{layer_class}' in the model, skipping.") + continue + else: + transformer_cls_to_wrap.add(transformer_cls) + if not transformer_cls_to_wrap: + raise Exception("Could not find any transformer layer class to wrap in the model.") + + transformer_policy = functools.partial( + transformer_auto_wrap_policy, + transformer_layer_cls=transformer_cls_to_wrap, + ) + policies.append(transformer_policy) + + if len(policies) > 0: + auto_wrap_policy = functools.partial(_or_policy, policies=policies) + + return auto_wrap_policy + + +@torch.no_grad() +def offload_fsdp_model_to_cpu(model: FSDP, empty_cache: bool = True): + if fsdp_version(model) == 2: + offload_fsdp2_model_to_cpu(model, empty_cache) + return + + assert isinstance(model, FSDP) + # lazy init FSDP model + _lazy_init(model, model) + assert model._is_root, "Only support root model offloading to CPU" + for handle in model._all_handles: + if handle._offload_params: + continue + flat_param = handle.flat_param + assert ( + flat_param.data.data_ptr() == flat_param._local_shard.data_ptr() + and id(flat_param.data) != id(flat_param._local_shard) + and flat_param.data.size() == flat_param._local_shard.size() + ) + handle.flat_param_to(torch.device("cpu"), non_blocking=True) + # the following still keeps id(._local_shard) != id(.data) + flat_param._local_shard = flat_param.data + assert id(flat_param._local_shard) != id(flat_param.data) + if empty_cache: + torch.cuda.empty_cache() + + +@torch.no_grad() +def offload_fsdp2_model_to_cpu(model, empty_cache: bool = True): + model.to("cpu", non_blocking=True) + if empty_cache: + torch.cuda.empty_cache() + + +@torch.no_grad() +def load_fsdp_model_to_gpu(model: FSDP): + if fsdp_version(model) == 2: + load_fsdp2_model_to_gpu(model) + return + + assert isinstance(model, FSDP) + # lazy init FSDP model + _lazy_init(model, model) + assert model._is_root, "Only support root model loading to GPU" + device_id = torch.cuda.current_device() + for handle in model._all_handles: + if handle._offload_params: + continue + flat_param = handle.flat_param + handle.flat_param_to(torch.device(f"cuda:{device_id}"), non_blocking=True) + # the following still keeps id(._local_shard) != id(.data) + flat_param._local_shard = flat_param.data + + +@torch.no_grad() +def load_fsdp2_model_to_gpu(model): + device = torch.cuda.current_device() + model.to(device, non_blocking=True) + + +@torch.no_grad() +def offload_fsdp_optimizer(optimizer): + if not optimizer.state: + return + for param_group in optimizer.param_groups: + for param in param_group["params"]: + state = optimizer.state[param] + for key, value in state.items(): + if isinstance(value, torch.Tensor): + state[key] = value.to("cpu", non_blocking=True) + + +@torch.no_grad() +def load_fsdp_optimizer(optimizer, device_id): + if not optimizer.state: + return + for param_group in optimizer.param_groups: + for param in param_group["params"]: + state = optimizer.state[param] + for key, value in state.items(): + if isinstance(value, torch.Tensor): + state[key] = value.to(device_id, non_blocking=True) + + +def fsdp_version(model): + if isinstance(model, FSDP): + return 1 + elif FSDPModule is not None and isinstance(model, FSDPModule): + return 2 + else: + return 0 + + +def get_fsdp_state_ctx(model, state_type, state_cfg, optim_cfg): + if fsdp_version(model) == 1: + return FSDP.state_dict_type(model, state_type, state_cfg, optim_cfg) + else: + return nullcontext() + + +# Fsdp2 load full state dict from `accelerate` +# Reference: https://github.com/huggingface/accelerate/blob/0af621bbecc0e43f5d43766a4945d3d2236bb8a9/src/accelerate/utils/fsdp_utils.py#L455 +# NOTE (sumanthrh): The original code from `accelerate` assumes init on meta device - with cpu init only on rank 0, but the code is compatible with cpu init on all ranks. +def fsdp2_load_full_state_dict(model: torch.nn.Module, full_sd: dict, cpu_offload=None): + """ + Loads the full state dict (could be only on rank 0) into the sharded model. This is done by broadcasting the + parameters from rank 0 to all other ranks. This function modifies the model in-place. + + Args: + model (`torch.nn.Module`): + The model to load the state dict into, expected to be on meta device or a VRAM spike can occur + full_sd (`dict`): The full state dict to load, can be only on rank 0 + """ + import torch.distributed as dist + from torch.distributed.tensor import distribute_tensor + + # Model was previously copied to meta device + meta_sharded_sd = model.state_dict() + sharded_sd = {} + + # Rank 0 distributes the full state dict to other ranks + def _infer_parameter_dtype(model, param_name, empty_param): + try: + old_param = model.get_parameter_or_buffer(param_name) + except AttributeError: + # Need this for LORA, as there some params are not *parameters* of sorts + base_param_name, local_param_name = param_name.rsplit(".", 1) + submodule = model.get_submodule(base_param_name) + old_param = getattr(submodule, local_param_name) + + is_torch_e4m3fn_available = hasattr(torch, "float8_e4m3fn") + casting_dtype = None + is_param_float8_e4m3fn = is_torch_e4m3fn_available and empty_param.dtype == torch.float8_e4m3fn + + if empty_param.dtype.is_floating_point and not is_param_float8_e4m3fn: + casting_dtype = old_param.dtype + + return old_param is not None and old_param.is_contiguous(), casting_dtype + + def _cast_and_contiguous(tensor, to_contiguous, dtype): + if dtype is not None: + tensor = tensor.to(dtype=dtype) + if to_contiguous: + tensor = tensor.contiguous() + return tensor + + if dist.get_rank() == 0: + for (param_name, full_param), sharded_param in zip(full_sd.items(), meta_sharded_sd.values()): + full_param = full_param.detach().cuda() + mesh = sharded_param.device_mesh + dist.broadcast(full_param, src=0) + sharded_tensor = distribute_tensor(full_param, mesh, sharded_param.placements) + to_contiguous, casting_dtype = _infer_parameter_dtype( + model, + param_name, + full_param, + ) + sharded_tensor = _cast_and_contiguous(sharded_tensor, to_contiguous, casting_dtype) + sharded_sd[param_name] = sharded_tensor + # We need this else to have a matching `broadcast` for all of the ranks, else we deadlock + else: + for param_name, sharded_param in meta_sharded_sd.items(): + full_tensor = torch.empty(sharded_param.size(), device="cuda", dtype=sharded_param.dtype) + mesh = sharded_param.device_mesh + dist.broadcast(full_tensor, src=0) + sharded_tensor = distribute_tensor(full_tensor, mesh, sharded_param.placements) + to_contiguous, casting_dtype = _infer_parameter_dtype( + model, + param_name, + full_tensor, + ) + sharded_tensor = _cast_and_contiguous(sharded_tensor, to_contiguous, casting_dtype) + sharded_sd[param_name] = sharded_tensor + + # we set `assign=True` because our params can be on meta device + model.load_state_dict(sharded_sd, assign=True) + + # If we don't offload FSDP2 Module to CPU and then back to GPU, + # it will occupy a large amount of reserved GPU memory，which can not be released using torch.cuda.empty_cache() + # even if we are using cpu_offload + # TODO (erictang000): this requires an additional offload + backload, see if this can be avoided + # Credit: https://github.com/volcengine/verl/pull/1667 + offload_fsdp2_model_to_cpu(model) + + torch.cuda.synchronize() + torch.cuda.empty_cache() + + if not cpu_offload: + load_fsdp2_model_to_gpu(model) + return model + + +def fsdp2_get_full_state_dict(model: torch.nn.Module, cpu_offload=True, rank0_only=True): + """ + Get the full state dict from an FSDP2 model using proper PyTorch FSDP2 APIs. + This function will gather the complete state dict on rank 0 only by default. + + Args: + model (`torch.nn.Module`): The FSDP2 model to get state dict from + cpu_offload (`bool`): Whether to offload to CPU + rank0_only (`bool`): Whether to gather full state dict only on rank 0 + + Returns: + dict: The full state dict (only on rank 0 if rank0_only=True, empty dict on other ranks) + """ + from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict + + # All ranks must participate in the collective operation + options = StateDictOptions( + full_state_dict=True, cpu_offload=cpu_offload, broadcast_from_rank0=False # We want to get, not set + ) + + # This must be called on all ranks for the collective operation to work + state_dict = get_model_state_dict(model, options=options) + + # If rank0_only is True, clear the state_dict on non-rank-0 processes + if rank0_only and dist.get_rank() != 0: + # Clear the state dict on non-rank-0 processes to save memory + state_dict.clear() + + return state_dict + + +def apply_fsdp2(model, fsdp_kwargs, config): + """model: AutoModelForCausalLM""" + assert CPUOffloadPolicy is not None, "PyTorch version >= 2.4 is required for using fully_shard API (FSDP2)" + default_transformer_cls_names_to_wrap = getattr(model, "_no_split_modules", None) + fsdp_transformer_layer_cls_to_wrap = config.get("wrap_policy", {}).get( + "transformer_layer_cls_to_wrap", default_transformer_cls_names_to_wrap + ) + + if isinstance(fsdp_transformer_layer_cls_to_wrap, str): + fsdp_transformer_layer_cls_to_wrap = [fsdp_transformer_layer_cls_to_wrap] + elif isinstance(fsdp_transformer_layer_cls_to_wrap, set): + fsdp_transformer_layer_cls_to_wrap = list(fsdp_transformer_layer_cls_to_wrap) + + assert len(fsdp_transformer_layer_cls_to_wrap) > 0 and fsdp_transformer_layer_cls_to_wrap[0] is not None + + modules = [] + for name, module in model.named_modules(): + if module.__class__.__name__ in fsdp_transformer_layer_cls_to_wrap or ( + isinstance(module, nn.Embedding) and not model.config.tie_word_embeddings + ): + modules.append(module) + + for idx, module in enumerate(modules): + fully_shard(module, **fsdp_kwargs) + fully_shard(model, **fsdp_kwargs) # fsdp2 will not reshard_after_forward for root module + + +def fsdp2_clip_grad_norm_(parameters, max_norm, norm_type=2.0, error_if_nonfinite=False, foreach=None): + """torch.nn.utils.clip_grad_norm_ can't run on cpu parameter DTensor""" + from torch.nn.utils.clip_grad import _clip_grads_with_norm_, _get_total_norm + + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + else: + # prevent generators from being exhausted + parameters = list(parameters) + grads = [p.grad for p in parameters if p.grad is not None] + total_norm = _get_total_norm(grads, norm_type, error_if_nonfinite, foreach) + total_norm = total_norm.to(torch.cuda.current_device(), non_blocking=True) + _clip_grads_with_norm_(parameters, max_norm, total_norm, foreach) + return total_norm + + +def create_device_mesh(world_size, fsdp_size): + if fsdp_size < 0 or fsdp_size >= world_size: + device_mesh = init_device_mesh("cuda", mesh_shape=(world_size,), mesh_dim_names=["fsdp"]) + else: + device_mesh = init_device_mesh( + "cuda", mesh_shape=(world_size // fsdp_size, fsdp_size), mesh_dim_names=["ddp", "fsdp"] + ) + return device_mesh + + +def get_sharding_strategy(device_mesh): + from torch.distributed.fsdp import ShardingStrategy + + if device_mesh.ndim == 1: + sharding_strategy = ShardingStrategy.FULL_SHARD + elif device_mesh.ndim == 2: + sharding_strategy = ShardingStrategy.HYBRID_SHARD + else: + raise NotImplementedError(f"Get device mesh ndim={device_mesh.ndim}, but only support 1 or 2") + return sharding_strategy + + +""" +Adapted from Cruise. +""" + +HALF_LIST = [16, "16", "fp16", "float16", torch.float16] +FLOAT_LIST = [32, "32", "fp32", "float32", torch.float32] +BFLOAT_LIST = ["bf16", "bfloat16", torch.bfloat16] + + +class PrecisionType: + """Type of precision used. + + >>> PrecisionType.HALF == 16 + True + >>> PrecisionType.HALF in (16, "16") + True + """ + + HALF = "16" + FLOAT = "32" + FULL = "64" + BFLOAT = "bf16" + MIXED = "mixed" + + @staticmethod + def supported_type(precision: Union[str, int]) -> bool: + return any(x == precision for x in PrecisionType) + + @staticmethod + def supported_types() -> list[str]: + return [x.value for x in PrecisionType] + + @staticmethod + def is_fp16(precision): + return precision in HALF_LIST + + @staticmethod + def is_fp32(precision): + return precision in FLOAT_LIST + + @staticmethod + def is_bf16(precision): + return precision in BFLOAT_LIST + + @staticmethod + def to_dtype(precision): + if precision in HALF_LIST: + return torch.float16 + elif precision in FLOAT_LIST: + return torch.float32 + elif precision in BFLOAT_LIST: + return torch.bfloat16 + else: + raise RuntimeError(f"unexpected precision: {precision}") + + @staticmethod + def to_str(precision): + if precision == torch.float16: + return "fp16" + elif precision == torch.float32: + return "fp32" + elif precision == torch.bfloat16: + return "bf16" + else: + raise RuntimeError(f"unexpected precision: {precision}") + + +# Reference: https://github.com/volcengine/verl/blob/main/verl/utils/fsdp_utils.py +def layered_summon_lora_params(fsdp_module) -> OrderedDict: + + def __prefix_submodules(module, prefix): + for name, submodule in module.named_modules(): + if name.startswith(prefix) and "." not in name[len(prefix) :]: + yield name, submodule + + lora_params = OrderedDict() + prefix_list = [ + # fsdp + "_fsdp_wrapped_module.base_model.model.", + "_fsdp_wrapped_module.base_model.model.model.", + "_fsdp_wrapped_module.base_model.model.model.layers.", + "_fsdp_wrapped_module.base_model.model.model.language_model.layers.", + # fsdp2 + "base_model.model.", + "base_model.model.model.", + "base_model.model.model.layers.", + "base_model.model.model.language_model.layers.", + ] + peft_model = getattr(fsdp_module, "_fsdp_wrapped_module", fsdp_module) + for prefix in prefix_list: + for name, submodule in __prefix_submodules(fsdp_module, prefix): + prefix = name.replace("_fsdp_wrapped_module.base_model.model.", "base_model.model.") + if name.endswith(".model") or name.endswith(".layers"): + continue + if fsdp_version(submodule) > 0: + with FSDP.summon_full_params(submodule, writeback=False): + sub_lora_params = get_peft_model_state_dict(peft_model, state_dict=submodule.state_dict()) + sub_lora_params = { + f"{prefix}.{name}": ( + param.full_tensor().detach().cpu() + if hasattr(param, "full_tensor") + else param.detach().cpu() + ) + for name, param in sub_lora_params.items() + } + lora_params.update(sub_lora_params) + submodule._is_root = False + torch.cuda.empty_cache() + return lora_params + + +def collect_lora_params(module: FSDP) -> OrderedDict: + """ + collect lora params or full params if base model is not ready in vllm + requires `module._fsdp_wrapped_module` to be a `PeftModel` + """ + lora_params = OrderedDict() + peft_model = getattr(module, "_fsdp_wrapped_module", module) + if fsdp_version(module) > 0: + with FSDP.summon_full_params(module, writeback=False): + # If base model is synced, we can get the full state dict from peft model + lora_params = get_peft_model_state_dict(peft_model) + lora_params = { + name: param.full_tensor().detach().cpu() if hasattr(param, "full_tensor") else param.detach().cpu() + for name, param in lora_params.items() + } + torch.cuda.empty_cache() + else: + lora_params = get_peft_model_state_dict(peft_model) + return lora_params diff --git a/skyrl-train/skyrl_train/distributed/megatron/megatron_strategy.py b/skyrl-train/skyrl_train/distributed/megatron/megatron_strategy.py new file mode 100644 index 0000000000..6884feb2ca --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/megatron/megatron_strategy.py @@ -0,0 +1,334 @@ +import os +import random +from datetime import timedelta +from typing import List, Union, Optional +from jaxtyping import Float + +import numpy as np +import torch +import torch.nn as nn +from torch import optim +from torch import distributed as dist + +from skyrl_train.distributed.strategy import DistributedStrategy +from skyrl_train.distributed.utils import ModelOrModelOptimPair +from skyrl_train.utils.io import io +from skyrl_train.workers.megatron.megatron_model_wrapper import MegatronModelWrapper +import megatron.core.parallel_state as mpu +from skyrl_train.distributed.megatron.megatron_utils import ( + offload_megatron_model_to_cpu, + load_megatron_model_to_gpu, + offload_megatron_optimizer, + load_megatron_optimizer, + offload_megatron_grads_to_cpu, + load_megatron_grads_to_gpu, +) + +from megatron.core.dist_checkpointing.strategies import base as ckpt_base +from megatron.core.dist_checkpointing.strategies.async_utils import AsyncCallsQueue +from megatron.core import dist_checkpointing +from megatron.core.dist_checkpointing.serialization import ( + get_default_load_sharded_strategy, + get_default_save_sharded_strategy, +) +from megatron.core.dist_checkpointing.strategies.fully_parallel import ( + FullyParallelLoadStrategyWrapper, + FullyParallelSaveStrategyWrapper, +) +from transformers import PreTrainedTokenizer +from megatron.core.optimizer import DistributedOptimizer +from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler + + +class MegatronStrategy(DistributedStrategy): + """ + The strategy for training with Megatron. + """ + + def __init__( + self, + megatron_config, + optimizer_config=None, + seed: int = 42, + is_lora: bool = False, + ) -> None: + super().__init__() + self.megatron_config = megatron_config + self.optimizer_config = optimizer_config + self.seed = seed + self.hf_config = None # Set by the megatron worker once configs are initialized. + self.is_lora = is_lora + + # NOTE: Set Megatron dist checkpoint async backend to persistent to avoid `os.fork()`-ing + # short-lived background workers, which does not work well with Ray. + ckpt_base.async_calls = AsyncCallsQueue(persistent=True) + + def set_seed(self, seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if torch.cuda.device_count() > 0: + from megatron.core import tensor_parallel + + tensor_parallel.model_parallel_cuda_manual_seed(seed) + + def setup_distributed(self, timeout=timedelta(minutes=30)) -> None: + local_rank = int(os.environ.get("LOCAL_RANK", "-1")) + if local_rank != -1: + torch.cuda.set_device(local_rank) + + mpu.initialize_model_parallel( + tensor_model_parallel_size=self.megatron_config.tensor_model_parallel_size, + pipeline_model_parallel_size=self.megatron_config.pipeline_model_parallel_size, + expert_model_parallel_size=self.megatron_config.expert_model_parallel_size, + expert_tensor_parallel_size=self.megatron_config.expert_tensor_parallel_size, + use_sharp=False, + context_parallel_size=self.megatron_config.context_parallel_size, + nccl_communicator_config_path=None, + ) + self.set_seed(self.seed) + self.world_size = dist.get_world_size() + + def offload_to_cpu( + self, model, optimizer, pin_memory=True, non_blocking=True, offload_optimizer=True, offload_model=True + ): + """ + Offload model weights and optimizer to CPU memory. + """ + if offload_model: + offload_megatron_model_to_cpu(model) + if optimizer and offload_optimizer: + offload_megatron_grads_to_cpu(model) + offload_megatron_optimizer(optimizer) + torch.cuda.synchronize() + torch.cuda.empty_cache() + + def backload_to_gpu(self, model, optimizer, non_blocking=True, backload_optimizer=True, backload_model=True): + """Reload model weights back to GPU.""" + if backload_model: + load_megatron_model_to_gpu(model) + if optimizer and backload_optimizer: + load_megatron_grads_to_gpu(model) + load_megatron_optimizer(optimizer) + torch.cuda.synchronize() + + def backward(self, loss: torch.Tensor, model, optimizer: optim.Optimizer, **kwargs) -> None: + raise NotImplementedError() + + def optimizer_step( + self, + optimizer: optim.Optimizer, + model, + scheduler, + name="model", + **kwargs, + ) -> Optional[Float[torch.Tensor, "1"]]: + """Perform optimizer step""" + _, grad_norm, _ = optimizer.step() + scheduler.step(1) + optimizer.zero_grad() + return grad_norm + + def prepare( + self, *models_or_model_optim_pairs: ModelOrModelOptimPair + ) -> Union[List[ModelOrModelOptimPair], ModelOrModelOptimPair]: + raise NotImplementedError() + + def save_checkpoint( + self, + model: MegatronModelWrapper, + ckpt_dir: str, + node_local_rank: int, + optimizer: Optional[DistributedOptimizer] = None, + scheduler: Optional[OptimizerParamScheduler] = None, + tokenizer: Optional[PreTrainedTokenizer] = None, + ): + # Extract base model. + model: List[nn.Module] = model.actor_module + assert len(model) == 1, "Megatron virtual pipeline parallel is not yet supported" + unwrapped_model = model[0] + while hasattr(unwrapped_model, "module"): + unwrapped_model = unwrapped_model.module + + # Create checkpoint directory if it doesn't exist. + if node_local_rank == 0: + io.makedirs(ckpt_dir, exist_ok=True) + + # All ranks wait for the checkpoint directory to be created before saving. + dist.barrier() + + # Collect the sharded state dicts for model and optimizer, and full state dict for the scheduler. + sharded_state_dict = {} + model_sharded_state_dict = unwrapped_model.sharded_state_dict() + if not self.is_lora: + sharded_state_dict["model"] = model_sharded_state_dict + if optimizer: + sharded_state_dict["optimizer"] = optimizer.sharded_state_dict(model_sharded_state_dict) + if scheduler: + sharded_state_dict["lr_scheduler"] = scheduler.state_dict() + + # Save RNG state. + sharded_state_dict["rng"] = self.get_rng_state() + + # Save the checkpoint across ranks in parallel. + save_strategy = get_default_save_sharded_strategy("torch_dist") + save_strategy = FullyParallelSaveStrategyWrapper( + save_strategy, mpu.get_data_parallel_group(with_context_parallel=True) + ) + + with io.local_work_dir(ckpt_dir) as work_dir: + # TODO(tgriggs): Support configurable async saves. + async_save_request = dist_checkpointing.save( + sharded_state_dict=sharded_state_dict, + checkpoint_dir=work_dir, + sharded_strategy=save_strategy, + async_sharded_save=False, + validate_access_integrity=True, + ) + assert async_save_request is None, "Async save is not yet supported for Megatron" + + # Only global rank 0 saves the Huggingface config and tokenizer. + if self.is_rank_0(): + hf_dir = os.path.join(work_dir, "huggingface") + self.save_hf_configs(self.hf_config, hf_dir, tokenizer) + + if self.is_lora: + self._save_lora_adapters(unwrapped_model, ckpt_dir) + + dist.barrier() + ckpt_base.async_calls.close() + ckpt_base.async_calls = AsyncCallsQueue(persistent=True) + self.print(f"Checkpoint successfully saved to {ckpt_dir}") + + def _get_rank_path(self, ckpt_dir): + tp_rank = mpu.get_tensor_model_parallel_rank() + pp_rank = mpu.get_pipeline_model_parallel_rank() + cp_rank = mpu.get_context_parallel_rank() + dp_rank = mpu.get_data_parallel_rank() + ep_rank = mpu.get_expert_model_parallel_rank() + etp_rank = mpu.get_expert_tensor_parallel_rank() + + return os.path.join( + ckpt_dir, f"adapter_tp{tp_rank}_pp{pp_rank}_cp{cp_rank}_dp{dp_rank}_ep{ep_rank}_etp{etp_rank}.pt" + ) + + def _save_lora_adapters(self, model, ckpt_dir): + """Save LoRA adapters to checkpoint.""" + if not self.is_lora: + return + + assert isinstance(model, nn.Module), "Model must be a nn.Module" + + model_state_dict = {} + for name, param in model.named_parameters(): + if ".adapter" in name.lower(): + model_state_dict[name] = param.data + + with io.local_work_dir(ckpt_dir) as work_dir: + adapter_path = self._get_rank_path(work_dir) + torch.save({"model_state_dict": model_state_dict}, adapter_path) + self.print(f"Saved {len(model_state_dict)} LoRA adapter parameters to {adapter_path}") + + def load_checkpoint( + self, + model: MegatronModelWrapper, + ckpt_dir: str, + optimizer: Optional[DistributedOptimizer] = None, + scheduler: Optional[OptimizerParamScheduler] = None, + load_module_strict: bool = True, + load_optimizer_states: bool = True, + load_lr_scheduler_states: bool = True, + ): + if not ckpt_dir or not io.exists(ckpt_dir): + raise FileNotFoundError(f"Checkpoint directory not found: {ckpt_dir}") + + # Extract base model. + model: List[nn.Module] = model.actor_module + assert len(model) == 1, "Megatron virtual pipeline parallel is not yet supported" + unwrapped_model = model[0] + while hasattr(unwrapped_model, "module"): + unwrapped_model = unwrapped_model.module + + # Extract sharded state dicts. + sharded_state_dict = {} + model_sharded_state_dict = unwrapped_model.sharded_state_dict() + if not self.is_lora: + sharded_state_dict["model"] = model_sharded_state_dict + if optimizer and load_optimizer_states: + sharded_state_dict["optimizer"] = optimizer.sharded_state_dict(model_sharded_state_dict) + if scheduler and load_lr_scheduler_states: + sharded_state_dict["lr_scheduler"] = scheduler.state_dict() + + with io.local_read_dir(ckpt_dir) as read_dir: + # Load the checkpoint in parallel. + load_strategy = get_default_load_sharded_strategy(read_dir) + load_strategy = FullyParallelLoadStrategyWrapper( + load_strategy, mpu.get_data_parallel_group(with_context_parallel=True) + ) + state_dict = dist_checkpointing.load( + sharded_state_dict=sharded_state_dict, checkpoint_dir=read_dir, sharded_strategy=load_strategy + ) + if not self.is_lora: + # Load the model, optimizer, and scheduler state dicts. + assert ( + "model" in state_dict + ), f"Model state dict not found in checkpoint loaded from {ckpt_dir}. Available keys: {state_dict.keys()}" + model[0].load_state_dict(state_dict["model"], strict=load_module_strict) + self.print("Loaded model state dict.") + else: + self._load_lora_adapters(unwrapped_model, ckpt_dir) + + if optimizer and load_optimizer_states: + assert ( + "optimizer" in state_dict + ), f"Optimizer state dict not found in checkpoint loaded from {ckpt_dir}. Available keys: {state_dict.keys()}" + optimizer.load_state_dict(state_dict["optimizer"]) + self.print("Loaded optimizer state dict.") + + if scheduler and load_lr_scheduler_states: + assert ( + "lr_scheduler" in state_dict + ), f"LR scheduler state dict not found in checkpoint loaded from {ckpt_dir}. Available keys: {state_dict.keys()}" + scheduler.load_state_dict(state_dict["lr_scheduler"]) + self.print("Loaded LR scheduler state dict.") + + # Load RNG state, if present. + if "rng" in state_dict: + self.load_rng_state(state_dict["rng"]) + + return ckpt_dir, {} + + def _load_lora_adapters(self, model, ckpt_dir): + """Load LoRA adapters from checkpoint.""" + # TODO (erictang000): Update this logic once LoRA checkpointing is upstreamed to Megatron-Bridge + if not self.is_lora: + return + + assert isinstance(model, nn.Module), "Model must be a nn.Module" + + with io.local_read_dir(ckpt_dir) as read_dir: + adapter_path = self._get_rank_path(read_dir) + state_dict = torch.load(adapter_path, map_location="cpu") + _, unexpected = model.load_state_dict(state_dict["model_state_dict"], strict=False) + if len(unexpected) > 0: + raise ValueError(f"Unexpected keys in LoRA adapter state dict: {unexpected}") + self.print(f"Loaded {len(state_dict['model_state_dict'])} LoRA adapters from {adapter_path}.") + + def save_hf_model(self, bridge, model: MegatronModelWrapper, output_dir: str, tokenizer=None, **kwargs) -> None: + # Create checkpoint directory if it doesn't exist. + if self.is_rank_0(): + io.makedirs(output_dir, exist_ok=True) + dist.barrier() + + # All ranks call into bridge. + with io.local_work_dir(output_dir) as work_dir: + bridge.save_hf_weights(model.actor_module, work_dir) + self.print(f"Successfully saved HF safetensors model to {output_dir}") + + # Only rank 0 saves the Huggingface config and tokenizer. + if self.is_rank_0(): + self.save_hf_configs(self.hf_config, work_dir, tokenizer) + self.print(f"Successfully saved HF config and tokenizer to {output_dir}") + + dist.barrier() diff --git a/skyrl-train/skyrl_train/distributed/megatron/megatron_utils.py b/skyrl-train/skyrl_train/distributed/megatron/megatron_utils.py new file mode 100644 index 0000000000..f09504ecaa --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/megatron/megatron_utils.py @@ -0,0 +1,607 @@ +# Utils ported from Verl +# https://github.com/volcengine/verl/blob/e1603dc97f3c20c58feed1f5be34acd5c72a830c/verl/utils/megatron_utils.py#L4 +# https://github.com/volcengine/verl/blob/dfa3933ac44b545fca1f6a8519fd07394a2cde1c/verl/models/mcore/util.py +# The original copyright is reproduced below: + +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2023-2024 SGLang Team +# Copyright 2025 ModelBest Inc. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import torch +import torch.nn as nn +from loguru import logger +from megatron.core.distributed import DistributedDataParallel as DDP +from megatron.core.transformer.module import Float16Module +from megatron.core.optimizer import ChainedOptimizer +from megatron.core import parallel_state as mpu +from megatron.core.utils import get_attr_wrapped_model +from megatron.core.packed_seq_params import PackedSeqParams + +ALL_MODULE_WRAPPER_CLASSNAMES = (DDP, Float16Module) + + +def make_batch_generator(batches, vpp_size): + """ + Creates a batch generator suitable for Megatron pipeline parallelism, + handling virtual pipeline parallelism (VPP). + + If VPP is used (vpp_size > 1), it duplicates the batch iterator for each + virtual pipeline stage. Otherwise, it returns a single iterator. + + Args: + batches: An iterable (e.g., list) of micro-batches. + vpp_size (int): The virtual pipeline model parallel size. + + Returns: + An iterator or a list of iterators over the micro-batches. + """ + if vpp_size > 1: + # has vpp + batch_generator = [batches] * vpp_size # number of vpp chunks + batch_generator = [iter(b) for b in batch_generator] + else: + # no vpp + batch_generator = iter(batches) + return batch_generator + + +def print_model_size(model: nn.Module, name: str = None): + n_params, scale = get_model_size(model, scale="auto") + if name is None: + name = model.__class__.__name__ + logger.info(f"{name} contains {n_params:.2f}{scale} parameters") + + +def get_model_size(model: nn.Module, scale="auto"): + n_params = sum(p.numel() for p in model.parameters()) + + if scale == "auto": + if n_params > 1e9: + scale = "B" + elif n_params > 1e6: + scale = "M" + elif n_params > 1e3: + scale = "K" + else: + scale = "" + + if scale == "B": + n_params = n_params / 1e9 + elif scale == "M": + n_params = n_params / 1e6 + elif scale == "K": + n_params = n_params / 1e3 + elif scale == "": + pass + else: + raise NotImplementedError(f"Unknown scale {scale}") + + return n_params, scale + + +def freeze_moe_router(model): + for layer in model.decoder.layers: + if hasattr(layer.mlp, "router"): + if hasattr(layer.mlp.router, "weight"): + layer.mlp.router.weight.requires_grad = False + if hasattr(layer.mlp.router, "bias"): + layer.mlp.router.bias.requires_grad = False + if hasattr(layer.mlp, "shared_experts"): + if hasattr(layer.mlp.shared_experts, "gate_weight"): + layer.mlp.shared_experts.gate_weight.requires_grad = False + if hasattr(layer.mlp.shared_experts, "gate_bias"): + layer.mlp.shared_experts.gate_bias.requires_grad = False + + +@torch.no_grad() +def offload_megatron_grads_to_cpu(models): + for model_chunk in models: + if isinstance(model_chunk, DDP): + model_chunk_all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers] + for buffers in model_chunk_all_buffers: + for buffer in buffers: + if buffer.grad_data.storage().size() > 0: + buffer.grad_data_size = buffer.grad_data.storage().size() + buffer.grad_data.storage().resize_(0) + else: + # we need this for ref module + for _, param in model_chunk.named_parameters(): + if param.grad is not None: + param.grad = param.grad.to("cpu", non_blocking=True) + gc.collect() + torch.cuda.empty_cache() + + +@torch.no_grad() +def load_megatron_grads_to_gpu(models): + for model_chunk in models: + if isinstance(model_chunk, DDP): + model_chunk_all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers] + for buffers in model_chunk_all_buffers: + for buffer in buffers: + buffer.grad_data.storage().resize_(buffer.grad_data_size) + buffer.grad_data.zero_() + else: + # we need this for ref module + for _, param in model_chunk.named_parameters(): + if param.grad is not None: + param.grad = param.grad.to(torch.cuda.current_device(), non_blocking=True) + gc.collect() + torch.cuda.empty_cache() + + +@torch.no_grad() +def offload_megatron_model_to_cpu(models): + """ + In megatron, the model and optimizer storage are: + - bf16 parameter data chunked in model parallel group + - fp32 grad chunked in model parallel group + - fp32 main_parameter chunked in model and dp group + - fp32 optimizer state chunked in model and dp group + """ + for model_chunk in models: + if isinstance(model_chunk, DDP): + model_chunk_all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers] + for buffers in model_chunk_all_buffers: + for buffer in buffers: + # offload parameters from fused Megatron buffers + if buffer.param_data.storage().size() > 0: + buffer.param_data.cpu_data = buffer.param_data.data.cpu().pin_memory() + buffer.param_data_size = buffer.param_data.storage().size() + buffer.param_data.storage().resize_(0) + + assert buffer.param_data_size == buffer.param_data.cpu_data.storage().size() + + # lora aware offloading - if using lora,offload non-lora base weights, since megatron fused buffers do not include the HF/bridge "to_wrap" weights + for name, param in model_chunk.named_parameters(): + if ( + param.is_cuda + and not param.requires_grad + and "adapter" not in name + and param.data.storage().size() > 0 + ): + # Always refresh the CPU copy and release GPU storage. + cpu_tensor = param.data.detach().cpu().pin_memory() + param._offload_cpu_data = cpu_tensor + param._offload_cuda_numel = param.data.numel() + # Release GPU storage while keeping dtype/device metadata. + empty_cuda = torch.empty( + 0, + dtype=param.data.dtype, + device=param.data.device, + ) + param.data = empty_cuda + else: + # we need this for ref module + for _, param in model_chunk.named_parameters(): + param.data = param.data.to("cpu", non_blocking=True) + gc.collect() + torch.cuda.empty_cache() + + +@torch.no_grad() +def load_megatron_model_to_gpu(models): + for model_chunk in models: + if isinstance(model_chunk, DDP): + model_chunk_all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers] + for buffers in model_chunk_all_buffers: + for buffer in buffers: + if buffer.param_data.storage().size() == 0: + buffer.param_data.storage().resize_(buffer.param_data_size) + # copy data from cpu to cuda + buffer.param_data.copy_(buffer.param_data.cpu_data, non_blocking=True) + + # Restore any LoRA-frozen base weights that were offloaded above. + device_id = torch.cuda.current_device() + for name, param in model_chunk.named_parameters(): + if hasattr(param, "_offload_cpu_data") and param.data.storage().size() == 0: + restored = param._offload_cpu_data.to(device_id, non_blocking=True) + param.data = restored + else: + # we need this for ref module + device_id = torch.cuda.current_device() + for _, param in model_chunk.named_parameters(): + param.data = param.data.to(device_id, non_blocking=True) + gc.collect() + torch.cuda.empty_cache() + + +@torch.no_grad() +def offload_megatron_copy_params(optimizers): + """ + Offload optimizer parameters to CPU. Supports both Megatron optimizers + and `ChainedOptimizer`, which wraps a list of underlying optimizers. + + Args: + optimizers: The optimizer or ChainedOptimizer instance. + """ + + def _iter_opts(opt): + if isinstance(opt, ChainedOptimizer): + return opt.chained_optimizers + return [opt] + + def offload_tensor_to_cpu(tensor): + if tensor is None: + return + tensor.data = tensor.data.to("cpu", non_blocking=True) + + def offload_group_to_cpu(group): + if group is None: + return + + if isinstance(group, list): + for param_group in group: + if isinstance(param_group, list): + for param in param_group: + offload_tensor_to_cpu(param) + else: + offload_tensor_to_cpu(param_group) + else: + offload_tensor_to_cpu(group) + + # Offload all parameter groups to CPU for each underlying optimizer + + for _opt in _iter_opts(optimizers): + if hasattr(_opt, "shard_fp32_from_float16_groups"): + offload_group_to_cpu(_opt.shard_fp32_from_float16_groups) + + +@torch.no_grad() +def load_megatron_copy_params(optimizers): + """ + Load optimizer parameters back to GPU. Handles ChainedOptimizer. + + Args: + optimizers: Optimizer or ChainedOptimizer instance. + """ + + def _iter_opts(opt): + if isinstance(opt, ChainedOptimizer): + return opt.chained_optimizers + return [opt] + + def load_tensor_to_gpu(tensor): + if tensor is None: + return + device_id = torch.cuda.current_device() + tensor.data = tensor.data.to(device_id, non_blocking=True) + + def load_group_to_gpu(group): + if group is None: + return + + if isinstance(group, list): + for param_group in group: + if isinstance(param_group, list): + for param in param_group: + load_tensor_to_gpu(param) + else: + load_tensor_to_gpu(param_group) + else: + load_tensor_to_gpu(group) + + # Load all parameter groups to GPU for each underlying optimizer + + for _opt in _iter_opts(optimizers): + if hasattr(_opt, "shard_fp32_from_float16_groups"): + load_group_to_gpu(_opt.shard_fp32_from_float16_groups) + + +@torch.no_grad() +def offload_megatron_optimizer(optimizers): + def _iter_opts(opt): + if isinstance(opt, ChainedOptimizer): + return opt.chained_optimizers + return [opt] + + for _opt in _iter_opts(optimizers): + offload_megatron_copy_params(_opt) + opt_state_dict_values = _opt.optimizer.state.values() + for v in opt_state_dict_values: + if "exp_avg" in v: + v["exp_avg"] = v["exp_avg"].to("cpu", non_blocking=True) + if "exp_avg_sq" in v: + v["exp_avg_sq"] = v["exp_avg_sq"].to("cpu", non_blocking=True) + gc.collect() + torch.cuda.empty_cache() + + +@torch.no_grad() +def load_megatron_optimizer(optimizers): + def _iter_opts(opt): + if isinstance(opt, ChainedOptimizer): + return opt.chained_optimizers + return [opt] + + for _opt in _iter_opts(optimizers): + load_megatron_copy_params(_opt) + # if we are using HybridDeviceOptimizer, we need to only move gpu optimizer state to gpu + if hasattr(_opt.optimizer, "_move_new_state_to_right_device"): + _opt.optimizer._move_new_state_to_right_device() + else: + opt_state_dict_values = _opt.optimizer.state.values() + for v in opt_state_dict_values: + if "exp_avg" in v: + v["exp_avg"] = v["exp_avg"].to(torch.cuda.current_device(), non_blocking=True) + if "exp_avg_sq" in v: + v["exp_avg_sq"] = v["exp_avg_sq"].to(torch.cuda.current_device(), non_blocking=True) + gc.collect() + torch.cuda.empty_cache() + + +def preprocess_packed_seqs( + input_ids: torch.Tensor, attention_mask: torch.Tensor, pre_process: bool = True +) -> tuple[torch.Tensor, PackedSeqParams]: + """ + Preprocess packed sequences + CP splits sequence into CP*2 chunks, and each GPU gets 2 chunks (GPU0 gets first and last chunks, GPU1 + gets second and second last chunks, and so on), this is for load balancing with causal masking. + See https://github.com/NVIDIA/TransformerEngine/issues/1368 + """ + batch_size = input_ids.shape[0] + + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + tp_size = mpu.get_tensor_model_parallel_world_size() + cp_size = mpu.get_context_parallel_world_size() + cp_rank = mpu.get_context_parallel_rank() + align_size = tp_size * cp_size * 2 if cp_size > 1 else tp_size + + pad_size = (align_size - seqlens_in_batch % align_size) % align_size + seqlens_in_batch_padded = seqlens_in_batch + pad_size + + cu_seqlens = torch.zeros(batch_size + 1, dtype=torch.int32, device=input_ids.device) + cu_seqlens[1:] = torch.cumsum(seqlens_in_batch, dim=0) + cu_seqlens_padded = torch.zeros(batch_size + 1, dtype=torch.int32, device=input_ids.device) + cu_seqlens_padded[1:] = torch.cumsum(seqlens_in_batch_padded, dim=0) + + # ---------------------------------------------------------------------------- + # Move the index information needed in the subsequent loop to the CPU at once, + # to avoid frequent .item() calls in the loop that cause D2H synchronization + # ---------------------------------------------------------------------------- + seqlens_in_batch_cpu: list[int] = seqlens_in_batch.tolist() # original valid lengths + seqlens_in_batch_padded_cpu: list[int] = seqlens_in_batch_padded.tolist() # lengths after padding + cu_seqlens_padded_cpu: list[int] = cu_seqlens_padded.tolist() # start positions (after padding) + + # Pure Python int calculation to avoid further synchronization + max_seqlen_in_batch = max(seqlens_in_batch_padded_cpu) + + shape = list(input_ids.shape[1:]) + shape[0] = sum(seqlens_in_batch_padded_cpu) // cp_size + if pre_process: + input_ids_rmpad = torch.zeros(shape, dtype=input_ids.dtype, device=input_ids.device) + for i in range(batch_size): + # Use Python int, so no GPU→CPU sync in the loop + if cp_size <= 1: + seqlen = seqlens_in_batch_cpu[i] + start_idx = cu_seqlens_padded_cpu[i] + input_ids_rmpad[start_idx : start_idx + seqlen] = input_ids[i, attention_mask[i]] + continue + + seqlen_padded_i = seqlens_in_batch_padded_cpu[i] + seqlen = seqlen_padded_i // cp_size + half_seqlen = seqlen // 2 + start_idx = cu_seqlens_padded_cpu[i] // cp_size + # split to 2 chunks + d = input_ids[i, attention_mask[i]] + input_ids_rmpad[start_idx : start_idx + half_seqlen] = d[ + half_seqlen * cp_rank : half_seqlen * (cp_rank + 1) + ] + + remain_start = seqlen_padded_i - half_seqlen * (cp_rank + 1) + remain_end = seqlen_padded_i - half_seqlen * cp_rank + remain_end = min(remain_end, d.shape[0]) + remain_len = remain_end - remain_start + if remain_len > 0: + input_ids_rmpad[start_idx + half_seqlen : start_idx + half_seqlen + remain_len] = d[ + remain_start:remain_end + ] + + packed_seq_params = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens_padded, + max_seqlen_q=max_seqlen_in_batch, + cu_seqlens_kv=cu_seqlens_padded, + max_seqlen_kv=max_seqlen_in_batch, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + ) + if pre_process: + return input_ids_rmpad.unsqueeze(0), packed_seq_params + else: + return input_ids, packed_seq_params + + +def postprocess_packed_seqs( + output: torch.Tensor, + packed_seq_params: PackedSeqParams, + attention_mask: torch.Tensor, + batch_size: int, + seq_len: int, + post_process: bool = True, +) -> torch.Tensor: + """ + Postprocess packed sequences + """ + if not post_process: + return output + + # ------------------------------------------------------------------------- + # Move the lengths and offsets needed for subsequent Python-level indexing to the CPU in advance, + # to avoid a large number of .item() calls in the loop + # ------------------------------------------------------------------------- + cu_padded_cpu: list[int] = packed_seq_params.cu_seqlens_q_padded.tolist() + seq_lens_cpu: list[int] = attention_mask.sum(dim=1, dtype=torch.int32).cpu().tolist() + + shape = [batch_size, seq_len] + list(output.shape[2:]) # 1,packed, dim -> batch_size, seq_len, dim + output_new = torch.zeros(shape, dtype=output.dtype, device=output.device) + + cp_size = mpu.get_context_parallel_world_size() + # all gather output across context parallel group + if cp_size > 1: + # output shape: [1, packed_len, hidden_dim] + # need to gather across cp group and concatenate in sequence dimension + output_list = [torch.empty_like(output) for _ in range(cp_size)] + torch.distributed.all_gather(output_list, output.detach(), group=mpu.get_context_parallel_group()) + output_list[mpu.get_context_parallel_rank()] = output + else: + output_list = [output] + for i in range(batch_size): + if cp_size <= 1: + s = seq_lens_cpu[i] + start_idx = cu_padded_cpu[i] + output_new[i, attention_mask[i]] = output[0][start_idx : start_idx + s] + continue + s_len_padded_chunk = (cu_padded_cpu[i + 1] - cu_padded_cpu[i]) // cp_size + half_seqlen = s_len_padded_chunk // 2 + s_len = seq_lens_cpu[i] + s_len_padded = s_len_padded_chunk * cp_size + tmp = torch.empty(s_len_padded, *output.shape[2:], device=output.device) + for j in range(cp_size): + o = output_list[j][0] + # split to 2 chunks + packed_start_idx = cu_padded_cpu[i] // cp_size + o0, o1 = ( + o[packed_start_idx : packed_start_idx + half_seqlen], + o[packed_start_idx + half_seqlen : packed_start_idx + s_len_padded_chunk], + ) + tmp[j * half_seqlen : (j + 1) * half_seqlen] = o0 + tmp[s_len_padded - (j + 1) * half_seqlen : s_len_padded - j * half_seqlen] = o1 + output_new[i, attention_mask[i]] = tmp[:s_len] + + return output_new + + +def remove_left_padding( + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + position_ids: torch.Tensor, + pre_process: bool = True, +): + """ + Remove left padding from input_ids, attention_mask and position_ids + return new_input_ids, new_attention_mask, new_position_ids + """ + assert attention_mask.ndim == 2 + assert position_ids.ndim == 2 + cp_size = mpu.get_context_parallel_world_size() + assert cp_size == 1, "Context parallel size without seq_pack is not supported" + batch_size = input_ids.shape[0] + shape = list(input_ids.shape) # batch_size, seq_len,... + seq_lens = attention_mask.sum(dim=1) + seq_len = seq_lens.max().item() + if mpu.get_tensor_model_parallel_world_size() > 1: + sp_world_size = mpu.get_tensor_model_parallel_world_size() + pad_size = (sp_world_size - seq_len % sp_world_size) % sp_world_size + seq_len = seq_len + pad_size + shape[1] = seq_len + if pre_process: + new_input_ids = torch.zeros(dtype=input_ids.dtype, device=input_ids.device, size=shape) + new_attention_mask = torch.zeros( + dtype=attention_mask.dtype, device=attention_mask.device, size=(batch_size, seq_len) + ) + new_position_ids = torch.zeros(dtype=position_ids.dtype, device=position_ids.device, size=(batch_size, seq_len)) + for i in range(batch_size): + if pre_process: + new_input_ids[i, : seq_lens[i]] = input_ids[i, attention_mask[i]] + new_attention_mask[i, : seq_lens[i]] = attention_mask[i, attention_mask[i]] + new_position_ids[i, : seq_lens[i]] = position_ids[i, attention_mask[i]] + if pre_process: + return new_input_ids, new_attention_mask, new_position_ids + else: + return input_ids, new_attention_mask, new_position_ids + + +def recover_left_padding( + result, + attention_mask: torch.Tensor, + original_attention_mask: torch.Tensor, + origin_seqlen: int, + post_process: bool = True, +): + """ + Recover left padding from result + return result + """ + if not post_process: + return result + shape = list(result.shape) + batch_size = shape[0] + shape[1] = origin_seqlen + new_result = torch.zeros(dtype=result.dtype, device=result.device, size=shape) + for i in range(batch_size): + new_result[i, original_attention_mask[i]] = result[i, attention_mask[i]] + return new_result + + +def get_model_config(model): + return get_attr_wrapped_model(model, "config", allow_none=False) + + +def broadcast_object_across_pp_ranks(obj): + """Broadcast an object across pipeline parallel ranks. + + From Nemo-RL: https://github.com/NVIDIA-NeMo/RL/blob/0a769cc3553a265dd1ca4648de0a7d0b1ad5ece6/nemo_rl/models/policy/megatron_policy_worker.py#L136 + + This utility function handles broadcasting an object from the rank that owns it + to all other pipeline parallel ranks. If only one rank has the object (non-None), + it will be broadcast to all other ranks. + + Args: + obj: The object to broadcast. Can be None on ranks that don't own it. + + Returns: + The object on all ranks (either the original or the broadcast copy). + + Raises: + ValueError: If the object doesn't exist on any pipeline parallel rank. + """ + pp_size = mpu.get_pipeline_model_parallel_world_size() + pp_group = mpu.get_pipeline_model_parallel_group() + + if pp_size == 1: + return obj + + # ------------------------------------------------------------------ + # 1. Gather presence flags from all PP ranks to find the source rank + # ------------------------------------------------------------------ + has_obj = obj is not None + obj_flags = [None] * pp_size + torch.distributed.all_gather_object(obj_flags, has_obj, group=pp_group) + + # ------------------------------------------------------------------ + # 2. Identify the owning rank (the only rank with True flag) + # ------------------------------------------------------------------ + src_rank = None # Rank *inside* the PP group + for rank, flag in enumerate(obj_flags): + if flag: + src_rank = rank + break + + if src_rank is None: + raise ValueError("Object must exist on at least one PP rank") + + # ------------------------------------------------------------------ + # 3. Broadcast the object from the source rank to all ranks + # ------------------------------------------------------------------ + # Use broadcast_object_list which is more robust than all_gather_object + obj_list = [obj] + pp_ranks = torch.distributed.get_process_group_ranks(pp_group) + global_src = pp_ranks[src_rank] + torch.distributed.broadcast_object_list(obj_list, src=global_src, group=pp_group) + + return obj_list[0] diff --git a/skyrl-train/skyrl_train/distributed/megatron/model_utils.py b/skyrl-train/skyrl_train/distributed/megatron/model_utils.py new file mode 100644 index 0000000000..a59f22cf5e --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/megatron/model_utils.py @@ -0,0 +1,590 @@ +# Utils ported from NeMo-Aligner by way of NeMo-RL +# https://github.com/NVIDIA-NeMo/RL/blob/9301d36cbf847212430b84a27cfe6990f773b7cf/nemo_rl/distributed/model_utils.py#L4 +# The original copyright is reproduced below: + +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Optional + +import torch +import torch.distributed as dist +import megatron.core.parallel_state as mpu + + +@torch.no_grad() +def _compute_distributed_log_softmax( + vocab_parallel_logits: torch.Tensor, group: torch.distributed.ProcessGroup +) -> torch.Tensor: + """Compute a stable distributed log softmax across tensor parallel workers. + + Taken from: https://github.com/NVIDIA/NeMo-Aligner/blob/9faab404f21994a7eb1d6ed5890b76152b941636/nemo_aligner/utils/distributed.py#L265 + + Args: + vocab_parallel_logits (torch.Tensor): Logits tensor with shape [batch_size, seq_length, vocab_size//TP] + where TP is the tensor parallel size. + group (torch.distributed.ProcessGroup): Process group for the all-reduce operations. + + Returns: + torch.Tensor: Log softmax output with the same shape as input, but values represent + log probabilities normalized across the full vocabulary dimension. + """ + logits_max = torch.amax(vocab_parallel_logits, dim=-1, keepdim=True) + torch.distributed.all_reduce( + logits_max, + op=torch.distributed.ReduceOp.MAX, + group=group, + ) + + # Subtract the maximum value. + vocab_parallel_logits = vocab_parallel_logits - logits_max + + sum_exp_logits = vocab_parallel_logits.exp().sum(-1, keepdim=True).float() + + torch.distributed.all_reduce( + sum_exp_logits, + op=torch.distributed.ReduceOp.SUM, + group=group, + ) + + return vocab_parallel_logits - sum_exp_logits.log_().to(vocab_parallel_logits.dtype) + + +class DistributedLogprob(torch.autograd.Function): + """Custom autograd function for computing log probabilities in a distributed setting. + + Taken from https://github.com/NVIDIA/NeMo-Aligner/blob/9faab404f21994a7eb1d6ed5890b76152b941636/nemo_aligner/utils/distributed.py#L286 + """ + + @staticmethod + def forward( # pyrefly: ignore[bad-override] Always ignore torch.autograd.Function.forward's type since it's always more specific than the base class + ctx: Any, + vocab_parallel_logits: torch.Tensor, + target: torch.Tensor, + vocab_start_index: int, + vocab_end_index: int, + group: torch.distributed.ProcessGroup, + inference_only: bool = False, + ) -> torch.Tensor: + # Create a mask of valid vocab ids (1 means it needs to be masked). + target_mask = (target < vocab_start_index) | (target >= vocab_end_index) + masked_target = target - vocab_start_index + masked_target[target_mask] = 0 + + vocab_parallel_logits = vocab_parallel_logits.to(dtype=torch.float32) + + log_probs = _compute_distributed_log_softmax(vocab_parallel_logits, group=group) + softmax_output = log_probs.exp() + + log_probs = torch.gather(log_probs, -1, masked_target.unsqueeze(-1)).squeeze(-1) + log_probs[target_mask] = 0.0 + + torch.distributed.all_reduce( + log_probs, + op=torch.distributed.ReduceOp.SUM, + group=group, + ) + + if not inference_only: + # only save for backward when we have inference only=False + ctx.save_for_backward(softmax_output, target_mask, masked_target) + + return log_probs + + @staticmethod + def backward( + ctx: Any, + *grad_outputs: torch.Tensor, + ) -> tuple[torch.Tensor, None, None, None, None, None, None]: + grad_output = grad_outputs[0] + softmax, target_mask, masked_target = ctx.saved_tensors + + if softmax.ndim == 3: + B, S, V = softmax.shape + + # skip `torch.nn.functional.one_hot` + row = torch.arange(B, device=softmax.device).view(-1, 1).expand(-1, S).reshape(-1) + col = torch.arange(S, device=softmax.device).expand(B, -1).reshape(-1) + flat_idx = (row * S + col) * V + + flat_chosen = flat_idx.masked_select(~target_mask.reshape(-1)) + masked_target.masked_select(~target_mask) + + # `neg` is zero-copy + grad_input = softmax.neg() + grad_input = grad_input.mul_(grad_output.unsqueeze(-1)) + + grad_output_selected = grad_output.masked_select(~target_mask) + grad_input.view(-1).scatter_add_(0, flat_chosen, grad_output_selected) + else: + V = softmax.size(-1) + is_chosen = (~target_mask).unsqueeze(-1) * torch.nn.functional.one_hot(masked_target, num_classes=V) + grad_input = is_chosen.float().sub_(softmax) + grad_input.mul_(grad_output.unsqueeze(-1)) + + # if you add an argument to the forward method, then you must add a corresponding None here + return grad_input, None, None, None, None, None, None + + +class ChunkedDistributedLogprob(torch.autograd.Function): + """Custom autograd function for computing log probabilities in a distributed setting. + + The log probabilities computation is chunked in the sequence dimension + to mitigate GPU OOM (especially during backward pass). + In addition, logits casting from float16 or bfloat16 -> float32 is performed + inside the chunk loop to avoid materializing a whole float32 logits tensor. + + Adapted from https://github.com/NVIDIA/NeMo-Aligner/blob/9faab404f21994a7eb1d6ed5890b76152b941636/nemo_aligner/utils/distributed.py#L286 + """ + + @staticmethod + def forward( # pyrefly: ignore[bad-override] Always ignore torch.autograd.Function.forward's type since it's always more specific than the base class + ctx: Any, + vocab_parallel_logits: torch.Tensor, + target: torch.Tensor, + vocab_start_index: int, + vocab_end_index: int, + chunk_size: int, + tp_group: torch.distributed.ProcessGroup, + inference_only: bool = False, + ) -> torch.Tensor: + # Create a mask of valid vocab ids (1 means it needs to be masked). + target_mask = (target < vocab_start_index) | (target >= vocab_end_index) + masked_target = target - vocab_start_index + masked_target[target_mask] = 0 + + seq_size = int(vocab_parallel_logits.shape[1]) + num_chunks = (seq_size + chunk_size - 1) // chunk_size + all_log_probs = [] + + for chunk_idx in range(num_chunks): + chunk_start = chunk_idx * chunk_size + chunk_end = min(seq_size, (chunk_idx + 1) * chunk_size) + + logits = vocab_parallel_logits[:, chunk_start:chunk_end, :] + logits = logits.to(dtype=torch.float32) + + log_probs = _compute_distributed_log_softmax( + logits, + group=tp_group, + ) + + log_probs = torch.gather(log_probs, -1, masked_target[:, chunk_start:chunk_end].unsqueeze(-1)).squeeze(-1) + log_probs[target_mask[:, chunk_start:chunk_end]] = 0.0 + + torch.distributed.all_reduce( + log_probs, + op=torch.distributed.ReduceOp.SUM, + group=tp_group, + ) + + all_log_probs.append(log_probs) + + log_probs = torch.cat(all_log_probs, dim=1) + + if not inference_only: + # only save for backward when we have inference only=False + ctx.save_for_backward(vocab_parallel_logits, target_mask, masked_target) + ctx.chunk_size = chunk_size + ctx.tp_group = tp_group + + return log_probs + + @staticmethod + def backward( + ctx: Any, + *grad_outputs: torch.Tensor, + ) -> tuple[torch.Tensor, None, None, None, None, None, None]: + grad_output = grad_outputs[0] + vocab_parallel_logits, target_mask, masked_target = ctx.saved_tensors + chunk_size = ctx.chunk_size + tp_group = ctx.tp_group + + partition_vocab_size = int(vocab_parallel_logits.shape[-1]) + seq_size = int(vocab_parallel_logits.shape[1]) + num_chunks = (seq_size + chunk_size - 1) // chunk_size + + all_grad_input = [] + + for chunk_idx in range(num_chunks): + chunk_start = chunk_idx * chunk_size + chunk_end = min(seq_size, (chunk_idx + 1) * chunk_size) + + logits = vocab_parallel_logits[:, chunk_start:chunk_end, :] + logits = logits.to(dtype=torch.float32) + + softmax_output = _compute_distributed_log_softmax( + logits, + group=tp_group, + ) + softmax_output = softmax_output.exp() + + # 1 if it's the chosen log prob, 0 otherwise + is_chosen = (~(target_mask[:, chunk_start:chunk_end])).unsqueeze(-1) * torch.nn.functional.one_hot( + masked_target[:, chunk_start:chunk_end], + num_classes=partition_vocab_size, + ) + + grad_input = is_chosen.float().sub_(softmax_output) + + grad_input.mul_(grad_output[:, chunk_start:chunk_end].unsqueeze(dim=-1)) + + all_grad_input.append(grad_input) + + grad_input = torch.cat(all_grad_input, dim=1) + + # if you add an argument to the forward method, then you must add a corresponding None here + return grad_input, None, None, None, None, None, None + + +def from_parallel_logits_to_logprobs( + vocab_parallel_logits: torch.Tensor, + target: torch.Tensor, + vocab_start_index: int, + vocab_end_index: int, + tp_group: torch.distributed.ProcessGroup, + inference_only: bool = False, + cp_group: Optional[torch.distributed.ProcessGroup] = None, + chunk_size: Optional[int] = None, +) -> torch.Tensor: + """Get log probabilities from TP+CP sharded vocab logits. + + Args: + vocab_parallel_logits (torch.Tensor): Logits tensor with shape [batch_size, seq_len // CP, vocab_size // TP] + where TP is the tensor parallel size. + target (torch.Tensor): Target token indices with shape [batch_size, seq_len]. + NOTE: Must be the unmodified targets as this function will shift them internally. + vocab_start_index (int): Starting vocabulary index for this worker's partition. + vocab_end_index (int): Ending vocabulary index for this worker's partition. + tp_group (torch.distributed.ProcessGroup): Process group for distributed communication. + inference_only (bool, optional): If True, tensors won't be saved for backward pass. Defaults to False. + cp_group (torch.distributed.ProcessGroup, optional): Context parallelism process group. Defaults to None. + chunk_size (int, optional): Sequence dimension chunk size for computing the log probabilities. + + Returns: + torch.Tensor: Log probabilities tensor with shape [batch_size, seq_len-1]. + The sequence dimension is reduced by 1 due to the target shifting. + + Taken from: https://github.com/NVIDIA/NeMo-Aligner/blob/9faab404f21994a7eb1d6ed5890b76152b941636/nemo_aligner/utils/distributed.py#L354 + """ + target = target.roll(shifts=-1, dims=-1) + cp_size = 1 if cp_group is None else torch.distributed.get_world_size(cp_group) + pad_len = 0 + # if cp_size > 1: + # Pad the targets to local size * cp_size + pad_len = vocab_parallel_logits.shape[1] * cp_size - target.shape[1] + if pad_len > 0: + target = torch.nn.functional.pad(target, (0, pad_len), value=0) + + # Shard the targets by context parallelism + cp_rank = torch.distributed.get_rank(cp_group) + target = _get_tokens_on_this_cp_rank(target, cp_rank, cp_size, seq_dim=1) + + if chunk_size is not None: + logprobs: torch.Tensor = ChunkedDistributedLogprob.apply( # type: ignore + vocab_parallel_logits, + target, + vocab_start_index, + vocab_end_index, + chunk_size, + tp_group, + inference_only, + ).contiguous() + else: + logprobs: torch.Tensor = DistributedLogprob.apply( # type: ignore + vocab_parallel_logits, + target, + vocab_start_index, + vocab_end_index, + tp_group, + inference_only, + ).contiguous() + + if cp_size > 1: + # we need to gather the logits by context parallelism + logprobs = allgather_cp_sharded_tensor(logprobs, cp_group, seq_dim=1) # , unpadded_seqlen=target.shape[1]) + + if pad_len > 0: + logprobs = logprobs[:, :-pad_len] + + return logprobs[:, :-1] + + +def from_parallel_logits_to_logprobs_packed_sequences( + vocab_parallel_logits: torch.Tensor, + target: torch.Tensor, + cu_seqlens_padded: torch.Tensor, + unpacked_seqlen: int, + vocab_start_index: int, + vocab_end_index: int, + group: torch.distributed.ProcessGroup, + inference_only: bool = False, + cp_group: Optional[torch.distributed.ProcessGroup] = None, + chunk_size: Optional[int] = None, +) -> torch.Tensor: + """Get log probabilities from TP sharded vocab logits for packed sequences. + + Args: + vocab_parallel_logits (torch.Tensor): Packed logits tensor with shape [1, T // CP, vocab_size//TP] + where T is the total number of tokens across all packed sequences. + target (torch.Tensor): Packed target token indices with shape [1, T]. + NOTE: Must be the unmodified targets as this function will shift them internally. + cu_seqlens (torch.Tensor): Cumulative sequence lengths tensor with shape [batch_size + 1]. + cu_seqlens[i] indicates the start position of sequence i in the packed format. + unpacked_seqlen (int): The length of the unpacked sequence tensor. + vocab_start_index (int): Starting vocabulary index for this worker's partition. + vocab_end_index (int): Ending vocabulary index for this worker's partition. + group (torch.distributed.ProcessGroup): Process group for distributed communication. + inference_only (bool, optional): If True, tensors won't be saved for backward pass. Defaults to False. + cp_group (torch.distributed.ProcessGroup, optional): Context parallelism process group. Defaults to None. + chunk_size (int, optional): Sequence dimension chunk size for computing the log probabilities. + + Returns: + torch.Tensor: Unpacked log probabilities tensor with shape [batch_size, unpacked_seqlen-1]. + The total length is reduced by batch_size due to target shifting (one token per sequence). + """ + # Remove batch dimension to work with [T, vocab_size] and [T] + vocab_parallel_logits = vocab_parallel_logits.squeeze(0) + target = target.squeeze(0) + + batch_size = cu_seqlens_padded.shape[0] - 1 + cp_size = 1 if cp_group is None else torch.distributed.get_world_size(cp_group) + cp_rank = 0 if cp_group is None else torch.distributed.get_rank(cp_group) + + # Roll each sequence individually + rolled_targets = torch.zeros(target.shape[0] // cp_size, dtype=target.dtype, device=target.device) + for i in range(batch_size): + start_idx = cu_seqlens_padded[i].item() + end_idx = cu_seqlens_padded[i + 1].item() + + # Get the sequence targets and roll by -1 + seq_targets = target[start_idx:end_idx] + rolled_seq_targets = seq_targets.roll(shifts=-1, dims=0) + rolled_targets[start_idx // cp_size : end_idx // cp_size] = _get_tokens_on_this_cp_rank( + rolled_seq_targets, cp_rank, cp_size, seq_dim=0 + ) + + # Add batch dimension back for DistributedLogprob + rolled_targets = rolled_targets.unsqueeze(0) + vocab_parallel_logits = vocab_parallel_logits.unsqueeze(0) + + # Apply distributed log probability computation + if chunk_size is not None: + probs: torch.Tensor = ChunkedDistributedLogprob.apply( # type: ignore + vocab_parallel_logits, + rolled_targets, + vocab_start_index, + vocab_end_index, + chunk_size, + group, + inference_only, + ).contiguous() + else: + probs: torch.Tensor = DistributedLogprob.apply( # type: ignore + vocab_parallel_logits, + rolled_targets, + vocab_start_index, + vocab_end_index, + group, + inference_only, + ).contiguous() + + # Remove batch dimension for filtering + probs = probs.squeeze(0) + + # Ensure probs is 1D after squeezing + if probs.dim() != 1: + raise ValueError( + f"Expected probs to be 1D after squeezing, but got shape {probs.shape}. " + f"Original shape before squeeze: {probs.unsqueeze(0).shape}" + ) + + if cp_size > 1: + # per-sequence cp_allgather + final_probs = torch.zeros(probs.shape[0] * cp_size, device=probs.device) + for i in range(batch_size): + start_idx = cu_seqlens_padded[i].item() + end_idx = cu_seqlens_padded[i + 1].item() + final_probs[start_idx:end_idx] = allgather_cp_sharded_tensor( + probs[start_idx // cp_size : end_idx // cp_size], cp_group, seq_dim=0 + ) + probs = final_probs + + out_logprobs = torch.zeros((batch_size, unpacked_seqlen - 1), dtype=probs.dtype, device=probs.device) + # Filter out the last token of each sequence + for i in range(batch_size): + start_idx = cu_seqlens_padded[i].item() + end_idx = cu_seqlens_padded[i + 1].item() + + # Exclude the last position (which has the rolled target from position 0) + if end_idx - start_idx > 0: + seq_probs = probs[start_idx : end_idx - 1] + # Ensure seq_probs is 1D + if seq_probs.dim() > 1: + seq_probs = seq_probs.squeeze() + + # Ensure we don't exceed the unpacked sequence length + seq_len = min(seq_probs.shape[0], unpacked_seqlen - 1) + if seq_len > 0: + out_logprobs[i, :seq_len] = seq_probs[:seq_len] + + return out_logprobs + + +def _get_tokens_on_this_cp_rank( + input_ids: torch.Tensor, + cp_rank: int, + cp_size: int, + seq_dim: int = 1, +) -> torch.Tensor: + """Get tokens on this context parallelism rank. + + Assumes that input_ids are already padded to a multiple of cp_size * 2 or cp_size == 1. + + Args: + input_ids: Input token IDs [seq_length, ] + cp_rank: Context parallelism rank + cp_size: Context parallelism size + + Returns: + Tokens on this context parallelism rank [1, seq_length // cp_size] + """ + if cp_size == 1: + return input_ids + + # load balance for causal attention + shard_size = input_ids.shape[seq_dim] // (cp_size * 2) + shard_inds = (cp_rank, (cp_size * 2) - cp_rank - 1) + + # Create slices for each dimension + slices = [slice(None)] * input_ids.dim() + ids_chunks = [] + + for ind in shard_inds: + slices[seq_dim] = slice(ind * shard_size, (ind + 1) * shard_size) + ids_chunks.append(input_ids[slices]) + + ids = torch.cat(ids_chunks, dim=seq_dim) + return ids + + +def allgather_cp_sharded_tensor(tensor, cp_group, seq_dim=1): # , unpadded_seqlen=None): + return AllGatherCPTensor.apply(tensor, cp_group, seq_dim) # , unpadded_seqlen) + + +class AllGatherCPTensor(torch.autograd.Function): + def forward( + ctx, tensor, cp_group: torch.distributed.ProcessGroup, seq_dim=1 + ): # , unpadded_seqlen: Optional[int] = None): + cp_size = torch.distributed.get_world_size(cp_group) + cp_rank_chunks = [] + for _ in range(cp_size): + cp_rank_chunks.append(torch.empty_like(tensor)) + + torch.distributed.all_gather(tensor_list=cp_rank_chunks, tensor=tensor, group=cp_group) + + # undo the CP load balancing chunking + tensor_chunks = [] + for logit_chunk in cp_rank_chunks: + tensor_chunks.extend(torch.chunk(logit_chunk, chunks=2, dim=seq_dim)) + + chunk_indices = [] + for cp_rank in range(cp_size): + chunk_indices.append(cp_rank) + chunk_indices.append(2 * cp_size - cp_rank - 1) + + chunks_and_indices = list(zip(tensor_chunks, chunk_indices)) + chunks_and_indices = sorted(chunks_and_indices, key=lambda tup: tup[1]) + ret_tensor = [chunk for chunk, _ in chunks_and_indices] + ret_tensor = torch.cat(ret_tensor, dim=seq_dim) + + ctx.seq_dim = seq_dim + ctx.cp_group = cp_group + # ctx.unpadded_seqlen = unpadded_seqlen + + return ret_tensor + + def backward(ctx, grad_output): + cp_size = torch.distributed.get_world_size(ctx.cp_group) + cp_rank = torch.distributed.get_rank(ctx.cp_group) + torch.distributed.all_reduce(grad_output, group=ctx.cp_group) + + # chunk the seqdim in 2*cp chunks, and select with a CP load balanced indexing + seq_dim = ctx.seq_dim + # if ctx.unpadded_seqlen is not None: + # # Zero out grad_output along the seq_dim after unpadded_seqlen + # slicer = [slice(None)] * grad_output.dim() + # slicer[seq_dim] = slice(ctx.unpadded_seqlen, None) + # grad_output[tuple(slicer)] = 0 + + grad_output = grad_output.view( + *grad_output.shape[0:seq_dim], + 2 * cp_size, + grad_output.shape[seq_dim] // (2 * cp_size), + *grad_output.shape[(seq_dim + 1) :], + ) + + index = torch.tensor([cp_rank, (2 * cp_size - cp_rank - 1)], device="cpu", pin_memory=True).cuda( + non_blocking=True + ) + + grad_input = grad_output.index_select(seq_dim, index) + grad_input = grad_input.view(*grad_input.shape[0:seq_dim], -1, *grad_input.shape[(seq_dim + 2) :]) + + return grad_input, None, None # , None + + +# Below ported from https://github.com/volcengine/verl/blob/main/verl/utils/megatron/tensor_parallel.py#L109 +class _VocabParallelEntropy(torch.autograd.Function): + @staticmethod + def forward(ctx, vocab_parallel_logits: torch.Tensor) -> torch.Tensor: + @torch.compile(dynamic=True) + def mul_reduce(a, b): + return (a * b).sum(dim=-1, keepdim=True) + + logits_max = vocab_parallel_logits.max(dim=-1, keepdim=True).values + dist.all_reduce(logits_max, op=dist.ReduceOp.MAX, group=mpu.get_tensor_model_parallel_group()) + normalized_vocab_parallel_logits = vocab_parallel_logits - logits_max + normalized_exp_logits = normalized_vocab_parallel_logits.exp_() + normalized_sum_exp_logits = normalized_exp_logits.sum(dim=-1, keepdim=True) + dist.all_reduce(normalized_sum_exp_logits, group=mpu.get_tensor_model_parallel_group()) + softmax_logits = normalized_exp_logits.div_(normalized_sum_exp_logits) + sum_softmax_times_logits = mul_reduce(softmax_logits, vocab_parallel_logits) + dist.all_reduce(sum_softmax_times_logits, group=mpu.get_tensor_model_parallel_group()) + entropy = logits_max + normalized_sum_exp_logits.log() - sum_softmax_times_logits + ctx.save_for_backward(vocab_parallel_logits, softmax_logits, sum_softmax_times_logits) + return entropy.squeeze(dim=-1) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor) -> torch.Tensor: + vocab_parallel_logits, softmax_logits, sum_softmax_times_logits = ctx.saved_tensors + # reuse softmax_logits as grad + vocab_parallel_logits.sub_(sum_softmax_times_logits) + softmax_logits.mul_(vocab_parallel_logits) + softmax_logits.mul_(grad_output.unsqueeze(dim=-1)) + # recover vocab_parallel_logits + vocab_parallel_logits.add_(sum_softmax_times_logits) + softmax_logits.mul_(-1) + return softmax_logits + + +def vocab_parallel_entropy(vocab_parallel_logits: torch.Tensor) -> torch.Tensor: + """Compute entropy when the logits are sharded in tp ranks + + Args: + vocab_parallel_logits: (total_nnz, vocab_size // tp_size) + + Returns: (total_nnz,) + + """ + return _VocabParallelEntropy.apply(vocab_parallel_logits) diff --git a/skyrl-train/skyrl_train/distributed/megatron/optimizer.py b/skyrl-train/skyrl_train/distributed/megatron/optimizer.py new file mode 100644 index 0000000000..b0cd929a10 --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/megatron/optimizer.py @@ -0,0 +1,100 @@ +# Utils ported from Verl +# https://github.com/volcengine/verl/blob/e1603dc97f3c20c58feed1f5be34acd5c72a830c/verl/utils/megatron/optimizer.py#L4 +# The original copyright is reproduced below: + +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from megatron.core.optimizer import OptimizerConfig +from megatron.core.optimizer import get_megatron_optimizer as get_megatron_optimizer_native +from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler + + +def init_megatron_optim_config(optim_config: dict, optimizer_config_kwargs: dict) -> OptimizerConfig: + optim_args = { + "optimizer": optim_config.get("optimizer", "adam"), + "lr": optim_config.get("lr"), + "min_lr": optim_config.get("min_lr", 0.0), + "clip_grad": optim_config.get("max_grad_norm", 1.0), + "weight_decay": optim_config.get("weight_decay", 0.01), + "bf16": True, + "params_dtype": torch.bfloat16, + "use_distributed_optimizer": True, + } + + optim_args.update(optimizer_config_kwargs) + + config = OptimizerConfig(**optim_args) + return config + + +def get_megatron_optimizer( + model, + config: OptimizerConfig, +): + # Base optimizer. + return get_megatron_optimizer_native( + config=config, + model_chunks=model, + ) + + +def get_megatron_optimizer_param_scheduler( + optimizer, + config, + num_training_steps: int = 1e9, # default to a large number for constant lr/wd +): + """ + Get the optimizer parameter scheduler for Megatron. + """ + # TODO: support other schedulers for Megatron + if config.get("scheduler", "constant_with_warmup") != "constant_with_warmup": + raise ValueError("Only constant_with_warmup scheduler is supported for Megatron") + + lr_warmup_steps = config.num_warmup_steps + if config.get("lr_decay_steps", None) is None: + lr_decay_steps = num_training_steps + if config.get("lr_warmup_steps_ratio", None) is not None and ( + config.get("lr_warmup_steps", None) is None or config.lr_warmup_steps <= 0 + ): + lr_warmup_steps = int(config.lr_warmup_steps_ratio * lr_decay_steps) + + opt_param_scheduler = OptimizerParamScheduler( + optimizer, + init_lr=config.get("lr_warmup_init", 0.0), + max_lr=config.lr, + min_lr=config.get("min_lr", 0.0), + lr_warmup_steps=lr_warmup_steps, + lr_decay_steps=lr_decay_steps, + lr_decay_style="constant", + start_wd=config.weight_decay, + end_wd=config.weight_decay, + wd_incr_steps=num_training_steps, + wd_incr_style="constant", + use_checkpoint_opt_param_scheduler=False, + override_opt_param_scheduler=True, + wsd_decay_steps=None, + lr_wsd_decay_style="exponential", + ) + + return opt_param_scheduler + + +def get_megatron_last_lr(optimizer): + """ + Get the last learning rate from the optimizer parameter scheduler. + """ + return optimizer.param_groups[0]["lr"] diff --git a/skyrl-train/skyrl_train/distributed/strategy.py b/skyrl-train/skyrl_train/distributed/strategy.py new file mode 100644 index 0000000000..acceccb451 --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/strategy.py @@ -0,0 +1,161 @@ +import random +from abc import ABC, abstractmethod + +from loguru import logger +import numpy as np +import torch +from torch import distributed as dist +from typing import Optional, Dict, Any, Union, TypeVar +import torch.optim as optim +from jaxtyping import Float +from transformers import GenerationConfig, PretrainedConfig, PreTrainedTokenizer +from skyrl_train.utils.io import io + + +DataT = TypeVar("DataT", bound=Union[Dict[str, Any], torch.Tensor]) + + +class DistributedStrategy(ABC): + @abstractmethod + def setup_distributed(self): + pass + + @abstractmethod + def backward(self, loss: torch.Tensor, model, optimizer: optim.Optimizer, **kwargs): + """Perform backward pass""" + pass + + @abstractmethod + def optimizer_step( + self, + optimizer: optim.Optimizer, + model, + scheduler, + name="model", + **kwargs, + ) -> Optional[Float[torch.Tensor, "1"]]: + """Perform optimizer step""" + pass + + @abstractmethod + def save_checkpoint(self, model, ckpt_dir, node_local_rank, optimizer, scheduler, tokenizer): + """Save checkpoint""" + pass + + @abstractmethod + def load_checkpoint( + self, model, ckpt_dir, optimizer, scheduler, load_module_strict, load_optimizer_states, load_lr_scheduler_states + ): + """Load checkpoint""" + pass + + @abstractmethod + def save_hf_model(self, model, output_dir: str, tokenizer=None, **kwargs): + """Save model in HuggingFace safetensors format""" + pass + + def print(self, *msg): + """Print only on rank 0""" + if self.is_rank_0(): + print(*msg) + + def is_rank_0(self) -> bool: + """Check if current process is rank 0""" + return dist.get_rank() == 0 + + def get_rank(self) -> int: + """Get current process rank""" + return dist.get_rank() + + def all_reduce(self, data: DataT, op="mean") -> DataT: + """Perform all_reduce across all processes""" + assert op in ("mean", "max", "sum") + if isinstance(data, dict): + ret = {} + for k, v in data.items(): + ret[k] = self.all_reduce(v, op) + return ret + else: + is_tensor = True + if not isinstance(data, torch.Tensor): + data = torch.Tensor([data]) + is_tensor = False + is_cpu_tensor = data.device.type == "cpu" + + if is_cpu_tensor: + data = data.to(torch.cuda.current_device()) + if op == "mean": + data /= self.world_size + dist.all_reduce(data, op=dist.ReduceOp.MAX if op == "max" else dist.ReduceOp.SUM) + if is_cpu_tensor: + data = data.cpu() + return data.item() if not is_tensor else data + + def all_gather(self, data: DataT) -> DataT: + """Perform all_gather across all processes""" + if isinstance(data, dict): + ret = {} + for k, v in data.items(): + ret[k] = self.all_gather(v) + return ret + else: + if not isinstance(data, torch.Tensor): + data = torch.Tensor([data]) + is_cpu_tensor = data.device.type == "cpu" + + ret = [torch.zeros_like(data).to(torch.cuda.current_device()) for _ in range(self.world_size)] + dist.all_gather(ret, data.to(torch.cuda.current_device())) + return torch.cat(ret).cpu() if is_cpu_tensor else torch.cat(ret) + + def save_hf_configs(self, model_config: PretrainedConfig, hf_dir: str, tokenizer: PreTrainedTokenizer = None): + """ + Save model and tokenizer configs to hf_dir + + Args: + model_config: PretrainedConfig - huggingface model config + hf_dir: str - the directory to save the huggingface configs to + tokenizer: PreTrainedTokenizer - tokenizer to save + """ + io.makedirs(hf_dir, exist_ok=True) + + with io.local_work_dir(hf_dir) as work_dir: + model_config.save_pretrained(work_dir) + if tokenizer: + tokenizer.save_pretrained(work_dir) + + if hasattr(model_config, "name_or_path") and model_config.name_or_path: + try: + # Some model's name_or_path is empty if not initialized from pretrained, + # in this cases, we don't save generation config. + generation_config = GenerationConfig.from_pretrained(model_config.name_or_path) + # with io.local_work_dir(hf_config_tokenizer_path) as work_dir: + generation_config.save_pretrained(work_dir) + except Exception as e: + # if the generation config isn't available, we don't save it + logger.warning(f"Could not save generation config for '{model_config.name_or_path}'. Error: {e}") + + @staticmethod + def get_rng_state(): + """Get current RNG state for reproducibility""" + rng_state = { + "cpu": torch.get_rng_state(), + "numpy": np.random.get_state(), + "random": random.getstate(), + } + + # Only save CUDA RNG state if CUDA is available and being used + if torch.cuda.is_available() and torch.cuda.device_count() > 0: + rng_state["cuda"] = torch.cuda.get_rng_state() + + return rng_state + + @staticmethod + def load_rng_state(rng_state): + """Load RNG state for reproducibility""" + torch.set_rng_state(rng_state["cpu"]) + np.random.set_state(rng_state["numpy"]) + random.setstate(rng_state["random"]) + + # Only restore CUDA RNG state if it was saved and CUDA is available + if "cuda" in rng_state and torch.cuda.is_available() and torch.cuda.device_count() > 0: + torch.cuda.set_rng_state(rng_state["cuda"]) diff --git a/skyrl-train/skyrl_train/distributed/ulysses/__init__.py b/skyrl-train/skyrl_train/distributed/ulysses/__init__.py new file mode 100644 index 0000000000..f972706006 --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/ulysses/__init__.py @@ -0,0 +1,4 @@ +from .utils import set_ulysses_sequence_parallel_group +from .monkey_patch import apply_monkey_patch + +__all__ = ["set_ulysses_sequence_parallel_group", "apply_monkey_patch"] diff --git a/skyrl-train/skyrl_train/distributed/ulysses/monkey_patch.py b/skyrl-train/skyrl_train/distributed/ulysses/monkey_patch.py new file mode 100644 index 0000000000..608ca1019b --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/ulysses/monkey_patch.py @@ -0,0 +1,171 @@ +# Transformers monkey patch for seq parallelism from VERL. +# We reproduce the original copyright below. +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Apply monkey-patch function to models +""" + +import importlib.metadata +import sys +from functools import lru_cache +from typing import Optional + +from loguru import logger +import torch +from packaging import version +from transformers.modeling_flash_attention_utils import _flash_attention_forward +from transformers.modeling_utils import PreTrainedModel + +from skyrl_train.distributed.ulysses.utils import ( + gather_heads_scatter_seq, + gather_seq_scatter_heads, + get_ulysses_sequence_parallel_group, + get_ulysses_sequence_parallel_world_size, +) + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=2, repeats=n_rep). The hidden states go from (batch, + seqlen, num_key_value_heads, head_dim) to (batch, seqlen, num_attention_heads, head_dim) + """ + batch, slen, num_key_value_heads, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, :, None, :].expand(batch, slen, num_key_value_heads, n_rep, head_dim) + return hidden_states.reshape(batch, slen, num_key_value_heads * n_rep, head_dim) + + +# TODO (sumanthrh): For some reason, VERL all-gathers position_ids but not attention_mask. +# I've added attention_mask all gather here, but this is pending correctness checks for sp_size > 1. +def _ulysses_flash_attention_forward( + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + *args, + position_ids: Optional[torch.Tensor] = None, + **kwargs, +): + """Insert all-to-all before and after flash attention. + DeepSpeed-Ulysses: https://arxiv.org/pdf/2309.14509 + + Args: + query_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads, head_dim) + key_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads_k, head_dim) + value_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads_k, head_dim) + position_ids (torch.Tensor, optional): (batch_size, seqlen/sp_size) + + Returns: + torch.Tensor: (batch_size, seqlen/sp_size, nheads, head_dim) + """ + ulysses_sp_size = get_ulysses_sequence_parallel_world_size() + + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + assert position_ids is not None, "position_ids is required for Ulysses sequence parallelism" + # NOTE: repeat kv heads to be divided by sequence parallel. Instead of repeating nheads_q//nheads_k, + # we choose to repeat sp_size//nheads_k, since flash_attention supports MQA/GQA. + # For example: + # - nheads_k=4, sp=8, repeats=2 + # - nheads_k=8, sp=8, repeats=1 + # - nheads_k=16, sp=8, repeats=1 + repeats = max(ulysses_sp_size // key_states.size(2), 1) + key_states = repeat_kv(key_states, repeats) + value_states = repeat_kv(value_states, repeats) + + # (bsz, seq_len/n, n_head, head_dim) -> (bsz, seq_len, n_head/n, head_dim) + query_states = gather_seq_scatter_heads(query_states, seq_dim=1, head_dim=2) + key_states = gather_seq_scatter_heads(key_states, seq_dim=1, head_dim=2) + value_states = gather_seq_scatter_heads(value_states, seq_dim=1, head_dim=2) + + # TODO: all_gather position_ids because `prepare_fa2_from_position_ids` needs it, we can eliminate + # this all_gather by passing cu_seq_lens_q, cu_seq_lens_k, max_length_k, max_length_q explicitly. + # https://github.com/huggingface/transformers/pull/33932 + # (bsz, seq_len/n) -> (bsz, seq_len) + position_ids_list = [torch.empty_like(position_ids) for _ in range(ulysses_sp_size)] + torch.distributed.all_gather(position_ids_list, position_ids, group=get_ulysses_sequence_parallel_group()) + position_ids = torch.concat(position_ids_list, dim=-1) + + if attention_mask is not None: + # all gather attention mask + # (bsz, seq_len/n) -> (bsz, seq_len) + attention_mask_list = [torch.empty_like(attention_mask) for _ in range(ulysses_sp_size)] + torch.distributed.all_gather(attention_mask_list, attention_mask, group=get_ulysses_sequence_parallel_group()) + attention_mask = torch.concat(attention_mask_list, dim=-1) + + # NOTE: we need to recompute this parameter otherwise the output shape will be wrong with + # a non-null attention mask + if "query_length" in kwargs: + kwargs["query_length"] = query_states.size(1) + + # (bsz, seq_len, n_head/n, head_dim) + attn_output = _flash_attention_forward( + query_states, key_states, value_states, attention_mask, *args, position_ids=position_ids, **kwargs + ) + + ########## AlltoAll for Ulysses ########## + if ulysses_sp_size > 1: + # (bsz, seq_len, n_head/n, head_dim) -> (bsz, seq_len/n, n_head, head_dim) + attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) + + return attn_output + + +def apply_monkey_patch( + model: PreTrainedModel, + ulysses_sp_size: int = 1, + use_remove_padding: bool = True, + use_fused_kernels: bool = False, + use_parent_class: bool = False, +): + """Replace _flash_attention_forward to _ulysses_flash_attention_forward""" + if use_parent_class: + parent_cls = type(model).__bases__[0] + logger.info(f"Applying monkey patch to {parent_cls.__name__} in {parent_cls.__module__}") + module = sys.modules[parent_cls.__module__] + else: + module = sys.modules[model.__module__] + + num_attention_heads, num_key_value_heads = model.config.num_attention_heads, model.config.num_key_value_heads + assert ( + num_attention_heads % ulysses_sp_size == 0 + ), f"num_attention_heads {num_attention_heads} must be divisible by ulysses_sp_size {ulysses_sp_size}" + assert ( + num_key_value_heads % ulysses_sp_size == 0 or ulysses_sp_size % num_key_value_heads == 0 + ), f"num_key_value_heads {num_key_value_heads} must be divisible by ulysses_sp_size {ulysses_sp_size}or vise versa. Upon ulysses_sp_size % num_key_value_heads == 0,kv heads are repeated to ensure correctness." + # transformers<=4.47.1 + if use_remove_padding or ulysses_sp_size > 1: + if hasattr(module, "_flash_attention_forward"): + module._flash_attention_forward = _ulysses_flash_attention_forward + logger.info(f"Monkey patch _flash_attention_forward in {model.__module__}") + else: + # transformers>=4.48.0 + from transformers.integrations import flash_attention + + flash_attention._flash_attention_forward = _ulysses_flash_attention_forward + logger.info(f"Monkey patch _flash_attention_forward in {flash_attention.__name__}") + + +@lru_cache +def is_transformers_version_in_range(min_version: str, max_version: str) -> bool: + try: + # Get the installed version of the transformers library + transformers_version = importlib.metadata.version("transformers") + except importlib.metadata.PackageNotFoundError as e: + raise ModuleNotFoundError("The `transformers` package is not installed.") from e + + # Check if the version is within the specified range + return version.parse(min_version) <= version.parse(transformers_version) <= version.parse(max_version) diff --git a/skyrl-train/skyrl_train/distributed/ulysses/utils.py b/skyrl-train/skyrl_train/distributed/ulysses/utils.py new file mode 100644 index 0000000000..f51105b053 --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/ulysses/utils.py @@ -0,0 +1,329 @@ +# DeepSpeed Ulysses Sequence Parallelism borrowed from VERL +# The original copyright is reproduced below: +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utilities for DeepSpeed Ulysses Sequence Parallelism. +DeepSpeed Ulysses Paper: https://arxiv.org/abs/2309.14509 +Inspired from: https://github.com/deepspeedai/DeepSpeed/blob/master/deepspeed/sequence/layer.py +""" + +from typing import Any, Optional, Tuple + +import torch +import torch.distributed as dist +from loguru import logger +from torch import Tensor +from torch.distributed import ProcessGroup + +_ULYSSES_SEQUENCE_PARALLEL_GROUP = None + + +def set_ulysses_sequence_parallel_group(group: dist.ProcessGroup): + """ + Set ulysses sequence parallel process group. + """ + global _ULYSSES_SEQUENCE_PARALLEL_GROUP + _ULYSSES_SEQUENCE_PARALLEL_GROUP = group + + +def get_ulysses_sequence_parallel_group() -> Optional[dist.ProcessGroup]: + """ + Get ulysses sequence parallel process group. + """ + global _ULYSSES_SEQUENCE_PARALLEL_GROUP + return _ULYSSES_SEQUENCE_PARALLEL_GROUP + + +def get_ulysses_sequence_parallel_world_size(group: ProcessGroup = None) -> int: + """ + Get ulysses sequence parallel world size. + """ + group = get_ulysses_sequence_parallel_group() if group is None else group + return dist.get_world_size(group) if group else 1 + + +def get_ulysses_sequence_parallel_rank(group: ProcessGroup = None) -> int: + """ + Get ulysses sequence parallel rank. + """ + group = get_ulysses_sequence_parallel_group() if group is None else group + return dist.get_rank(group) if group else 0 + + +def gather_seq_scatter_heads( + x: Tensor, + seq_dim: int, + head_dim: int, + unpadded_dim_size: int = 0, + group: ProcessGroup = None, +) -> Tensor: + """ + A func to sync embedding input with alltoall in sequence parallel + gather sequence dimension and scatter head dim: + e.g. seq_dim: 1, head_dim: 2 + [bsz, seq/n, h, ...] -> [bsz, seq, h/n, ...] + """ + group = get_ulysses_sequence_parallel_group() if group is None else group + if not group: + return x + sp_world = get_ulysses_sequence_parallel_world_size(group) + x = SeqAllToAll.apply(group, x, head_dim, seq_dim) + if unpadded_dim_size and unpadded_dim_size % sp_world != 0: + padding_size = x.size(seq_dim) - unpadded_dim_size + x = _unpad_tensor(x, seq_dim, padding_size) + return x + + +def gather_heads_scatter_seq(x: Tensor, head_dim: int, seq_dim: int, group: ProcessGroup = None) -> Tensor: + """ + A func to sync attention result with alltoall in sequence parallel + gather head dimension and scatter seq dim: + e.g. seq_dim: 1, head_dim: 2 + [bsz, seq, h/n, ...] -> [bsz, seq/n, h, ...] + """ + group = get_ulysses_sequence_parallel_group() if group is None else group + if not group: + logger.info("No group found for gather_heads_scatter_seq") + return x + dim_size = x.size(seq_dim) + sp_world = get_ulysses_sequence_parallel_world_size(group) + if dim_size % sp_world != 0: + padding_size = sp_world - (dim_size % sp_world) + x = _pad_tensor(x, seq_dim, padding_size) + return SeqAllToAll.apply(group, x, seq_dim, head_dim, False) + + +def _pad_tensor(x: Tensor, dim: int, padding_size: int) -> Tensor: + shape = list(x.shape) + shape[dim] = padding_size + pad = torch.zeros(shape, dtype=x.dtype, device=x.device) + return torch.cat([x, pad], dim=dim) + + +def _unpad_tensor(x: Tensor, dim: int, padding_size: int) -> Tensor: + slc = [slice(None)] * len(x.shape) + slc[dim] = slice(0, -padding_size) + return x[slc] + + +def slice_input_tensor(x: Tensor, dim: int, padding: bool = True, group: ProcessGroup = None) -> Tensor: + group = get_ulysses_sequence_parallel_group() if group is None else group + sp_world_size = dist.get_world_size(group) + sp_rank = get_ulysses_sequence_parallel_rank() + dim_size = x.size(dim) + # pad before slice + if padding and dim_size % sp_world_size: + padding_size = sp_world_size - (dim_size % sp_world_size) + x = _pad_tensor(x, dim, padding_size) + # slice the input tensor + parts = x.size(dim) // sp_world_size + slc = [slice(None)] * len(x.shape) + slc[dim] = slice(sp_rank * parts, (sp_rank + 1) * parts) + return x[slc].contiguous() + + +def all_to_all_tensor( + local_input: Tensor, + scatter_dim: int, + gather_dim: int, + group: Optional[dist.ProcessGroup] = None, + async_op: bool = False, +): + group = get_ulysses_sequence_parallel_group() if group is None else group + seq_world_size = dist.get_world_size(group) + input_list = [t.contiguous() for t in torch.tensor_split(local_input, seq_world_size, scatter_dim)] + output_list = [torch.empty_like(input_list[0]) for _ in range(seq_world_size)] + comm = dist.all_to_all(output_list, input_list, group=group, async_op=async_op) + if async_op: + + def wait(): + comm.wait() + return torch.cat(output_list, dim=gather_dim).contiguous() + + return wait + return torch.cat(output_list, dim=gather_dim).contiguous() + + +def all_gather_tensor(local_tensor: Tensor, group: Optional[dist.ProcessGroup] = None, async_op: bool = False): + group = get_ulysses_sequence_parallel_group() if group is None else group + sp_world_size = dist.get_world_size(group=group) + output_shape = list(local_tensor.shape) + output_shape[0] = output_shape[0] * sp_world_size + output = torch.empty(output_shape, dtype=local_tensor.dtype, device=local_tensor.device) + dist.all_gather_into_tensor(output, local_tensor, group=group, async_op=async_op) + return output + + +class SeqAllToAll(torch.autograd.Function): + @staticmethod + def forward( + ctx: Any, + group: dist.ProcessGroup, + local_input: Tensor, + scatter_dim: int, + gather_dim: int, + async_op: bool = False, + ) -> Tensor: + ctx.group = group + ctx.scatter_dim = scatter_dim + ctx.gather_dim = gather_dim + ctx.async_op = async_op + return all_to_all_tensor(local_input, scatter_dim, gather_dim, group, async_op) + + @staticmethod + def backward(ctx: Any, *grad_output: Tensor) -> Tuple[None, Tensor, None, None]: + input_t = torch.cat(grad_output[1:], dim=ctx.gather_dim).contiguous() if ctx.async_op else grad_output[0] + return ( + None, + all_to_all_tensor(input_t, ctx.gather_dim, ctx.scatter_dim, ctx.group, False), + None, + None, + None, + None, + ) + + +class Gather(torch.autograd.Function): + @staticmethod + def forward( + ctx: Any, + group: dist.ProcessGroup, + local_tensor: Tensor, + gather_dim: int, + grad_scaler: bool = True, + async_op=False, + ) -> Tensor: + ctx.group = group + ctx.gather_dim = gather_dim + ctx.grad_scaler = grad_scaler + ctx.async_op = async_op + + sp_world_size = dist.get_world_size(group=group) + ctx.sp_world_size = sp_world_size + + sp_rank = dist.get_rank(group=group) + ctx.sp_rank = sp_rank + + local_shape = list(local_tensor.size()) + split_size = local_shape[0] + part_size = local_shape[gather_dim] # store original size + ctx.part_size = part_size + + output = all_gather_tensor(local_tensor, group, async_op) + return torch.cat(output.split(split_size, dim=0), dim=gather_dim) + + @staticmethod + def backward(ctx: Any, grad_output: Tensor) -> Any: + if ctx.grad_scaler: + grad_output = grad_output * ctx.sp_world_size + return ( + None, + grad_output.split(ctx.part_size, dim=ctx.gather_dim)[ctx.sp_rank].contiguous(), + None, + None, + None, + None, + ) + + +def gather_outputs_and_unpad( + x: Tensor, + gather_dim: int, + unpad_dim: int = None, + padding_size: int = 0, + grad_scaler: bool = True, + group: Optional[dist.ProcessGroup] = None, +): + group = get_ulysses_sequence_parallel_group() if group is None else group + if group is None: + logger.info("No group found for gather_outputs_and_unpad") + return x + x = Gather.apply(group, x, gather_dim, grad_scaler) + if unpad_dim is not None: + assert isinstance(padding_size, int), "padding size is not given or is not an integer" + if padding_size == 0: + return x + x = _unpad_tensor(x, unpad_dim, padding_size) + return x + + +# NOTE (sumanthrh): This is an extension of the same helper function in VERL. +# We further add attention_mask slicing +def ulysses_pad_and_slice_inputs( + input_ids_rmpad: torch.Tensor, + position_ids_rmpad: Optional[torch.Tensor] = None, + attention_mask_rmpad: Optional[torch.Tensor] = None, + sp_size: int = 1, +): + """ + Pad and slice input_ids to be divisible by sp_size + Pad position_ids to be divisible by sp_size. + + Note both input_ids_rmpad and position_ids_rmpad will be padded and sliced. + + The is the utility of pre-forward for ulysses sequence parallelism + + Args: + input_ids_rmpad: shape of [bsz, seqlen] + position_ids_rmpad: shape of [bsz, seqlen] + sp_size (int): ulysses sequence parallelism size + + Returns: + torch.Tensor: padded and sliced input_ids + torch.Tensor: padded and sliced position_ids + int: pad size + """ + + if sp_size <= 1: + return input_ids_rmpad, position_ids_rmpad, attention_mask_rmpad, 0 + + # sp_size > 1. assert that a group exists + group = get_ulysses_sequence_parallel_group() + if group is None: + raise ValueError("`sp_size` > 1 but no ulysses sequence parallel group set.") + _, total_seq_len = input_ids_rmpad.shape + pad_size = (sp_size - total_seq_len % sp_size) % sp_size + if pad_size > 0: + input_ids_rmpad = torch.nn.functional.pad(input_ids_rmpad, (0, pad_size), value=0) + if position_ids_rmpad is not None: + pad_pos_ids = ( + torch.arange(pad_size, device=position_ids_rmpad.device) + .unsqueeze(0) + .repeat(position_ids_rmpad.size(0), 1) + ) + position_ids_rmpad = torch.cat((position_ids_rmpad, pad_pos_ids), dim=-1) + + if attention_mask_rmpad is not None: + pad_attn_mask = ( + torch.zeros(pad_size, device=attention_mask_rmpad.device) + .unsqueeze(0) + .repeat(attention_mask_rmpad.size(0), 1) + ) + attention_mask_rmpad = torch.cat((attention_mask_rmpad, pad_attn_mask), dim=-1) + + input_ids_rmpad = slice_input_tensor(input_ids_rmpad, dim=1, padding=False) + if position_ids_rmpad is not None: + position_ids_rmpad = slice_input_tensor(position_ids_rmpad, dim=1, padding=False) + if attention_mask_rmpad is not None: + attention_mask_rmpad = slice_input_tensor(attention_mask_rmpad, dim=1, padding=False) + return input_ids_rmpad, position_ids_rmpad, attention_mask_rmpad, pad_size + + +def validate_ulysses_config(num_heads, ulysses_sequence_size): + if ulysses_sequence_size > 1: + assert ( + num_heads % ulysses_sequence_size == 0 + ), f"num_heads ({num_heads}) must be divisible by ulysses sequence size({ulysses_sequence_size})" diff --git a/skyrl-train/skyrl_train/distributed/utils.py b/skyrl-train/skyrl_train/distributed/utils.py new file mode 100644 index 0000000000..3749d4cdd3 --- /dev/null +++ b/skyrl-train/skyrl_train/distributed/utils.py @@ -0,0 +1,221 @@ +"""Communication utilities. + +This file currently provides the following functionalities, with code mainly +sourced from PyTorch: + +1. Provides init process group capability without being restricted by default + process group. PyTorch assumes users always use its default process group. +2. Provides CUDA IPC capability, which allows bypassing torch's multiprocessing + to use GPU shared memory, for example to communicate with vllm workers using + shared memory. +""" + +from __future__ import annotations + +import socket +from datetime import timedelta +from typing import Any, Optional, Union, Tuple + +import torch +from torch.distributed.distributed_c10d import ( + Backend, + PrefixStore, + Store, + _new_process_group_helper, + _world, + default_pg_timeout, + rendezvous, +) +from torch.multiprocessing.reductions import rebuild_cuda_tensor +import torch.nn as nn +from torch.optim import Optimizer + +ModelOptimPair = Tuple[nn.Module, Optimizer] +ModelOrModelOptimPair = Union[nn.Module, ModelOptimPair] + + +def get_free_port(): + with socket.socket() as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + + +# Copy from pytorch to allow creating multiple main groups. +# https://github.com/pytorch/pytorch/blob/main/torch/distributed/distributed_c10d.py +# https://github.com/OpenRLHF/OpenRLHF/blob/main/openrlhf/utils/distributed_util.py +def init_custom_process_group( + backend: Union[str, Backend] = None, + init_method: Optional[str] = None, + timeout: Optional[timedelta] = None, + world_size: int = -1, + rank: int = -1, + store: Optional[Store] = None, + group_name: str = None, + pg_options: Optional[Any] = None, +): + assert (store is None) or (init_method is None), "Cannot specify both init_method and store." + + if store is not None: + assert world_size > 0, "world_size must be positive if using store" + assert rank >= 0, "rank must be non-negative if using store" + elif init_method is None: + init_method = "env://" + + if backend: + backend = Backend(backend) + else: + backend = Backend("undefined") + + if timeout is None: + timeout = default_pg_timeout + + # backward compatible API + if store is None: + rendezvous_iterator = rendezvous(init_method, rank, world_size, timeout=timeout) + store, rank, world_size = next(rendezvous_iterator) + store.set_timeout(timeout) + + # Use a PrefixStore to avoid accidental overrides of keys used by + # different systems (e.g. RPC) in case the store is multi-tenant. + store = PrefixStore(group_name, store) + + # NOTE: The pg_options parameter was renamed into backend_options in PyTorch 2.6.0 + # https://github.com/pytorch/pytorch/commit/a0c7029a75628cd5fa8df83c0de0ea98ee7fd844 + # We need to determine the appropriate parameter name based on PyTorch version + pg_options_param_name = "backend_options" if str(torch.__version__) >= "2.6" else "pg_options" + pg, _ = _new_process_group_helper( + world_size, + rank, + [], + backend, + store, + group_name=group_name, + **{pg_options_param_name: pg_options}, + timeout=timeout, + ) + + _world.pg_group_ranks[pg] = {i: i for i in range(world_size)} + + return pg + + +def get_optimizer_grouped_parameters( + model, + weight_decay, + no_decay_name_list=["bias", "layer_norm.weight", "layernorm.weight", "norm.weight", "ln_f.weight"], +): + optimizer_grouped_parameters = [ + { + "params": [ + p + for n, p in model.named_parameters() + if (not any(nd in n for nd in no_decay_name_list) and p.requires_grad) + ], + "weight_decay": weight_decay, + }, + { + "params": [ + p + for n, p in model.named_parameters() + if (any(nd in n for nd in no_decay_name_list) and p.requires_grad) + ], + "weight_decay": 0.0, + }, + ] + return optimizer_grouped_parameters + + +class CUDAIPCHandle: + def __init__( + self, + tensor_type: type, + size: tuple, + stride: tuple, + offset: int, + storage_type: type, + dtype: torch.dtype, + device: torch.device, + handle: bytes, + storage_size_bytes: bytes, + storage_offset_bytes: bytes, + requires_grad: bool, + ref_counter_handle: bytes, + ref_counter_offset: bytes, + event_handle: bytes, + event_sync_required: bool, + ): + self.tensor_type = tensor_type + self.size = size + self.stride = stride + self.offset = offset + self.storage_type = storage_type + self.dtype = dtype + self.device = device + self.handle = handle + self.storage_size_bytes = storage_size_bytes + self.storage_offset_bytes = storage_offset_bytes + self.requires_grad = requires_grad + self.ref_counter_handle = ref_counter_handle + self.ref_counter_offset = ref_counter_offset + self.event_handle = event_handle + self.event_sync_required = event_sync_required + + def rebuild(self) -> torch.Tensor: + # NOTE: Rebuild within the same process is not thread-safe and will + # likely crash (segfault core dump). + return rebuild_cuda_tensor( + self.tensor_type, + self.size, + self.stride, + self.offset, + self.storage_type, + self.dtype, + self.device, + self.handle, + self.storage_size_bytes, + self.storage_offset_bytes, + self.requires_grad, + self.ref_counter_handle, + self.ref_counter_offset, + self.event_handle, + self.event_sync_required, + ) + + @staticmethod + def from_tensor(tensor: torch.Tensor) -> CUDAIPCHandle: + if not tensor.is_cuda: + raise ValueError("Tensor must be on CUDA device to use CUDA IPC") + tensor = tensor.share_memory_() + storage = tensor._typed_storage() + ( + device, + handle, + storage_size_bytes, + storage_offset_bytes, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, + ) = storage._share_cuda_() + tensor_offset = tensor.storage_offset() + return CUDAIPCHandle( + tensor_type=type(tensor), + size=tensor.size(), + stride=tensor.stride(), + # tensor offset in its storage + offset=tensor_offset, + storage_type=type(storage), + dtype=tensor.dtype, + device=device, + # identifier which CUDA allocation is the storage in. + handle=handle, + # size(in bytes) of the storage + storage_size_bytes=storage_size_bytes, + # offset(in bytes) of the storage in the CUDA allocation + storage_offset_bytes=storage_offset_bytes, + requires_grad=tensor.requires_grad, + ref_counter_handle=ref_counter_handle, + ref_counter_offset=ref_counter_offset, + event_handle=event_handle, + event_sync_required=event_sync_required, + ) diff --git a/skyrl-train/skyrl_train/entrypoints/main_base.py b/skyrl-train/skyrl_train/entrypoints/main_base.py new file mode 100644 index 0000000000..59026cf35c --- /dev/null +++ b/skyrl-train/skyrl_train/entrypoints/main_base.py @@ -0,0 +1,321 @@ +""" +Main entrypoint for training. +""" + +from ray.util.placement_group import placement_group, PlacementGroup + +from transformers import AutoTokenizer, PreTrainedTokenizerBase +from skyrl_train.dataset import PromptDataset +from skyrl_train.utils import validate_cfg + +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.remote_inference_engine import create_remote_inference_engines +from skyrl_train.utils.utils import initialize_ray, get_ray_pg_ready_with_timeout +from skyrl_train.env_vars import SKYRL_RAY_PG_TIMEOUT_IN_S +from skyrl_train.generators.base import GeneratorInterface +from omegaconf import OmegaConf, DictConfig +from pathlib import Path +import ray + +import os +import hydra +from loguru import logger +from skyrl_train.utils.tracking import Tracking +import multiprocessing as mp +import asyncio + +# NOTE (sumanthrh): We use ray heavily and thus disable `fork` start method. +# forking within ray leads to undefined behaviour and often causes hard to debug +# memory leaks. See: https://docs.ray.io/en/latest/ray-core/patterns/fork-new-processes.html +# A common culprit is Pytorch dataloaders which use `fork` by default. +mp.set_start_method("spawn", force=True) + +config_dir = str(Path(__file__).parent.parent / "config") +__all__ = ["BasePPOExp", "config_dir"] + + +def create_ray_wrapped_inference_engines_from_config(cfg: DictConfig, colocate_pg, tokenizer: PreTrainedTokenizerBase): + from skyrl_train.inference_engines.ray_wrapped_inference_engine import create_ray_wrapped_inference_engines + + engine_kwargs = { + "num_inference_engines": cfg.generator.num_inference_engines, + "tensor_parallel_size": cfg.generator.inference_engine_tensor_parallel_size, + "pipeline_parallel_size": cfg.generator.inference_engine_pipeline_parallel_size, + "model_dtype": cfg.generator.model_dtype, + "pretrain": cfg.trainer.policy.model.path, + "seed": cfg.trainer.seed, + "vllm_v1_disable_multiproc": cfg.generator.vllm_v1_disable_multiproc, + "enable_prefix_caching": cfg.generator.enable_prefix_caching, + "enforce_eager": cfg.generator.enforce_eager, + "expert_parallel_size": cfg.generator.inference_engine_expert_parallel_size, + "data_parallel_size": cfg.generator.inference_engine_data_parallel_size, + "shared_pg": colocate_pg, + "gpu_memory_utilization": cfg.generator.gpu_memory_utilization, + "inference_engine_enable_sleep": cfg.trainer.placement.colocate_all, + "async_engine": cfg.generator.async_engine, + "max_num_batched_tokens": cfg.generator.max_num_batched_tokens, + "max_num_seqs": cfg.generator.max_num_seqs, + "tokenizer": tokenizer, + "backend": cfg.generator.backend, + "engine_init_kwargs": cfg.generator.engine_init_kwargs, + } + + # Conditionally add LoRA parameters if LoRA is enabled + if cfg.trainer.policy.model.lora.rank > 0 and cfg.trainer.strategy != "megatron": + engine_kwargs["enable_lora"] = True + engine_kwargs["max_lora_rank"] = cfg.trainer.policy.model.lora.rank + engine_kwargs["sleep_level"] = 1 + engine_kwargs["max_loras"] = 1 + engine_kwargs["fully_sharded_loras"] = cfg.generator.fully_sharded_loras + + # TODO(devpatel): Bandaid solution, replace this once we have a better + # solution for LoRA performance degradation on the vLLM side + if cfg.generator.enforce_eager and cfg.generator.backend == "vllm": + logger.warning( + "LoRA is enabled but generator.enforce_eager=true. " + "This combination causes significant performance degradation (2-3x slower generation). " + "Automatically setting enforce_eager=false for better performance. " + ) + engine_kwargs["enforce_eager"] = False + + if (rope_scaling := cfg.generator.get("rope_scaling", None)) is not None: + engine_kwargs["rope_scaling"] = rope_scaling + if (rope_theta := cfg.generator.get("rope_theta", None)) is not None: + engine_kwargs["rope_theta"] = rope_theta + + return create_ray_wrapped_inference_engines(**engine_kwargs) + + +def create_remote_inference_engines_from_config(cfg: DictConfig, tokenizer: PreTrainedTokenizerBase): + # TODO(tgriggs): We may want a separate config for the model name in case + # it's different from the name used in the OpenAI API + return create_remote_inference_engines( + urls=cfg.generator.remote_inference_engine_urls, + model_name=cfg.trainer.policy.model.path, + engine_backend=cfg.generator.backend, + tokenizer=tokenizer, + tensor_parallel_size=cfg.generator.inference_engine_tensor_parallel_size, + pipeline_parallel_size=cfg.generator.inference_engine_pipeline_parallel_size, + data_parallel_size=cfg.generator.inference_engine_data_parallel_size, + expert_parallel_size=cfg.generator.inference_engine_expert_parallel_size, + ) + + +class BasePPOExp: + def __init__(self, cfg: DictConfig): + """ + Initializes a PPO experiment. + + The `cfg` passed here will be the final config from Hydra, including CLI overrides. + """ + self.cfg = cfg + self.tokenizer = self.get_tokenizer() + self.train_dataset = self.get_train_dataset() + self.eval_dataset = self.get_eval_dataset() + self.colocate_pg = self.get_colocate_pg() + + @staticmethod + def get_cfg_as_str(dict_cfg: DictConfig) -> str: + return OmegaConf.to_yaml(dict_cfg) + + def get_tokenizer(self, padding_side="left"): + """Initializes a tokenizer for the given model.""" + tokenizer = AutoTokenizer.from_pretrained( + self.cfg.trainer.policy.model.path, + trust_remote_code=True, + use_fast=not self.cfg.trainer.disable_fast_tokenizer, + ) + tokenizer.padding_side = padding_side + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + tokenizer.pad_token_id = tokenizer.eos_token_id + return tokenizer + + def get_train_dataset(self): + """Initializes the training dataset. + + Returns: + PromptDataset: The training dataset. + """ + prompts_dataset = PromptDataset( + datasets=self.cfg.data.train_data, + tokenizer=self.tokenizer, + max_prompt_length=self.cfg.trainer.max_prompt_length, + num_workers=8, + ) + # make sure the dataset is large enough to train on + assert ( + len(prompts_dataset) >= self.cfg.trainer.train_batch_size + ), f"dataset should be at least as large as `train_batch_size` {self.cfg.trainer.train_batch_size}, got size {len(prompts_dataset)}" + return prompts_dataset + + def get_eval_dataset(self): + """Initializes the evaluation dataset. + + Returns: + PromptDataset: The evaluation dataset. + """ + if self.cfg.trainer.eval_interval > 0 and self.cfg.data.val_data: + prompts_dataset = PromptDataset( + datasets=self.cfg.data.val_data, + tokenizer=self.tokenizer, + max_prompt_length=self.cfg.trainer.max_prompt_length, + num_workers=8, + ) + return prompts_dataset + return None + + def get_colocate_pg(self, timeout: int = SKYRL_RAY_PG_TIMEOUT_IN_S) -> PlacementGroup: + """Initializes a placement group for colocated training. + + A single placement group that packs all the inference engines together is created. + + Args: + timeout (int): The timeout for the placement group to be ready. + + Returns: + PlacementGroup: The placement group for colocated training. + """ + if self.cfg.trainer.placement.colocate_all: + pg = placement_group( + [{"GPU": 1, "CPU": 1}] + * self.cfg.generator.num_inference_engines + * self.cfg.generator.inference_engine_tensor_parallel_size + * self.cfg.generator.inference_engine_pipeline_parallel_size + * self.cfg.generator.inference_engine_data_parallel_size, + strategy="PACK", + ) + get_ray_pg_ready_with_timeout(pg, timeout=timeout) + return pg + else: + return None + + def get_generator(self, cfg, tokenizer, inference_engine_client): + """Initializes the generator. + + Returns: + GeneratorInterface: The generator. + """ + from skyrl_train.generators.skyrl_gym_generator import SkyRLGymGenerator + + return SkyRLGymGenerator( + generator_cfg=cfg.generator, + skyrl_gym_cfg=cfg.environment.skyrl_gym, + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + model_name=cfg.trainer.policy.model.path, + ) + + def get_trainer( + self, + cfg, + tracker, + tokenizer, + train_dataset, + eval_dataset, + inference_engine_client, + generator: GeneratorInterface, + colocate_pg, + ): + """Initializes the trainer. + + Returns: + RayPPOTrainer: The trainer. + """ + return RayPPOTrainer( + cfg=cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=colocate_pg, + ) + + def get_tracker(self): + """Initializes the tracker for experiment tracking. + + Returns: + Tracking: The tracker. + """ + return Tracking( + project_name=self.cfg.trainer.project_name, + experiment_name=self.cfg.trainer.run_name, + backends=self.cfg.trainer.logger, + config=self.cfg, + ) + + def _setup_trainer(self): + """Setup and return the trainer. + + Instantiates the trainer and all the associated models for training. + + Returns: + RayPPOTrainer: The trainer. + """ + logger.info(self.get_cfg_as_str(self.cfg)) + os.makedirs(self.cfg.trainer.export_path, exist_ok=True) + os.makedirs(self.cfg.trainer.ckpt_path, exist_ok=True) + + if self.cfg.trainer.strategy in ("fsdp", "fsdp2"): + from skyrl_train.workers.fsdp.fsdp_worker import PolicyWorker, CriticWorker, RefWorker + elif self.cfg.trainer.strategy == "megatron": + from skyrl_train.workers.megatron.megatron_worker import PolicyWorker, CriticWorker, RefWorker + else: + raise ValueError(f"Unknown strategy type: {self.cfg.trainer.strategy}") + + # NOTE (sumanthrh): Instantiate tracker before trainer init. + # We have custom validation before this step to give better error messages. + tracker = self.get_tracker() + + tokenizer = self.tokenizer + if self.cfg.generator.run_engines_locally: + inference_engines = create_ray_wrapped_inference_engines_from_config(self.cfg, self.colocate_pg, tokenizer) + else: + inference_engines = create_remote_inference_engines_from_config(self.cfg, tokenizer) + + inference_engine_client = InferenceEngineClient(inference_engines, tokenizer, self.cfg) + + generator: GeneratorInterface = self.get_generator(self.cfg, tokenizer, inference_engine_client) + + trainer = self.get_trainer( + cfg=self.cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=self.train_dataset, + eval_dataset=self.eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=self.colocate_pg, + ) + + # Build the models + trainer.build_models(PolicyWorker, CriticWorker, RefWorker) + return trainer + + def run(self): + trainer = self._setup_trainer() + # Start the training loop + asyncio.run(trainer.train()) + + +@ray.remote(num_cpus=1) +def skyrl_entrypoint(cfg: DictConfig): + # make sure that the training loop is not run on the head node. + exp = BasePPOExp(cfg) + exp.run() + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + # validate the arguments + validate_cfg(cfg) + + initialize_ray(cfg) + ray.get(skyrl_entrypoint.remote(cfg)) + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/skyrl_train/entrypoints/main_generate.py b/skyrl-train/skyrl_train/entrypoints/main_generate.py new file mode 100644 index 0000000000..a211ce0373 --- /dev/null +++ b/skyrl-train/skyrl_train/entrypoints/main_generate.py @@ -0,0 +1,73 @@ +""" +Main entrypoint for evaluation-only. +""" + +import asyncio + +import hydra +import ray +from loguru import logger +from omegaconf import DictConfig +from typing import Any + +from skyrl_train.entrypoints.main_base import ( + BasePPOExp, + config_dir, + create_ray_wrapped_inference_engines_from_config, + create_remote_inference_engines_from_config, +) +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.utils.utils import validate_generator_cfg, initialize_ray +from skyrl_train.evaluate import evaluate +from skyrl_train.utils.trainer_utils import build_dataloader + + +class EvalOnlyEntrypoint(BasePPOExp): + def get_train_dataset(self): + """Override to avoid requiring a train dataset for eval-only runs.""" + return None + + async def run(self) -> dict[str, Any]: + assert self.eval_dataset is not None, "The evaluation only entrypoint requires an eval dataset is provided" + + tokenizer = self.tokenizer + + if self.cfg.generator.run_engines_locally: + inference_engines = create_ray_wrapped_inference_engines_from_config(self.cfg, self.colocate_pg, tokenizer) + else: + inference_engines = create_remote_inference_engines_from_config(self.cfg, tokenizer) + + inference_engine_client = InferenceEngineClient(inference_engines, tokenizer, self.cfg) + await inference_engine_client.wake_up() + generator = self.get_generator(self.cfg, tokenizer, inference_engine_client) + + results: dict[str, Any] = await evaluate( + eval_dataloader=build_dataloader(self.cfg, self.eval_dataset, is_train=False), + generator=generator, + cfg=self.cfg, + global_step=None, + tokenizer=self.tokenizer, + ) + + tracker = self.get_tracker() + tracker.log(results, step=0, commit=True) + + return results + + +@ray.remote(num_cpus=1) +def eval_entrypoint(cfg: DictConfig) -> dict: + exp = EvalOnlyEntrypoint(cfg) + return asyncio.run(exp.run()) + + +@hydra.main(config_path=config_dir, config_name="ppo_base_config", version_base=None) +def main(cfg: DictConfig) -> None: + validate_generator_cfg(cfg) + initialize_ray(cfg) + metrics = ray.get(eval_entrypoint.remote(cfg)) + logger.info(f"Metrics from eval only run: {metrics}") + + +if __name__ == "__main__": + main() diff --git a/skyrl-train/skyrl_train/env_vars.py b/skyrl-train/skyrl_train/env_vars.py new file mode 100644 index 0000000000..52e68896ce --- /dev/null +++ b/skyrl-train/skyrl_train/env_vars.py @@ -0,0 +1,51 @@ +""" +Environment variable configuration for SkyRL. + +All environment variables used by SkyRL should be defined here for discoverability. +""" + +import os + +# ───────────────────────────────────────────────────────────────────────────── +# Ray / Placement Group +# ───────────────────────────────────────────────────────────────────────────── + +SKYRL_RAY_PG_TIMEOUT_IN_S = int(os.environ.get("SKYRL_RAY_PG_TIMEOUT_IN_S", 180)) +""" +Timeout for allocating the placement group for different actors in SkyRL. +""" + +# ───────────────────────────────────────────────────────────────────────────── +# Worker / NCCL +# ───────────────────────────────────────────────────────────────────────────── + +SKYRL_WORKER_NCCL_TIMEOUT_IN_S = int(os.environ.get("SKYRL_WORKER_NCCL_TIMEOUT_IN_S", 600)) +""" +Timeout for initializing the NCCL process group for the worker, defaults to 10 minutes. +""" + +# ───────────────────────────────────────────────────────────────────────────── +# Runtime Environment Exports +# ───────────────────────────────────────────────────────────────────────────── + +SKYRL_LD_LIBRARY_PATH_EXPORT = str(os.environ.get("SKYRL_LD_LIBRARY_PATH_EXPORT", "False")).lower() in ( + "true", + "1", + "yes", +) +""" +Whether to export ``LD_LIBRARY_PATH`` environment variable from the driver to the workers with Ray's runtime env. + +For example, if you are using RDMA, you may need to customize the ``LD_LIBRARY_PATH`` to include the RDMA libraries (Ex: EFA on AWS). +""" + +SKYRL_PYTHONPATH_EXPORT = str(os.environ.get("SKYRL_PYTHONPATH_EXPORT", "False")).lower() in ( + "true", + "1", + "yes", +) +""" +Whether to export ``PYTHONPATH`` environment variable from the driver to the workers with Ray's runtime env. + +See https://github.com/ray-project/ray/issues/56697 for details on why this is needed. +""" diff --git a/skyrl-train/skyrl_train/evaluate.py b/skyrl-train/skyrl_train/evaluate.py new file mode 100644 index 0000000000..2bdc11d114 --- /dev/null +++ b/skyrl-train/skyrl_train/evaluate.py @@ -0,0 +1,283 @@ +import torch +from tqdm import tqdm +from typing import Dict, List, Any +from pathlib import Path +from loguru import logger +from collections import defaultdict + +from skyrl_train.utils import Timer + +from skyrl_train.generators.utils import ( + concatenate_generator_outputs, + prepare_generator_input, +) +from skyrl_train.generators.base import ( + GeneratorOutput, + GeneratorInterface, +) +from skyrl_train.utils.trainer_utils import ( + calculate_per_dataset_metrics, + dump_per_dataset_eval_results, + validate_generator_output, +) +from skyrl_train.metrics import compute_reward_metrics +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.utils.logging_utils import log_example + +from omegaconf import DictConfig +from torchdata.stateful_dataloader import StatefulDataLoader +from transformers import AutoTokenizer + + +@torch.no_grad() +async def evaluate( + eval_dataloader: StatefulDataLoader, + generator: GeneratorInterface, + cfg: DictConfig, + global_step: int | None, + tokenizer: AutoTokenizer, +) -> Dict[str, float]: + """Runs generation and evaluation of trajectories. + + Args: + eval_dataloader (StatefulDataLoader): dataloader of the eval dataset + generator (GeneratorInterface): generator to use + cfg (DictConfig): config + global_step (int | None): current global step, or + `None` to indicate a non-training context (e.g., eval-only) + tokenizer (AutoTokenizer): tokenizer to use + + Returns: + Dict[str, float]: evaluation metrics + """ + + # 1. Get all generator outputs + generator_outputs: List[GeneratorOutput] = [] + concat_all_envs: List[str] = [] + concat_env_extras: List[Dict[str, Any]] = [] + concat_uids: List[str] = [] + sampling_params = cfg.generator.eval_sampling_params + pbar = tqdm(total=len(eval_dataloader), initial=0, desc="Evaluation Progress") + for _, prompts in enumerate(eval_dataloader): + pbar.update(1) + generator_input, uids = prepare_generator_input( + prompts, + cfg.generator.eval_n_samples_per_prompt, + get_sampling_params_for_backend(cfg.generator.backend, sampling_params), + cfg.environment.env_class, + "eval", + global_step, + ) + generator_output: GeneratorOutput = await generator.generate(generator_input) + validate_generator_output(len(generator_input["prompts"]), generator_output) + generator_outputs.append(generator_output) + concat_all_envs.extend(generator_input["env_classes"]) + concat_env_extras.extend(generator_input["env_extras"]) + concat_uids.extend(uids) + concat_generator_outputs: GeneratorOutput = concatenate_generator_outputs(generator_outputs) + + # Extract data_sources from env_extras + concat_data_sources = [env_extra.get("data_source") for env_extra in concat_env_extras] + vis = tokenizer.decode(generator_output["response_ids"][0]) + log_example( + logger, + prompt=generator_input["prompts"][0], + response=vis, + reward=generator_output["rewards"][0], + ) + + # 2. Group data by data source and calculate per-dataset metrics + eval_metrics = calculate_per_dataset_metrics( + concat_generator_outputs, concat_uids, concat_data_sources, cfg.generator.eval_n_samples_per_prompt + ) + + # 3. Calculate overall metrics across all datasets using shared module + overall_metrics = compute_reward_metrics( + rewards=concat_generator_outputs["rewards"], + uids=concat_uids, + n_samples_per_prompt=cfg.generator.eval_n_samples_per_prompt, + ) + eval_metrics.update( + { + f"eval/all/pass_at_{cfg.generator.eval_n_samples_per_prompt}": overall_metrics[ + f"pass_at_{cfg.generator.eval_n_samples_per_prompt}" + ], + "eval/all/variance_per_prompt": overall_metrics["variance_per_prompt"], + "eval/all/mean_positive_reward": overall_metrics["mean_positive_reward"], + } + ) + + # 4. Prepare dumping data and upload to S3 + if cfg.trainer.dump_eval_results: + with Timer("dump_eval_results"): + data_save_dir = ( + Path(cfg.trainer.export_path) + / "dumped_evals" + / ("eval_only" if global_step is None else f"global_step_{global_step}_evals") + ) + data_save_dir.mkdir(parents=True, exist_ok=True) + dump_per_dataset_eval_results( + data_save_dir, + tokenizer, + concat_generator_outputs, + concat_data_sources, + concat_all_envs, + concat_env_extras, + eval_metrics, + ) + + # Upload to S3 if credentials are available + try: + from integrations.fleet.s3_checkpoints import upload_eval_results_to_s3 + + run_name = getattr(cfg.trainer, "run_name", None) + if run_name: + upload_eval_results_to_s3( + local_dir=str(data_save_dir), + run_name=run_name, + global_step=global_step, + delete_local=False, # Keep local copy + ) + except ImportError: + pass # S3 upload not available + except Exception as e: + logger.warning(f"Failed to upload eval results to S3: {e}") + + return eval_metrics + + +@torch.no_grad() +async def evaluate_step_wise( + eval_dataloader: StatefulDataLoader, + generator: GeneratorInterface, + cfg: DictConfig, + global_step: int | None, + tokenizer: AutoTokenizer, +) -> Dict[str, float]: + """Runs generation and evaluation of trajectories for step-wise training. + + Currently assumes that the rewards are assigned to the last step of each trajectory. + + Args: + eval_dataloader (StatefulDataLoader): dataloader of the eval dataset + generator (GeneratorInterface): generator to use + cfg (DictConfig): config + global_step (int | None): current global step, or + `None` to indicate a non-training context (e.g., eval-only) + tokenizer (AutoTokenizer): tokenizer to use + + Returns: + Dict[str, float]: evaluation metrics + """ + + # 1. Get all generator outputs + generator_outputs: List[GeneratorOutput] = [] + concat_all_envs: List[str] = [] + concat_env_extras: List[Dict[str, Any]] = [] + concat_uids: List[str] = [] + sampling_params = cfg.generator.eval_sampling_params + pbar = tqdm(total=len(eval_dataloader), initial=0, desc="Evaluation Progress") + for _, prompts in enumerate(eval_dataloader): + pbar.update(1) + generator_input, uids = prepare_generator_input( + prompts, + cfg.generator.eval_n_samples_per_prompt, + get_sampling_params_for_backend(cfg.generator.backend, sampling_params), + cfg.environment.env_class, + "eval", + global_step, + ) + generator_output: GeneratorOutput = await generator.generate(generator_input) + traj_id_to_input = { + traj_id.instance_id: {"env_class": env_class, "env_extras": env_extra} + for traj_id, env_class, env_extra in zip( + generator_input["trajectory_ids"], generator_input["env_classes"], generator_input["env_extras"] + ) + } + for traj_id in generator_output["trajectory_ids"]: + assert traj_id.instance_id in traj_id_to_input, f"Trajectory ID {traj_id.instance_id} not found in input" + concat_all_envs.append(traj_id_to_input[traj_id.instance_id]["env_class"]) + concat_env_extras.append(traj_id_to_input[traj_id.instance_id]["env_extras"]) + concat_uids.append(traj_id.instance_id) + # validate_generator_output(generator_input, generator_output) + generator_outputs.append(generator_output) + concat_generator_outputs: GeneratorOutput = concatenate_generator_outputs(generator_outputs) + + # Extract data_sources from env_extras + concat_data_sources = [env_extra.get("data_source") for env_extra in concat_env_extras] + vis = tokenizer.decode(generator_output["response_ids"][0]) + logger.info(f"Eval output example: {vis}") + + # Only use the final step metrics + generator_output_last_step = defaultdict(list) + is_last_step_mask = concat_generator_outputs["is_last_step"] + for key in concat_generator_outputs: + if isinstance(concat_generator_outputs[key], list): + assert len(concat_generator_outputs[key]) == len( + is_last_step_mask + ), f"Length mismatch: {len(concat_generator_outputs[key])} != {len(is_last_step_mask)} for key {key}" + generator_output_last_step[key] = [ + val for val, is_last_step in zip(concat_generator_outputs[key], is_last_step_mask) if is_last_step + ] + uids_last_step = [uid for uid, is_last_step in zip(concat_uids, is_last_step_mask) if is_last_step] + data_sources_last_step = [ + data_source for data_source, is_last_step in zip(concat_data_sources, is_last_step_mask) if is_last_step + ] + + # 2. Group data by data source and calculate per-dataset metrics + eval_metrics = calculate_per_dataset_metrics( + generator_output_last_step, uids_last_step, data_sources_last_step, cfg.generator.eval_n_samples_per_prompt + ) + # 3. Calculate overall metrics across all datasets using shared module + overall_metrics = compute_reward_metrics( + rewards=generator_output_last_step["rewards"], + uids=uids_last_step, + n_samples_per_prompt=cfg.generator.eval_n_samples_per_prompt, + ) + eval_metrics.update( + { + f"eval/all/pass_at_{cfg.generator.eval_n_samples_per_prompt}": overall_metrics[ + f"pass_at_{cfg.generator.eval_n_samples_per_prompt}" + ], + "eval/all/variance_per_prompt": overall_metrics["variance_per_prompt"], + "eval/all/mean_positive_reward": overall_metrics["mean_positive_reward"], + } + ) + + # 4. Prepare dumping data and upload to S3 + if cfg.trainer.dump_eval_results: + with Timer("dump_eval_results"): + data_save_dir = ( + Path(cfg.trainer.export_path) + / "dumped_evals" + / ("eval_only" if global_step is None else f"global_step_{global_step}_evals") + ) + data_save_dir.mkdir(parents=True, exist_ok=True) + dump_per_dataset_eval_results( + data_save_dir, + tokenizer, + concat_generator_outputs, + concat_data_sources, + concat_all_envs, + concat_env_extras, + eval_metrics, + ) + + # Upload to S3 if credentials are available + try: + from integrations.fleet.s3_checkpoints import upload_eval_results_to_s3 + + run_name = getattr(cfg.trainer, "run_name", None) + if run_name: + upload_eval_results_to_s3( + local_dir=str(data_save_dir), + run_name=run_name, + global_step=global_step, + delete_local=False, # Keep local copy + ) + except ImportError: + pass # S3 upload not available + except Exception as e: + logger.warning(f"Failed to upload eval results to S3: {e}") + + return eval_metrics diff --git a/skyrl-train/skyrl_train/fully_async_trainer.py b/skyrl-train/skyrl_train/fully_async_trainer.py new file mode 100644 index 0000000000..b7a64beae2 --- /dev/null +++ b/skyrl-train/skyrl_train/fully_async_trainer.py @@ -0,0 +1,703 @@ +""" +Implementation of fully async training in SkyRL. + +For details, see https://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html. + +High-level notes: +- The global_step in each training loop iteration denotes the "current step being worked on", so +`global_step - 1` is the number of steps the model has finished training. +- We do not do any cross-epoch asynchrony here, so all the control logics like + generation workers and data buffer are initialized per-epoch. The async dataloader + and staleness manager are also reset / validated at the end of each epoch. +""" + +import asyncio +import os +import torch +import traceback +import sys +from loguru import logger +from skyrl_train.trainer import RayPPOTrainer +from tqdm import tqdm +from skyrl_train.utils import Timer +from skyrl_train.utils.ppo_utils import normalize_advantages_dict +from skyrl_train.training_batch import TrainingInputBatch +from skyrl_train.generators.base import GeneratorOutput +from skyrl_train.utils.trainer_utils import ResumeMode, build_dataloader +from skyrl_train.utils.alerts import check_tool_error_rate_alert +from skyrl_train.utils.io import io +from skyrl_train.generators.utils import prepare_generator_input, concatenate_generator_outputs +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from dataclasses import dataclass +from torchdata.stateful_dataloader import StatefulDataLoader +from typing import List, Tuple, Iterable, Set +import inspect + + +@dataclass +class GeneratedOutputGroup: + """ + The GeneratorOutput for a single group of rollouts, along with the metadata. + + Attributes: + generator_output (GeneratorOutput): The GeneratorOutput for a single group of rollouts. + That is, the output to the same prompt, but `n_samples_per_prompt` of it. + + uid (str): The uid of the group. Underlyingly, it is the index of the data in train_dataloader.dataset. + + global_step_when_scheduled (int): The global step when the group was scheduled for generation, + used for validating the staleness control. + + data_source (str): Optional data source identifier for per-environment metrics tracking. + """ + + generator_output: GeneratorOutput + uid: str + global_step_when_scheduled: int + data_source: str = None + + +@dataclass +class _RolloutStat: + """ + Global statistics of the trajectories used for staleness control in `_AsyncStalenessManager`. + + Note that these statistics are not per-epoch, but accumulates across all epochs. + + Attributes: + submitted (int): The number of trajectories submitted to all generation workers, only used + for logging purposes (e.g. compute submitted / accepted ratio to see how many + trajectories failed). This is strictly increasing. + accepted (int): The number of trajectories finished generation (can be either consumed by, + or about to be consumed by the training worker). This is strictly increasing. + running (int): The number of trajectories currently being generated by the generation workers. + + For details, see https://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html#async-staleness-manager + """ + + submitted: int = 0 + accepted: int = 0 + running: int = 0 + + +class _AsyncStalenessManager: + """ + A controller that manages the capacity of the generation workers based on staleness control. + + The goal is to never submit more trajectories to the generation workers than the training worker + can consume, so that the trajectories are not too stale (relative to max_staleness_steps). + This is enforced via a capacity rule, not a hard **per-group** staleness guarantee: we bound + the **aggregate** number of groups that can be ahead of training so that, in **steady state**, + staleness remains within the configured budget of `max_staleness_steps`. + + In pathological cases (e.g., very long-running trajectories), an individual group may take + more than `max_staleness_steps` of training steps of time to finish generation. For such rare + cases, we still accept the trajectory and log the staleness metrics with a warning. + + The key capacity formula is implemented in `_compute_capacity_unlocked`. For details and caveats, + see https://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html#async-staleness-manager. + + Reference: + - Modeled after AReal's StalenessManager: https://github.com/inclusionAI/AReaL/blob/b755c4447c2fff97889d8828293ee85f17a806f9/areal/core/staleness_manager.py + - The idea of this controller is from section 5.1 of AReal's paper: https://arxiv.org/pdf/2505.24298v3 + """ + + def __init__(self, max_concurrent_generation_groups: int, mini_batch_size: int, max_staleness_steps: int): + self.max_concurrent_generation_groups = max_concurrent_generation_groups + self.mini_batch_size = mini_batch_size + self.max_staleness_steps = max_staleness_steps + + # Control logics. + self._stat = _RolloutStat() + self._cond = asyncio.Condition() + + # The current version that is being worked on, i.e. FullyAsyncRayPPOTrainer.global_step. + # `self._current_global_step - 1` is the number of steps the model has finished training. + self._current_global_step = 1 + + def load_state_from_checkpoint(self, global_step: int) -> None: + """ + Load the state from a checkpoint. + """ + self._current_global_step = global_step + # trainer has already consumed (and hence submitted) this many trajectories. + self._stat.accepted = (global_step - 1) * self.mini_batch_size + self._stat.submitted = self._stat.accepted + + async def validate_state_at_epoch_end(self, global_step: int) -> None: + """ + Check that the current version and accepted rollouts are consistent with the global step. + + Args: + global_step: The global step we are about to train on (after incrementing). + """ + async with self._cond: + assert self._stat.running == 0, "We expect no rollouts are running at end of an epoch." + assert ( + self._stat.submitted == self._stat.accepted + ), "We expect all submitted rollouts to be accepted at end of an epoch." + consumed = (global_step - 1) * self.mini_batch_size + assert ( + self._stat.accepted == consumed + ), f"Unexpected number of accepted rollouts. Got {self._stat.accepted} != {consumed}." + assert ( + self._current_global_step == global_step + ), f"Unexpected current version. Got {self._current_global_step} != {global_step}." + assert self._stat.submitted == self._stat.accepted, ( + "We expect all submitted rollouts to be accepted at end of an epoch. " + f"Got {self._stat.submitted} != {self._stat.accepted}." + ) + + def _compute_capacity_unlocked(self) -> int: + # NOTE(Charlie): do not need a self._current_global_step + 1 here unlike AReal because our + # `_current_global_step` is "the version being worked on", not already finished steps. + consumer_capacity = (self.max_staleness_steps + self._current_global_step) * self.mini_batch_size + producer_staleness_capacity = consumer_capacity - (self._stat.accepted + self._stat.running) + producer_concurrency_capacity = self.max_concurrent_generation_groups - self._stat.running + return min(producer_concurrency_capacity, producer_staleness_capacity) + + async def acquire_submission_slot(self) -> None: + """Block until there is capacity, then reserve a slot (increments submitted and running). + + This method always uses the latest current version tracked internally, which is + updated by `notify_capacity_change(new_global_step)`. + """ + async with self._cond: + while self._compute_capacity_unlocked() <= 0: + await self._cond.wait() + # Reserve slot + self._stat.submitted += 1 + self._stat.running += 1 + + async def on_rollout_accepted(self) -> None: + async with self._cond: + self._stat.accepted += 1 + self._stat.running -= 1 + self._cond.notify_all() + + async def on_rollout_rejected(self) -> None: + """ + Called when a generation is not accepted, or generation worker runs into error while generating a trajectory. + + Currently, we do not call this method but instead raise errors. We might need to use this when we want to + filter out trajectories. + """ + async with self._cond: + self._stat.running -= 1 + self._cond.notify_all() + + async def notify_capacity_change(self, new_global_step: int) -> None: + # Called when current_global_step changes (e.g., after a training step) + async with self._cond: + self._current_global_step = int(new_global_step) + self._cond.notify_all() + + +class _AsyncDataloader: + """ + A train dataloader wrapper that accommodates the need of fully async training, including: + - Thread-safe dataloader iteration with a lock, as there are multiple parallel generation workers polling data. + - Records consumed data UIDs for checkpointing to avoid training on the same data upon resuming. + - Set the effective dataloader length to be divisible by mini-batch size, since we cannot rely on `drop_last` + because the batch size is 1 in fully async training. + """ + + def __init__(self, train_dataloader: StatefulDataLoader, mini_batch_size: int): + self._train_dataloader = train_dataloader + self._train_dataloader_initial_state = train_dataloader.state_dict() + self._effective_dataloader_length = len(self._train_dataloader) // mini_batch_size * mini_batch_size + self._iter = enumerate(self._train_dataloader) + self._lock: asyncio.Lock = asyncio.Lock() + self._consumed_data_uids: Set[str] = set() + self._exhausted: bool = False # currently not used. + + def load_state_from_checkpoint(self, consumed_data_uids_set: Set[str]) -> None: + """ + Load the state from a checkpoint. + """ + self._consumed_data_uids = consumed_data_uids_set + + # Reset in case the dataloader loaded the state from the checkpoint, which we do not want. + self._train_dataloader.load_state_dict(self._train_dataloader_initial_state) + + async def reset_at_epoch_end(self) -> None: + async with self._lock: + self._train_dataloader.load_state_dict(self._train_dataloader_initial_state) # reset to initial state + self._iter = enumerate(self._train_dataloader) + self._consumed_data_uids.clear() + self._exhausted = False + + async def get_next_non_consumed_data(self): + """ + Return the next batch of training data. + + If we loaded from a checkpoint, it will skip the already-consumed data. Returns None if the dataloader is exhausted. + """ + assert self._iter is not None and self._lock is not None, "Dataloader not initialized; call reset() first" + async with self._lock: + try: + while True: + # Keep polling until we get a non-consumed data or the dataloader is exhausted. + iter_idx, rand_prompts = next(self._iter) + if iter_idx >= self._effective_dataloader_length: + raise StopIteration + uid = rand_prompts[0]["uid"] + if uid not in self._consumed_data_uids: + return rand_prompts + except StopIteration: + self._exhausted = True + return None + + async def mark_consumed_uids(self, uids: Iterable[str]) -> None: + assert self._lock is not None, "Dataloader not initialized; call reset() first" + async with self._lock: + for uid in uids: + assert uid not in self._consumed_data_uids, "Duplicate UID found in mini-batch" + self._consumed_data_uids.add(uid) + + def get_consumed_uids_list(self) -> List[str]: + return list(self._consumed_data_uids) + + +class FullyAsyncRayPPOTrainer(RayPPOTrainer): + + def __init__(self, *args, **kwargs): + # Extract cfg before base init so we can initialize async-specific knobs used by our overrides. + cfg = kwargs.get("cfg", args[0] if len(args) > 0 else None) + assert cfg is not None, "cfg must be provided to FullyAsyncRayPPOTrainer" + + # Initialize async-specific knobs + self.num_parallel_generation_workers = cfg.trainer.fully_async.num_parallel_generation_workers + self.mini_batch_size = cfg.trainer.policy_mini_batch_size + self.max_staleness_steps = cfg.trainer.fully_async.max_staleness_steps + + assert ( + # otherwise wasted throughput + self.mini_batch_size <= self.num_parallel_generation_workers + and + # otherwise would never use all workers due to capacity constraint + self.num_parallel_generation_workers <= self.mini_batch_size * (self.max_staleness_steps + 1) + ), ( + "Invalid num_parallel_generation_workers, the following must hold: " + "mini_batch_size <= num_parallel_generation_workers <= mini_batch_size * (max_staleness_steps + 1). Got: " + f"{self.mini_batch_size=}, {self.num_parallel_generation_workers=}, {self.max_staleness_steps=}" + ) + + # Initialize base trainer + super().__init__(*args, **kwargs) + + # Some async-specific validations + assert ( + self.cfg.trainer.train_batch_size == self.cfg.trainer.policy_mini_batch_size + ), "train_batch_size must equal policy_mini_batch_size for fully async training" + assert ( + self.cfg.trainer.algorithm.dynamic_sampling.type is None + ), "dynamic sampling is not supported for fully async training yet." + assert ( + not self.cfg.generator.batched + ), "batched is not supported for fully async training since a batched generate() call does not support pause/continue." + assert self.cfg.generator.async_engine, "async_engine must be True for fully async training." + # TODO(Charlie): we can support it, just multi-turn partial rollout but synchronous. + assert not self.colocate_all, "colocate_all is not supported for async training yet." + + # TODO(Charlie): need to assert we are doing TIS and returning logprobs + + # Async-specific states + self.async_train_dataloader = _AsyncDataloader(self.train_dataloader, self.mini_batch_size) + self._staleness_manager = _AsyncStalenessManager( + max_concurrent_generation_groups=self.num_parallel_generation_workers, + mini_batch_size=self.mini_batch_size, + max_staleness_steps=self.max_staleness_steps, + ) + + def _build_train_dataloader_and_compute_training_steps(self): + """ + Overrides to build dataloader for fully async training. See `_AsyncDataloader` for more details. + """ + self.train_dataloader = build_dataloader(self.cfg, self.train_dataset, is_train=True, is_fully_async=True) + self.num_steps_per_epoch = len(self.train_dataloader) // self.mini_batch_size + self.total_training_steps = self.num_steps_per_epoch * self.cfg.trainer.epochs + logger.info(f"Length of train_dataloader: {len(self.train_dataloader)}") + logger.info(f"Number of steps per epoch: {self.num_steps_per_epoch}") + logger.info(f"Total training steps: {self.total_training_steps}") + + async def train(self): + """ + Main fully async training loop for PPO + """ + self.global_step = 0 + + # Load checkpoint state if resumption is enabled. Also load the data UIDs that are already trained on. + if self.resume_mode != ResumeMode.NONE: + with Timer("load_checkpoints"): + self.global_step, _, loaded_consumed_data_uids_set = self.load_checkpoints() + logger.info(f"Resumed training from global_step {self.global_step}") + if self.global_step > 0: + # Set async dataloader manager and staleness manager to the loaded state. + self.async_train_dataloader.load_state_from_checkpoint(loaded_consumed_data_uids_set) + self._staleness_manager.load_state_from_checkpoint( + self.global_step + 1 + ) # +1 due to we haven't incremented yet + steps_completed_in_epoch = self.global_step % self.num_steps_per_epoch + if steps_completed_in_epoch == 0 and len(loaded_consumed_data_uids_set) > 0: + # When resuming mid-epoch at the boundary, treat modulo 0 as a full epoch. + steps_completed_in_epoch = self.num_steps_per_epoch + expected_consumed_in_epoch = self.mini_batch_size * steps_completed_in_epoch + assert len(loaded_consumed_data_uids_set) == expected_consumed_in_epoch, ( + "Unexpected number of consumed data UIDs. Got: " + f"{len(loaded_consumed_data_uids_set)} != {expected_consumed_in_epoch}" + ) + + # Initialize weight sync state + with Timer("init_weight_sync_state"): + self.init_weight_sync_state() + + # sync weights to inference engines + with Timer("sync_weights_to_inference_engines"): + await self.async_sync_policy_weights_to_inference_engines() + + # Eval before training - always log at step 0 regardless of resumed checkpoint + if self.cfg.trainer.eval_interval > 0 and self.cfg.trainer.eval_before_train: + with Timer("eval", self.all_timings): + eval_metrics = await self.eval() + self.tracker.log(eval_metrics, step=0) + + # main training loop + pbar = tqdm(total=self.total_training_steps, initial=self.global_step, desc="Training Step Progress") + start_epoch = self.global_step // self.num_steps_per_epoch + self.global_step += 1 # start training at global_step 1 + for epoch in range(start_epoch, self.cfg.trainer.epochs): + # 0. Per-epoch prologue. Note that we do not do any cross-epoch asynchrony here. + + # Buffer of completed generation, size bounded by capacity - consumed = B * (max_staleness_steps + 1) + generation_output_group_buffer = asyncio.Queue[GeneratedOutputGroup]( + maxsize=self.mini_batch_size * (self.max_staleness_steps + 1) + ) + + # Maintain self.num_parallel_generation_workers concurrent group-generation workers + generator_tasks = [ + asyncio.create_task(self._run_generate_for_a_group_loop(generation_output_group_buffer)) + for _ in range(self.num_parallel_generation_workers) + ] + + for _ in range(self.global_step, (1 + epoch) * self.num_steps_per_epoch + 1): + with Timer("step", self.all_timings): + # 1. Wait until we have enough groups buffered. + cur_generation_group_mini_batch: List[GeneratedOutputGroup] = [] + with Timer("wait_for_generation_buffer", self.all_timings): + buffer_pbar = tqdm( + total=self.mini_batch_size, + initial=0, + desc="Generation Buffer Progress", + position=1, + ) + # NOTE(Charlie): we currently trim the train_dataloader to make it perfectly divisible by + # self.mini_batch_size, and assume that all trajectories succeed (just like sync training), + # so we always get a full mini-batch. Otherwise (e.g. want to drop stale trajectories), we + # should handle the case where the dataloader is exhausted and the buffer is empty, or + # else this loop will never exit. + while len(cur_generation_group_mini_batch) < self.mini_batch_size: + # We do finish-time FIFO here (not schedule-time FIFO) + cur_generation_group_mini_batch.append(await generation_output_group_buffer.get()) + buffer_pbar.update(1) + buffer_pbar.set_postfix({"buffer qsize": generation_output_group_buffer.qsize()}) + buffer_pbar.close() + + # 2. Post-process the generated groups, aggregating to a single GeneratorOutput, and convert to training format. + with Timer("convert_to_training_input", self.all_timings): + training_input = await asyncio.to_thread( + self.convert_generation_group_mini_batch_to_training_input, cur_generation_group_mini_batch + ) + + # 3. Run training and update consumed UIDs. + with Timer("run_training", self.all_timings): + status = await self._run_training(training_input) + await self.async_train_dataloader.mark_consumed_uids( + [g.uid for g in cur_generation_group_mini_batch] + ) + + # 4. After training: pause generation, sync weights, resume. + with Timer("sync_weights", self.all_timings): + await self.inference_engine_client.pause_generation() + await self.async_sync_policy_weights_to_inference_engines() + await self.inference_engine_client.resume_generation() + + # 5. Set logs for this training step. + logger.info(status) + self.all_metrics.update({"trainer/epoch": epoch, "trainer/global_step": self.global_step}) + self.tracker.log(self.all_metrics, step=self.global_step) + + # Check for high tool error rate and alert via Slack + run_name = self.cfg.trainer.get("experiment_name", None) + check_tool_error_rate_alert(self.all_metrics, self.global_step, threshold=0.5, run_name=run_name) + + self.all_metrics = {} + pbar.update(1) + + # 6. Eval and checkpointing if needed. + # NOTE(Charlie): eval does not overlap with training, but can overlap with generation. Is it fine? + if self.cfg.trainer.eval_interval > 0 and ( + self.global_step % self.cfg.trainer.eval_interval == 0 + or self.global_step == self.total_training_steps + ): + with Timer("eval", self.all_timings): + eval_metrics = await self.eval() + self.all_metrics.update(eval_metrics) + if self.cfg.trainer.ckpt_interval > 0 and self.global_step % self.cfg.trainer.ckpt_interval == 0: + with Timer("save_checkpoints", self.all_timings): + await asyncio.to_thread(self.save_checkpoints) + if self.cfg.trainer.hf_save_interval > 0 and self.global_step % self.cfg.trainer.hf_save_interval == 0: + with Timer("save_hf_model", self.all_timings): + await asyncio.to_thread(self.save_models) + self.tracker.log({"timing/" + k: v for k, v in self.all_timings.items()}, step=self.global_step) + self.all_timings = {} + self.global_step += 1 + + # 7. Notify generation workers that the capacity has increased, unblocking them. + await self._staleness_manager.notify_capacity_change(self.global_step) + steps_completed_in_epoch = (self.global_step - 1) % self.num_steps_per_epoch + if steps_completed_in_epoch == 0: + # At the end of an epoch, modulo becomes 0 but we've consumed a full epoch. + steps_completed_in_epoch = self.num_steps_per_epoch + expected_consumed_in_epoch = self.mini_batch_size * steps_completed_in_epoch + actual_consumed_in_epoch = len(self.async_train_dataloader.get_consumed_uids_list()) + assert actual_consumed_in_epoch == expected_consumed_in_epoch, ( + "Unexpected number of consumed data UIDs. Got: " + f"{actual_consumed_in_epoch} != {expected_consumed_in_epoch}" + ) + + # 8. Per-epoch epilogue. + if self.cfg.trainer.update_ref_every_epoch and self.ref_model is not None: + with Timer("update_ref_with_policy", self.all_timings): + await asyncio.to_thread(self.update_ref_with_policy) + + # Cancel generator tasks for this epoch + for t in generator_tasks: + t.cancel() + try: + await asyncio.gather(*generator_tasks, return_exceptions=True) + except Exception: + pass + + # Per-epoch reset/validation for data loading and staleness management + assert all( + t.done() for t in generator_tasks + ), "Generator tasks must be done before resetting the dataloader manager and validating the staleness manager." + assert ( + generation_output_group_buffer.qsize() == 0 + ), f"We expect all generation output to be consumed by the training worker at end of an epoch, got {generation_output_group_buffer.qsize()}." + await self.async_train_dataloader.reset_at_epoch_end() + await self._staleness_manager.validate_state_at_epoch_end(self.global_step) + + # End of an epoch. + pbar.close() + if self.cfg.trainer.ckpt_interval > 0: + with Timer("save_checkpoints", self.all_timings): + await asyncio.to_thread(self.save_checkpoints) + logger.info("Saved final checkpoint.") + if self.cfg.trainer.hf_save_interval > 0: + with Timer("save_hf_model", self.all_timings): + await asyncio.to_thread(self.save_models) + logger.info("Saved final model.") + logger.info("Training done!") + + async def _run_training(self, training_input: TrainingInputBatch): + # TODO(Charlie): share this code with the one-step-off async trainer. + # inference and calculate values, log probs, rewards, kl divergence + with Timer("fwd_logprobs_values_reward", self.all_timings): + training_input = await asyncio.to_thread(self.fwd_logprobs_values_reward, training_input) + + # calculate kl divergence and create experiences + if self.cfg.trainer.algorithm.use_kl_in_reward: + with Timer("apply_reward_kl_penalty", self.all_timings): + training_input = self.apply_reward_kl_penalty(training_input) + + # calculate advantages and returns / along with tensorboard logging + with Timer("compute_advantages_and_returns", self.all_timings): + training_input = self.compute_advantages_and_returns(training_input) + # remove some unwanted keys + for key in ["rewards"]: + training_input.pop(key) + training_input.metadata.pop("uids") + + if self.cfg.trainer.algorithm.advantage_batch_normalize: + training_input = normalize_advantages_dict(training_input) + + if self.cfg.trainer.dump_data_batch: + # dump data to file + with Timer("dump_data_batch"): + self.dump_data(training_input, file_name=f"global_step_{self.global_step}_training_input") + + # train policy/critic model + with Timer("train_critic_and_policy", self.all_timings): + status = await asyncio.to_thread(self.train_critic_and_policy, training_input) + + return status + + async def _run_generate_for_a_group_loop(self, generation_output_group_buffer: asyncio.Queue): + """ + Generator worker: repeatedly pulls the next prompt (possibly blocked by staleness control), + generates one single generation group, respecting a pause/resume event, and enqueues the result. + """ + try: + while True: + # 0. Pull next batch from dataloader. If returns None, then dataloader is exhausted. + rand_prompts = await self.async_train_dataloader.get_next_non_consumed_data() + if rand_prompts is None: + return + + # 1. Prepare generator input + assert len(rand_prompts) == 1 + generator_input, uids = prepare_generator_input( + rand_prompts, + self.cfg.generator.n_samples_per_prompt, + get_sampling_params_for_backend(self.cfg.generator.backend, self.cfg.generator.sampling_params), + self.cfg.environment.env_class, + "train", + self.global_step, + ) + assert all(uid == uids[0] for uid in uids), "Expect all uids to be the same" + + # 2. Acquire capacity slot. + slot_acquired = False + await self._staleness_manager.acquire_submission_slot() + slot_acquired = True + + # 3. Generate one rollout group + global_step_at_start = self.global_step # for staleness control + + if "disable_tqdm" in inspect.signature(self.generator.generate).parameters: + # A workaround to disable tqdm for the SkyRLGymGenerator.generate method which will + # blast the console with each worker's progress bar. + cur_generator_output: GeneratorOutput = await self.generator.generate( + generator_input, disable_tqdm=True + ) + else: + cur_generator_output: GeneratorOutput = await self.generator.generate(generator_input) + + # 4. Enqueue the completed group and mark accepted to free capacity slot. + # Extract data_source for per-environment metrics + env_extras = generator_input.get("env_extras", []) + data_source = env_extras[0].get("data_source") if env_extras else None + try: + generation_output_group_buffer.put_nowait( + GeneratedOutputGroup( + generator_output=cur_generator_output, + uid=uids[0], + global_step_when_scheduled=global_step_at_start, + data_source=data_source, + ) + ) + except asyncio.QueueFull: + raise AssertionError("Generation buffer should never be full given staleness control.") + await self._staleness_manager.on_rollout_accepted() + except asyncio.CancelledError: + # If a slot was acquired but we exit early, release running count + try: + if "slot_acquired" in locals() and slot_acquired: + raise RuntimeError("Generation workers should only be cancelled when they finish running.") + finally: + return + except Exception as e: + logger.error(f"Generator worker errored out with exception: {e}") + logger.error(f"Traceback: \n{traceback.format_exc()}") + if "slot_acquired" in locals() and slot_acquired: + raise RuntimeError("Generation workers should only run into error when they finish running.") + sys.exit(1) + + async def async_sync_policy_weights_to_inference_engines(self): + return await self.policy_model.async_run_method( + "pass_through", "broadcast_to_inference_engines", self.inference_engine_client + ) + + def convert_generation_group_mini_batch_to_training_input( + self, cur_generation_group_mini_batch: List[GeneratedOutputGroup] + ) -> TrainingInputBatch: + """Given a mini-batch of generated groups, concatenate them into a single GeneratorOutput, then convert to a TrainingInputBatch.""" + generator_outputs = [] + uids = [] + data_sources = [] + stalenesses = [] + staleness_violation_count = 0 + group_size = len(cur_generation_group_mini_batch[0].generator_output["response_ids"]) + for cur_generated_output_group in cur_generation_group_mini_batch: + cur_staleness = self.global_step - cur_generated_output_group.global_step_when_scheduled + stalenesses.append(cur_staleness) + generator_outputs.append(cur_generated_output_group.generator_output) + uids.extend([cur_generated_output_group.uid] * group_size) + data_sources.extend([cur_generated_output_group.data_source] * group_size) + + # Check staleness violation. + if cur_staleness > self.max_staleness_steps: + # TODO(Charlie): should we drop, drop and resample, or just log? + logger.warning( + "Staleness control violated despite using AsyncStalenessManager: " + f"cur_staleness={cur_staleness}, max_staleness_steps={self.max_staleness_steps}.\n" + "If this happens too often, consider increasing max_staleness_steps, adjusting " + "trainer.fully_async.num_parallel_generation_workers, or adjusting generation-training GPU allocation.\n" + "See https://skyrl.readthedocs.io/en/latest/tutorials/fully_async.html#async-staleness-manager for more details." + ) + staleness_violation_count += 1 + + generator_output = concatenate_generator_outputs(generator_outputs) + assert generator_output["rollout_metrics"] is not None, "Rollout metrics should be non-null." + self.all_metrics.update(generator_output["rollout_metrics"]) + + # Log staleness statistics for this step + self.all_metrics.update( + { + "async/staleness_mean": sum(stalenesses) / len(stalenesses), + "async/staleness_max": max(stalenesses), + "async/staleness_min": min(stalenesses), + "async/staleness_ratio": sum(1 for s in stalenesses if s > 0) / len(stalenesses), + "async/staleness_violation_count": staleness_violation_count, + } + ) + + # Convert rewards to per-token form and compute reward metrics before training conversion + generator_output = self.postprocess_generator_output(generator_output, uids, data_sources) + + # print example just for debugging + vis = self.tokenizer.decode(generator_output["response_ids"][0]) + logger.info(f"Example generated: {vis}") + + return self.convert_to_training_input(generator_output, uids) + + def save_checkpoints(self): + """ + Extend base checkpointing by recording consumed UIDs for fully-async training. + + Otherwise, when resuming, there is no way to know which data has been trained on. + """ + consumed_uids_list = ( + self.async_train_dataloader.get_consumed_uids_list() + ) # read first to prevent race condition + # The base method will save the model, dataloader path, trainer_state, and latest_ckpt_global_step.txt. + super().save_checkpoints() + # In addition, we need to save the consumed UIDs -- the data that we have already trained on. + global_step_folder = os.path.join(self.cfg.trainer.ckpt_path, f"global_step_{self.global_step}") + fully_async_state_path = os.path.join(global_step_folder, "fully_async_state.pt") + fully_async_state = { + "consumed_uids": consumed_uids_list, + } + with io.open_file(fully_async_state_path, "wb") as f: + torch.save(fully_async_state, f) + logger.info(f"Saved fully-async state to {fully_async_state_path}") + + def load_checkpoints(self) -> Tuple[int, str, Set[str]]: + """ + Load the base checkpoint without loading the dataloader state, and load the fully-async state. + + Returns the global step to resume from, the checkpoint path, and the consumed data UIDs. + """ + global_step, checkpoint_path = super().load_checkpoints() + if global_step == 0: + return 0, checkpoint_path, None + fully_async_state_path = os.path.join(checkpoint_path, "fully_async_state.pt") + assert io.exists(fully_async_state_path), f"Fully-async state file not found at {fully_async_state_path}" + with io.open_file(fully_async_state_path, "rb") as f: + fully_async_state = torch.load(f, map_location="cpu", weights_only=False) + assert "consumed_uids" in fully_async_state, "consumed_uids key not found in fully-async state" + consumed_data_uids_set = set(fully_async_state["consumed_uids"]) + logger.info(f"Loaded fully-async state with {len(consumed_data_uids_set)} consumed UIDs") + return global_step, checkpoint_path, consumed_data_uids_set diff --git a/skyrl-train/skyrl_train/generators/__init__.py b/skyrl-train/skyrl_train/generators/__init__.py new file mode 100644 index 0000000000..19b9122a90 --- /dev/null +++ b/skyrl-train/skyrl_train/generators/__init__.py @@ -0,0 +1,4 @@ +from .base import GeneratorInterface, GeneratorInput, GeneratorOutput +from .skyrl_gym_generator import SkyRLGymGenerator + +__all__ = ["GeneratorInterface", "GeneratorInput", "GeneratorOutput", "SkyRLGymGenerator"] diff --git a/skyrl-train/skyrl_train/generators/base.py b/skyrl-train/skyrl_train/generators/base.py new file mode 100644 index 0000000000..b0c677f852 --- /dev/null +++ b/skyrl-train/skyrl_train/generators/base.py @@ -0,0 +1,64 @@ +from typing import List, Dict, Any, TypedDict, Optional, Union, Literal +from abc import ABC, abstractmethod +from dataclasses import dataclass +from skyrl_train.inference_engines.base import ConversationType + +TrainingPhase = Literal["train", "eval"] + + +@dataclass +class TrajectoryID: + instance_id: str # Unique identifier for the instance in the dataset + repetition_id: int # Which sample/repetition for this UID (0, 1, 2... for GRPO) + + def to_string(self) -> str: + return f"{self.instance_id}_{self.repetition_id}" + + +@dataclass +class BatchMetadata: + global_step: int + training_phase: TrainingPhase + + +class GeneratorInput(TypedDict): + prompts: List[ConversationType] + env_classes: List[str] + env_extras: Optional[List[Dict[str, Any]]] + sampling_params: Optional[Dict[str, Any]] + trajectory_ids: Optional[List[TrajectoryID]] + batch_metadata: Optional[BatchMetadata] + + +class GeneratorOutput(TypedDict): + prompt_token_ids: List[List[int]] + response_ids: List[List[int]] + rewards: Union[List[float], List[List[float]]] + loss_masks: List[List[int]] + stop_reasons: Optional[List[str]] + rollout_metrics: Optional[Dict[str, Any]] + rollout_logprobs: Optional[List[List[float]]] + trajectory_ids: Optional[List[TrajectoryID]] + # Applicable only for step-wise training + is_last_step: Optional[List[bool]] + + +class MetricsOutput(TypedDict): + avg_score: Optional[float] + pass_at_n: Optional[float] + mean_positive_reward: Optional[float] + + +class GeneratorInterface(ABC): + @abstractmethod + async def generate(self, input_batch: GeneratorInput) -> GeneratorOutput: + """Generate trajectories for the input batch. + + Returns outputs in the same order as the input batch. + + Args: + input_batch (GeneratorInput): Input batch + Returns: + GeneratorOutput: Generated trajectories + """ + raise NotImplementedError diff --git a/skyrl-train/skyrl_train/generators/skyrl_gym_generator.py b/skyrl-train/skyrl_train/generators/skyrl_gym_generator.py new file mode 100644 index 0000000000..690d18d832 --- /dev/null +++ b/skyrl-train/skyrl_train/generators/skyrl_gym_generator.py @@ -0,0 +1,1390 @@ +""" +This file implements ``SkyRLGymGenerator``, an implementation of the `GeneratorInterface` that +uses SkyRL-Gym as the environment. + +For details, see https://skyrl.readthedocs.io/en/latest/tutorials/skyrl_gym_generator.html +""" + +import asyncio +import copy +import time +from collections import defaultdict +from uuid import uuid4 +import skyrl_gym +from typing import List, Dict, Any, Optional, Union, Tuple +from concurrent.futures import ThreadPoolExecutor +from tqdm.asyncio import tqdm +from dataclasses import dataclass +from loguru import logger + +from skyrl_train.generators.base import GeneratorInterface, GeneratorInput, GeneratorOutput, TrajectoryID +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.base import InferenceEngineInput, ConversationType +from omegaconf import DictConfig +from skyrl_gym.envs.base_text_env import BaseTextEnvStepOutput +from skyrl_train.generators.utils import ( + get_custom_chat_template, + get_generation_prompt_ids, + apply_overlong_filtering, + get_rollout_metrics, +) + + +class TrajectoryTimeoutError(Exception): + """Raised when a trajectory exceeds its timeout.""" + + pass + + +@dataclass +class TrajectoryOutput: + """Output from a single agent_loop execution.""" + + response_ids: List[int] + reward: Union[List[float], float] + stop_reason: str + loss_mask: List[int] + prompt_ids: List[int] + rollout_logprobs: Optional[List[float]] + env_metrics: Dict[str, Any] + + +@dataclass +class StepWiseOutput: + """Output from a single agent_loop execution for step-wise training.""" + + step_outputs: List[TrajectoryOutput] + + +@dataclass +class AgentLoopState: + chat_history: ConversationType + input_ids: List[int] + loss_mask: List[int] + rollout_logprobs: Optional[List[float]] + response_end_idx: Optional[int] + done: bool + + +@dataclass +class TurnOutput: + output: str + output_ids: List[int] + output_logprobs: Optional[List[float]] + new_obs: ConversationType + obs_ids: List[int] + reward: Optional[float] + added_eos: bool = False + + def get_turn_loss_mask(self) -> List[int]: + """ + Get loss mask for this turn's tokens. + + Returns: + List[int]: Loss mask where 1 indicates tokens to include in loss computation and 0 indicates + tokens to exclude. If `added_eos` is True, the EOS token is masked out (set to 0). + Observation tokens are always masked out (set to 0). + """ + # if `added_eos` is `True`, then the EOS token was not generated and only added in the + # `agent_loop` function. For consistency with other entities like logprobs , we ignore it in the loss + # mask + return ([1] * len(self.output_ids) if not self.added_eos else [1] * (len(self.output_ids) - 1) + [0]) + [ + 0 + ] * len(self.obs_ids) + + def get_turn_rollout_logprobs(self) -> Optional[List[float]]: + """ + Get rollout logprobs for this turn's tokens. + + Returns: + Optional[List[float]]: Logprobs for output tokens followed by dummy values (0.0) for + observation tokens. Returns None if output_logprobs is None. + """ + if not self.output_logprobs: + return None + return self.output_logprobs + [0.0] * len(self.obs_ids) + + +class SkyRLGymGenerator(GeneratorInterface): + def __init__( + self, + generator_cfg: DictConfig, + skyrl_gym_cfg: DictConfig, + inference_engine_client: InferenceEngineClient, + tokenizer, + model_name: str, + ): + """ + Args: + generator_cfg: DictConfig object containing the generator configuration + inference_engine_client: InferenceEngineClient object for interacting with the inference engines + tokenizer: tokenizer object for encoding and decoding text + """ + self.generator_cfg = generator_cfg + self.skyrl_gym_cfg = skyrl_gym_cfg + self.inference_engine_client = inference_engine_client + self.tokenizer = tokenizer + self.max_turns = generator_cfg.max_turns + self.trajectory_timeout_seconds = getattr(generator_cfg, "trajectory_timeout_seconds", 600) + self.batched = generator_cfg.batched + self.use_conversation_multi_turn = generator_cfg.use_conversation_multi_turn + # optionally use custom chat template to get loss masks (i.e. for Qwen3) + self.custom_chat_template = get_custom_chat_template(generator_cfg.chat_template) + # get generation prompt ids for the tokenizer if needed + self.generation_prompt_ids = get_generation_prompt_ids(tokenizer) if self.use_conversation_multi_turn else None + if self.skyrl_gym_cfg.max_env_workers > 0: + self.env_executor = ThreadPoolExecutor( + max_workers=self.skyrl_gym_cfg.max_env_workers, thread_name_prefix="skyrl-gym-env-" + ) + else: + self.env_executor = None + + self._validate_cfg(generator_cfg) + + # base_conversation is used when `use_conversation_multi_turn==True and custom_chat_template==None` to + # correctly format and tokenize observations into `observation_ids`. + # Follows https://jybsuper.github.io/posts/multiturn_tokenization/#the-breakthrough-fixed-base-approach + self.base_conversation = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "I am a user."}, + ] + self.base_conversation_token_ids = tokenizer.apply_chat_template( + self.base_conversation, + add_generation_prompt=False, + tokenize=True, + return_dict=False, + **self.generator_cfg.chat_template_kwargs, + ) + # We remove tokens after the last EOS token so that it can be captured in `observation_ids`. + # For details, see https://skyrl.readthedocs.io/en/latest/tutorials/skyrl_gym_generator.html#multi-turn-tokenization-and-ti-to + if self.tokenizer.eos_token_id in self.base_conversation_token_ids: + last_eos_token_index = ( + len(self.base_conversation_token_ids) + - 1 + - self.base_conversation_token_ids[::-1].index(self.tokenizer.eos_token_id) + ) + self.base_conversation_token_ids = self.base_conversation_token_ids[: last_eos_token_index + 1] + + def _validate_cfg(self, generator_cfg: DictConfig): + if len(generator_cfg.chat_template_kwargs) and generator_cfg.batched: + raise ValueError( + "`chat_template_kwargs` is not compatible with `batched=True` since the chat templating is handled by the inference engine" + ) + + if self.generator_cfg.step_wise_trajectories: + if self.batched: + raise ValueError("`step_wise_trajectories` doesn't support `batched=True`") + + if self.custom_chat_template is not None: + raise ValueError( + f"`step_wise_trajectories` doesn't support custom chat template, got {generator_cfg.chat_template}" + ) + + if not self.use_conversation_multi_turn: + raise ValueError("`step_wise_trajectories` doesn't support `use_conversation_multi_turn=False`") + + async def _run_in_executor_if_available(self, func, *args, **kwargs): + if (executor := self.env_executor) is not None: + loop = asyncio.get_running_loop() + return await loop.run_in_executor(executor, func, *args, **kwargs) + else: + return func(*args, **kwargs) + + async def _env_init(self, env, *args, **kwargs): + """Call env.init, using async path if available to enable cancellation.""" + if hasattr(env, "init_async"): + return await env.init_async(*args, **kwargs) + return await self._run_in_executor_if_available(env.init, *args, **kwargs) + + async def _env_step(self, env, action): + """Call env.step, using async path if available to enable cancellation.""" + if hasattr(env, "step_async"): + return await env.step_async(action) + return await self._run_in_executor_if_available(env.step, action) + + async def _env_close(self, env): + """Call env.close, using async path if available to enable cancellation.""" + if hasattr(env, "close_async"): + return await env.close_async() + return await self._run_in_executor_if_available(env.close) + + async def agent_loop( + self, + prompt: ConversationType, + env_class: str, + env_extras: Dict[str, Any], + max_tokens: int, + max_input_length: int, + sampling_params: Optional[Dict[str, Any]] = None, + trajectory_id: Optional[TrajectoryID] = None, + ) -> Union[TrajectoryOutput, StepWiseOutput]: + """ + Multi-turn generation loop that executes a single trajectory. + + Note: + We ensure token-in-token-out generation. With two exceptions: + - When calling Env.step() and BaseTextEnvStepOutput["postprocessed_action"] is not None. + This will likely be deprecated soon. + - When custom_chat_template = True and use_conversation_multi_turn = True. We always + re-tokenize the entire chat history every turn and at the end. This is used for cases + like removing Qwen3 thinking tokens in non-last-round assistant message. + + Args: + prompt: ConversationType + env_extras: Dict[str, Any] + max_tokens: int + max_input_length: int + sampling_params: Optional[Dict[str, Any]] + Returns: + response_ids: List[int] + reward: Union[float, List[float]] + stop_reason: str + loss_mask: List[int] + prompt_token_ids: List[int] + rollout_logprobs: Optional[List[float]] + """ + # NOTE: `custom_chat_template` was mainly for getting accurate loss masks for thinking models. + # This is no longer needed now given that step wise training is supported + # TODO (sumanthrh): This path can be deprecated + retokenize_chat_history = self.use_conversation_multi_turn and self.custom_chat_template + + # Create a new environment instance + env_extras["max_turns"] = self.max_turns # TODO(shu): move this to config + env_config = self.skyrl_gym_cfg.get(env_class, DictConfig({})) + + session_id = ( + f"{trajectory_id.instance_id}_{trajectory_id.repetition_id}" if trajectory_id is not None else uuid4().hex + ) + + # Instantiate chat_history and chat_end_index, which are only used if `retokenize_chat_history==True`. + # Need copy here since the prompt is a list of messages and we are going to modify it. + chat_history = copy.deepcopy(prompt) + + # Extract env_key for logging (data_source in dataset maps to env_key) + env_key = env_extras.get("data_source", "unknown") + + # Try to create and initialize the environment. If it fails (e.g., Fleet health check failure), + # return a zero-reward trajectory instead of crashing the entire training run. + try: + env = skyrl_gym.make(env_class, env_config=env_config, extras=env_extras) + # init() returns the first prompt to be given to the model, and optional metadata dict + chat_history, _ = await self._env_init(env, chat_history) + except Exception as e: + logger.warning( + f"[env={env_key}] Environment init failed for session {session_id}: {e}. Returning zero-reward trajectory." + ) + # Return a minimal failed trajectory with zero reward + prompt_ids = self.tokenizer.apply_chat_template( + chat_history, + add_generation_prompt=True, + tokenize=True, + return_dict=False, + **self.generator_cfg.chat_template_kwargs, + ) + # Use token-level reward format [0.0] to match normal trajectories when custom_chat_template is None + # This ensures consistent reward types across all trajectories in a batch + zero_reward = 0.0 if self.custom_chat_template else [0.0] + init_fail_output = TrajectoryOutput( + response_ids=[self.tokenizer.eos_token_id], + reward=zero_reward, + stop_reason="env_init_failed", + loss_mask=[0], # Don't learn from failed trajectories + prompt_ids=prompt_ids, + rollout_logprobs=[0.0], # Match response_ids length for validation + env_metrics={"env_init_failed": 1.0, f"env_init_failed/{env_key}": 1.0}, + ) + # Return correct type for step-wise mode + if self.generator_cfg.step_wise_trajectories: + return StepWiseOutput(step_outputs=[init_fail_output]) + return init_fail_output + initial_chat_history_length = len(chat_history) + initial_input_ids = self.tokenizer.apply_chat_template( + chat_history, + # If retokenize_chat_history==True, avoid including the generation prompt in both the + # prompt_ids and response_ids due to how `response_encodings["input_ids"]` works. + add_generation_prompt=not retokenize_chat_history, + chat_template=self.custom_chat_template if retokenize_chat_history else None, + tokenize=True, + return_dict=False, + **self.generator_cfg.chat_template_kwargs, + ) + + initial_prompt_length = len(initial_input_ids) + loss_mask = [] # this excludes the prompt + rollout_logprobs = None + + current_sampling_params = sampling_params if sampling_params is not None else self.generator_cfg.sampling_params + + # Accumulate per-step rewards. Format: (reward, response_end_token_idx) + per_step_rewards: List[Tuple[float, Optional[int]]] = [] + + is_step_wise = self.generator_cfg.step_wise_trajectories + + agent_loop_output = StepWiseOutput(step_outputs=[]) if is_step_wise else None + + get_logprobs = self.generator_cfg.sampling_params.logprobs is not None + agent_loop_state = AgentLoopState( + chat_history=chat_history, + input_ids=initial_input_ids, + loss_mask=[], + rollout_logprobs=[] if get_logprobs else None, + response_end_idx=None, + done=False, + ) + + # Track trajectory start time for timeout + trajectory_start_time = time.time() + + try: + while not agent_loop_state.done: + # Check for trajectory timeout at the start of each turn + if self.trajectory_timeout_seconds > 0: + elapsed = time.time() - trajectory_start_time + if elapsed > self.trajectory_timeout_seconds: + logger.warning( + f"Trajectory timeout after {elapsed:.1f}s (limit: {self.trajectory_timeout_seconds}s) " + f"for session {session_id}. Returning zero-reward trajectory." + ) + await self._env_close(env) + # Use token-level reward format [0.0] to match normal trajectories when custom_chat_template is None + zero_reward = 0.0 if self.custom_chat_template else [0.0] + early_timeout_output = TrajectoryOutput( + response_ids=[self.tokenizer.eos_token_id], + reward=zero_reward, + stop_reason="timeout", + loss_mask=[0], + prompt_ids=initial_input_ids, + rollout_logprobs=[0.0], + env_metrics={"trajectory_timeout": 1.0}, + ) + # Return correct type for step-wise mode + if is_step_wise: + return StepWiseOutput(step_outputs=[early_timeout_output]) + return early_timeout_output + + if len(agent_loop_state.input_ids) > max_input_length: + stop_reason = "length" + break + + # 1. Generate output + if is_step_wise or retokenize_chat_history: + # re-apply whole chat template so length check is correct + agent_loop_state.input_ids = self.tokenizer.apply_chat_template( + chat_history, + chat_template=self.custom_chat_template if retokenize_chat_history else None, + add_generation_prompt=True, + tokenize=True, + return_dict=False, + **self.generator_cfg.chat_template_kwargs, + ) + agent_loop_state.loss_mask = [] + agent_loop_state.rollout_logprobs = None + # Check length again after re-tokenization (step-wise can jump significantly) + if len(agent_loop_state.input_ids) > max_input_length: + stop_reason = "length" + break + + engine_input = InferenceEngineInput( + prompt_token_ids=[agent_loop_state.input_ids], + session_ids=[session_id], + sampling_params=sampling_params, + ) + try: + remaining = ( + self.trajectory_timeout_seconds - (time.time() - trajectory_start_time) + if self.trajectory_timeout_seconds > 0 + else None + ) + engine_output = await asyncio.wait_for( + self.inference_engine_client.generate(engine_input), timeout=remaining + ) + except asyncio.TimeoutError: + raise TrajectoryTimeoutError( + f"Timeout during generate after {time.time() - trajectory_start_time:.1f}s" + ) + output = engine_output["responses"][0] + output_ids = engine_output["response_ids"][0] + stop_reason = engine_output["stop_reasons"][0] + response_logprobs = engine_output.get("response_logprobs", None) + if response_logprobs is not None: + response_logprobs = response_logprobs[0] + if self.custom_chat_template is not None: + raise ValueError("Response Logprobs bookkeeping is not supported with custom chat template") + + # Append eos when sampling_params.stop is not None. Does not affect 3.a as chat templates add eos_token. + # sampling_params is not None for eval, but None for training (which uses engine.sampling_params which are from cfg) + stop_strs = current_sampling_params.get("stop", None) + added_eos = False + if ( + stop_strs is not None + and self.generator_cfg.append_eos_token_after_stop_str_in_multi_turn + and self.use_conversation_multi_turn + ): + if output.endswith(tuple(stop_strs)) and output_ids[-1] != self.tokenizer.eos_token_id: + output_ids.append(self.tokenizer.eos_token_id) + # dummy logprobs for EOS token id. It will be loss masked with 0 in TurnOutput.get_turn_loss_mask + if response_logprobs is not None: + response_logprobs.append(0.0) + added_eos = True + + # 2. Environment step + try: + remaining = ( + self.trajectory_timeout_seconds - (time.time() - trajectory_start_time) + if self.trajectory_timeout_seconds > 0 + else None + ) + env_step_output: BaseTextEnvStepOutput = await asyncio.wait_for( + self._env_step(env, output), timeout=remaining + ) + except asyncio.TimeoutError: + raise TrajectoryTimeoutError( + f"Timeout during env.step after {time.time() - trajectory_start_time:.1f}s" + ) + new_obs = env_step_output["observations"] + step_reward: float = env_step_output["reward"] + agent_loop_state.done = env_step_output["done"] + + # Inject context status into observation if enabled + # This helps models learn when to use context management tools + if getattr(self.generator_cfg, "inject_context_status", False) and new_obs: + current_tokens = len(agent_loop_state.input_ids) + len(output_ids) + percentage = min(100, int((current_tokens / max_input_length) * 100)) + # Get turn count from env metadata (FleetTaskEnv tracks this) + step_metadata = env_step_output.get("metadata", {}) + current_turn = step_metadata.get("turn", 0) + status_line = f"\n[Context: {current_tokens:,}/{max_input_length:,} tokens ({percentage}%), Turn {current_turn}/{self.generator_cfg.max_turns}]" + # Append to last observation's content + if isinstance(new_obs[-1], dict) and "content" in new_obs[-1]: + new_obs[-1]["content"] += status_line + + if env_step_output.get("postprocessed_action", None) is not None: + # TODO(Charlie): come back to this, we should deprecate postprocessed action + logger.warning( + "WARNING: postprocessed action may violate token-in-token-out. Ideally you " + "post-process it in the token space rather than string space. " + "A better solution coming soon." + ) + output = env_step_output["postprocessed_action"] + output_ids = self.tokenizer.encode(output, add_special_tokens=False) + + obs_ids = self.get_obs_ids_from_obs(new_obs, agent_loop_state.done) + + # final turn output containing generated response and environment observations + turn_output = TurnOutput( + output=output, + output_ids=output_ids, + output_logprobs=response_logprobs, + new_obs=new_obs, + reward=step_reward, + obs_ids=obs_ids, + added_eos=added_eos, + ) + + if is_step_wise: + # current response + observation ids + turn_response_ids = turn_output.output_ids + turn_output.obs_ids + turn_prompt_ids = agent_loop_state.input_ids + + # agent loop only tracks loss mask and rollout logprobs for this turn with step_wise training + turn_loss_mask = turn_output.get_turn_loss_mask() + turn_response_logprobs: Optional[List[float]] = turn_output.get_turn_rollout_logprobs() + + per_step_output = TrajectoryOutput( + response_ids=turn_response_ids, + reward=step_reward, + loss_mask=turn_loss_mask, + prompt_ids=turn_prompt_ids, + rollout_logprobs=turn_response_logprobs, + stop_reason=stop_reason, + env_metrics=env.get_metrics() if agent_loop_state.done else {}, + ) + agent_loop_output.step_outputs.append(per_step_output) + + # 3. Update states: input ids, loss_mask, chat_history, etc. + # Three ways of managing input + if retokenize_chat_history: + # a. custom chat template + agent_loop_state = self._update_agent_state_by_retokenizing_chat_history( + agent_loop_state, turn_output + ) + elif self.use_conversation_multi_turn: + # b. Token-in-token-out. Follow multi-turn chat history format. + agent_loop_state = self._update_agent_loop_state_with_multiturn_chat_template( + agent_loop_state, turn_output + ) + else: + # c. Token-in-token-out. All steps/observations are appended to a single assistant message. + agent_loop_state = self._update_agent_loop_state_with_singleturn_chat_template( + agent_loop_state, turn_output + ) + + # If env modified chat_history (e.g., context management dropped turns), + # replace our copy. Must happen AFTER _update_* methods to avoid duplication. + step_metadata = env_step_output.get("metadata", {}) + if step_metadata.get("modified_chat_history") is not None: + agent_loop_state.chat_history = step_metadata["modified_chat_history"] + + per_step_rewards.append((step_reward, agent_loop_state.response_end_idx)) + + except TrajectoryTimeoutError as e: + logger.warning(f"Trajectory timeout for session {session_id}: {e}. Returning zero-reward trajectory.") + await self._env_close(env) + zero_reward = 0.0 if self.custom_chat_template else [0.0] + timeout_output = TrajectoryOutput( + response_ids=[self.tokenizer.eos_token_id], + reward=zero_reward, + stop_reason="timeout", + loss_mask=[0], + prompt_ids=initial_input_ids, + rollout_logprobs=[0.0], + env_metrics={"trajectory_timeout": 1.0}, + ) + # Return correct type for step-wise mode + if is_step_wise: + return StepWiseOutput(step_outputs=[timeout_output]) + return timeout_output + + # Close the environment (runs verifier for orphaned rollouts like context overflow) + await self._env_close(env) + # Get environment-specific metrics after close (includes final_reward from verifier) + env_metrics = env.get_metrics() + + # If trajectory was stopped early (context overflow) but verifier found the task + # was completed, use the actual reward instead of the default 0.0 + final_reward = env_metrics.get("final_reward") + if isinstance(final_reward, (int, float)) and per_step_rewards: + last_reward, last_idx = per_step_rewards[-1] + per_step_rewards[-1] = (final_reward, last_idx) + + # Guard: if prompt exceeded max_input_length before any generation happened, + # response_end_idx is still None. Return a zero-reward trajectory. + if agent_loop_state.response_end_idx is None and not per_step_rewards: + logger.warning( + f"[env={env_key}] Prompt too long ({len(initial_input_ids)} tokens > {max_input_length}) " + f"for session {session_id}. Returning zero-reward trajectory." + ) + zero_reward = 0.0 if self.custom_chat_template else [0.0] + overlong_prompt_output = TrajectoryOutput( + response_ids=[self.tokenizer.eos_token_id], + reward=zero_reward, + stop_reason="length", + loss_mask=[0], + prompt_ids=initial_input_ids, + rollout_logprobs=[0.0], + env_metrics=env_metrics, + ) + if is_step_wise: + return StepWiseOutput(step_outputs=[overlong_prompt_output]) + return overlong_prompt_output + + prompt_ids = agent_loop_state.input_ids[:initial_prompt_length] + rollout_logprobs = None + response_ids = None + + # Prepare the final loss_mask, response_ids and rollout_logprobs . + # We remove the final observation messages /token IDs here + # Note that during the agent loop, we still add the final observation messages/ tokens because we terminate the agent loop if the input length + # exceeds the maximum + if retokenize_chat_history: + response_encodings = self.tokenizer.apply_chat_template( + agent_loop_state.chat_history[ + initial_chat_history_length : len(agent_loop_state.chat_history) - len(new_obs) + ], + chat_template=self.custom_chat_template, + add_generation_prompt=False, + return_dict=True, + return_assistant_tokens_mask=True, + tokenize=True, + **self.generator_cfg.chat_template_kwargs, + ) + loss_mask = response_encodings["assistant_masks"] + response_ids = response_encodings["input_ids"] + elif not self.generator_cfg.step_wise_trajectories: + assert not any( + agent_loop_state.loss_mask[agent_loop_state.response_end_idx - initial_prompt_length + 1 :] + ), "loss_mask at index after response end should be all 0" + loss_mask = agent_loop_state.loss_mask[: agent_loop_state.response_end_idx - initial_prompt_length + 1] + response_ids = agent_loop_state.input_ids[initial_prompt_length : agent_loop_state.response_end_idx + 1] + if agent_loop_state.rollout_logprobs is not None: + rollout_logprobs = agent_loop_state.rollout_logprobs[ + : agent_loop_state.response_end_idx - initial_prompt_length + 1 + ] + # fix index for per_step_rewards + per_step_rewards = [(reward, idx - initial_prompt_length) for reward, idx in per_step_rewards] + assert len(loss_mask) == len( + response_ids + ), f"loss_mask and response_ids should have the same length, got {len(loss_mask)} and {len(response_ids)}" + + appended_eos_token = False + if not self.use_conversation_multi_turn: + assert response_ids is not None and loss_mask is not None + if stop_reason != "length" and response_ids and response_ids[-1] != self.tokenizer.eos_token_id: + response_ids.append(self.tokenizer.eos_token_id) + # TODO(Charlie): this should be 0? Otherwise logprobs will be extremely off. But if it is loss + # masked with 0, why bother adding it? + loss_mask.append(1) + if rollout_logprobs is not None: + rollout_logprobs.append(0.0) + appended_eos_token = True + + if self.generator_cfg.step_wise_trajectories: + for per_step_output, (reward, resp_end_idx) in zip(agent_loop_output.step_outputs, per_step_rewards): + per_token_reward = [0.0] * len(per_step_output.response_ids) + per_token_reward[resp_end_idx] = float(reward) + # in-place update to per-token reward + per_step_output.reward = per_token_reward + else: + reward_out = self._build_per_token_rewards(per_step_rewards, response_ids, appended_eos_token) + + agent_loop_output = TrajectoryOutput( + response_ids=response_ids, + reward=reward_out, + stop_reason=stop_reason, + loss_mask=loss_mask, + prompt_ids=prompt_ids, + rollout_logprobs=rollout_logprobs, + env_metrics=env_metrics, + ) + + return agent_loop_output + + def _build_per_token_rewards( + self, per_step_rewards: List[Tuple[float, Optional[int]]], response_ids: List[int], appended_eos_token: bool + ) -> Union[float, List[float]]: + """ + Build reward output from per-step rewards. + + Args: + per_step_rewards: List of (reward, response_end_token_idx) tuples for each step + response_ids: List of response token IDs + appended_eos_token: Whether an EOS token was manually appended at the end + + Returns: + Union[float, List[float]]: If custom_chat_template is used, returns the last step's reward (float). + Otherwise, returns a list of token-level rewards (List[float]). + """ + if self.custom_chat_template: + # TODO(Charlie): Currently, the possible response truncation will not affect the reward + # in the if branch, but some final rewards may be lost in the else branch. Fix this + # when we support turn-level rewards for the `retokenize_chat_history` codepath. + reward_out = per_step_rewards[-1][0] + else: + # Build token-level rewards placed at assistant turn boundaries + token_level_rewards: List[float] = [0.0] * len(response_ids) + for i, (step_reward, idx) in enumerate(per_step_rewards): + assert step_reward is not None + if idx >= len(response_ids): + break + if appended_eos_token and i == len(per_step_rewards) - 1: + # NOTE(Charlie): If we appended the eos token, we need to place + # the reward at the last token (the manually appended eos token) + # rather than the last turn's assistant-generated token. This matches + # the logic in trainer.py::postprocess_generator_output when rewards are List[float]. + token_level_rewards[-1] = step_reward + else: + token_level_rewards[idx] += step_reward + reward_out = token_level_rewards + return reward_out + + def get_obs_ids_from_obs(self, new_obs: ConversationType, is_done: bool) -> List[int]: + """ + Returns observation token ids from observation messages for a turn. + + Args: + new_obs: Observation messages from the environment step + is_done: Whether the agent loop has terminated + + Returns: + List[int]: Observation token IDs. For multi-turn mode, includes chat template formatting. + For single-turn mode, returns directly encoded observation tokens. + """ + if self.use_conversation_multi_turn: + # 2. apply chat template for observations, also generate generation prompt for next turn + obs_ids_to_add = [] + if len(new_obs) > 0: + # For Qwen, this will generate `\n<|user|>Some observation<|im_end|>\n`. Note that the + # first `\n` is generated since we stripped it in ``base_conversation_token_ids``. + obs_ids_to_add = self.tokenizer.apply_chat_template( + [*self.base_conversation, *new_obs], + add_generation_prompt=not is_done, + tokenize=True, + return_dict=False, + **self.generator_cfg.chat_template_kwargs, + )[len(self.base_conversation_token_ids) :] + elif not is_done: + obs_ids_to_add = self.generation_prompt_ids + else: + # Build observation token ids (encoded directly, not using chat template) + # no generation prompt is added in this case + obs_ids_to_add = [] + if len(new_obs) > 0: + for obs in new_obs: + obs_tokens = self.tokenizer.encode(obs["content"], add_special_tokens=False) + obs_ids_to_add.extend(obs_tokens) + return obs_ids_to_add + + def _update_chat_history( + self, + chat_history: ConversationType, + output: str, + new_obs: ConversationType, + ) -> ConversationType: + """ + Update chat history with assistant response and new observations. + + Args: + chat_history: Current conversation history + output: Assistant's response text + new_obs: New observation messages from the environment + + Returns: + ConversationType: Updated chat history with assistant response and observations appended. + The EOS token is removed from output if present, as it will be reapplied by the chat template. + """ + # remove eos token from end of output if it exists, since it will be reapplied by the chat template + if output.endswith(self.tokenizer.eos_token): + output = output[: -len(self.tokenizer.eos_token)] + + # Add assistant response to chat history + chat_history += [{"role": "assistant", "content": output}] + + # Add observations to chat history + if len(new_obs) > 0: + chat_history += new_obs + return chat_history + + async def generate_batched( + self, + prompts: List[ConversationType], + env_classes: List[str], + env_extras: List[Dict[str, Any]], + max_tokens: int, + sampling_params: Optional[Dict[str, Any]] = None, + ) -> GeneratorOutput: + """ + Single-turn batched generation (can use the synchronous offline engine) + + Args: + prompts: List[ConversationType] + env_classes: List[str] + env_extras: List[Dict[str, Any]] + max_tokens: int + max_input_length: int --> Currently unused as we assume batched is used only for single-turn. + sampling_params: Optional[Dict[str, Any]] + Returns: + GeneratorOutput + """ + envs = [] + init_prompts = [] + for env_class, env_extra, prompt in zip(env_classes, env_extras, prompts): + env_extra["max_turns"] = self.max_turns + env_config = self.skyrl_gym_cfg.get(env_class, DictConfig({})) + env = skyrl_gym.make(env_class, env_config=env_config, extras=env_extra) + init_prompt, _ = await self._env_init(env, prompt) + init_prompts.append(init_prompt) + envs.append(env) + + # For single-turn generation, we can use text-in-token-out, since we do not need to re-tokenize. + engine_input = InferenceEngineInput(prompts=init_prompts, sampling_params=sampling_params) + engine_output = await self.inference_engine_client.generate(engine_input) + outputs = engine_output["responses"] + responses = engine_output["response_ids"] + stop_reasons = engine_output["stop_reasons"] + logprobs = engine_output.get("response_logprobs", None) + + truncated_responses = [] + rewards = [] + loss_masks = [] + env_metrics = [] + truncated_logprobs: Optional[List[List[float]]] = [] if logprobs is not None else None + + for i, (output, response, env, env_class) in enumerate(zip(outputs, responses, envs, env_classes)): + # step on environment and compute reward + env_step_output: BaseTextEnvStepOutput = await self._env_step(env, output) + reward = env_step_output["reward"] + rewards.append(reward) + + if len(response) > max_tokens: + response = response[:max_tokens] + loss_masks.append([1] * len(response)) + truncated_responses.append(response) + if logprobs is not None: + sample_logprobs = logprobs[i][: len(response)] + truncated_logprobs.append(sample_logprobs) + + # Get environment-specific metrics + env_metrics.append(env.get_metrics()) + # Close the environment + await self._env_close(env) + + prompt_token_ids = self.tokenizer.apply_chat_template( + init_prompts, + add_generation_prompt=True, + tokenize=True, + return_dict=False, + ) + rollout_metrics = get_rollout_metrics(responses, rewards, env_metrics, env_classes) + + if self.generator_cfg.apply_overlong_filtering: + loss_masks = apply_overlong_filtering(loss_masks, responses, self.tokenizer.eos_token_id) + + generator_output: GeneratorOutput = { + "prompt_token_ids": prompt_token_ids, + "response_ids": truncated_responses, + "rewards": rewards, + "loss_masks": loss_masks, + "stop_reasons": stop_reasons, + "rollout_metrics": rollout_metrics, + "rollout_logprobs": truncated_logprobs, + } + + return generator_output + + async def _run_hint_augmentation( + self, + all_outputs: List[TrajectoryOutput], + prompts: List[ConversationType], + env_classes: List[str], + env_extras: List[Dict[str, Any]], + trajectory_ids: List[TrajectoryID], + max_tokens: int, + max_input_length: int, + sampling_params: Optional[Dict[str, Any]], + hint_cfg: DictConfig, + ) -> Tuple[List[TrajectoryOutput], List[TrajectoryID], List[str]]: + """Run hinted rollouts for prompts where all raw samples failed. + + Groups raw outputs by instance_id, identifies groups where max_reward < threshold, + builds hint text from verifier feedback, and launches additional hinted rollouts. + + Returns: + Tuple of (hinted_outputs, hinted_trajectory_ids, hinted_env_classes) + """ + from integrations.fleet.env import FleetTaskEnv + + # 1. Group outputs by instance_id + groups: Dict[str, List[Tuple[int, TrajectoryOutput]]] = defaultdict(list) + for i, output in enumerate(all_outputs): + iid = trajectory_ids[i].instance_id + groups[iid].append((i, output)) + + # 2. Identify prompts needing hints + hint_tasks = [] + hint_tids = [] + hint_envs = [] + orig_prompt_ids = [] # unhinted prompt_ids for RLTF-SD + prompts_hinted = 0 + + for iid, items in groups.items(): + rewards = [] + for _, output in items: + r = output.reward + rewards.append(r if isinstance(r, (int, float)) else sum(r)) + max_reward = max(rewards) + + if max_reward > hint_cfg.hint_reward_threshold: + continue # at least one raw sample has signal + + # Find best raw rollout (highest partial reward) for feedback + best_idx = max(range(len(items)), key=lambda j: rewards[j]) + best_orig_idx, best_output = items[best_idx] + metrics = best_output.env_metrics + + # Build hint from verifier feedback + hint_text = FleetTaskEnv.build_hint_text( + verifier_stdout=metrics.get("verifier_stdout"), + verifier_error=metrics.get("verifier_error"), + tool_error_messages=metrics.get("tool_error_messages"), + ) + if not hint_text: + continue + + logger.info( + f"Hint for instance {iid} (best_reward={rewards[best_idx]:.3f}, " + f"verifier_stdout={bool(metrics.get('verifier_stdout'))}, " + f"verifier_error={bool(metrics.get('verifier_error'))}):\n{hint_text}" + ) + prompts_hinted += 1 + + # Create hinted agent_loop tasks (new env instances) + base_rep_id = max(item[0] for item in items) + 1 + n_hint = hint_cfg.get("n_hint_samples", 2) + for h in range(n_hint): + hinted_extras = dict(env_extras[best_orig_idx]) + hinted_extras["hint"] = hint_text + hinted_extras["is_hinted"] = True + tid = TrajectoryID(instance_id=iid, repetition_id=base_rep_id + h) + hint_tasks.append( + self.agent_loop( + prompts[best_orig_idx], + env_classes[best_orig_idx], + hinted_extras, + max_tokens, + max_input_length, + sampling_params=sampling_params, + trajectory_id=tid, + ) + ) + hint_tids.append(tid) + hint_envs.append(env_classes[best_orig_idx]) + orig_prompt_ids.append(best_output.prompt_ids) + + # 3. Run all hinted rollouts in parallel + if hint_tasks: + logger.info( + f"Hint augmentation: {prompts_hinted} prompts need hints, " + f"launching {len(hint_tasks)} hinted rollouts" + ) + hint_outputs = await tqdm.gather( + *hint_tasks, + desc="Hinted Rollouts", + miniters=1, + mininterval=5, + ) + # RLTF-SD: strip hint from training prompt. Replace hinted prompt_ids + # with the original unhinted prompt_ids so the model learns to produce + # hint-quality outputs conditioned on the original prompt alone: + # ∇θ log π(y_hint | x_0) instead of ∇θ log π(y_hint | x_0 + hint) + hint_outputs = list(hint_outputs) + for i, output in enumerate(hint_outputs): + hinted_len = len(output.prompt_ids) + output.prompt_ids = orig_prompt_ids[i] + logger.debug( + f"RLTF-SD: replaced hinted prompt ({hinted_len} tokens) " + f"with original prompt ({len(output.prompt_ids)} tokens)" + ) + return hint_outputs, hint_tids, hint_envs + + return [], [], [] + + async def generate(self, input_batch: GeneratorInput, disable_tqdm: bool = False) -> GeneratorOutput: + """ + Generate trajectories for the input batch. + + Returns outputs in the same order as the input batch. + Args: + input_batch: GeneratorInput + disable_tqdm: bool + Returns: + GeneratorOutput + """ + prompts = input_batch["prompts"] + env_classes = input_batch["env_classes"] + env_extras = input_batch["env_extras"] + trajectory_ids = input_batch.get("trajectory_ids", None) + if self.generator_cfg.step_wise_trajectories: + assert trajectory_ids is not None, "`trajectory_ids` is a required field for step wise training" + sampling_params: Optional[dict] = input_batch.get("sampling_params", None) + max_tokens = self.generator_cfg.sampling_params.max_generate_length + max_input_length = self.generator_cfg.max_input_length + + if self.batched: + return await self.generate_batched(prompts, env_classes, env_extras, max_tokens, sampling_params) + + # Async agent loop to generate trajectories in parallel. + tasks = [] + for i in range(len(prompts)): + tasks.append( + self.agent_loop( + prompts[i], + env_classes[i], + env_extras[i], + max_tokens, + max_input_length, + sampling_params=sampling_params, + trajectory_id=trajectory_ids[i] if trajectory_ids is not None else None, + ) + ) + + all_outputs = await tqdm.gather( + *tasks, + desc="Generating Trajectories", + miniters=max(1, len(tasks) // 10), + mininterval=5, + disable=disable_tqdm, + ) + + # --- Hint augmentation: rescue GRPO signal on dead prompts --- + # Only during training; eval should not run hints. + n_raw = len(all_outputs) + batch_metadata = input_batch.get("batch_metadata") + is_training = batch_metadata is not None and batch_metadata.training_phase == "train" + hint_cfg = self.skyrl_gym_cfg.get("fleet_task", DictConfig({})) + if ( + hint_cfg.get("enable_hints", False) + and not self.generator_cfg.step_wise_trajectories + and trajectory_ids is not None + and is_training + ): + hint_outputs, hint_tids, hint_env_classes = await self._run_hint_augmentation( + all_outputs=list(all_outputs), + prompts=prompts, + env_classes=env_classes, + env_extras=env_extras, + trajectory_ids=trajectory_ids, + max_tokens=max_tokens, + max_input_length=max_input_length, + sampling_params=sampling_params, + hint_cfg=hint_cfg, + ) + if hint_outputs: + all_outputs = list(all_outputs) + hint_outputs + # Extend in-place so input_batch references are updated (trainer reads these) + trajectory_ids.extend(hint_tids) + env_classes.extend(hint_env_classes) + # Also extend prompts and env_extras arrays to stay aligned + for tid in hint_tids: + # Find original prompt index for this instance_id + for orig_i, orig_tid in enumerate(input_batch.get("trajectory_ids", [])): + if orig_tid.instance_id == tid.instance_id: + prompts.append(prompts[orig_i]) + env_extras.append(env_extras[orig_i]) + break + + # Build is_hinted array: raw samples are False, hint-augmented samples are True + is_hinted = [False] * n_raw + [True] * (len(all_outputs) - n_raw) + + if self.generator_cfg.step_wise_trajectories: + responses = [] + rewards = [] + stop_reasons = [] + loss_masks = [] + prompt_token_ids = [] + env_metrics = [] + is_last_step = [] + out_trajectory_ids = [] + out_env_classes = [] + out_is_hinted = [] + for i, output in enumerate(all_outputs): + for j, step_output in enumerate(output.step_outputs): + responses.append(step_output.response_ids) + rewards.append(step_output.reward) + stop_reasons.append(step_output.stop_reason) + loss_masks.append(step_output.loss_mask) + prompt_token_ids.append(step_output.prompt_ids) + env_metrics.append(step_output.env_metrics) + is_last_step.append(j == len(output.step_outputs) - 1) + out_trajectory_ids.append(trajectory_ids[i]) + out_env_classes.append(env_classes[i]) + out_is_hinted.append(is_hinted[i]) + env_classes = out_env_classes + is_hinted = out_is_hinted + else: + responses = [output.response_ids for output in all_outputs] + rewards = [output.reward for output in all_outputs] + stop_reasons = [output.stop_reason for output in all_outputs] + loss_masks = [output.loss_mask for output in all_outputs] + prompt_token_ids = [output.prompt_ids for output in all_outputs] + env_metrics = [output.env_metrics for output in all_outputs] + is_last_step = None + out_trajectory_ids = trajectory_ids + + if sampling_params is not None: + # sampling params will be a dict in the format of the inference engine backend + # TODO: this might have to change when we support logprobs for sglang + get_logprobs = sampling_params.get("logprobs", None) is not None + else: + get_logprobs = self.generator_cfg.sampling_params.logprobs is not None + + if get_logprobs: + if self.generator_cfg.step_wise_trajectories: + rollout_logprobs = sum( + [[step_output.rollout_logprobs for step_output in output.step_outputs] for output in all_outputs], + [], + ) + else: + rollout_logprobs = [output.rollout_logprobs for output in all_outputs] + else: + rollout_logprobs = None + + rollout_metrics = get_rollout_metrics(responses, rewards, env_metrics, env_classes) + + # Log hint augmentation metrics + hinted_metrics = [m for m in env_metrics if isinstance(m, dict) and m.get("is_hinted")] + if hinted_metrics: + n_hinted = len(hinted_metrics) + hinted_rewards = [] + for m in hinted_metrics: + r = m.get("final_reward", 0.0) + hinted_rewards.append(r if r is not None else 0.0) + n_success = sum(1 for r in hinted_rewards if r > 0) + rollout_metrics["hint/total_hinted_rollouts"] = n_hinted + rollout_metrics["hint/hint_success_rate"] = n_success / n_hinted if n_hinted > 0 else 0.0 + # Count unique prompts that were hinted (by instance_id) + hinted_iids = set() + for i, m in enumerate(env_metrics): + if m.get("is_hinted") and trajectory_ids is not None and i < len(trajectory_ids): + hinted_iids.add(trajectory_ids[i].instance_id) + rollout_metrics["hint/prompts_hinted"] = len(hinted_iids) + # Signal rescued: prompts where at least 1 hinted sample scored > 0 + rescued = 0 + for iid in hinted_iids: + for j, m in enumerate(env_metrics): + if m.get("is_hinted") and trajectory_ids is not None and j < len(trajectory_ids): + if trajectory_ids[j].instance_id == iid: + r = m.get("final_reward", 0.0) + if r is not None and r > 0: + rescued += 1 + break + rollout_metrics["hint/signal_rescued"] = rescued / len(hinted_iids) if hinted_iids else 0.0 + + # Log count of failed env inits by environment + env_init_failures = sum(1 for sr in stop_reasons if sr == "env_init_failed") + if env_init_failures > 0: + # Count failures by env_class + failures_by_env: Dict[str, int] = {} + for env_class, stop_reason in zip(env_classes, stop_reasons): + if stop_reason == "env_init_failed": + failures_by_env[env_class] = failures_by_env.get(env_class, 0) + 1 + + failure_details = ", ".join(f"{env}: {count}" for env, count in sorted(failures_by_env.items())) + logger.warning( + f"Env init failures: {env_init_failures}/{len(stop_reasons)} trajectories " + f"({100 * env_init_failures / len(stop_reasons):.1f}%) - by env: {failure_details}" + ) + rollout_metrics["generate/env_init_failures"] = env_init_failures + # Per-env failure counts for WandB + for env, count in failures_by_env.items(): + rollout_metrics[f"environment/{env}/env_init_failures"] = count + + # Log count of trajectory timeouts by environment + trajectory_timeouts = sum(1 for sr in stop_reasons if sr == "timeout") + if trajectory_timeouts > 0: + # Count timeouts by env_class + timeouts_by_env: Dict[str, int] = {} + for env_class, stop_reason in zip(env_classes, stop_reasons): + if stop_reason == "timeout": + timeouts_by_env[env_class] = timeouts_by_env.get(env_class, 0) + 1 + + timeout_details = ", ".join(f"{env}: {count}" for env, count in sorted(timeouts_by_env.items())) + logger.warning( + f"Trajectory timeouts: {trajectory_timeouts}/{len(stop_reasons)} trajectories " + f"({100 * trajectory_timeouts / len(stop_reasons):.1f}%) - by env: {timeout_details}" + ) + rollout_metrics["generate/trajectory_timeouts"] = trajectory_timeouts + # Per-env timeout counts for WandB + for env, count in timeouts_by_env.items(): + rollout_metrics[f"environment/{env}/trajectory_timeouts"] = count + + if self.generator_cfg.zero_reward_on_non_stop: + # set reward to 0 if the stop reason is not "stop" + rewards = self._zero_reward_if_not_stop(rewards, stop_reasons) + + if self.generator_cfg.apply_overlong_filtering: + loss_masks = apply_overlong_filtering(loss_masks, responses, self.tokenizer.eos_token_id) + + generator_output: GeneratorOutput = { + "prompt_token_ids": prompt_token_ids, + "response_ids": responses, + "rewards": rewards, + "loss_masks": loss_masks, + "stop_reasons": stop_reasons, + "rollout_metrics": rollout_metrics, + "rollout_logprobs": rollout_logprobs, + "trajectory_ids": out_trajectory_ids, + "is_last_step": is_last_step, + "is_hinted": is_hinted, + } + + return generator_output + + def _zero_reward_if_not_stop( + self, rewards: List[Union[float, List[float]]], stop_reasons: List[str] + ) -> List[Union[float, List[float]]]: + """ + Sets the reward to 0 if the stop reason is not "stop". + + This can be useful in cases where the LLM generation was truncated or aborted, but the environment still assigns non-zero reward. + Often, we have format rewards for the LLM to follow, but in cases where the LLM didn't finish the response, + we typically don't want to reward it. This is a general setting for all environments. + + Args: + rewards: List of rewards (can be float or List[float] for per-token rewards) + stop_reasons: List of stop reasons for each trajectory + + Returns: + List[Union[float, List[float]]]: Modified rewards with non-"stop" cases set to 0 + """ + for i, stop_reason in enumerate(stop_reasons): + if stop_reason != "stop": + if isinstance(rewards[i], list): + rewards[i] = [0.0] * len(rewards[i]) + else: + rewards[i] = 0.0 + return rewards + + # ---------------------------------------------------------------------------- + # Three methods of managing chat history and input ids in `agent_loop()` + # ---------------------------------------------------------------------------- + def _update_agent_state_by_retokenizing_chat_history( + self, + agent_loop_state: AgentLoopState, + turn_output: TurnOutput, + ) -> AgentLoopState: + """ + Update the chat history and input ids given a new model response and observation by retokenizing + the entire chat history. Hence token-in-token-out is not followed. + + This method is used when `use_conversation_multi_turn=True` and `custom_chat_template` is set. + It re-tokenizes the entire chat history every turn, which is useful for cases like removing + Qwen3 thinking tokens in non-last-round assistant messages. + + Args: + agent_loop_state: Current agent loop state containing chat history and input IDs + turn_output: Turn output containing the model's response and new observations + + Returns: + AgentLoopState: Updated agent loop state with retokenized chat history and input IDs. + Note: loss_mask, response_end_idx, and rollout_logprobs are set to None as they + are computed at the end with the custom chat template. + """ + assert self.use_conversation_multi_turn and self.custom_chat_template + + agent_loop_state.chat_history = self._update_chat_history( + agent_loop_state.chat_history, turn_output.output, turn_output.new_obs + ) + + # `loss_mask` is computed at the end with `custom_chat_template` + agent_loop_state.loss_mask = None + # untracked state + agent_loop_state.response_end_idx = None + # `logprobs` are not computed because retokenizing breaks token-in-token-out + agent_loop_state.rollout_logprobs = None + return agent_loop_state + + def _update_agent_loop_state_with_multiturn_chat_template( + self, + agent_loop_state: AgentLoopState, + turn_output: TurnOutput, + ) -> AgentLoopState: + """ + Update the loss mask and input ids given a new model response and observation, following + token-in-token-out. + + This function is used if `use_conversation_multi_turn` is True. It assumes that the input to the LLM is formatted as a list of messages, with observations + stored in user messages. + + For example (using the Qwen 2.5 chat template), a trajectory for multi-turn generation would look like: + <|im_start|>system + ... + <|im_end|> + <|im_start|>user + question goes here + <|im_end|> + <|im_start|>assistant + turn 1 model response goes here + ... + ... + <|im_end|> + <|im_start|>user + turn 1 env observation goes here + ... + <|im_end|> + ... + + The chat template is applied without tokenization before and after the chat history is appended to + in order to get new token ids in the chat template format (but without re-tokenizing the entire chat history every turn). + + Args: + agent_loop_state: Current agent loop state containing chat history, input IDs, loss mask, etc. + turn_output: Turn output containing the model's response, output IDs, logprobs, and observations + + Returns: + AgentLoopState: Updated agent loop state with appended turn IDs, loss mask, and logprobs. + For step-wise training, only response_end_idx is updated; loss_mask and rollout_logprobs + are set to None as they are tracked per-step. + """ + agent_loop_state.chat_history = self._update_chat_history( + agent_loop_state.chat_history, turn_output.output, turn_output.new_obs + ) + + loss_mask_for_turn = turn_output.get_turn_loss_mask() + rollout_logprobs_for_turn = turn_output.get_turn_rollout_logprobs() + + if self.generator_cfg.step_wise_trajectories: + # cumulative input_ids is not tracked for step wise training + agent_loop_state.response_end_idx = len(turn_output.output_ids) - 1 + # no running loss_mask or `rollout_logprobs` are tracked for step-wise training + agent_loop_state.loss_mask = None + agent_loop_state.rollout_logprobs = None + else: + # Directly append turn output + turn_ids = turn_output.output_ids + turn_output.obs_ids + agent_loop_state.response_end_idx = len(agent_loop_state.input_ids) + len(turn_output.output_ids) - 1 + agent_loop_state.input_ids += turn_ids + agent_loop_state.loss_mask += loss_mask_for_turn + if agent_loop_state.rollout_logprobs is not None and rollout_logprobs_for_turn is not None: + agent_loop_state.rollout_logprobs += rollout_logprobs_for_turn + + return agent_loop_state + + def _update_agent_loop_state_with_singleturn_chat_template( + self, + agent_loop_state: AgentLoopState, + turn_output: TurnOutput, + ) -> AgentLoopState: + """ + Update the loss mask and input ids given a new model response and observation, following + token-in-token-out. + + This function is used if `use_conversation_multi_turn` is False. It assumes that the input to the LLM is a list of token ids + and that the multi-turn conversation happens in a single assistant message. + + For example (using the Qwen 2.5 chat template), a trajectory for single-turn generation would look like: + <|im_start|>system + ... + <|im_end|> + <|im_start|>user + question goes here + <|im_end|> + <|im_start|>assistant + turn 1 model response goes here + ... + ... + + turn 1 env observation goes here + ... + + turn 2 model response goes here: + ... + ... + + Args: + agent_loop_state: Current agent loop state containing chat history, input IDs, loss mask, etc. + turn_output: Turn output containing the model's response, output IDs, logprobs, and observations + + Returns: + AgentLoopState: Updated agent loop state with appended turn IDs, loss mask, and logprobs. + The EOS token is removed from response tokens (if present) since we are continuing + the current assistant message. Observations are encoded directly without chat template formatting. + """ + agent_loop_state.chat_history = self._update_chat_history( + agent_loop_state.chat_history, turn_output.output, turn_output.new_obs + ) + + obs_ids_to_add = turn_output.obs_ids + + # Remove EOS token from response tokens since we are continuing the current assistant message + new_resp_tokens = turn_output.output_ids.copy() + if new_resp_tokens and new_resp_tokens[-1] == self.tokenizer.eos_token_id: + new_resp_tokens = new_resp_tokens[:-1] + + turn_ids = new_resp_tokens + obs_ids_to_add + loss_mask_for_turn = [1] * len(new_resp_tokens) + [0] * len(obs_ids_to_add) + rollout_logprobs_for_turn = None + if turn_output.output_logprobs is not None: + # For response tokens, use actual logprobs + # for obs tokens, use dummy values + rollout_logprobs_for_turn = turn_output.output_logprobs[: len(new_resp_tokens)] + [0.0] * len( + obs_ids_to_add + ) + + # Directly append turn output + agent_loop_state.response_end_idx = len(agent_loop_state.input_ids) + len(new_resp_tokens) - 1 + agent_loop_state.input_ids += turn_ids + agent_loop_state.loss_mask += loss_mask_for_turn + if agent_loop_state.rollout_logprobs is not None and rollout_logprobs_for_turn is not None: + agent_loop_state.rollout_logprobs += rollout_logprobs_for_turn + + return agent_loop_state diff --git a/skyrl-train/skyrl_train/generators/utils.py b/skyrl-train/skyrl_train/generators/utils.py new file mode 100644 index 0000000000..3ba27f6954 --- /dev/null +++ b/skyrl-train/skyrl_train/generators/utils.py @@ -0,0 +1,543 @@ +import os +import torch +from typing import List, Tuple, Union, Optional, Dict, Any +from collections import defaultdict +import numpy as np +from skyrl_train.generators.base import ( + GeneratorOutput, + GeneratorInput, + TrajectoryID, + BatchMetadata, + TrainingPhase, + MetricsOutput, +) +from skyrl_train.inference_engines.base import ConversationType +from omegaconf import DictConfig +from loguru import logger +from skyrl_gym.metrics import aggregate_for_environment + + +def _validate_template_file_path(file_path: str) -> str: + """ + Validate and sanitize a template file path. + + NOTE(Charlie): this is vibe coded to address comment https://github.com/NovaSky-AI/SkyRL/pull/890#discussion_r2699773416. + Could be too strict. + """ + # Resolve to absolute path first + resolved_path = os.path.abspath(os.path.expanduser(file_path)) + + # Check for path traversal attempts in the original path + # Normalize path separators for consistent checking + normalized_input = os.path.normpath(file_path) + + # Check if the path contains parent directory references that could indicate traversal + # After normpath, legitimate paths won't start with .. but malicious ones trying to escape might + if normalized_input.startswith(".."): + raise ValueError( + f"Invalid template file path '{file_path}': Path traversal sequences are not allowed. " + "Please use an absolute path or a path relative to the current working directory." + ) + + # Additional check: ensure the path doesn't contain null bytes (which could bypass checks) + if "\x00" in file_path: + raise ValueError(f"Invalid template file path '{file_path}': Null bytes are not allowed in paths.") + + # Ensure the resolved path is a regular file (not a directory, symlink to sensitive location, etc.) + if os.path.exists(resolved_path): + if not os.path.isfile(resolved_path): + raise ValueError(f"Invalid template file path '{file_path}': Path must point to a regular file.") + + # Check that the file has a reasonable size (prevent reading very large files) + file_size = os.path.getsize(resolved_path) + max_template_size = 1024 * 1024 # 1MB should be more than enough for any chat template + if file_size > max_template_size: + raise ValueError( + f"Template file '{file_path}' is too large ({file_size} bytes). " + f"Maximum allowed size is {max_template_size} bytes." + ) + + return resolved_path + + +CUSTOM_CHAT_TEMPLATES = { + # chat template for qwen3 that preserves thinking tokens + "qwen3_with_thinking": ( + "{% for message in messages %}" + "{% if (message['role'] != 'assistant') %}" + "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}" + "{% elif (message['role'] == 'assistant')%}" + "{{'<|im_start|>' + message['role'] + '\n'}}" + "{% generation %}" + "{{message['content'] + '<|im_end|>'}}" + "{% endgeneration %}" + "{{'\n'}}" + "{% endif %}" + "{% endfor %}" + ), + # chat template for qwen3 that strips non-last-turn thinking tokens (same as the official Qwen3 chat + # template but we add `generation` and `endgeneration` tags) + "qwen3_without_thinking": ( + "{% for message in messages %}" + "{% if (message['role'] != 'assistant') %}" + "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}" + "{% elif (message['role'] == 'assistant')%}" + "{{'<|im_start|>' + message['role'] + '\n'}}" + "{% generation %}" + "{% set full_content = message['content'] %}" + "{% set mycontent = message['content'] %}" + "{% set is_last_message = loop.last and messages[-1]['role'] == 'assistant' %}" + "{% if '' in full_content and not is_last_message %}" + "{% set mycontent = full_content.split('')[-1].lstrip('\n') %}" + "{% endif %}" + "{{mycontent + '<|im_end|>'}}" + "{% endgeneration %}" + "{{'\n'}}" + "{% endif %}" + "{% endfor %}" + ), +} + + +def get_custom_chat_template(chat_template_config: Optional[Union[dict, DictConfig]] = None) -> Optional[str]: + """ + Get custom chat template based on the new config structure. + + Args: + chat_template_config: Config dict with 'source' and 'name_or_path' fields. + + Returns: + Chat template string or None + """ + if chat_template_config is None: + return None + + source = chat_template_config.get("source") + if not source: + raise ValueError("'source' is required in chat_template_config") + + name_or_path = chat_template_config.get("name_or_path") + if not name_or_path: + return None # if name_or_path is not provided, use the default chat template from the tokenizer + + if source == "name": + if name_or_path in CUSTOM_CHAT_TEMPLATES: + return CUSTOM_CHAT_TEMPLATES[name_or_path] + else: + raise ValueError( + f"Template name '{name_or_path}' not found. Available templates: {list(CUSTOM_CHAT_TEMPLATES.keys())}" + ) + elif source == "file": + # Validate and sanitize the file path to prevent path traversal attacks + validated_path = _validate_template_file_path(name_or_path) + try: + with open(validated_path, "r", encoding="utf-8") as f: + return f.read() + except FileNotFoundError as e: + raise ValueError(f"Template file '{name_or_path}' not found") from e + except OSError as e: + raise ValueError(f"Error reading template file '{name_or_path}': {e}") from e + else: + raise ValueError(f"Invalid source '{source}'. Must be 'name' or 'file'") + + +def get_generation_prompt_ids(tokenizer) -> List[int]: + """ + Helper function to get the generation prompt ids for a given tokenizer. + """ + empty_user = tokenizer.apply_chat_template([{"role": "user", "content": ""}], tokenize=True, return_dict=False) + empty_user_with_generation_prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": ""}], add_generation_prompt=True, tokenize=True, return_dict=False + ) + + generation_prompt_ids = empty_user_with_generation_prompt[len(empty_user) :] + return generation_prompt_ids + + +@torch.no_grad() +def get_metrics_from_generator_output(generator_output: GeneratorOutput, uids: List[str]) -> MetricsOutput: + """ + Get `mean_raw_reward` (or avg_score), `pass_at_n`, and `mean_positive_reward` from generator output. + + The `n` in `pass_at_n` is the number of trajectories we generate for each example. It is + calculated as `len(generator_output["rewards"]) / len(uids)`, where `len(uids)` is the number of + unique examples. + + Rewards can be either per-trajectory or per-token, and metrics are computed correspondingly. + + Note: For per-group metrics (e.g., per-environment), use the shared functions in + skyrl_train.metrics.reward_metrics which provide consistent metric calculation + across SkyRL trainer and Tinker integration. + """ + rewards: Union[List[float], List[List[float]]] = generator_output["rewards"] + if not len(rewards): + raise ValueError(f"`rewards` must be a non-empty list, got {rewards}") + + # TODO: We should make metrics customizable by the environment. + # Map from the example's uid to each trajectory's reward on that same example + uid_to_trajectory_rewards = defaultdict(list) + if isinstance(rewards[0], list): + # Token-level rewards: rewards is List[List[float]] + # For each trajectory, we sum over the token rewards for `mean_raw_reward` computation + mean_raw_reward = float(np.mean([sum(trajectory_rewards) for trajectory_rewards in rewards])) + + # For each trajectory, we sum over the positive token rewards for mean_positive_reward computation + mean_positive_reward = float( + np.mean([sum(max(r, 0) for r in trajectory_rewards) for trajectory_rewards in rewards]) + ) + + # Assume the last token's reward signifies the trajectory's reward for `pass_at_n` computation + for i, cur_trajectory_rewards in enumerate(rewards): + if len(cur_trajectory_rewards) == 0: + raise ValueError("Token-level rewards must be a non-empty list.") + uid_to_trajectory_rewards[uids[i]].append(cur_trajectory_rewards[-1]) + else: + mean_raw_reward = float(np.mean(rewards)) + mean_positive_reward = float(np.mean(np.maximum(rewards, 0.0))) + for i, reward in enumerate(rewards): + uid_to_trajectory_rewards[uids[i]].append(reward) + + # For each trajectory, if the reward is a full success (>= 1.0), then it's a "pass". + # Partial rewards (e.g. 0.3 from partial_reward mode) do not count as a pass. + pass_at_n = sum(1 for v in uid_to_trajectory_rewards.values() if any(r >= 1.0 for r in v)) / len( + uid_to_trajectory_rewards + ) + + return MetricsOutput( + avg_score=mean_raw_reward, + pass_at_n=pass_at_n, + mean_positive_reward=mean_positive_reward, + ) + + +def concatenate_generator_outputs(generator_outputs: List[GeneratorOutput]) -> GeneratorOutput: + """ + Concatenate the generator outputs of multiple batches. + + We only aggregate rollout metrics the can deduced by responses and rewards, but not + those that use `env_metrics` or `env_classes`. + """ + assert len(generator_outputs) > 0 + has_rollout_logprobs = [output.get("rollout_logprobs") is not None for output in generator_outputs] + if any(has_rollout_logprobs) and not all(has_rollout_logprobs): + raise ValueError( + "generator outputs are expected to all have null rollout_logprobs or all non-null, but received a mix" + ) + result: GeneratorOutput = { + "prompt_token_ids": sum([output["prompt_token_ids"] for output in generator_outputs], []), + "response_ids": sum([output["response_ids"] for output in generator_outputs], []), + "rewards": sum([output["rewards"] for output in generator_outputs], []), + "loss_masks": sum([output["loss_masks"] for output in generator_outputs], []), + "stop_reasons": ( + sum([output["stop_reasons"] for output in generator_outputs], []) + if "stop_reasons" in generator_outputs[0] and generator_outputs[0]["stop_reasons"] is not None + else None + ), + "rollout_logprobs": ( + sum([output["rollout_logprobs"] for output in generator_outputs], []) + if generator_outputs[0]["rollout_logprobs"] is not None + else None + ), + } + + # propagate additional keys with list values as-is + additional_keys = [ + key for key in generator_outputs[0] if key not in result and isinstance(generator_outputs[0][key], list) + ] + if len(additional_keys): + logger.info(f"Attempting to concatenate values for additional keys {additional_keys}") + for key in additional_keys: + result[key] = sum([generator_output[key] for generator_output in generator_outputs], []) + + # Re-aggregate rollout metrics + rollout_metrics = get_rollout_metrics(result["response_ids"], result["rewards"]) + result["rollout_metrics"] = rollout_metrics + + # Validate the generator output using the number of prompts + # Import here to avoid circular dependency. + from skyrl_train.utils.trainer_utils import validate_generator_output + + num_prompts = len(result["prompt_token_ids"]) + validate_generator_output(num_prompts, result) + + return result + + +def apply_overlong_filtering( + loss_masks: List[List[int]], + response_ids: List[List[int]], + eos_token_id: int, +) -> List[List[int]]: + """ + Implements DAPO Overlong Filtering: zero-out every token's mask whenever + the response does not end with the eos token id (i.e. truncated). + + Returns: + - The loss masks with tokens zeroed out for truncated responses + """ + assert len(loss_masks) == len(response_ids), "loss_masks and response_ids must have the same length" + return [ + [0] * len(mask) if not response or response[-1] != eos_token_id else mask + for mask, response in zip(loss_masks, response_ids) + ] + + +def get_rollout_metrics( + responses: List[List[int]], + rewards: Union[List[float], List[List[float]]], + env_metrics: Optional[List[Dict[str, Any]]] = None, + env_classes: Optional[List[str]] = None, +): + """ + Computes rollout metrics including token statistics and optional environment-specific metrics. + + Args: + responses: List of token ID sequences for each response + rewards: List of rewards (either per-trajectory or per-token) + env_metrics: Optional list of environment-specific metrics for each trajectory + env_classes: Optional list of environment class names for each trajectory + + Returns: + Dictionary of aggregated metrics + """ + num_tokens_arr = np.array([len(response) for response in responses]) + # Support both response-level and token-level rewards + flat_rewards = [] + for r in rewards: + if isinstance(r, list): + flat_rewards.append(float(np.sum(r))) + else: + flat_rewards.append(float(r)) + flat_rewards_arr = np.array(flat_rewards) + non_zero_rewards_arr = flat_rewards_arr > 0.0 + zero_rewards_arr = flat_rewards_arr == 0.0 + # average tokens for non zero rewards + avg_tokens_non_zero_rewards = ( + np.mean(num_tokens_arr[non_zero_rewards_arr]) if non_zero_rewards_arr.sum() > 0 else np.zeros(1) + ) + # average tokens for zero rewards + avg_tokens_zero_rewards = np.mean(num_tokens_arr[zero_rewards_arr]) if zero_rewards_arr.sum() > 0 else np.zeros(1) + + rollout_metrics = { + "generate/min_num_tokens": np.min(num_tokens_arr).item(), + "generate/max_num_tokens": np.max(num_tokens_arr).item(), + "generate/avg_num_tokens": np.mean(num_tokens_arr).item(), + "generate/std_num_tokens": np.std(num_tokens_arr).item(), + "generate/avg_tokens_non_zero_rewards": avg_tokens_non_zero_rewards.item(), + "generate/avg_tokens_zero_rewards": avg_tokens_zero_rewards.item(), + } + + if env_metrics is not None and env_classes is not None: + env_to_metrics = defaultdict(list) + for i, metrics in enumerate(env_metrics): + env_to_metrics[env_classes[i]].append(metrics) + for env_name, metrics in env_to_metrics.items(): + # Aggregate metrics across all trajectories for the same environment + agg = aggregate_for_environment(env_name, metrics) + for key, value in agg.items(): + rollout_metrics[f"environment/{key}"] = value + + return rollout_metrics + + +def prepare_generator_input( + prompts: List[Any], + n_samples_per_prompt: int, + sampling_params: Dict[str, Any], + default_env_class: str, + training_phase: TrainingPhase, + global_step: int, +) -> Tuple[GeneratorInput, List[str]]: + """Prepares the generator input for training and eval + + Args: + prompts (List[Any]): list of prompts + n_samples_per_prompt (int): how many samples to create per prompt + sampling_params (Dict[str, Any]): sampling parameters + default_env_class (str): env class to use if env class missing from prompts + training_phase (TrainingPhase): training or eval + global_step (int): current global step + + Returns: + Tuple[GeneratorInput, List[str]]: generator input and list of uuids + """ + + all_prompts = [prompt["prompt"] for prompt in prompts for _ in range(n_samples_per_prompt)] + + all_envs = [ + prompt["env_class"] if prompt["env_class"] is not None else default_env_class + for prompt in prompts + for _ in range(n_samples_per_prompt) + ] + + # all the other columns are env_extras + env_extras = [prompt["env_extras"] for prompt in prompts for _ in range(n_samples_per_prompt)] + + # Create TrajectoryID objects - one UID per row, repetition_id for multiple samples + trajectory_ids = [] + uids = [] + for _, prompt in enumerate(prompts): + uid: str = prompt["uid"] + + # Create TrajectoryID for each repetition + for repetition_id in range(n_samples_per_prompt): + trajectory_ids.append(TrajectoryID(instance_id=uid, repetition_id=repetition_id)) + uids.append(uid) + + generator_input: GeneratorInput = { + "prompts": all_prompts, + "env_classes": all_envs, + "env_extras": env_extras, + "sampling_params": sampling_params, + "trajectory_ids": trajectory_ids, + "batch_metadata": BatchMetadata(global_step=global_step, training_phase=training_phase), + } + + return generator_input, uids + + +def encode_messages_subset(messages: ConversationType, tokenizer): + """Encodes a subset of messages from a multi-turn conversation using the fixed base approach. + + This function tokenizes messages as if they are part of a larger conversation, ensuring + no additional default system messages are prepended by the tokenizer's chat template + + The "fixed base approach" works by: + - Creating a dummy base conversation to establish context + - Appending the target messages to this base + - Tokenizing the full conversation and extracting only the tokens for the target messages + + For simple chat templates without complex token splitting behavior, this produces the same + result as directly tokenizing the messages. For templates like Qwen's ChatML format where + a default system prompt can be appended, this ensures correct tokenization. + + In addition, for Qwen3, this function will keep all the thinking tokens from the messages. + + Reference: https://jybsuper.github.io/posts/multiturn_tokenization/#the-breakthrough-fixed-base-approach + + Args: + messages: List of message dicts with 'role' and 'content' keys. Must contain at least + one message. These are assumed to be a subset from a larger conversation. + tokenizer: HuggingFace tokenizer with chat_template support and eos_token_id defined. + + Returns: + List[int]: Token IDs for the given messages, with proper multi-turn context handling. + """ + assert len(messages), "messages list cannot be empty" + # Follows https://jybsuper.github.io/posts/multiturn_tokenization/#the-breakthrough-fixed-base-approach + base_conversation = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "I am a user."}, + ] + base_conversation_token_ids = tokenizer.apply_chat_template( + base_conversation, + add_generation_prompt=False, + tokenize=True, + return_dict=False, + ) + + full_conversation = base_conversation + messages + full_conversation_token_ids = tokenizer.apply_chat_template( + full_conversation, + add_generation_prompt=False, + tokenize=True, + return_dict=False, + ) + conversation_token_ids = full_conversation_token_ids[len(base_conversation_token_ids) :] + return conversation_token_ids + + +def get_response_ids_and_loss_mask_from_messages(messages: ConversationType, tokenizer, assistant_logprobs=None): + """ + Get the response ids and loss mask from a list of messages. + + We encode each message one by one, using a fixed base approach, building response token IDs, loss mask, + and rollout logprobs if provided. For Qwen3, this function will keep all the thinking tokens from the messages. + + Args: + messages: List of message dicts with 'role' and 'content' keys. Must contain at least + one message. + tokenizer: HuggingFace tokenizer with chat_template support and eos_token_id defined. + assistant_logprobs: Optional list of logprobs for each assistant message. In the format of + `[[logprobs for assistant msg 1], [logprobs for assistant msg 2], ...]`. + + Returns: + Tuple[List[int], List[int], Optional[List[float]]]: response ids, loss mask, and rollout logprobs + """ + assert len(messages), "messages list cannot be empty" + + # Needed to correctly mask it zero for assistant messages. + generation_prompt_ids = get_generation_prompt_ids(tokenizer) + + # 1. Initalize the things to accumulate + response_ids = [] + loss_mask = [] + rollout_logprobs = None if assistant_logprobs is None else [] + assistant_msg_idx = 0 + + for i in range(len(messages)): + # 2. Use fixed base approach to encode the message and accumulate + cur_message = messages[i] + cur_token_ids = encode_messages_subset([cur_message], tokenizer) + response_ids.extend(cur_token_ids) + + # 3. Set loss mask and rollout logprobs. + # Regardless of the message role, each message is responsible for adding its own generation + # prompt, and we apply the correct masking. + if cur_message["role"] == "user": + # 3.1. For user messages, it is simply zeros + loss_mask.extend([0] * len(cur_token_ids)) + if assistant_logprobs: + rollout_logprobs.extend([0.0] * len(cur_token_ids)) + elif cur_message["role"] == "assistant": + # 3.2. For assistant messages, we need to separate out: + # 1) generation prompt IDs -- mask is 0 + # 2) tokens actually generated by the assistant (including the EOS) -- mask is 1 + # 3) tokens after the EOS token (the `\n` in Qwen models) -- mask is 0 + assert cur_token_ids[: len(generation_prompt_ids)] == generation_prompt_ids, ( + f"Assistant message tokens should start with generation prompt. " + f"Expected {generation_prompt_ids}, got {cur_token_ids[:len(generation_prompt_ids)]}" + ) + if tokenizer.eos_token_id in cur_token_ids: + last_eos_token_index = len(cur_token_ids) - 1 - cur_token_ids[::-1].index(tokenizer.eos_token_id) + generated_token_ids = cur_token_ids[len(generation_prompt_ids) : last_eos_token_index + 1] + tokens_after_eos = cur_token_ids[last_eos_token_index + 1 :] + else: + generated_token_ids = cur_token_ids[len(generation_prompt_ids) :] + tokens_after_eos = [] + assert len(generation_prompt_ids) + len(generated_token_ids) + len(tokens_after_eos) == len( + cur_token_ids + ), "The sum of the lengths of the generation prompt IDs, the generated tokens, and the tokens after the EOS token should equal the length of the current token IDs" + + # 3.2.1. Add the generation prompt IDs. + loss_mask.extend([0] * len(generation_prompt_ids)) + if assistant_logprobs: + rollout_logprobs.extend([0.0] * len(generation_prompt_ids)) + + # 3.2.2. Add what the assistant actually generated + loss_mask.extend([1] * len(generated_token_ids)) + if assistant_logprobs: + if assistant_msg_idx >= len(assistant_logprobs): + raise ValueError( + f"Missing logprobs for assistant message #{assistant_msg_idx + 1}. Provided {len(assistant_logprobs)} logprob lists." + ) + msg_logprobs = assistant_logprobs[assistant_msg_idx] + if len(msg_logprobs) != len(generated_token_ids): + raise ValueError( + f"Logprobs count ({len(msg_logprobs)}) does not match token count ({len(generated_token_ids)}) for assistant message #{assistant_msg_idx + 1}." + ) + rollout_logprobs.extend(msg_logprobs) + + # 3.2.3. Add the tokens after the EOS token. + loss_mask.extend([0] * len(tokens_after_eos)) + if assistant_logprobs: + rollout_logprobs.extend([0.0] * len(tokens_after_eos)) + + assistant_msg_idx += 1 + else: + raise ValueError(f"Expected message role to be 'user' or 'assistant', got {cur_message['role']}") + + assert len(loss_mask) == len(response_ids) + assert len(rollout_logprobs) == len(response_ids) if rollout_logprobs is not None else True + + return response_ids, loss_mask, rollout_logprobs diff --git a/skyrl-train/skyrl_train/inference_engines/__init__.py b/skyrl-train/skyrl_train/inference_engines/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/skyrl_train/inference_engines/base.py b/skyrl-train/skyrl_train/inference_engines/base.py new file mode 100644 index 0000000000..392e2100eb --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/base.py @@ -0,0 +1,116 @@ +from abc import ABC, abstractmethod +from typing import List, Dict, TypedDict, Any, Optional, Hashable, TYPE_CHECKING + +if TYPE_CHECKING: + from skyrl_train.weight_sync.transfer_strategy import WeightSyncInitInfo + from skyrl_train.weight_sync import WeightUpdateRequest + +MessageType = Dict[str, str] +ConversationType = List[MessageType] + + +class InferenceEngineInput(TypedDict): + # Either prompts or prompt_token_ids must be provided, but not both. + prompts: Optional[List[ConversationType]] + prompt_token_ids: Optional[List[List[int]]] + sampling_params: Optional[Dict[str, Any]] + session_ids: Optional[List[Hashable]] + + +class InferenceEngineOutput(TypedDict): + # We always return both tokens and text outputs. The tokens are the outputs + # of inference engine, and the text is the decoded text output. Therefore, + # it is guaranteed that tokenizer.decode(response_token_ids, skip_special_tokens=True) == responses, + # but the reverse is not guaranteed, since there are multiple ways to + # represent the same text with tokens. Therefore, for multi-turn generation, + # please use token-in-token-out to ensure correctness. + # `skip_special_tokens=True` is needed because string responses do not include EOS tokens like `<|im_end|>` + responses: List[str] + response_ids: List[List[int]] + stop_reasons: List[str] + response_logprobs: Optional[List[List[float]]] + + +class InferenceEngineInterface(ABC): + + @abstractmethod + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + raise NotImplementedError + + @abstractmethod + async def chat_completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + """Handles OpenAI-compatible HTTP endpoint. + + Accepts a JSON payload: {"json": , "headers": }. + The request body will be used to construct a ChatCompletionRequest. + Returns a plain dict, either a ChatCompletionResponse or an ErrorResponse. + The specific fields of the response/request depend on the engine's backend (e.g. for vllm + these are defined in vllm.entrypoints.openai.protocol). + """ + raise NotImplementedError + + @abstractmethod + async def completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + """Handles OpenAI-compatible HTTP endpoint. + + Accepts a JSON payload: {"json": , "headers": }. + The request body will be used to construct a CompletionRequest. + Returns a plain dict, either a CompletionResponse or an ErrorResponse. + The specific fields of the response/request depend on the engine's backend (e.g. for vllm + these are defined in vllm.entrypoints.openai.protocol). + """ + raise NotImplementedError + + @abstractmethod + async def wake_up(self, *args: Any, **kwargs: Any): + raise NotImplementedError + + @abstractmethod + async def sleep(self, *args: Any, **kwargs: Any): + raise NotImplementedError + + @abstractmethod + async def init_weight_update_communicator(self, init_info: "WeightSyncInitInfo"): + """Initialize weight update communicator from init info. + + Args: + init_info: WeightSyncInitInfo from the sender containing all info needed + to create the appropriate receiver. + """ + raise NotImplementedError() + + @abstractmethod + async def update_named_weights(self, request: "WeightUpdateRequest"): + raise NotImplementedError() + + @abstractmethod + async def teardown(self): + raise NotImplementedError + + @abstractmethod + async def reset_prefix_cache(self): + raise NotImplementedError + + @abstractmethod + def tp_size(self) -> int: + """Return the tensor parallel size of this inference engine.""" + raise NotImplementedError + + @abstractmethod + def pp_size(self) -> int: + """Return the pipeline parallel size of this inference engine.""" + raise NotImplementedError + + @abstractmethod + def dp_size(self) -> int: + """Return the data parallel size of this inference engine.""" + raise NotImplementedError + + @abstractmethod + async def abort_generation(self) -> None: + """ + Abort all running and waiting requests, which make the ongoing requests return the + already-generated tokens with a stop_reason of "abort". If the request was waiting, + it returns a response with zero completion tokens. + """ + raise NotImplementedError diff --git a/skyrl-train/skyrl_train/inference_engines/inference_engine_client.py b/skyrl-train/skyrl_train/inference_engines/inference_engine_client.py new file mode 100644 index 0000000000..ae4ffc2510 --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/inference_engine_client.py @@ -0,0 +1,754 @@ +from __future__ import annotations + +from skyrl_train.inference_engines.base import ( + InferenceEngineInterface, + InferenceEngineInput, + InferenceEngineOutput, +) +from skyrl_train.inference_engines.inference_engine_client_http_endpoint import ErrorResponse, ErrorInfo +from transformers import PreTrainedTokenizerBase +import asyncio +from typing import List, Any, Optional, Dict, Union, TYPE_CHECKING +from skyrl_train.inference_engines.utils import ( + route_prompts_to_engines, + hash_with_sha256, + postprocess_completion_request, + aggregate_completion_usage_info, +) +from omegaconf import DictConfig +import threading +from loguru import logger +import random +from dataclasses import dataclass, field + +if TYPE_CHECKING: + from skyrl_train.weight_sync import WeightUpdateRequest + from skyrl_train.weight_sync.transfer_strategy import WeightSyncInitInfo + +ABORT_GENERATION_GRACE_PERIOD_SECONDS = 5 + + +class InferenceEngineClient(InferenceEngineInterface): + """ + Client to talk to a set of InferenceEngines. + + Note that InferenceEngineClient sub-classes InferenceEngineInterface so it can be used as if talking to a single + engine. + """ + + def __init__( + self, engines: List[InferenceEngineInterface], tokenizer: PreTrainedTokenizerBase, full_config: DictConfig + ): + """ + Args: + engines: List[InferenceEngineInterface] - The inference engines, remote or local. + tokenizer: PreTrainedTokenizerBase - The tokenizer to use. + full_config: DictConfig - See ppo_base_config.yaml + """ + self.engines = engines + self.tokenizer = tokenizer + self.model_name = full_config.trainer.policy.model.path + self.backend = full_config.generator.backend + self.enable_http_endpoint = full_config.generator.enable_http_endpoint + self.http_endpoint_host = full_config.generator.http_endpoint_host + self.http_endpoint_port = full_config.generator.http_endpoint_port + self.generation_paused_event = threading.Event() + if self.enable_http_endpoint: + self._spin_up_http_endpoint() + + logger.info(f"InferenceEngineClient initialized with {len(engines)} engines.") + + async def _run_on_all_engines(self, method_name: str, *args, **kwargs): + """ + Call a method on all engines concurrently and gather the results. + """ + assert len(self.engines) > 0, "No engines to call method on" + + awaitables = [getattr(engine, method_name)(*args, **kwargs) for engine in self.engines] + return await asyncio.gather(*awaitables) + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + # 0. Extract input + prompts = input_batch.get("prompts") + prompt_token_ids = input_batch.get("prompt_token_ids") + session_ids = input_batch.get("session_ids") + sampling_params = input_batch.get("sampling_params") + + if (prompts is None and prompt_token_ids is None) or (prompts is not None and prompt_token_ids is not None): + raise ValueError("Either `prompts` or `prompt_token_ids` must be provided, but not both.") + if prompt_token_ids is None: + prompt_token_ids = self.tokenizer.apply_chat_template( + prompts, + add_generation_prompt=True, + add_special_tokens=False, + return_dict=True, + tokenize=True, + )["input_ids"] + + num_prompts = len(prompt_token_ids) + num_inference_engines = len(self.engines) + + # 1. Route prompts to engines + engine_idx_to_prompt_ids: dict[int, list[int]] = route_prompts_to_engines( + num_prompts=num_prompts, + num_inference_engines=num_inference_engines, + session_ids=session_ids, + ) + + # We do a shortcut for non-batched requests, which can support pause/continue generation for + # in-flight weight updates. + if num_prompts == 1: + # Route to a single engine for this single prompt and use retry flow. + assert len(engine_idx_to_prompt_ids) == 1 + ((engine_idx, prompt_ids_list),) = engine_idx_to_prompt_ids.items() + assert prompt_ids_list == [0], "Single prompt should map to index [0]" + original_prompt_ids = prompt_token_ids[0] + return await self._generate_single_with_retry( + engine_idx=engine_idx, + original_prompt_ids=original_prompt_ids, + sampling_params=sampling_params, + ) + + # For batched generate(), pause/continue cannot be supported. + if self.generation_paused_event.is_set(): + raise RuntimeError("pause_generation is unsupported for batched InferenceEngineClient.generate().") + + # 2. Generate responses concurrently + tasks: list[asyncio.Task] = [] + indices_list: list[list[int]] = [] # the original prompt indices that each task works on + for engine_idx, prompt_ids in engine_idx_to_prompt_ids.items(): + # index prompt_token_ids with prompt_ids + cur_prompt_token_ids = [prompt_token_ids[i] for i in prompt_ids] + engine_input = InferenceEngineInput( + prompt_token_ids=cur_prompt_token_ids, + sampling_params=sampling_params, + ) + tasks.append(asyncio.create_task(self.engines[engine_idx].generate(engine_input))) + indices_list.append(prompt_ids) + + results = await asyncio.gather(*tasks) + + # 3. Reconstruct output in original order + n = len(prompt_token_ids) + responses: list[str] = [""] * n + stop_reasons: list[str] = [""] * n + response_logprobs: List[Optional[List[float]]] = [None for _ in range(n)] + response_ids: List[List[int]] = [[] for _ in range(n)] + # a bit hacky for now + add_resp_logprobs = False + + for indices, result in zip(indices_list, results): + for local_idx, original_idx in enumerate(indices): + responses[original_idx] = result["responses"][local_idx] + stop_reasons[original_idx] = result["stop_reasons"][local_idx] + response_ids[original_idx] = result["response_ids"][local_idx] + if result.get("response_logprobs", None): + add_resp_logprobs = True + response_logprobs[original_idx] = result["response_logprobs"][local_idx] + + return InferenceEngineOutput( + responses=responses, + stop_reasons=stop_reasons, + response_ids=response_ids, + response_logprobs=response_logprobs if add_resp_logprobs else None, + ) + + async def _generate_single_with_retry( + self, engine_idx: int, original_prompt_ids: List[int], sampling_params: Optional[Dict[str, Any]] + ) -> InferenceEngineOutput: + """ + Generate a single response with retry mechanism. + + This method is equivalent to `_chat_completion_with_retry()` but for the `generate()` codepath. + We keep sending `generate` requests (with previous responses accumulated) until the finish_reason + is not "abort". It is intended to be used in combination with `pause_generation()` and `resume_generation()` for + in-flight weight updates and partial rollouts. + + This method is equivalent to a single `generate()` call if we do not use `pause_generation()`. + + Since we operate purely in the token space, it is token-in-token-out, unlike `_chat_completion_with_retry()` + which re-encodes in each new request. + + For subsequent retry requests (`InferenceEngineInput`), we: + - Update the `InferenceEngineInput.prompt_token_ids` with the accumulated output tokens. + - Skip accumulating `InferenceEngineOutput.responses` since we decode the final output. + - Adjust remaining max tokens if `max_tokens` or `max_completion_tokens` is present. + + For the final response, we return `InferenceEngineOutput` with: + - `responses`: decoded at the end from `response_ids` if generation is completed in > 1 turns, otherwise + the text response of the first turn. + - `response_ids`: the accumulated output tokens + - `stop_reasons`: the stop reason of the final response + - `response_logprobs`: the accumulated logprobs + """ + if sampling_params is None: + sampling_params = {} + + # 1. First determine original max tokens key and value (if any) + max_key = None + if "max_tokens" in sampling_params: + max_key = "max_tokens" + elif "max_completion_tokens" in sampling_params: + max_key = "max_completion_tokens" + original_max_tokens: Optional[int] = sampling_params.get(max_key) if max_key else None + + # 2. Initialize fields we want to accumulate or update in each loop iteration + accum_response_ids: List[int] = [] + accum_response_logprobs: List[float] = [] + stop_reason: str = "abort" + + # We only use it if generation is completed in one turn to maintain original behavior with no retry. + text_response: Optional[str] = None + num_turns = 0 + + # 3. Loop until geneartion is completed. + while stop_reason == "abort": + await self._wait_for_generation_to_resume() + + # 3.1. Prepare the request payload. + cur_sampling_params = sampling_params.copy() + if original_max_tokens is not None: + new_max_tokens = original_max_tokens - len(accum_response_ids) + assert new_max_tokens >= 0, f"Expect new_max_tokens to be non-negative, but got {new_max_tokens}" + cur_sampling_params[max_key] = new_max_tokens + new_prompt_ids = original_prompt_ids + accum_response_ids + engine_input = InferenceEngineInput( + prompt_token_ids=[new_prompt_ids], + sampling_params=cur_sampling_params, + ) + + # 3.2. Send the request. + logger.debug(f"generate() request sent (including potential retries): {engine_input}") + partial_response: InferenceEngineOutput = await self.engines[engine_idx].generate(engine_input) + + # 3.3. Parse the partial response. + assert len(partial_response["response_ids"]) == 1, "Expected exactly one response." + new_response_ids: List[int] = partial_response["response_ids"][0] + text_response = partial_response["responses"][0] + stop_reason = partial_response["stop_reasons"][0] + new_response_logprobs: Optional[List[float]] = None + new_response_logprobs_list: Optional[List[List[float]]] = partial_response.get("response_logprobs", None) + if new_response_logprobs_list is not None and len(new_response_logprobs_list) > 0: + new_response_logprobs = new_response_logprobs_list[0] + + # 3.4 Aborted without generating tokens, so partial_response is useless. + if stop_reason == "abort" and len(new_response_ids) == 0: + continue + + # 3.5 Accumulate outputs + accum_response_ids.extend(new_response_ids) + if new_response_logprobs is not None: + accum_response_logprobs.extend(new_response_logprobs) + num_turns += 1 + + # 4. Build the final response and return. + if num_turns == 1: + final_text_response = text_response + else: + final_text_response = self.tokenizer.decode(accum_response_ids, skip_special_tokens=True) + return InferenceEngineOutput( + responses=[final_text_response], + stop_reasons=[stop_reason], + response_ids=[accum_response_ids], + response_logprobs=[accum_response_logprobs] if len(accum_response_logprobs) > 0 else None, + ) + + async def _chat_completion_with_retry( + self, engine_idx: int, original_request_payload: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Keep sending `chat_completion` requests (with previous responses accumulated) until the finish_reason is not + "abort". + + The retry mechanism is intended to be used in combination with `pause_generation()` and `resume_generation()` + for in-flight weight updates and partial rollouts. + + This method is equivalent to a single `chat_completion()` call if we do not use `pause_generation()`. + + For subsequent retry requests, we can reuse the original request with the following exceptions: + - Update the last assistant message content to accumulated content, where the role uses the first non-empty + response's role. + - Set continue_final_message=True and add_generation_prompt=False. + - Adjust remaining max tokens if `max_tokens` or `max_completion_tokens` is present. + - If no tokens have been generated yet, resend the original request unchanged. + + For the final response, we maintain all the first non-empty response's fields (i.e. prefilled already), + with the following exceptions: + - Accumulate the following across retry requests: + - `choices[0]["logprobs"]["content"]` + - `choices[0]["token_ids"]` + - `choices[0]["message"]["content"]` + - Use the last response's finish_reason and stop_reason + """ + original_request_json: Dict[str, Any] = original_request_payload.get("json", {}).copy() + headers: Dict[str, str] = original_request_payload.get("headers", {}).copy() + + assert not original_request_json.get( + "continue_final_message", False + ), "continue_final_message must be False for /chat/completions requests" + + # Accumulated fields for building subsequent requests and final response. It is inplace-updated + # in `_parse_partial_response_and_inplace_update_accum()`. + accum = AccumulatedResponse() + + # First non-empty response (i.e. the response that prefilled the prompt) to copy meta from. + base_response: Optional[Dict[str, Any]] = None + + # Determine original max tokens key and value (if any) + max_key = None + if "max_tokens" in original_request_json: + max_key = "max_tokens" + elif "max_completion_tokens" in original_request_json: + max_key = "max_completion_tokens" + orig_max_tokens: Optional[int] = original_request_json.get(max_key) if max_key else None + + # Fields to be updated in each loop iteration + finish_reason: str = "abort" + stop_reason: Optional[str] = None + response_role: Optional[str] = None + + # 1. Loop until the generation is completed. + while finish_reason == "abort": + await self._wait_for_generation_to_resume() + + # 1.1. Prepare the request payload. + cur_request_json = _prepare_retry_request( + original_request_json=original_request_json, + accum=accum, + response_role=response_role, + orig_max_tokens=orig_max_tokens, + max_key=max_key, + ) + + # 1.2. Send the request. + logger.debug(f"/chat/completions request sent (including potential retries): {cur_request_json}") + partial_response = await self.engines[engine_idx].chat_completion( + {"json": cur_request_json, "headers": headers} + ) + + # 1.3. Parse partial response and in-place update accumulators. + ( + finish_reason, + stop_reason, + response_role, + aborted_without_generating, + ) = _parse_partial_response_and_inplace_update_accum( + partial_response=partial_response, + accum=accum, + response_role=response_role, + ) + + # 1.4. Aborted without generating tokens, so partial_response is useless. + if aborted_without_generating: + continue + + # At this point, either some tokens were generated and/or request completed with a non-"abort" finish_reason + + # 1.5. Update base response if it is the first non-empty response + if base_response is None: + if finish_reason != "abort": + # If we only made one request and it is not aborted, return the partial result directly. + # This is the codepath that will hit when we do not use `pause_generation()` + # or `resume_generation()`. + return partial_response + # NOTE(Charlie): not doing deepcopy here to avoid copying large logprobs, so be careful when + # modifying this. + base_response = partial_response.copy() + + # 2. Build final response by combining fields + assert base_response is not None, "Expected at least one non-empty response to build final response" + return _build_final_response( + base_response=base_response, + accum=accum, + finish_reason=finish_reason, + stop_reason=stop_reason, + ) + + async def chat_completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + session_id = request_payload["json"].pop("session_id", None) + if session_id is None: + # if session_id is not provided, we'll use a random engine + engine_idx = random.randint(0, len(self.engines) - 1) + else: + assert isinstance(session_id, (str, int)), "Session ID must be an integer or string for `/chat/completions`" + engine_idx = hash_with_sha256(str(session_id)) % len(self.engines) + + # Always use the retry loop which also issues the first request inside + return await self._chat_completion_with_retry(engine_idx, request_payload) + + async def completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + """ + Handles an OpenAI /completions request. + + Since `request["prompt"]` can be `Union[list[int], list[list[int]], str, list[str]]`, + (i.e. {batched, single} x {string, token IDs}), we need to route the request to engines + differently, based on whether it's a single or batched request, and whether `request["session_id"]` + is provided. This is similar to `generate()` method. + + For single, we do the same routing logic as `chat_completion()`. For batched, we route by + `request["session_id"]` if present, and if not we split evenly across engines. + + Regardless, the order will be maintained, i.e. `output["choices"][i]` corresponds to `request["prompt"][i]`. + """ + if self.generation_paused_event.is_set(): + raise RuntimeError("pause_generation is unsupported for /completions requests.") + body = request_payload.get("json", {}) + + # NOTE(Charlie): do not reuse headers here as the single request may become various new requests + headers = {"Content-Type": "application/json"} + + # 1. Postprocess prompt, session_id, and validate request. + prompt = body.get("prompt") + session_id_value = body.pop("session_id", None) + ret = postprocess_completion_request(prompt, session_id_value) + session_id_list: Optional[Union[List[int], List[str], ErrorResponse]] = ret[0] + prompt: Union[List[List[int]], List[str]] = ret[1] + if isinstance(session_id_list, ErrorResponse): + return session_id_list.model_dump() + + num_prompts = len(prompt) + num_inference_engines = len(self.engines) + assert num_prompts > 0, "Number of prompts must be greater than 0" + + # 1. Route prompts to engines + engine_idx_to_prompt_ids: dict[int, list[int]] = route_prompts_to_engines( + num_prompts=num_prompts, + num_inference_engines=num_inference_engines, + session_ids=session_id_list, + ) + + # 2. Generate responses concurrently + tasks: list[asyncio.Task] = [] + indices_list: list[list[int]] = [] # the original prompt indices that each task works on + for engine_idx, prompt_ids in engine_idx_to_prompt_ids.items(): + cur_prompt = [prompt[i] for i in prompt_ids] + # reuse the exact same request except for the prompt + cur_json = dict(body) + cur_json["prompt"] = cur_prompt + coro = self.engines[engine_idx].completion({"json": cur_json, "headers": headers}) + tasks.append(asyncio.create_task(coro)) + indices_list.append(prompt_ids) + + results = await asyncio.gather(*tasks) + + # 3. Check for errors. + # results can be ErrorResponse or CompletionResponse. If one of the sub-requests fails, we + # return an error response. That is, there is no partial success, following vLLM and SGLang's behavior. + for result in results: + if "error" in result or result.get("object", "") == "error": + # former is vllm format, latter is sglang format + error_details = result.get("error", result) # resolves vllm/sglang format difference + error_code = error_details["code"] + error_type = error_details["type"] + error_message = error_details["message"] + return ErrorResponse( + error=ErrorInfo( + message=f"In one of the engines that SkyRL manages, an error occurred: {error_message}", + type=error_type, + code=error_code, + ), + ).model_dump() + + # 4. Combine choices and preserve original order. + # If there is only one result, we return it directly. + if len(results) == 1: + return results[0] + + # Use the first result as base response. There are some fields that cannot be shared + # across sub-requests. For now it is just the usage field. + final_response = dict(results[0]) + final_response["usage"] = aggregate_completion_usage_info(results, self.backend) + + # Aggregate choices. TODO(Charlie): improve logic when we need to support n > 1 + # vLLM sets index positions per sub-batch, so we reset indices to be 0..n-1 for the combined response. + combined_choices: list[Dict[str, Any]] = [None] * num_prompts + for indices, result in zip(indices_list, results): + # indices are the original prompt indices that the task's response corresponds to + for local_idx, original_idx in enumerate(indices): + choice = result["choices"][local_idx] + choice["index"] = original_idx # overwrite index with the global position + combined_choices[original_idx] = choice + + # sanity check that the index is correct + for new_idx in range(len(combined_choices)): + assert combined_choices[new_idx]["index"] == new_idx + + final_response["choices"] = combined_choices + return final_response + + async def wake_up(self, *args: Any, **kwargs: Any): + return await self._run_on_all_engines("wake_up", *args, **kwargs) + + async def sleep(self, *args: Any, **kwargs: Any): + return await self._run_on_all_engines("sleep", *args, **kwargs) + + async def init_weight_update_communicator(self, init_info: "WeightSyncInitInfo"): + """Initialize weight update communicator on all engines. + + Args: + init_info: WeightSyncInitInfo from the sender. + + Note: + Per-engine adjustments (e.g., rank_offset for broadcast) are handled + by init_info.for_engine(). + """ + tasks = [] + for i, engine in enumerate(self.engines): + engine_init_info = init_info.for_engine(i, engine.tp_size(), engine.pp_size()) + tasks.append(engine.init_weight_update_communicator(engine_init_info)) + await asyncio.gather(*tasks) + + async def update_named_weights(self, request: WeightUpdateRequest): + return await self._run_on_all_engines("update_named_weights", request=request) + + async def reset_prefix_cache(self): + return await self._run_on_all_engines("reset_prefix_cache") + + async def teardown(self): + return await self._run_on_all_engines("teardown") + + def tp_size(self) -> int: + raise NotImplementedError("InferenceEngineClient does not implement tp_size()") + + def pp_size(self) -> int: + raise NotImplementedError("InferenceEngineClient does not implement pp_size()") + + def dp_size(self) -> int: + raise NotImplementedError("InferenceEngineClient does not implement dp_size()") + + # ---------------------------- + # Generation pause and resume + # ---------------------------- + async def _wait_for_generation_to_resume(self) -> None: + """Waits for generation to be resumed, intended for in-flight weight updates and partial rollouts.""" + while self.generation_paused_event.is_set(): + await asyncio.sleep(0.5) + + async def pause_generation(self) -> None: + """ + Pauses generation for all engines, intended for in-flight weight updates and partial rollouts. + + Currently only supported for `/chat/completions` and not `/completions` or `generate()`. + + Both in-flight and incoming requests will be blocked until `resume_generation` is called. + 1. Set the paused event to avoid new requests from being submitted while aborting requests. + 2. Wait for a grace period to ensure all in-flight requests have entered the engine's + scheduler and hence can be aborted. Otherwise, there can be requests already submitted + but not yet entered the scheduler, which can miss the abort request. + 3. Finally, we abort requests on all engines. This will cause the requests sent from + InferenceEngineClient to `InferenceEngineClient.engines` to return the already-generated tokens. + The request to `InferenceEngineClient` will not yet return until requests are completed with + stop reason that is not `abort`. + """ + if self.generation_paused_event.is_set(): + raise RuntimeError("Generation is already paused, cannot pause again.") + self.generation_paused_event.set() + await asyncio.sleep(ABORT_GENERATION_GRACE_PERIOD_SECONDS) + await self._run_on_all_engines("abort_generation") + + async def resume_generation(self) -> None: + """ + Resumes generation for all engines, intended for in-flight weight updates and partial rollouts. + + Resume all in-flight requests with the previously-generated tokens, and unblock incoming requests + that were blocked by `pause_generation()`. + """ + if not self.generation_paused_event.is_set(): + raise RuntimeError("Generation is not paused, cannot resume.") + self.generation_paused_event.clear() + + async def abort_generation(self) -> None: + raise NotImplementedError( + "InferenceEngineClient does not implement abort_generation(), but calls " + "`abort_generation` on all engines in `pause_generation()`." + ) + + # ---------------------------- + # HTTP endpoint related methods + # ---------------------------- + + def __del__(self): + """ + Destructor to shut down the HTTP endpoint if it was started. + """ + # TODO(Charlie): __del__ is not guaranteed to be called in general. Add to `teardown` method + # when the `_handle_termination` flow is implemented. See `skyrl_train/workers/worker.py` + # comments on `_handle_termination` for more details. + if ( + self.enable_http_endpoint + and hasattr( + self, "_server_thread" + ) # don't want to shut down the server when it is pickled as a ray method argument. + and self._server_thread is not None + ): + try: + from skyrl_train.inference_engines.inference_engine_client_http_endpoint import shutdown_server + + shutdown_server( + host=self.http_endpoint_host, + port=self.http_endpoint_port, + max_wait_seconds=10, + ) + if hasattr(self, "_server_thread") and self._server_thread.is_alive(): + self._server_thread.join(timeout=10) + except Exception as e: + logger.error(f"Error shutting down HTTP endpoint: {e}") + + def __getstate__(self): + """ + Override to avoid pickling the server thread and the threading.Event object, which are not picklable. + Needed when passing InferenceEngineClient as an argument to async_run_ray_method(), mainly for + invoking `init_weight_sync_state()` and `broadcast_to_inference_engines()`, which do + not need these attributes. + """ + state = self.__dict__.copy() + state["_server_thread"] = None + state["generation_paused_event"] = None + return state + + def _spin_up_http_endpoint(self): + from skyrl_train.inference_engines.inference_engine_client_http_endpoint import ( + serve, + wait_for_server_ready, + ) + + self._server_thread = threading.Thread( + target=serve, + args=(self,), + kwargs={ + "host": self.http_endpoint_host, + "port": self.http_endpoint_port, + "log_level": "warning", + }, + daemon=True, + ) + self._server_thread.start() + wait_for_server_ready( + host=self.http_endpoint_host, + port=self.http_endpoint_port, + max_wait_seconds=30, + ) + logger.info( + f"InferenceEngineClient HTTP endpoint started on {self.http_endpoint_host}:{self.http_endpoint_port}" + ) + + +# ---------------------------------------------- +# Helper methods for _chat_completion_with_retry +# ---------------------------------------------- + + +@dataclass +class AccumulatedResponse: + content: str = "" + logprobs_content: List[Any] = field(default_factory=list) + token_ids: List[int] = field(default_factory=list) + completion_tokens: int = 0 + + +def _prepare_retry_request( + original_request_json: Dict[str, Any], + accum: AccumulatedResponse, + response_role: Optional[str], + orig_max_tokens: Optional[int], + max_key: Optional[str], +) -> Dict[str, Any]: + """Build the per-iteration request payload. + + If no tokens have been generated yet, resend the original request unchanged. + Otherwise, build a continuation request that appends the accumulated content + and adjusts remaining max tokens if present. + """ + if accum.completion_tokens == 0: + return original_request_json.copy() + + assert accum.content != "", "accum.content must be non-empty for a continuation request" + assert response_role is not None, "response_role must be set for a continuation request" + + cur_request_json = original_request_json.copy() + cur_request_json["messages"] = original_request_json["messages"] + [ + {"role": response_role, "content": accum.content} + ] + cur_request_json["continue_final_message"] = True + cur_request_json["add_generation_prompt"] = False + if orig_max_tokens is not None: + assert ( + orig_max_tokens - accum.completion_tokens >= 0 + ), "orig_max_tokens - accum.completion_tokens must be non-negative" + assert max_key is not None + cur_request_json[max_key] = orig_max_tokens - accum.completion_tokens + + return cur_request_json + + +def _parse_partial_response_and_inplace_update_accum( + partial_response: Dict[str, Any], + accum: AccumulatedResponse, + response_role: Optional[str], +) -> tuple[str, Optional[str], Optional[str], bool]: + """Parse the partial response and in-place update accumulators. + + Returns (finish_reason, stop_reason, response_role, aborted_without_generating). + """ + choice = partial_response["choices"][0] + finish_reason: str = choice["finish_reason"] + stop_reason: Optional[str] = choice.get("stop_reason", None) + new_content: str = choice["message"]["content"] or "" + + assert ( + partial_response["usage"] is not None and partial_response["usage"]["completion_tokens"] is not None + ), "partial_response['usage']['completion_tokens'] must be present" + new_completion_tokens: int = partial_response["usage"]["completion_tokens"] + + if response_role is None: + response_role = choice["message"]["role"] + else: + assert response_role == choice["message"]["role"], "response_role must be the same across retries" + + # If aborted without generating tokens, ignore this partial response. + aborted_without_generating = finish_reason == "abort" and new_completion_tokens == 0 + if not aborted_without_generating: + accum.content += new_content + logprobs = choice.get("logprobs") + if logprobs is not None and logprobs.get("content") is not None: + accum.logprobs_content.extend(logprobs["content"]) + if choice.get("token_ids") is not None: + accum.token_ids.extend(choice["token_ids"]) + accum.completion_tokens += new_completion_tokens + + return finish_reason, stop_reason, response_role, aborted_without_generating + + +def _build_final_response( + base_response: Dict[str, Any], + accum: AccumulatedResponse, + finish_reason: str, + stop_reason: Optional[str], +) -> Dict[str, Any]: + """Construct the final aggregated response from the base and accumulators.""" + # NOTE(Charlie): not doing deepcopy for performance. Be careful when re-using this method + # as it mutates base_response. + final_response = base_response + + # Combine usage: prompt_tokens from base, completion_tokens summed, total_tokens accordingly + base_usage = final_response["usage"] + prompt_tokens = base_usage["prompt_tokens"] + final_usage = base_usage.copy() + final_usage["completion_tokens"] = accum.completion_tokens + final_usage["total_tokens"] = prompt_tokens + accum.completion_tokens + final_response["usage"] = final_usage + + # Set accumulated content, logprobs, token_ids. + final_choice = final_response["choices"][0] + final_choice["message"]["content"] = accum.content + if final_choice.get("logprobs", None) is not None: + final_choice["logprobs"]["content"] = accum.logprobs_content + if final_choice.get("token_ids", None) is not None: + final_choice["token_ids"] = accum.token_ids + + # Set last response's finish_reason and stop_reason. + final_choice["finish_reason"] = finish_reason + if stop_reason is not None: + final_choice["stop_reason"] = stop_reason + + return final_response diff --git a/skyrl-train/skyrl_train/inference_engines/inference_engine_client_http_endpoint.py b/skyrl-train/skyrl_train/inference_engines/inference_engine_client_http_endpoint.py new file mode 100644 index 0000000000..9bd3535cf5 --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/inference_engine_client_http_endpoint.py @@ -0,0 +1,362 @@ +""" +OpenAI-compatible HTTP endpoint using InferenceEngineClient as backend. + +This module provides a FastAPI-based HTTP endpoint that exposes OpenAI's chat completion API +while routing requests to our internal InferenceEngineClient system. + +Main functions: +- serve(): Start the HTTP endpoint. +- wait_for_server_ready(): Wait for server to be ready. +- shutdown_server(): Shutdown the server. +""" + +import asyncio +import json +import logging +import time +import requests +import traceback +from contextlib import asynccontextmanager +from http import HTTPStatus +from typing import Optional, TYPE_CHECKING, Dict, Any + +import fastapi +import uvicorn +from fastapi import Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from pydantic import BaseModel + + +if TYPE_CHECKING: + from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient + +logger = logging.getLogger(__name__) + +# Global state to hold the inference engine client and backend +_global_inference_engine_client: Optional["InferenceEngineClient"] = None +_global_uvicorn_server: Optional[uvicorn.Server] = None + + +# Adapted from vllm.entrypoints.openai.protocol.ErrorResponse +class ErrorInfo(BaseModel): + message: str + type: str + param: Optional[str] = None + code: int + + +class ErrorResponse(BaseModel): + error: ErrorInfo + + +def set_global_state(inference_engine_client: "InferenceEngineClient", uvicorn_server: uvicorn.Server): + """Set the global inference engine client.""" + global _global_inference_engine_client + global _global_uvicorn_server + _global_inference_engine_client = inference_engine_client + _global_uvicorn_server = uvicorn_server + + +def _validate_openai_request(request_json: Dict[str, Any], endpoint: str) -> Optional[ErrorResponse]: + """Common validation for /chat/completions and /completions endpoints.""" + assert endpoint in ["/completions", "/chat/completions"] + + if _global_inference_engine_client is None: + return ErrorResponse( + error=ErrorInfo( + message="Inference engine client not initialized", + type=HTTPStatus.INTERNAL_SERVER_ERROR.phrase, + code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + ), + ) + if "model" not in request_json: + return ErrorResponse( + error=ErrorInfo( + message=f"The field `model` is required in your `{endpoint}` request.", + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ) + if _global_inference_engine_client.model_name != request_json["model"]: + # TODO(Charlie): add a config similar to vllm's `served_model_name`. + # See https://github.com/NovaSky-AI/SkyRL/pull/238#discussion_r2326561295 + return ErrorResponse( + error=ErrorInfo( + message=f"Model name mismatch: loaded model name {_global_inference_engine_client.model_name} != model name in request {request_json['model']}", + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ) + if request_json.get("stream", False): + return ErrorResponse( + error=ErrorInfo( + message="Streaming is not supported in SkyRL yet, please set stream to False.", + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ) + if endpoint == "/completions" and "n" in request_json and request_json["n"] > 1: + # TODO(Charlie): this constraint can be removed when we leave DP routing to + # inference frameworks. Or we could try to resolve it when needed. + return ErrorResponse( + error=ErrorInfo( + message="n is not supported in SkyRL for /completions request yet, please set n to 1.", + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ) + if endpoint == "/chat/completions" and "messages" not in request_json: + return ErrorResponse( + error=ErrorInfo( + message="The field `messages` is required in your `/chat/completions` request.", + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ) + if endpoint == "/chat/completions" and request_json["messages"] == []: + return ErrorResponse( + error=ErrorInfo( + message="The field `messages` in `/chat/completions` cannot be an empty list.", + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ) + return None + + +async def handle_openai_request(raw_request: Request, endpoint: str) -> JSONResponse: + """Handle /completions or /chat/completions request.""" + assert endpoint in ["/completions", "/chat/completions"] + try: + request_json = await raw_request.json() + + # SkyRL-side validation + error_response = _validate_openai_request(request_json, endpoint=endpoint) + if error_response is not None: + return JSONResponse(content=error_response.model_dump(), status_code=error_response.error.code) + + # Serialize fastapi.Request because it is not pickable, which causes ray methods to fail. + payload = { + "json": request_json, + "headers": dict(raw_request.headers) if hasattr(raw_request, "headers") else {}, + } + if endpoint == "/chat/completions": + response = await _global_inference_engine_client.chat_completion(payload) + else: + response = await _global_inference_engine_client.completion(payload) + + if "error" in response or response.get("object", "") == "error": + # former is vllm format, latter is sglang format + error_code = response["error"]["code"] if "error" in response else response["code"] + return JSONResponse(content=response, status_code=error_code) + else: + return JSONResponse(content=response) + + except json.JSONDecodeError as e: + # To catch possible raw_request.json() errors + error_response = ErrorResponse( + error=ErrorInfo( + message=f"Invalid JSON error: {str(e)}", + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ) + return JSONResponse(content=error_response.model_dump(), status_code=HTTPStatus.BAD_REQUEST.value) + except Exception as e: + error_response = ErrorResponse( + error=ErrorInfo( + message=f"Error when handling {endpoint} request in SkyRL: {str(e)}", + type=HTTPStatus.INTERNAL_SERVER_ERROR.phrase, + code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + ), + ) + return JSONResponse(content=error_response.model_dump(), status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value) + + +def shutdown_server(host: str = "127.0.0.1", port: int = 8000, max_wait_seconds: int = 30) -> None: + """Shutdown the server. + + Args: + host: Server host. + port: Server port. + max_wait_seconds: How long to wait until the server stops listening. + + Raises: + Exception: If the server is still responding after *max_wait_seconds*. + """ + if _global_uvicorn_server is not None: + _global_uvicorn_server.should_exit = True + + health_url = f"http://{host}:{port}/health" + + for i in range(max_wait_seconds): + try: + # If this succeeds, server is still alive + requests.get(health_url, timeout=1) + except requests.exceptions.RequestException: + # A network error / connection refused means server is down. + logger.info(f"Server shut down after {i+1} seconds") + return + time.sleep(1) + + raise Exception(f"Server failed to shut down within {max_wait_seconds} seconds") + + +def wait_for_server_ready(host: str = "127.0.0.1", port: int = 8000, max_wait_seconds: int = 30) -> None: + """ + Wait for the HTTP endpoint to be ready by polling the health endpoint. + + Args: + host: Host where the server is running + port: Port where the server is running + max_wait_seconds: Maximum time to wait in seconds + + Raises: + Exception: If server doesn't become ready within max_wait_seconds + """ + max_retries = max_wait_seconds + health_url = f"http://{host}:{port}/health" + + for i in range(max_retries): + try: + response = requests.get(health_url, timeout=1) + if response.status_code == 200: + logger.info(f"Server ready after {i+1} attempts ({i+1} seconds)") + return + except (requests.exceptions.RequestException, requests.exceptions.ConnectionError): + if i == max_retries - 1: + raise Exception(f"Server failed to start within {max_wait_seconds} seconds") + time.sleep(1) # Wait 1 second between retries + + +def create_app() -> fastapi.FastAPI: + """Create the FastAPI application.""" + + @asynccontextmanager + async def lifespan(app: fastapi.FastAPI): + logger.info("Starting inference HTTP endpoint...") + yield + + app = fastapi.FastAPI( + title="InferenceEngine OpenAI-Compatible API", + description="OpenAI-compatible chat completion API using InferenceEngineClient", + version="1.0.0", + lifespan=lifespan, + ) + + # Add CORS middleware + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + @app.post("/v1/chat/completions") + async def chat_completion(raw_request: Request): + """ + Takes in OpenAI's `ChatCompletionRequest` and returns OpenAI's `ChatCompletionResponse`. + + Note that the specific fields inside the request and response depend on the backend you use. + If `config.generator.backend` is `vllm`, then the request and response will be vLLM's. + Same for SGLang. SkyRL does not perform field validation beyond `model` and `session_id`, + and otherwise depends on the underlying engines' validation. + + Make sure you add in `session_id` (a string or an integer) to ensure load balancing and + sticky routing. The same agentic rollout / session should share the same `session_id` so + they get routed to the same engine for better prefix caching. If unprovided, we will route + to a random engine which is not performant. + + API reference: + - https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html + - https://docs.sglang.ai/basic_usage/openai_api_completions.html + """ + return await handle_openai_request(raw_request, endpoint="/chat/completions") + + @app.post("/v1/completions") + async def completions(raw_request: Request): + """ + Takes in OpenAI's `CompletionRequest` and returns OpenAI's `CompletionResponse`. + + Note that the specific fields inside the request and response depend on the backend you use. + If `config.generator.backend` is `vllm`, then the request and response will be vLLM's. + SkyRL only validates the fields `model` and `session_id`, and otherwise offloads + field validation to the underlying engines. + + Make sure you add in `session_id` to ensure load balancing and sticky routing. Since + `request["prompt"]` can be `Union[list[int], list[list[int]], str, list[str]]`, i.e. + {batched, single} x {string, token IDs}, we follow the following logic for request routing: + - For batched request: `session_id`, if provided, must have the same length as `request["prompt"]` + so that each `request["prompt"][i]` is routed based on `session_id[i]`. + - For single request: `session_id`, if provided, must be a single integer or a singleton + list, where each `session_id` is a string or an integer. + + API reference: + - https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html + - https://docs.sglang.ai/basic_usage/openai_api_completions.html + """ + return await handle_openai_request(raw_request, endpoint="/completions") + + # Health check endpoint + # All inference engine replicas are initialized before creating `InferenceEngineClient`, and thus + # we can start receiving requests as soon as the FastAPI server starts + @app.get("/health") + async def health_check(): + return {"status": "healthy"} + + # This handler only catches unexpected server-side exceptions + @app.exception_handler(Exception) + async def general_exception_handler(request: Request, exc: Exception): + logger.error(f"Unhandled exception: {str(exc)}\n{traceback.format_exc()}") + error_response = ErrorResponse( + error=ErrorInfo( + message=f"Unhandled exception: {str(exc)}", + type=HTTPStatus.INTERNAL_SERVER_ERROR.phrase, + code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + ), + ) + return JSONResponse(content=error_response.model_dump(), status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value) + + return app + + +def serve( + inference_engine_client: "InferenceEngineClient", + host: str = "0.0.0.0", + port: int = 8000, + log_level: str = "info", +): + """ + Start the HTTP endpoint. + + Args: + inference_engine_client: The InferenceEngineClient to use as backend + host: Host to bind to (default: "0.0.0.0") + port: Port to bind to (default: 8000) + log_level: Logging level (default: "info") + """ + # Create app + app = create_app() + + # Configure logging + logging.basicConfig(level=getattr(logging, log_level.upper())) + + logger.info(f"Starting server on {host}:{port}") + + # Run server + config = uvicorn.Config(app, host=host, port=port, log_level=log_level, access_log=True) + server = uvicorn.Server(config) + + # Expose server for external shutdown control (tests) + set_global_state(inference_engine_client, server) + + try: + # Run until shutdown + asyncio.run(server.serve()) + except KeyboardInterrupt: + logger.info("Received keyboard interrupt, shutting down...") + except Exception as e: + logger.error(f"Server error: {e}") + raise diff --git a/skyrl-train/skyrl_train/inference_engines/ray_wrapped_inference_engine.py b/skyrl-train/skyrl_train/inference_engines/ray_wrapped_inference_engine.py new file mode 100644 index 0000000000..617b6842ce --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/ray_wrapped_inference_engine.py @@ -0,0 +1,315 @@ +import ray +from packaging import version +from ray.actor import ActorHandle +from typing import Any, List, Dict, TYPE_CHECKING + +if TYPE_CHECKING: + from skyrl_train.weight_sync.transfer_strategy import WeightSyncInitInfo +from ray.util.placement_group import PlacementGroupSchedulingStrategy, placement_group + +from skyrl_train.inference_engines.base import ( + InferenceEngineInterface, + InferenceEngineInput, + InferenceEngineOutput, +) +from skyrl_train.weight_sync import WeightUpdateRequest +from skyrl_train.inference_engines.utils import get_rendezvous_addr_port + + +class RayWrappedInferenceEngine(InferenceEngineInterface): + """ + A thin wrapper around a Ray ActorHandle to another InferenceEngineInterface. + This class implements the InferenceEngineInterface by delegating calls to the remote actor. + """ + + def __init__(self, inference_engine_actor: ActorHandle): + self.inference_engine_actor = inference_engine_actor + + def tp_size(self): + return ray.get(self.inference_engine_actor.tp_size.remote()) + + def pp_size(self): + return ray.get(self.inference_engine_actor.pp_size.remote()) + + def dp_size(self): + return ray.get(self.inference_engine_actor.dp_size.remote()) + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + return await self.inference_engine_actor.generate.remote(input_batch=input_batch) + + async def wake_up(self, *args: Any, **kwargs: Any): + return await self.inference_engine_actor.wake_up.remote(*args, **kwargs) + + async def sleep(self, *args: Any, **kwargs: Any): + return await self.inference_engine_actor.sleep.remote(*args, **kwargs) + + async def init_weight_update_communicator(self, init_info: "WeightSyncInitInfo"): + return await self.inference_engine_actor.init_weight_update_communicator.remote(init_info) + + async def update_named_weights(self, request: WeightUpdateRequest): + return await self.inference_engine_actor.update_named_weights.remote(request) + + async def teardown(self): + return await self.inference_engine_actor.teardown.remote() + + async def reset_prefix_cache(self): + return await self.inference_engine_actor.reset_prefix_cache.remote() + + async def chat_completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + return await self.inference_engine_actor.chat_completion.remote(request_payload) + + async def completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + return await self.inference_engine_actor.completion.remote(request_payload) + + async def abort_generation(self) -> None: + return await self.inference_engine_actor.abort_generation.remote() + + +def create_ray_wrapped_inference_engines( + num_inference_engines: int, + tensor_parallel_size: int, + model_dtype: str, + pretrain: str, + seed: int, + vllm_v1_disable_multiproc: bool, + enable_prefix_caching: bool, + enforce_eager: bool, + expert_parallel_size: int = 1, + pipeline_parallel_size: int = 1, + data_parallel_size: int = 1, + shared_pg=None, + gpu_memory_utilization=None, + inference_engine_enable_sleep=False, + async_engine=False, + max_num_batched_tokens=8192, + max_num_seqs=1024, + tokenizer=None, + backend="vllm", + sleep_level=2, # we only set to 1 for unit tests that do not explicitly sync weights or for LoRA + enable_lora=False, + max_lora_rank=64, + max_loras=1, + fully_sharded_loras=False, + engine_init_kwargs: Dict[str, Any] = {}, + rope_scaling: Dict[str, Any] = {}, + rope_theta: float | None = None, +) -> List[InferenceEngineInterface]: + """ + Create a list of RayWrappedInferenceEngine instances wrapping Ray actor handles to InferenceEngineInterface + instances. + """ + from skyrl_train.utils import ray_noset_visible_devices, get_all_env_variables, get_ray_pg_ready_with_timeout + from skyrl_train.env_vars import SKYRL_RAY_PG_TIMEOUT_IN_S + + if backend == "vllm": + import vllm + from skyrl_train.inference_engines.vllm.vllm_engine import VLLMRayActor, AsyncVLLMRayActor + + # if a dev version is being used, skip the version check + if "dev" not in vllm.__version__: + assert version.parse(vllm.__version__) >= version.parse("0.8.3"), "SkyRL-Train only supports vLLM >= 0.8.3" + elif backend == "sglang": + # We import SGLang later to avoid importing vllm. See `get_sglang_engine` for more. + pass + else: + raise ValueError(f"Unsupported backend: {backend}") + + inference_engine_actors = [] + noset_visible_devices = ray_noset_visible_devices(ray.get(get_all_env_variables.remote())) + # NOTE: we use the ray backend for tensor parallel size > 1 or pipeline parallel size > 1 + # to explicitly manage resource allocation + # TODO: we should be able to support mp backend by allocating resources at engine level + distributed_executor_backend = "uni" if (tensor_parallel_size == 1 and pipeline_parallel_size == 1) else "ray" + data_parallel_backend = "mp" + use_hybrid_engine = shared_pg is not None + num_gpus_per_actor = int(tensor_parallel_size == 1 and pipeline_parallel_size == 1) + + if use_hybrid_engine and tensor_parallel_size == 1 and pipeline_parallel_size == 1: + # Every worker will use 0.2 GPU, so that we can schedule + # inference and training workers on the same GPUs. + num_gpus_per_actor = 0.2 + + per_engine_gpu_count = tensor_parallel_size * pipeline_parallel_size * data_parallel_size + if not use_hybrid_engine: + # Create a big placement group to ensure that all inference engines are packed + bundles = [{"GPU": 1, "CPU": 1} for _ in range(num_inference_engines * per_engine_gpu_count)] + shared_pg = placement_group(bundles, strategy="PACK") + get_ray_pg_ready_with_timeout(shared_pg, timeout=SKYRL_RAY_PG_TIMEOUT_IN_S) + + for i in range(num_inference_engines): + base_pg_index = i * per_engine_gpu_count + + # Get DP group rendezvous (addr, port) on the same node as DP rank 0 for this engine. + data_parallel_address, data_parallel_rpc_port = get_rendezvous_addr_port(shared_pg, base_pg_index) + + if backend == "vllm": + if async_engine: + actor_class = AsyncVLLMRayActor + else: + actor_class = VLLMRayActor + + lora_kwargs = { + "enable_lora": enable_lora, + "max_lora_rank": max_lora_rank, + "max_loras": max_loras, + "fully_sharded_loras": fully_sharded_loras, + } + + rope_engine_kwargs = {} + # rope_scaling and rope_theta must be passed via hf_overrides.rope_parameters + # for vLLM >= 0.8.3. See: https://docs.vllm.ai/en/latest/examples/offline_inference/context_extension/ + # Convert to regular dict so we can pop keys safely (OmegaConf struct mode doesn't support .pop()) + engine_init_kwargs = dict(engine_init_kwargs) + hf_overrides = dict(engine_init_kwargs.pop("hf_overrides", {}) or {}) + if rope_scaling or rope_theta is not None: + # Convert to regular dict to avoid OmegaConf struct mode issues in vLLM + # vLLM expects rope_parameters, not rope_scaling + rope_parameters = dict(rope_scaling) if rope_scaling else {} + if rope_theta is not None: + rope_parameters["rope_theta"] = rope_theta + hf_overrides["rope_parameters"] = rope_parameters + + if rope_scaling and "max_model_len" not in engine_init_kwargs: + rope_factor = rope_scaling.get("factor", None) + rope_max_pos = rope_scaling.get("original_max_position_embeddings", None) + assert rope_factor is not None, "Please provide rope scaling `factor` to compute model max length" + assert ( + rope_max_pos is not None + ), "Please provide rope `original_max_position_embeddings` to compute model max length" + rope_engine_kwargs["max_model_len"] = int(rope_factor * rope_max_pos) + if hf_overrides: + rope_engine_kwargs["hf_overrides"] = hf_overrides + + # Launch one actor per DP rank + for dp_rank in range(data_parallel_size): + + # Contiguous TP*PP slice reserved for a single DP rank. + tp_pp_size = tensor_parallel_size * pipeline_parallel_size + base_dp_pg_index = base_pg_index + dp_rank * tp_pp_size + dp_rank_bundles = ( + list(range(base_dp_pg_index, base_dp_pg_index + tp_pp_size)) if tp_pp_size > 1 else None + ) + dp_rank_sched = PlacementGroupSchedulingStrategy( + placement_group=shared_pg, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=base_dp_pg_index, + ) + + dp_kwargs = ( + { + "data_parallel_backend": data_parallel_backend, + "data_parallel_size": data_parallel_size, + "data_parallel_rank": dp_rank, + "data_parallel_address": data_parallel_address, + "data_parallel_rpc_port": data_parallel_rpc_port, + } + if data_parallel_size > 1 + else {} + ) + + engine = actor_class.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + scheduling_strategy=dp_rank_sched, + ).remote( + model=pretrain, + enforce_eager=enforce_eager, + worker_extension_cls="skyrl_train.inference_engines.vllm.vllm_engine.WorkerWrap", + tensor_parallel_size=tensor_parallel_size, + pipeline_parallel_size=pipeline_parallel_size, + enable_expert_parallel=expert_parallel_size > 1, + distributed_executor_backend=distributed_executor_backend, + seed=seed + i * data_parallel_size + dp_rank, + enable_prefix_caching=enable_prefix_caching, + dtype=model_dtype, + trust_remote_code=True, + vllm_v1_disable_multiproc=vllm_v1_disable_multiproc, + gpu_memory_utilization=gpu_memory_utilization, + bundle_indices=dp_rank_bundles, + num_gpus=0.2 if use_hybrid_engine else 1, + enable_sleep_mode=inference_engine_enable_sleep, + noset_visible_devices=noset_visible_devices, + max_num_batched_tokens=max_num_batched_tokens, + max_num_seqs=max_num_seqs, + max_logprobs=1, # only need chosen-token logprobs + **dp_kwargs, + **engine_init_kwargs, + **lora_kwargs, + **rope_engine_kwargs, + ) + inference_engine_actors.append(engine) + elif backend == "sglang": + # NOTE: there is no async / sync engine distinction in SGLang + + bundle_indices = None + if per_engine_gpu_count > 1: + bundle_indices = list(range(i * per_engine_gpu_count, (i + 1) * per_engine_gpu_count)) + + scheduling_strategy = PlacementGroupSchedulingStrategy( + placement_group=shared_pg, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=i * per_engine_gpu_count, + ) + + # NOTE(Charlie): We need `torch.cuda.is_available()` to be True to import SGLang. Otherwise, it requires + # importing vllm. See https://github.com/sgl-project/sglang/blob/v0.4.8.post1/python/sglang/srt/layers/quantization/utils.py#L11-L17 + # Similar comment: https://github.com/volcengine/verl/blob/9cc307767b0c787e8f5ef581dac929f7bde044ef/verl/workers/fsdp_workers.py#L520-L527 + @ray.remote + def get_sglang_engine(): + # A workaround to avoid importing vllm is to give this task a GPU. + import os + + before_cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "") + os.environ["CUDA_VISIBLE_DEVICES"] = "0" + from skyrl_train.inference_engines.sglang.sglang_engine import SGLangRayActor + + os.environ["CUDA_VISIBLE_DEVICES"] = before_cuda_visible_devices + + actor_class = SGLangRayActor + engine = actor_class.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + scheduling_strategy=scheduling_strategy, + ).remote( + model_path=pretrain, + tp_size=tensor_parallel_size, + mem_fraction_static=gpu_memory_utilization, + random_seed=seed + i, + disable_radix_cache=not enable_prefix_caching, + dtype=model_dtype, + trust_remote_code=True, + max_prefill_tokens=max_num_batched_tokens, + max_running_requests=max_num_seqs, + # Borrowed from veRL's SGLang rollout + mm_attention_backend="fa3", + attention_backend="fa3", + enable_memory_saver=inference_engine_enable_sleep, + # Will be popped before instantiating sgl.Engine + distributed_executor_backend=distributed_executor_backend, + noset_visible_devices=noset_visible_devices, + bundle_indices=bundle_indices, + num_gpus=0.2 if use_hybrid_engine else 1, + tokenizer=tokenizer, + **engine_init_kwargs, + ) + return engine + + engine = ray.get(get_sglang_engine.remote()) + + inference_engine_actors.append(engine) + + engines = [RayWrappedInferenceEngine(actor_handle) for actor_handle in inference_engine_actors] + + if inference_engine_enable_sleep: + if backend == "vllm": + # NOTE(shu): set to 1 for LoRA + sleep_level = 1 if enable_lora else sleep_level + sleep_refs = [engine.inference_engine_actor.sleep.remote(level=sleep_level) for engine in engines] + elif backend == "sglang": + # NOTE(Charlie): we always need to sync weights after waking up, + # see: https://github.com/sgl-project/sglang/issues/7939 for more details. + assert sleep_level == 2, "SGLang always discards weights, so sleep_level is not applicable." + sleep_refs = [engine.inference_engine_actor.sleep.remote() for engine in engines] + ray.get(sleep_refs) + + return engines diff --git a/skyrl-train/skyrl_train/inference_engines/remote_inference_engine.py b/skyrl-train/skyrl_train/inference_engines/remote_inference_engine.py new file mode 100644 index 0000000000..7199441272 --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/remote_inference_engine.py @@ -0,0 +1,351 @@ +import aiohttp +from skyrl_train.inference_engines.base import ( + InferenceEngineInterface, + InferenceEngineInput, + InferenceEngineOutput, +) +from skyrl_train.weight_sync import WeightLoader, WeightUpdateRequest, BroadcastWeightUpdateRequest +from typing import List, Optional, Any, Dict, TYPE_CHECKING + +if TYPE_CHECKING: + from skyrl_train.weight_sync.transfer_strategy import WeightSyncInitInfo +import json +from transformers import PreTrainedTokenizerBase + + +class RemoteWeightLoader(WeightLoader): + """Loads weights into remote inference engine via HTTP. + + This loader coordinates weight updates with remote inference servers + (vLLM or SGLang) via their HTTP APIs. + """ + + def __init__(self, url: str, engine_backend: str) -> None: + """Initialize the loader. + + Args: + url: Base URL of the remote inference server. + engine_backend: Backend type ("vllm" or "sglang"). + """ + self._url = url + self._engine_backend = engine_backend + + async def init_communicator(self, init_info: "WeightSyncInitInfo") -> Dict[str, Any]: + """Initialize the distributed process group for syncing weights. + + Args: + init_info: WeightSyncInitInfo from the sender. + + Returns: + Response from the remote server. + + Raises: + ValueError: If init_info strategy is not BroadcastTransferStrategy (remote only supports broadcast). + """ + from skyrl_train.weight_sync import BroadcastTransferStrategy + + if init_info.strategy_type() is not BroadcastTransferStrategy: + raise ValueError( + f"Remote inference engines only support BroadcastTransferStrategy, got: {init_info.strategy_type().__name__}" + ) + + async with aiohttp.ClientSession() as session: + if self._engine_backend == "sglang": + # SGLang native API - only send fields it expects + async with session.post( + f"{self._url}/init_weights_update_group", + json={ + "master_address": init_info.master_addr, + "master_port": init_info.master_port, + "rank_offset": init_info.rank_offset, + "world_size": init_info.world_size, + "group_name": init_info.group_name, + "backend": init_info.backend, + }, + ) as response: + return await response.json() + elif self._engine_backend == "vllm": + # vLLM - our custom API, pass entire init_info + from dataclasses import asdict + + async with session.post( + f"{self._url}/init_weight_update_communicator", + json=asdict(init_info), + ) as response: + return await response.json() + else: + raise ValueError(f"Invalid engine backend: {self._engine_backend}") + + async def load_weights(self, request: WeightUpdateRequest) -> Dict[str, Any]: + """Load weights via HTTP to the remote inference server. + + Remote engines only support broadcast weight updates (no IPC). + Each request should contain a single weight to update. + + Args: + request: Weight update request. + + Returns: + Response from the remote server. + """ + async with aiohttp.ClientSession() as session: + if self._engine_backend == "sglang": + # SGLang native API expects singular name, dtype, shape + resp = await session.post( + f"{self._url}/update_weights_from_distributed", + json={ + "name": request.names[0], + "dtype": request.dtypes[0], + "shape": request.shapes[0], + }, + ) + return await resp.json() + elif self._engine_backend == "vllm": + # vLLM - our custom API, pass entire request + from dataclasses import asdict + + resp = await session.post( + f"{self._url}/update_weights", + json=asdict(request), + ) + return await resp.json() + else: + raise ValueError(f"Invalid engine backend: {self._engine_backend}") + + async def destroy_group(self) -> Dict[str, Any]: + """Destroy the weights update group. + + Returns: + Response from the remote server. + """ + async with aiohttp.ClientSession() as session: + resp = await session.post(f"{self._url}/destroy_weights_update_group") + return await resp.json() + + +class RemoteInferenceEngine(InferenceEngineInterface): + """ + Lightweight client to call into an OpenAI-compatible server over HTTP with a customizable backend. + """ + + def __init__( + self, + url: str, + model_name: str, + engine_backend: str, + tokenizer: PreTrainedTokenizerBase, + tp_size: Optional[int] = None, + pp_size: Optional[int] = None, + dp_size: Optional[int] = None, + ep_size: Optional[int] = None, + ): + """Initialize the InferenceEngine.""" + self.url = f"http://{url}" + self.model_name = model_name + self.engine_backend = engine_backend + self._tp_size = tp_size + self._pp_size = pp_size + self._dp_size = dp_size + self._ep_size = ep_size + self.tokenizer = tokenizer + + # Create weight loader for coordinating weight updates + self._weight_loader = RemoteWeightLoader(self.url, engine_backend) + + def tp_size(self) -> int: + return self._tp_size + + def pp_size(self) -> int: + return self._pp_size + + def dp_size(self) -> int: + return self._dp_size + + def ep_size(self) -> int: + return self._ep_size + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + # 1. Prepare inputs + prompts = input_batch.get("prompts") + prompt_token_ids: Optional[List[List[int]]] = input_batch.get("prompt_token_ids") + request_sampling_params = input_batch.get("sampling_params") + + assert ( + prompts is None and prompt_token_ids is not None + ), "RemoteInferenceEngine only accepts `prompt_token_ids`, not `prompts`." + + sampling_params = request_sampling_params if request_sampling_params is not None else {} + if "n" in sampling_params and sampling_params["n"] > 1: + raise ValueError( + "n is not supported yet for remote inference engines. " + "You can set `config.generator.n_samples_per_prompt` instead." + ) + + # 2. Send a batched request to the server + response = None + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session: + headers = {"Content-Type": "application/json"} + payload = {} + request_url = "" + if self.engine_backend == "vllm": + # vLLM does not support /generate, use /completions instead. It supports batch generation. + payload = sampling_params.copy() + payload["model"] = self.model_name + payload["prompt"] = prompt_token_ids + request_url = f"{self.url}/v1/completions" + elif self.engine_backend == "sglang": + # SGLang supports /generate, works exactly like its Python `async_generate()` method + # and can do batch generation. + payload = { + "input_ids": prompt_token_ids, + "sampling_params": sampling_params, + } + request_url = f"{self.url}/generate" + else: + raise ValueError(f"Invalid engine backend: {self.engine_backend}") + async with session.post(request_url, json=payload, headers=headers) as resp: + response = await resp.json() + + # 3. Parse outputs + outputs = [] + output_ids = [] + finish_reasons = [] + + if self.engine_backend == "vllm": + for i, choice in enumerate(response.get("choices", [])): + # Since n=1, index i represents the output for `prompt[i]` + assert choice["index"] == i, "Expect the choices to be ordered by index." + text = choice["text"] + outputs.append(text) + finish_reasons.append(choice["finish_reason"]) + # TODO(Charlie): this is not token-in-token-out because vLLM does not support + # returning token IDs via HTTP requests. Fix after this vLLM PR is merged: + # https://github.com/vllm-project/vllm/pull/22587 + output_ids.append(self.tokenizer.encode(text, add_special_tokens=False)) + elif self.engine_backend == "sglang": + # since prompt_token_ids is a list of lists, response is a list of dicts + for output in response: + cur_output_ids = output["output_ids"] + output_ids.append(cur_output_ids) + # SGLang only returns tokens not text when skip_tokenizer_init is True, so + # we manually decode it. + outputs.append(self.tokenizer.decode(cur_output_ids, skip_special_tokens=True)) + finish_reasons.append(output["meta_info"]["finish_reason"]["type"]) + else: + raise ValueError(f"Invalid engine backend: {self.engine_backend}") + + return InferenceEngineOutput( + responses=outputs, stop_reasons=finish_reasons, response_ids=output_ids, response_logprobs=None + ) + + async def chat_completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + body = request_payload.get("json", {}) + # NOTE(Charlie): cannot reuse payload["headers"] since we are posting a new request. + # Otherwise will lead to json decode error. + headers = {"Content-Type": "application/json"} + response = None + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session: + request_url = f"{self.url}/v1/chat/completions" + async with session.post(request_url, json=body, headers=headers) as resp: + response = await resp.json() + + return response + + async def completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + body = request_payload.get("json", {}) + headers = {"Content-Type": "application/json"} + response = None + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session: + request_url = f"{self.url}/v1/completions" + async with session.post(request_url, json=body, headers=headers) as resp: + response = await resp.json() + + return response + + async def wake_up(self, *args: Any, **kwargs: Any): + async with aiohttp.ClientSession() as session: + resp = await session.post(f"{self.url}/wake_up", json={"tags": kwargs.get("tags", 1)}) + return await resp.json() + + async def sleep(self, *args: Any, **kwargs: Any): + async with aiohttp.ClientSession() as session: + # TODO(Charlie): this is vLLM's API, not SGLang (which uses tags). Fix when need to + # support sleeping with remote engines. + resp = await session.post(f"{self.url}/sleep", json={"level": kwargs.get("level", 1)}) + return await resp.json() + + async def init_weight_update_communicator(self, init_info: "WeightSyncInitInfo"): + """Initialize the distributed process group for syncing weights. + + Args: + init_info: WeightSyncInitInfo from the sender. + + Note: Remote engines only support broadcast strategy. + """ + return await self._weight_loader.init_communicator(init_info) + + async def update_named_weights(self, request: WeightUpdateRequest): + if not isinstance(request, BroadcastWeightUpdateRequest): + raise ValueError( + "Remote inference engines do not support CUDA IPC weight updates. Only local engines support IPC." + ) + + assert ( + len(request) == 1 + ), f"Remote inference engines support only requests with a single named weight at a time, got request with {len(request)} entries" + + return await self._weight_loader.load_weights(request) + + # TODO(tgriggs): Come up with a (more) elegant way to handle text or json responses, and test it and handle errors. + async def reset_prefix_cache(self): + if self.engine_backend == "vllm": + reset_prefix_cache_method = "reset_prefix_cache" + elif self.engine_backend == "sglang": + reset_prefix_cache_method = "flush_cache" + else: + raise ValueError(f"Invalid engine backend: {self.engine_backend}") + + async with aiohttp.ClientSession() as session: + resp = await session.post(f"{self.url}/{reset_prefix_cache_method}") + text = await resp.text() + + # First try to parse it as JSON + try: + return json.loads(text) + except json.JSONDecodeError: + # If invalid JSON, return raw text plus status + return { + "status": resp.status, + "body": text, + } + + async def teardown(self): + await self._weight_loader.destroy_group() + + async def abort_generation(self) -> None: + raise NotImplementedError("Abort generation is not supported for remote inference engines.") + + +def create_remote_inference_engines( + urls: List[str], + model_name: str, + engine_backend: str, + tokenizer: PreTrainedTokenizerBase, + tensor_parallel_size: Optional[int] = None, + pipeline_parallel_size: Optional[int] = None, + data_parallel_size: Optional[int] = None, + expert_parallel_size: Optional[int] = None, +): + return [ + RemoteInferenceEngine( + url=url, + model_name=model_name, + tokenizer=tokenizer, + engine_backend=engine_backend, + tp_size=tensor_parallel_size, + pp_size=pipeline_parallel_size, + dp_size=data_parallel_size, + ep_size=expert_parallel_size, + ) + for url in urls + ] diff --git a/skyrl-train/skyrl_train/inference_engines/sglang/sglang_engine.py b/skyrl-train/skyrl_train/inference_engines/sglang/sglang_engine.py new file mode 100644 index 0000000000..f20c8ecc19 --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/sglang/sglang_engine.py @@ -0,0 +1,382 @@ +"""SGLang inference engine implementation.""" + +import torch +import os +from typing import List, Optional, Dict, Any +import ray +from loguru import logger +import multiprocessing as mp + +import sglang.srt.entrypoints.engine +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.utils import ( + assert_pkg_version, + is_cuda, + maybe_set_triton_cache_manager, + set_prometheus_multiproc_dir, + set_ulimit, + MultiprocessingSerializer, +) +from sglang.srt.managers.tokenizer_manager import ( + UpdateWeightsFromTensorReqInput, + UpdateWeightsFromDistributedReqInput, + InitWeightsUpdateGroupReqInput, + ReleaseMemoryOccupationReqInput, + ResumeMemoryOccupationReqInput, +) +from skyrl_train.inference_engines.base import ( + InferenceEngineInterface, + InferenceEngineInput, + InferenceEngineOutput, +) +from skyrl_train.weight_sync import ( + WeightLoader, + WeightUpdateRequest, + CudaIpcWeightUpdateRequest, + BroadcastWeightUpdateRequest, +) + + +# Patch SGLang's _set_envs_and_config to avoid signal handler issues in Ray actors +# Based on VERL's solution: https://github.com/sgl-project/sglang/issues/6723 +# https://github.com/volcengine/verl/blob/v0.4.1/verl/workers/rollout/sglang_rollout/sglang_rollout.py#L85 +def _patched_set_envs_and_config(server_args): + """Patched version of SGLang's _set_envs_and_config that removes signal handler registration.""" + # Set global environments + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["NCCL_NVLS_ENABLE"] = str(int(getattr(server_args, "enable_nccl_nvls", False))) + os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" + os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "4" + os.environ["CUDA_MODULE_LOADING"] = "AUTO" + + # Set prometheus env vars + if server_args.enable_metrics: + set_prometheus_multiproc_dir() + + # Set ulimit + set_ulimit() + + # Fix triton bugs + if server_args.tp_size * server_args.dp_size > 1: + # FIXME: remove this after https://github.com/triton-lang/triton/pull/4295 is used as a dependency. + maybe_set_triton_cache_manager() + + # Check flashinfer version + if server_args.attention_backend == "flashinfer": + assert_pkg_version( + "flashinfer_python", + "0.2.5", + "Please uninstall the old version and reinstall the latest version by following the instructions at https://docs.flashinfer.ai/installation.html.", + ) + if is_cuda(): + assert_pkg_version( + "sgl-kernel", + "0.1.1", + "Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`", + ) + + # Set mp start method + mp.set_start_method("spawn", force=True) + + # We do NOT register signal handlers here to avoid Ray actor issues + # Original SGLang code had: signal.signal(signal.SIGCHLD, sigchld_handler) + # But this fails in Ray actors since signal handlers only work in main thread + + +# Apply the patch +sglang.srt.entrypoints.engine._set_envs_and_config = _patched_set_envs_and_config + + +# TODO(charlie): duplicate of setup_envvars_for_vllm, is it needed? +def setup_envvars_for_sglang(kwargs, bundle_indices): + distributed_executor_backend = kwargs.pop("distributed_executor_backend", None) + noset_visible_devices = kwargs.pop("noset_visible_devices", None) + if distributed_executor_backend == "ray": + # a hack to make the script work. + # stop ray from manipulating *_VISIBLE_DEVICES + # at the top-level when the distributed_executor_backend is ray. + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + os.environ.pop("ROCR_VISIBLE_DEVICES", None) + os.environ.pop("HIP_VISIBLE_DEVICES", None) + pass + elif noset_visible_devices: + # We need to set CUDA_VISIBLE_DEVICES to the ray assigned GPU + # when the distributed_executor_backend is not rayargs and + # RAY_EXPERIMENTAL_NOSET_*_VISIBLE_DEVICES is set. + os.environ["CUDA_VISIBLE_DEVICES"] = str(ray.get_gpu_ids()[0]) + + +def sglang_custom_weight_loader(model, named_tensors): + """Custom weight loader for SGLang that handles CUDA IPC. + + This function is called by SGLang's model runner to load weights. + It reconstructs tensors from CudaIpcWeightUpdateRequest using CUDA IPC handles. + + Note: Broadcast path is not handled here. + Because unlike vLLM where we control WorkerWrap and can store the + process group there, SGLang's custom_weight_loader only receives (model, tensors). + The process group (_model_update_group) is stored in model_runner, which is not + accessible from the model object. We also cannot create the group lazily inside + custom_weight_loader because torch.distributed group creation requires coordination + (all processes must join at the same time), and by the time custom_weight_loader + is called, the training side has already completed its init. Therefore, broadcast + uses SGLang's native update_weights_from_distributed API which has internal access + to the process group. + """ + from skyrl_train.weight_sync import CudaIpcWeightUpdateRequest, CudaIpcWeightTransferReceiver + + # Extract tensor name and data + name, tensor = named_tensors[0] + if name != "ipc_request": + raise ValueError(f"Expected tensor name 'ipc_request', got: {name}") + + # Deserialize request from tensor + request = CudaIpcWeightUpdateRequest.deserialize(tensor.cpu().numpy().tobytes()) + + # Get model info and create receiver + model_dtype = next(model.parameters()).dtype + receiver = CudaIpcWeightTransferReceiver(model_dtype) + + # Receive weights via IPC + weights_to_load = list(receiver.receive_weights(request)) + model.load_weights(weights_to_load) + + +CUSTOM_WEIGHT_LOADER_PATH = "skyrl_train.inference_engines.sglang.sglang_engine.sglang_custom_weight_loader" + + +class SGLangWeightLoader(WeightLoader): + """Loads weights into SGLang engine, managing weight transfer coordination. + + This loader encapsulates the SGLang-specific weight loading logic for both + IPC and broadcast transfer paths: + - IPC: Uses update_weights_from_tensor with our custom_weight_loader + - Broadcast: Uses SGLang's native update_weights_from_distributed API + """ + + def __init__(self, engine: Any, tp_size: int) -> None: + """Initialize the loader. + + Args: + engine: The SGLang engine. + tp_size: Tensor parallel size. + """ + self._engine = engine + self._tp_size = tp_size + + async def init_communicator(self, init_info) -> None: + """Initialize the process group for broadcast weight sync. + + This is only needed for the broadcast path. IPC path does not require + a process group since it uses CUDA IPC handles directly. + + Args: + init_info: WeightSyncInitInfo from the sender. + """ + from skyrl_train.weight_sync import BroadcastTransferStrategy + + if init_info.strategy_type() is BroadcastTransferStrategy: + obj = InitWeightsUpdateGroupReqInput( + master_address=init_info.master_addr, + master_port=init_info.master_port, + rank_offset=init_info.rank_offset, + world_size=init_info.world_size, + group_name=init_info.group_name, + backend=init_info.backend, + ) + # NOTE(charlie): Call the async method on tokenizer_manager directly to avoid event loop + # conflicts. Same underlying implementation: https://github.com/sgl-project/sglang/blob/v0.4.8.post1/python/sglang/srt/model_executor/model_runner.py#L689 + success, message = await self._engine.tokenizer_manager.init_weights_update_group(obj, None) + if not success: + raise RuntimeError(f"Failed to initialize weight update group: {message}") + + async def load_weights(self, request: WeightUpdateRequest) -> None: + """Load weights by coordinating with SGLang's weight update APIs. + + Args: + request: Weight update request. + """ + if isinstance(request, CudaIpcWeightUpdateRequest): + await self._load_via_ipc(request) + elif isinstance(request, BroadcastWeightUpdateRequest): + await self._load_via_broadcast(request) + else: + raise TypeError(f"Unknown request type: {type(request).__name__}") + + async def _load_via_ipc(self, request: CudaIpcWeightUpdateRequest) -> None: + """Load weights via CUDA IPC using custom weight loader. + + Uses SGLangWeightTransferReceiver internally to receive weights + from IPC handles. + """ + tensor_array = torch.frombuffer(bytearray(request.serialize()), dtype=torch.uint8) + + # Use SGLang's API to update weights with our custom loader + request_tensor = [("ipc_request", tensor_array)] + obj = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=[ + MultiprocessingSerializer.serialize(request_tensor) for _ in range(self._tp_size) + ], + load_format=CUSTOM_WEIGHT_LOADER_PATH, + flush_cache=False, # TODO(charlie): flush cache on last weight update? + ) + + success, message = await self._engine.tokenizer_manager.update_weights_from_tensor(obj, None) + if not success: + raise RuntimeError(f"IPC weight update failed: {message}") + + async def _load_via_broadcast(self, request: BroadcastWeightUpdateRequest) -> None: + """Load weights via torch.distributed broadcast. + + Uses SGLang's native update_weights_from_distributed API which internally + uses the process group created during init_weights_update_group. + """ + assert len(request) == 1, f"Broadcast only supports a single weight at a time, got {len(request)} entries" + + obj = UpdateWeightsFromDistributedReqInput( + name=request.names[0], dtype=request.dtypes[0], shape=request.shapes[0] + ) + + success, message = await self._engine.tokenizer_manager.update_weights_from_distributed(obj, None) + if not success: + raise RuntimeError(f"Broadcast weight update failed: {message}") + + +class SGLangInferenceEngine(InferenceEngineInterface): + """SGLang inference engine that implements InferenceEngineInterface.""" + + def __init__(self, *args, bundle_indices: Optional[List[int]] = None, **kwargs): + setup_envvars_for_sglang(kwargs, bundle_indices) + + # Store common attributes + self._tp_size = kwargs.get("tp_size", 1) + if self._tp_size > 1: + raise ValueError( + "As of now, we don't support tensor parallel inference engine with SGLang. " + "Please set `inference_engine_tensor_parallel_size` to 1." + ) + self.tokenizer = kwargs.pop("tokenizer", None) + + # Unused kwargs + _ = kwargs.pop("num_gpus", 1) + + # Add custom weight loader + kwargs["custom_weight_loader"] = CUSTOM_WEIGHT_LOADER_PATH + + # Always use token-in-token-out SGLang engine + # NOTE(Charlie): unlike vLLM, SGLang cannot do token-in-token-out and + # token-in-text-out in the same engine config. + kwargs["skip_tokenizer_init"] = True + + # Create the SGLang engine (signal handler issue is now fixed by patching) + self.engine = Engine(**kwargs) + logger.info(f"Created SGLang engine with kwargs: {kwargs}") + + # Create weight loader for coordinating weight updates + self._weight_loader = SGLangWeightLoader(self.engine, self._tp_size) + + def tp_size(self): + return self._tp_size + + def pp_size(self): + # Pipeline parallelism not supported for SGLang + return 1 + + def dp_size(self): + # TODO(tgriggs): EP/DP not yet supported for SGLang + return 1 + + def _preprocess_prompts(self, input_batch: InferenceEngineInput): + """Preprocess prompts for SGLang generation.""" + prompts = input_batch.get("prompts") + prompt_token_ids = input_batch.get("prompt_token_ids") + request_sampling_params = input_batch.get("sampling_params") + + assert ( + prompts is None and prompt_token_ids is not None + ), "SGLangInferenceEngine only accepts `prompt_token_ids`, not `prompts`." + + # Use request sampling params if provided. + sampling_params = request_sampling_params if request_sampling_params is not None else {} + + return prompt_token_ids, sampling_params + + def _postprocess_outputs(self, outputs): + """Process SGLang outputs to match expected format.""" + responses: List[str] = [] + stop_reasons: List[str] = [] + response_ids: List[List[int]] = [] + + for output in outputs: + response_ids.append(output["output_ids"]) + responses.append(self.tokenizer.decode(output["output_ids"], skip_special_tokens=True)) + stop_reasons.append(output["meta_info"]["finish_reason"]["type"]) + + return InferenceEngineOutput( + responses=responses, + response_ids=response_ids, + stop_reasons=stop_reasons, + response_logprobs=None, + ) + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + """Generate responses using SGLang engine.""" + token_ids_prompts, sampling_params = self._preprocess_prompts(input_batch) + outputs = await self.engine.async_generate(input_ids=token_ids_prompts, sampling_params=sampling_params) + return self._postprocess_outputs(outputs) + + async def chat_completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + # TODO(charlie): implement this in the future + raise NotImplementedError() + + async def completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + # TODO(charlie): implement this in the future + raise NotImplementedError() + + async def init_weight_update_communicator(self, init_info): + """Initialize weight update communicator for SGLang. + + Args: + init_info: WeightSyncInitInfo from the sender. + """ + return await self._weight_loader.init_communicator(init_info) + + async def update_named_weights(self, request: WeightUpdateRequest) -> None: + """Update named weights in SGLang engine.""" + await self._weight_loader.load_weights(request) + + async def wake_up(self, tags: Optional[List[str]] = None): + """Wake up the engine. For multi-stage waking up, pass in `"weight"` or `"kv_cache"` to tags.""" + obj = ResumeMemoryOccupationReqInput(tags=tags) + # Call the underlying async method for the same reason as in `init_weight_update_communicator` + await self.engine.tokenizer_manager.resume_memory_occupation(obj, None) + logger.info( + f"From SGLang engine -- Free GPU memory after wake up with tags {tags if tags is not None else 'None'}: " + + f"{torch.cuda.mem_get_info()[0] / 1024**2:.1f} MB" + ) + + async def sleep(self, tags: Optional[List[str]] = None): + """Put engine to sleep.""" + obj = ReleaseMemoryOccupationReqInput(tags=tags) + # Call the underlying async method for the same reason as in `init_weight_update_communicator` + await self.engine.tokenizer_manager.release_memory_occupation(obj, None) + logger.info( + f"From SGLang engine -- Free GPU memory after sleep with tags {tags if tags is not None else 'None'}: " + + f"{torch.cuda.mem_get_info()[0] / 1024**2:.1f} MB" + ) + + async def teardown(self): + """Shutdown the SGLang engine.""" + self.engine.shutdown() + + async def reset_prefix_cache(self): + """Reset prefix cache in SGLang engine.""" + # Call the underlying async method for the same reason as in `init_weight_update_communicator` + return await self.engine.tokenizer_manager.flush_cache() + + async def abort_generation(self) -> None: + raise NotImplementedError("Abort generation is not supported for SGLang inference engines.") + + +SGLangRayActor = ray.remote(SGLangInferenceEngine) diff --git a/skyrl-train/skyrl_train/inference_engines/sglang/sglang_server.py b/skyrl-train/skyrl_train/inference_engines/sglang/sglang_server.py new file mode 100644 index 0000000000..365e46eb7f --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/sglang/sglang_server.py @@ -0,0 +1,27 @@ +import os +import sys + +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import prepare_server_args, ServerArgs +from sglang.srt.utils import kill_process_tree + + +class SGLangServer: + def __init__(self, server_args: ServerArgs): + self.server_args = server_args + + def run_server(self) -> None: + try: + launch_server(self.server_args) + finally: + kill_process_tree(os.getpid(), include_parent=False) + + +if __name__ == "__main__": + args = sys.argv[1:] + # SGLang requires `skip-tokenizer-init` to do token-in-token-out with `/generate` endpoint + if "--skip-tokenizer-init" not in args: + args.append("--skip-tokenizer-init") + server_args = prepare_server_args(args) + sglang_server = SGLangServer(server_args) + sglang_server.run_server() diff --git a/skyrl-train/skyrl_train/inference_engines/utils.py b/skyrl-train/skyrl_train/inference_engines/utils.py new file mode 100644 index 0000000000..180962d2c2 --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/utils.py @@ -0,0 +1,249 @@ +from typing import Dict, Any, Optional, Union +import random +import hashlib +from omegaconf import DictConfig, ListConfig +from skyrl_train.inference_engines.inference_engine_client_http_endpoint import ErrorResponse, ErrorInfo +from typing import List +from http import HTTPStatus +from typing import Tuple +import ray +from ray.util.placement_group import PlacementGroupSchedulingStrategy + + +def get_vllm_sampling_params(sampling_params: DictConfig) -> Dict[str, Any]: + stop_val = sampling_params.get("stop", None) + vllm_sampling_params = { + "min_tokens": 1, + "skip_special_tokens": True, + "include_stop_str_in_output": True, + "max_tokens": sampling_params.max_generate_length, + "temperature": sampling_params.temperature, + "top_p": sampling_params.top_p, + "top_k": sampling_params.top_k, + "min_p": sampling_params.min_p, + "logprobs": sampling_params.logprobs, + "stop": list(stop_val) if stop_val is not None else None, + } + exclude_keys = ["max_generate_length"] + for key, value in sampling_params.items(): + if key not in vllm_sampling_params and key not in exclude_keys: + # Convert OmegaConf ListConfig to regular list if needed + if isinstance(value, ListConfig): + value = list(value) + vllm_sampling_params[key] = value + return vllm_sampling_params + + +def get_sglang_sampling_params(sampling_params: DictConfig) -> Dict[str, Any]: + # `min_tokens` in vllm is equivalent to `min_new_tokens` in sglang. However `min_new_tokens` and + # `stop` are not supported when `skip_tokenizer_init` is True, which we need for token-in-token-out. + # See this issue for more: https://github.com/sgl-project/sglang/issues/9039#issuecomment-3218331087 + sglang_sampling_params = { + "skip_special_tokens": True, + "no_stop_trim": True, # equivalent to include_stop_str_in_output=True + "max_new_tokens": sampling_params.max_generate_length, + "temperature": sampling_params.temperature, + "top_p": sampling_params.top_p, + "top_k": sampling_params.top_k, + "min_p": sampling_params.min_p, + } + # logprobs not supported with sglang for now + exclude_keys = ["max_generate_length", "logprobs"] + for key, value in sampling_params.items(): + if key not in sglang_sampling_params and key not in exclude_keys: + # Convert OmegaConf ListConfig to regular list if needed + if isinstance(value, ListConfig): + value = list(value) + sglang_sampling_params[key] = value + return sglang_sampling_params + + +def get_sampling_params_for_backend(backend: str, sampling_params: DictConfig) -> Dict[str, Any]: + if backend == "vllm": + return get_vllm_sampling_params(sampling_params) + elif backend == "sglang": + return get_sglang_sampling_params(sampling_params) + else: + raise ValueError(f"Unsupported generation backend: {backend}") + + +def hash_with_sha256(x: Union[int, str]) -> int: + return int.from_bytes(hashlib.sha256(str(x).encode()).digest(), "big") + + +def route_prompts_to_engines( + num_prompts: int, num_inference_engines: int, session_ids: Optional[Union[list[int], list[str]]] +) -> dict[int, list[int]]: + """ + Given the number of prompts, number of inference engines, and the session_id, return a mapping + from engine index to the list of prompt IDs the engine will process. + + Args: + - num_prompts: int - The number of prompts. + - num_inference_engines: int - The number of inference engines. + - session_ids: Optional[Union[list[int], list[str]]] - The session IDs. + + Required: + - num_prompts > 0 + - num_inference_engines > 0 + - session_ids is a list of integers or strings if provided + - len(session_ids) == num_prompts if provided + + Returns: + - dict[int, list[int]] - A mapping from engine index to the list of prompt IDs the engine will process. + """ + # 0. Validation + assert num_prompts > 0, "Number of prompts must be greater than 0" + assert num_inference_engines > 0, "Number of inference engines must be greater than 0" + if session_ids is not None: + assert isinstance(session_ids, list) and all( + isinstance(sid, (int, str)) for sid in session_ids + ), "Session ID must be a list of integers or strings" + assert len(session_ids) == num_prompts, "Session ID must have the same length as the number of prompts" + + # 1. session_id not provided, with a single prompt: route to a random engine for a naive load balancing. + if session_ids is None and num_prompts == 1: + engine_idx = random.randint(0, num_inference_engines - 1) + return {engine_idx: [0]} + + # 2. session_id not provided, with a batched prompt: split evenly across engines. + engine_idx_to_prompt_ids: dict[int, list[int]] = {} + if session_ids is None: + dp_item_size = (num_prompts + num_inference_engines - 1) // num_inference_engines + for dp_rank in range(num_inference_engines): + start_idx = dp_rank * dp_item_size + end_idx = min((dp_rank + 1) * dp_item_size, num_prompts) + prompt_ids = list(range(start_idx, end_idx)) + if len(prompt_ids) > 0: + engine_idx_to_prompt_ids[dp_rank] = prompt_ids + return engine_idx_to_prompt_ids + + # 3. session_id provided, we route by session_id + for i, cur_sid in enumerate(session_ids): + engine_idx = hash_with_sha256(str(cur_sid)) % num_inference_engines + engine_idx_to_prompt_ids.setdefault(engine_idx, []).append(i) + return engine_idx_to_prompt_ids + + +def postprocess_completion_request( + prompt: Union[List[int], List[List[int]], List[str], str], + session_id_value: Optional[Union[List[int], List[str], int, str]], +) -> tuple[Optional[Union[List[int], List[str], ErrorResponse]], Union[List[List[int]], List[str]]]: + """ + Postprocess the session_id value and raise error if invalid. + + Returns a list of session_ids, or None if session_id_value is None, or ErrorResponse if invalid. + Also returns the processed prompt, where if the prompt is a single request, we make it + a singleton list of a single request. That is, List[int] becomes List[List[int]] of length 1, + and str becomes List[str] of length 1. + + Postconditions: + - If session_id_value is None, we return None. + - If session_id_value and prompt do not match, we return ErrorResponse. + - The returned session_id_list has the same length as the prompt. + - The returned prompt is either List[List[int]], or List[str], whether batched or not. + """ + + def _is_list_of_ints(x): + return isinstance(x, list) and all(isinstance(y, int) for y in x) + + # Determine if this is a single or batched request (a List[str] of length 1 is considered batched) + is_single = isinstance(prompt, str) or _is_list_of_ints(prompt) + if is_single: + prompt = [prompt] + + if session_id_value is None: + return None, prompt + + if isinstance(session_id_value, (int, str)): + session_id_value = [session_id_value] + + if len(session_id_value) != len(prompt): + return ( + ErrorResponse( + error=ErrorInfo( + message=( + "For /completions request with a single prompt, request.session_id must " + f"be a single integer/string or a singleton list.\nFor batched requests, " + f"request.session_id must have the same length as request.prompt." + f"However, received (len(session_id_value): {len(session_id_value)}, len(prompt): {len(prompt)})." + ), + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ), + prompt, + ) + + return session_id_value, prompt + + +def aggregate_completion_usage_info( + results: List[Dict[str, Any]], + backend: str, +) -> Dict[str, Any]: + """ + Aggregate the completion usage info and return the final usage info, since our + inference engine client breaks down a batched request into sub-requests and routes to engines, + where each engine only returns its sub-request's usage info. When we return the final response, + we need to aggregate the usage info. + + NOTE(Charlie): we don't explicitly import vllm here for ease of CPU test. Whether these fields + are still compatible with newer vllm versions can be checked in our GPU tests, where we explicitly + check the dictionary with `CompletionResponse.model_validate()`. + """ + if backend == "vllm": + # required fields + usage_info = { + "prompt_tokens": sum(result["usage"]["prompt_tokens"] for result in results), + "total_tokens": sum(result["usage"]["total_tokens"] for result in results), + } + # optional fields + if results[0]["usage"].get("completion_tokens") is not None: + usage_info["completion_tokens"] = sum(result["usage"]["completion_tokens"] for result in results) + if results[0]["usage"].get("prompt_tokens_details") is not None: + if results[0]["usage"]["prompt_tokens_details"].get("cached_tokens") is not None: + usage_info["prompt_tokens_details"] = { + "cached_tokens": sum( + result["usage"]["prompt_tokens_details"]["cached_tokens"] for result in results + ) + } + return usage_info + elif backend == "sglang": + raise NotImplementedError("SGLang is not supported yet") + else: + raise ValueError(f"Unsupported backend: {backend}") + + +def get_rendezvous_addr_port(placement_group, pg_index: int) -> Tuple[str, int]: + """ + Minimal helper to get a rendezvous addr:port in `placement_group`'s bundle at index `pg_index`. + """ + + @ray.remote(num_cpus=0, num_gpus=0) + def get_addr_port(): + try: + from ray.experimental.collective.util import get_address_and_port + except (ImportError, ModuleNotFoundError): + import socket + + def get_address_and_port(): + ip = ray.util.get_node_ip_address() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("", 0)) + port = sock.getsockname()[1] + sock.close() + return ip, port + + return get_address_and_port() + + master_sched = PlacementGroupSchedulingStrategy( + placement_group=placement_group, + placement_group_capture_child_tasks=True, + placement_group_bundle_index=pg_index, + ) + # Get DP group rendezvous (addr, port) on the same node as index `pg_index`'s bundle. + data_parallel_address, data_parallel_rpc_port = ray.get( + get_addr_port.options(scheduling_strategy=master_sched).remote() + ) + return data_parallel_address, data_parallel_rpc_port diff --git a/skyrl-train/skyrl_train/inference_engines/vllm/utils.py b/skyrl-train/skyrl_train/inference_engines/vllm/utils.py new file mode 100644 index 0000000000..6bb8f74412 --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/vllm/utils.py @@ -0,0 +1,33 @@ +from typing import Dict, Any + +from skyrl_train.generators.utils import get_custom_chat_template + + +def pop_openai_kwargs(engine_kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Normalize & remove OpenAI-serving-only kwargs from engine_kwargs. + """ + openai_kwargs: Dict[str, Any] = {} + + enable_auto_tools = engine_kwargs.pop("enable_auto_tools", engine_kwargs.pop("enable_auto_tool_choice", None)) + if enable_auto_tools is not None: + openai_kwargs["enable_auto_tools"] = bool(enable_auto_tools) + + tool_parser = engine_kwargs.pop("tool_parser", engine_kwargs.pop("tool_call_parser", None)) + if tool_parser is not None: + openai_kwargs["tool_parser"] = tool_parser + + reasoning_parser = engine_kwargs.pop("reasoning_parser", None) + if reasoning_parser is not None: + openai_kwargs["reasoning_parser"] = reasoning_parser + + # Since we use OpenAIServingChat() ourselves, which requires the content of + # the chat template, not the path (unlike --chat-template in vllm serve CLI args) + chat_template = engine_kwargs.pop("chat_template", None) + if chat_template is not None: + openai_kwargs["chat_template"] = get_custom_chat_template( + chat_template_config={"source": "file", "name_or_path": chat_template} + ) + assert openai_kwargs["chat_template"] is not None, "Failed to get custom chat template" + + return openai_kwargs diff --git a/skyrl-train/skyrl_train/inference_engines/vllm/vllm_engine.py b/skyrl-train/skyrl_train/inference_engines/vllm/vllm_engine.py new file mode 100644 index 0000000000..57345c46b4 --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/vllm/vllm_engine.py @@ -0,0 +1,693 @@ +import os +from typing import List, Any, Dict, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from skyrl_train.weight_sync.transfer_strategy import WeightSyncInitInfo +from dataclasses import dataclass +from http import HTTPStatus +import ray +import torch +import asyncio +import vllm +from types import SimpleNamespace +from vllm import SamplingParams +from vllm.inputs import TokensPrompt + +# vLLM API compatibility: nightly (0.14+) restructured module paths +# Try new paths first, fall back to old paths for stable releases +try: + # vLLM nightly (0.14+) - restructured into subdirectories + from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat + from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion + from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, + ) + from vllm.entrypoints.openai.completion.protocol import ( + CompletionRequest, + CompletionResponse, + ) + from vllm.entrypoints.openai.engine.protocol import ErrorResponse +except ImportError: + # vLLM stable (<=0.13) - flat module structure + from vllm.entrypoints.openai.serving_chat import OpenAIServingChat + from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion + from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels + from vllm.entrypoints.openai.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, + ErrorResponse, + CompletionRequest, + CompletionResponse, + ) + +from vllm.lora.request import LoRARequest +from uuid import uuid4 +import warnings +from skyrl_train.inference_engines.base import ( + InferenceEngineInterface, + InferenceEngineInput, + InferenceEngineOutput, +) +from skyrl_train.weight_sync import WeightLoader, WeightUpdateRequest +from skyrl_train.inference_engines.vllm.utils import pop_openai_kwargs +from loguru import logger +import time +from packaging import version + + +@dataclass +class Logprob: + logprob: float + rank: int + token_id: str + + +def setup_envvars_for_vllm(kwargs, bundle_indices): + noset_visible_devices = kwargs.pop("noset_visible_devices") + if kwargs.get("distributed_executor_backend") == "ray": + # a hack to make the script work. + # stop ray from manipulating *_VISIBLE_DEVICES + # at the top-level when the distributed_executor_backend is ray. + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + os.environ.pop("ROCR_VISIBLE_DEVICES", None) + os.environ.pop("HIP_VISIBLE_DEVICES", None) + elif noset_visible_devices: + # We need to set CUDA_VISIBLE_DEVICES to the ray assigned GPU + # when the distributed_executor_backend is not rayargs and + # RAY_EXPERIMENTAL_NOSET_*_VISIBLE_DEVICES is set. + os.environ["CUDA_VISIBLE_DEVICES"] = str(ray.get_gpu_ids()[0]) + + num_gpus = kwargs.pop("num_gpus") + if bundle_indices is not None: + os.environ["VLLM_RAY_PER_WORKER_GPUS"] = str(num_gpus) + os.environ["VLLM_RAY_BUNDLE_INDICES"] = ",".join(map(str, bundle_indices)) + logger.info(f"creating LLM with bundle_indices={bundle_indices}") + + +class WorkerWrap: + def test_rpc(self, *args, **kwargs): + """Test RPC call to worker""" + return args, kwargs + + def init_weight_update_communicator(self, init_info: bytes): + """Init weight update communicator from init info. + + Args: + init_info: Pickled bytes of WeightSyncInitInfo from the sender. + """ + import pickle + + assert torch.distributed.is_initialized(), "default torch process group must be initialized" + + # Unpickle init_info to restore the original object type + assert isinstance(init_info, bytes), f"Expected bytes, got {type(init_info).__name__}" + init_info = pickle.loads(init_info) + + strategy_cls = init_info.strategy_type() + + if hasattr(self, "_weight_receiver") and self._weight_receiver is not None: + # TODO(haochen): we should get rid of this flag and override existing receiver. + if init_info.override_existing_receiver: + self._weight_receiver.teardown() + self._weight_receiver = None + else: + warnings.warn( + "Detected an existing weight receiver. " + "For overriding, use `generator.override_existing_update_group=enable`" + ) + return + + self._weight_receiver = strategy_cls.create_receiver(init_info) + + def load_weights(self, request: bytes) -> None: + """Load weights using the receiver. + + This method is called via collective_rpc from VLLMWeightLoader. + + Args: + request: Pickled bytes of WeightUpdateRequest. + """ + import pickle + + # Unpickle request to restore the original object type + assert isinstance(request, bytes), f"Expected bytes, got {type(request).__name__}" + request = pickle.loads(request) + + weight_list = [] + for name, tensor in self._weight_receiver.receive_weights(request): + weight_list.append((name, tensor)) + + self.model_runner.model.load_weights(weights=weight_list) + + for weight in weight_list: + del weight + + # TODO (sumanthrh): Add destroy process group RPC as a atexit handler to Trainer code. + def teardown_weight_receiver(self): + if not hasattr(self, "_weight_receiver") or self._weight_receiver is None: + warnings.warn("No weight receiver to teardown") + return + self._weight_receiver.teardown() + + +class BaseVLLMInferenceEngine(InferenceEngineInterface): + """Base class containing shared logic between sync and async VLLM engines.""" + + def __init__(self, *args, bundle_indices: list = None, **kwargs): + setup_envvars_for_vllm(kwargs, bundle_indices) + vllm_v1_disable_multiproc = kwargs.pop("vllm_v1_disable_multiproc", False) + if vllm_v1_disable_multiproc or vllm.__version__ == "0.8.2": + # https://github.com/vllm-project/vllm/blob/effc5d24fae10b29996256eb7a88668ff7941aed/examples/offline_inference/reproduciblity.py#L11 + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" + + # Store common attributes + self._tp_size = kwargs.get("tensor_parallel_size", 1) + self._pp_size = kwargs.get("pipeline_parallel_size", 1) + self._dp_size = kwargs.get("data_parallel_size", 1) + self._is_lora = kwargs.get("enable_lora", False) + + # Let subclass create the appropriate engine + self.llm = self._create_engine(*args, **kwargs) + + # Weight loader is created by subclass after engine initialization + self._weight_loader = None + + def tp_size(self): + return self._tp_size + + def pp_size(self): + return self._pp_size + + def dp_size(self): + return self._dp_size + + def _create_engine(self, *args, **kwargs): + """Abstract method for subclasses to implement engine creation.""" + raise NotImplementedError("Subclasses must implement _create_engine") + + def _preprocess_prompts(self, input_batch: InferenceEngineInput): + """Common prompt preprocessing logic.""" + prompts = input_batch.get("prompts") + prompt_token_ids = input_batch.get("prompt_token_ids") + request_sampling_params = input_batch.get("sampling_params") + + assert ( + prompts is None and prompt_token_ids is not None + ), "VLLMInferenceEngine only accepts `prompt_token_ids`, not `prompts`." + + sampling_params = ( + SamplingParams(**request_sampling_params) if request_sampling_params is not None else SamplingParams() + ) + + return prompt_token_ids, sampling_params + + def _postprocess_outputs(self, outputs): + """Common output processing logic.""" + responses: List[str] = [] + stop_reasons: List[str] = [] + response_ids: List[List[int]] = [] + response_logprobs: Optional[List[List[float]]] = [] + + for output in outputs: + # TODO(tgriggs): Support n>1 sampling. + assert ( + len(output.outputs) == 1 + ), "Each prompt should have only one responses. n>1 sampling is supported by copying prompts." + resp = output.outputs[0] + responses.append(resp.text) + stop_reasons.append(resp.finish_reason) + response_ids.append(resp.token_ids) + _logprobs = None + if resp.logprobs: + _logprobs = [] + for i, token_logprobs in enumerate(resp.logprobs): + token_logprobs: Dict[str, Logprob] + token_id = resp.token_ids[i] + logprob = token_logprobs[token_id].logprob + _logprobs.append(logprob) + del token_logprobs + response_logprobs.append(_logprobs) + + if len(response_logprobs) and response_logprobs[0] is None: + response_logprobs = None # hack: assume uniform sampling params + + return InferenceEngineOutput( + responses=responses, + stop_reasons=stop_reasons, + response_ids=response_ids, + response_logprobs=response_logprobs, + ) + + def _get_engine(self): + """Get the underlying engine for RPC calls.""" + return self.llm.engine if hasattr(self.llm, "engine") else self.llm + + def reset_prefix_cache(self): + """Reset the prefix cache. Subclasses override for async version.""" + return self.llm.llm_engine.reset_prefix_cache() + + async def abort_generation(self) -> None: + raise NotImplementedError("Abort generation is only supported for AsyncVLLMInferenceEngine.") + + +class VLLMInferenceEngine(BaseVLLMInferenceEngine): + """Synchronous VLLM engine.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._weight_loader = VLLMWeightLoader(self.llm, is_async=False) + + def _create_engine(self, *args, **kwargs): + # Pipeline parallelism requires AsyncLLMEngine + if kwargs.get("pipeline_parallel_size", 1) > 1: + raise ValueError( + "Pipeline parallelism is only supported with AsyncVLLMInferenceEngine. " + "Please set `generator.async_engine=true` in your config." + ) + return vllm.LLM(*args, **kwargs) + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + prompt_token_ids, sampling_params = self._preprocess_prompts(input_batch) + + # Check if LoRA is enabled and create LoRA requests + lora_requests = None + if self._is_lora: + lora_int_ids = list(self.llm.llm_engine.list_loras()) + if len(lora_int_ids) > 0: + lora_int_id = lora_int_ids[0] + batch_size = len(prompt_token_ids) + # dummy_lora_path for placeholder (actual loading done in add_lora()) + lora_requests = [ + LoRARequest(lora_name=f"{lora_int_id}", lora_int_id=lora_int_id, lora_path="/dummy_lora_path") + ] * batch_size + + outputs = await asyncio.to_thread( + self.llm.generate, + prompts=[TokensPrompt(prompt_token_ids=r) for r in prompt_token_ids], + sampling_params=sampling_params, + lora_request=lora_requests, + ) + + return self._postprocess_outputs(outputs) + + async def chat_completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + """Only supported in AsyncVLLMInferenceEngine.""" + raise NotImplementedError("`chat_completion` is only supported in AsyncVLLMInferenceEngine.") + + async def completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + """Only supported in AsyncVLLMInferenceEngine.""" + raise NotImplementedError("`completion` is only supported in AsyncVLLMInferenceEngine.") + + async def wake_up(self, *args: Any, **kwargs: Any): + await asyncio.to_thread(self.llm.wake_up, tags=kwargs.get("tags", None)) + + async def sleep(self, *args: Any, **kwargs: Any): + engine = self._get_engine().llm_engine + output_processor = engine.output_processor + if output_processor.has_unfinished_requests(): + logger.warning( + "Calling sleep() with unfinished requests in vLLM engine. This is unexpected since all " + "generation should be done before sleep() is called. Check for potential failures or " + "dangling requests in your Generator/Env. Aborting all unfinished requests." + ) + unfinished_request_ids = list(output_processor.request_states.keys()) + await asyncio.to_thread(engine.abort_request, unfinished_request_ids) + + level = 1 if self._is_lora else kwargs.get("level", 2) + await asyncio.to_thread(self.llm.sleep, level=level) + + async def init_weight_update_communicator(self, init_info: "WeightSyncInitInfo"): + import pickle + + engine = self._get_engine() + # Pickle the init_info to preserve type through collective_rpc + pickled_init_info = pickle.dumps(init_info) + return await asyncio.to_thread( + engine.collective_rpc, + "init_weight_update_communicator", + args=(pickled_init_info,), + ) + + async def _load_lora_from_disk(self, lora_path: str): + """Load LoRA adapters from disk using vLLM's native add_lora method.""" + lora_id = int(time.time_ns() % 0x7FFFFFFF) + lora_request = LoRARequest(lora_name=f"{lora_id}", lora_int_id=lora_id, lora_path=lora_path) + result = self.llm.llm_engine.add_lora(lora_request) + return result + + async def update_named_weights(self, request: WeightUpdateRequest): + from skyrl_train.weight_sync import LoraLoadRequest + + # Handle LoRA disk loading request + if isinstance(request, LoraLoadRequest): + return await self._load_lora_from_disk(request.lora_path) + + if not len(request): + raise ValueError("Weight update request must not be empty") + + # Use the weight loader to coordinate weight transfer + return await self._weight_loader.load_weights(request) + + async def teardown(self): + await self._teardown_weight_receiver() + + async def reset_prefix_cache(self): + return await asyncio.to_thread(self.llm.llm_engine.reset_prefix_cache) + + async def _teardown_weight_receiver(self): + engine = self._get_engine() + return await asyncio.to_thread(engine.collective_rpc, "teardown_weight_receiver") + + +class AsyncVLLMInferenceEngine(BaseVLLMInferenceEngine): + """Asynchronous VLLM engine.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._weight_loader = VLLMWeightLoader(self.llm, is_async=True) + + def _create_engine(self, *args, **kwargs): + openai_kwargs = pop_openai_kwargs(kwargs) + # TODO (erictang000): potentially enable log requests for a debugging mode + if version.parse(vllm.__version__) >= version.parse("0.10.0"): + engine_args = vllm.AsyncEngineArgs(enable_log_requests=False, **kwargs) + else: + engine_args = vllm.AsyncEngineArgs(disable_log_requests=True, **kwargs) + engine = vllm.AsyncLLMEngine.from_engine_args(engine_args) + + # Adapted from https://github.com/volcengine/verl/blob/e90f18c40aa639cd25092b78a5ff7e2d2508c088/verl/workers/rollout/vllm_rollout/vllm_async_server.py#L327 + model_config = engine.model_config + model_path = kwargs.get("model") + # TODO(Charlie): add a config similar to vllm's `served_model_name`. See https://github.com/NovaSky-AI/SkyRL/pull/238#discussion_r2326561295 + model_name = model_path + + base_model_paths = [BaseModelPath(name=model_name, model_path=model_path)] + + # vllm >= 0.11.2 removed model_config from OpenAI serving APIs + is_new_api = version.parse(vllm.__version__) >= version.parse("0.11.2") + legacy_kwargs = {} + if is_new_api: + models = OpenAIServingModels(engine, base_model_paths) + else: + models = OpenAIServingModels(engine, model_config, base_model_paths) + legacy_kwargs["model_config"] = model_config + + # TODO(Charlie): revisit kwargs `enable_auto_tools` and `tool_parser` when we need to + # support OAI-style tool calling; and `request_logger` for better debugging. + self.openai_serving_chat = OpenAIServingChat( + engine_client=engine, + models=models, + response_role="assistant", + request_logger=None, + chat_template=openai_kwargs.pop("chat_template", None), # used to template /chat/completions requests + chat_template_content_format="auto", + **legacy_kwargs, + **openai_kwargs, + ) + + # TODO(Charlie): revisit kwargs `return_tokens_as_token_ids`, + # `enable_prompt_tokens_details`, `enable_force_include_usage`. + self.openai_serving_completion = OpenAIServingCompletion( + engine_client=engine, + models=models, + request_logger=None, + **legacy_kwargs, + ) + return engine + + async def _load_lora_from_disk(self, lora_path: str): + """Load LoRA adapters from disk using vLLM's native add_lora method.""" + lora_id = int(time.time_ns() % 0x7FFFFFFF) + lora_request = LoRARequest(lora_name=f"{lora_id}", lora_int_id=lora_id, lora_path=lora_path) + result = await self.llm.add_lora(lora_request) + return result + + async def _collect_outputs(self, prompt_token_ids, request_id: str, sampling_params: SamplingParams): + """Collect outputs for a single prompt.""" + # Check if LoRA is enabled and create LoRA request + final_output = None + lora_request = None + + if self._is_lora: + lora_int_ids = list(await self.llm.list_loras()) + if len(lora_int_ids) > 0: + lora_int_id = lora_int_ids[0] + # dummy_lora_path for placeholder (actual loading done in add_lora()) + lora_request = LoRARequest( + lora_name=f"{lora_int_id}", lora_int_id=lora_int_id, lora_path="/dummy_lora_path" + ) + + async for request_output in self.llm.generate( + prompt=TokensPrompt(prompt_token_ids=prompt_token_ids), + sampling_params=sampling_params, + request_id=request_id, + lora_request=lora_request, + ): + final_output = request_output + + return final_output + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + """Generate responses using vLLM's async engine.""" + prompt_token_ids, sampling_params = self._preprocess_prompts(input_batch) + + tasks = [] + for prompt in prompt_token_ids: + # Schedule the collection of outputs for each prompt. + # Avoid duplicate request_ids + request_id = str(uuid4().hex) + task = asyncio.create_task(self._collect_outputs(prompt, request_id, sampling_params)) + tasks.append(task) + outputs = await asyncio.gather(*tasks) + + return self._postprocess_outputs(outputs) + + async def wake_up(self, *args: Any, **kwargs: Any): + await self.llm.wake_up(tags=kwargs.get("tags", None)) + + async def sleep(self, *args: Any, **kwargs: Any): + engine = self._get_engine() + output_processor = engine.output_processor + # make sure that the engine is alive + engine.engine_core.ensure_alive() + if output_processor.has_unfinished_requests(): + logger.warning( + "Calling sleep() with unfinished requests in vLLM engine. This is unexpected since all " + "generation should be done before sleep() is called. Check for potential failures or " + "dangling requests in your Generator/Env. Aborting all unfinished requests." + ) + unfinished_request_ids = list(output_processor.request_states.keys()) + await engine.abort(unfinished_request_ids) + + # TODO(team): remove once vllm fixes this + # otherwise waking it up will output gibberish: https://github.com/vllm-project/vllm/issues/17103 + await self.reset_prefix_cache() + level = 1 if self._is_lora else kwargs.get("level", 2) + await self.llm.sleep(level=level) + + async def init_weight_update_communicator(self, init_info: "WeightSyncInitInfo"): + import pickle + + engine = self._get_engine() + # Pickle the init_info to preserve type through collective_rpc + pickled_init_info = pickle.dumps(init_info) + return await engine.collective_rpc( + "init_weight_update_communicator", + args=(pickled_init_info,), + ) + + async def update_named_weights(self, request: WeightUpdateRequest): + from skyrl_train.weight_sync import LoraLoadRequest + + # Check for LoRA disk loading request + if isinstance(request, LoraLoadRequest): + return await self._load_lora_from_disk(request.lora_path) + + if not len(request): + raise ValueError("Weight update request must not be empty") + + # Use the weight loader to coordinate weight transfer + return await self._weight_loader.load_weights(request) + + async def teardown(self): + await self._teardown_weight_receiver() + + async def reset_prefix_cache(self): + engine = self._get_engine() + await engine.reset_prefix_cache() + + async def _teardown_weight_receiver(self): + engine = self._get_engine() + return await engine.collective_rpc("teardown_weight_receiver") + + # ---------------------------------------- + # Methods for handling OpenAI API requests + # ---------------------------------------- + + async def _handle_openai_request(self, request_payload: Dict[str, Any], endpoint: str) -> Dict[str, Any]: + """Handle OpenAI API request.""" + assert endpoint in ["/chat/completions", "/completions"] + + body = request_payload.get("json", {}) + headers = request_payload.get("headers", {}) + + # 1. Build request + try: + if endpoint == "/chat/completions": + request = ChatCompletionRequest(**body) + else: + request = CompletionRequest(**body) + assert request.stream is False, "Streaming is not supported in SkyRL yet, please set stream to False." + except Exception as e: + if version.parse(vllm.__version__) >= version.parse("0.10.0"): + try: + from vllm.entrypoints.openai.engine.protocol import ErrorInfo + except ImportError: + from vllm.entrypoints.openai.protocol import ErrorInfo + + return ErrorResponse( + error=ErrorInfo( + message=str(e), + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ), + ).model_dump() + else: + return ErrorResponse( + message=str(e), + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ).model_dump() + + # 2. Call vllm engine + try: + # Create a minimal request-like object with attributes used by vLLM + minimal_request = _MinimalRequest(headers) + if endpoint == "/chat/completions": + generator = await self.openai_serving_chat.create_chat_completion(request, minimal_request) + assert isinstance(generator, (ChatCompletionResponse, ErrorResponse)) + else: + generator = await self.openai_serving_completion.create_completion(request, minimal_request) + assert isinstance(generator, (CompletionResponse, ErrorResponse)) + return generator.model_dump() + + except Exception as e: + # Handle it here so we can surface the error from a ray worker. + if version.parse(vllm.__version__) >= version.parse("0.10.0"): + try: + from vllm.entrypoints.openai.engine.protocol import ErrorInfo + except ImportError: + from vllm.entrypoints.openai.protocol import ErrorInfo + + return ErrorResponse( + error=ErrorInfo( + message=str(e), + type=HTTPStatus.INTERNAL_SERVER_ERROR.phrase, + code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + ), + ).model_dump() + else: + return ErrorResponse( + message=str(e), + type=HTTPStatus.INTERNAL_SERVER_ERROR.phrase, + code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + ).model_dump() + + async def chat_completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + """OpenAI-compatible HTTP endpoint for handling `/chat/completions` in Python vLLM engine. + + Accepts a JSON-serializable payload: {"json": , "headers": }. + Constructs a minimal request-like object for vLLM's openai_serving_chat. + Returns a plain dict, either a ChatCompletionResponse or an ErrorResponse, both defined + in vllm.entrypoints.openai.protocol. + """ + return await self._handle_openai_request(request_payload, endpoint="/chat/completions") + + async def completion(self, request_payload: Dict[str, Any]) -> Dict[str, Any]: + """OpenAI-compatible HTTP endpoint for handling `/completions` in Python vLLM engine. + + Accepts a JSON-serializable payload: {"json": , "headers": }. + Constructs a minimal request-like object for vLLM's openai_serving_completion. + Returns a plain dict, either a CompletionResponse or an ErrorResponse, both defined + in vllm.entrypoints.openai.protocol. + """ + return await self._handle_openai_request(request_payload, endpoint="/completions") + + async def abort_generation(self) -> None: + """ + Abort all running and waiting requests, which make the ongoing requests return the + already-generated tokens with a stop_reason of "abort". + """ + engine = self._get_engine() + # Collect all request IDs currently tracked by the scheduler/output processor + unfinished_request_ids = list(engine.output_processor.request_states.keys()) + if unfinished_request_ids: + await engine.abort(unfinished_request_ids) + await engine.reset_prefix_cache() # avoid KV-cache pollution + logger.info(f"abort_generation() finished, aborted {len(unfinished_request_ids)} requests") + + +class _MinimalRequest: + """ + Minimal request-like object for vLLM's openai_serving_chat and openai_serving_completion. + + We cannot use the original user Request object because it cannot be serialized and hence + cannot be a ray method argument. Instead we take the original request's headers and + reconstruct an instance of _MinimalRequest to mimic the FastAPI Request object. + + The fields depend on what vLLM accesses internally. + """ + + def __init__(self, headers): + self.headers = headers # Expect a mapping with .get support + self.state = SimpleNamespace() # vLLM sets raw_request.state.request_metadata + + +class VLLMWeightLoader(WeightLoader): + """Loads weights into vLLM engine, managing RPC coordination. + + This loader encapsulates the collective_rpc calls to workers. + Workers create the appropriate receiver locally for the actual weight transfer. + """ + + def __init__(self, engine: Any, is_async: bool = False) -> None: + """Initialize the loader. + + Args: + engine: The vLLM engine (LLM or AsyncLLMEngine). + is_async: Whether this is for AsyncVLLMInferenceEngine. + """ + self._engine = engine.engine if hasattr(engine, "engine") else engine + self._is_async = is_async + + async def load_weights(self, request: WeightUpdateRequest) -> None: + """Load weights by coordinating RPC to workers. + + Sends the request to workers via collective_rpc. Workers create + the receiver locally and use it to receive and load weights. + + Args: + request: Weight update request. + """ + import pickle + + # Pickle the request to preserve type through collective_rpc + pickled_request = pickle.dumps(request) + + if self._is_async: + await self._engine.collective_rpc( + "load_weights", + args=(pickled_request,), + ) + else: + await asyncio.to_thread( + self._engine.collective_rpc, + "load_weights", + args=(pickled_request,), + ) + + +VLLMRayActor = ray.remote(VLLMInferenceEngine) +AsyncVLLMRayActor = ray.remote(AsyncVLLMInferenceEngine) diff --git a/skyrl-train/skyrl_train/inference_engines/vllm/vllm_server.py b/skyrl-train/skyrl_train/inference_engines/vllm/vllm_server.py new file mode 100644 index 0000000000..85649fb69c --- /dev/null +++ b/skyrl-train/skyrl_train/inference_engines/vllm/vllm_server.py @@ -0,0 +1,156 @@ +import os +import signal +import uvloop +from vllm import AsyncLLMEngine +from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.system_utils import set_ulimit +from vllm.entrypoints.openai.cli_args import ( + make_arg_parser, + validate_parsed_serve_args, +) +from vllm.entrypoints.launcher import serve_http +from vllm.entrypoints.openai.api_server import ( + create_server_socket, + build_app, + init_app_state, +) +import vllm.envs as envs +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.usage.usage_lib import UsageContext +from fastapi import Request + + +# TODO(tgriggs): Handle errors and use best practices for vLLM server +# TODO(tgriggs): Return correct status codes. +class VllmServer: + def __init__(self, args): + self.server_args = args + + async def run_server(self, **uvicorn_kwargs) -> None: + sock_addr = (self.server_args.host or "", self.server_args.port) + sock = create_server_socket(sock_addr) + + set_ulimit() + + def signal_handler(*_) -> None: + # Interrupt server on sigterm while initializing + raise KeyboardInterrupt("terminated") + + signal.signal(signal.SIGTERM, signal_handler) + + # TODO(tgriggs): Move this elsewhere, make configurable. + os.environ["VLLM_USE_V1"] = "1" + engine_args = AsyncEngineArgs.from_cli_args(self.server_args) + engine = AsyncLLMEngine.from_engine_args( + engine_args=engine_args, + usage_context=UsageContext.OPENAI_API_SERVER, + ) + + sock_addr = (self.server_args.host or "", self.server_args.port) + sock = create_server_socket(sock_addr) + app = build_app(self.server_args) + + @app.post("/init_weight_update_communicator") + async def _init_weight_update_communicator(request: Request): + import pickle + from skyrl_train.weight_sync import BroadcastInitInfo + + data = await request.json() + init_info = BroadcastInitInfo(**data) + + # Pickle to preserve type through collective_rpc + pickled_init_info = pickle.dumps(init_info) + + await engine.collective_rpc( + "init_weight_update_communicator", + args=(pickled_init_info,), + ) + return {"status": "ok"} + + @app.post("/sleep") + async def _sleep(request: Request): + data = await request.json() + level = data.get("level") + + # TODO(team): remove once vllm fixes this + # otherwise waking it up will output gibberish: https://github.com/vllm-project/vllm/issues/17103 + await engine.reset_prefix_cache() + + await engine.sleep(level) + return {"status": "ok"} + + @app.post("/wake_up") + async def _wake_up(request: Request): + data = await request.json() + tags = data.get("tags") + await engine.wake_up(tags) + return {"status": "ok"} + + @app.post("/reset_prefix_cache") + async def _reset_prefix_cache(request: Request): + await engine.reset_prefix_cache() + return {"status": "ok"} + + @app.post("/update_weights") + async def _update_weights(request: Request): + import pickle + from skyrl_train.weight_sync import BroadcastWeightUpdateRequest + + # Convert the HTTP request to a BroadcastWeightUpdateRequest + # TODO(haochen): only the broadcast strategy is currently supported + # for the remote inference engine path. + # To support other strategies, we'll need to add a "strategy=xxx" + # parameter in the HTTP request. + data = await request.json() + weight_request = BroadcastWeightUpdateRequest(**data) + + # Pickle to preserve type through collective_rpc + pickled_request = pickle.dumps(weight_request) + + await engine.collective_rpc( + "load_weights", + args=(pickled_request,), + ) + return {"status": "ok"} + + @app.post("/destroy_weights_update_group") + async def _destroy_weights_update_group(request: Request): + data = await request.json() # noqa: F841 + await engine.collective_rpc( + "teardown_weight_receiver", + args=(), + ) + return {"status": "ok"} + + await init_app_state(engine, app.state, args) + + shutdown_task = await serve_http( + app, + sock, + host=self.server_args.host, + port=self.server_args.port, + log_level=self.server_args.uvicorn_log_level, + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE, + ssl_keyfile=self.server_args.ssl_keyfile, + ssl_certfile=self.server_args.ssl_certfile, + ssl_ca_certs=self.server_args.ssl_ca_certs, + ssl_cert_reqs=self.server_args.ssl_cert_reqs, + **uvicorn_kwargs, + ) + + await shutdown_task + + sock.close() + + def run_server_uvloop(self, **uvicorn_kwargs) -> None: + uvloop.run(self.run_server(**uvicorn_kwargs)) + + +if __name__ == "__main__": + parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible RESTful API server.") + parser = make_arg_parser(parser) + args = parser.parse_args() + validate_parsed_serve_args(args) + + vllm_server = VllmServer(args) + vllm_server.run_server_uvloop() diff --git a/skyrl-train/skyrl_train/metrics/__init__.py b/skyrl-train/skyrl_train/metrics/__init__.py new file mode 100644 index 0000000000..29201f5eaf --- /dev/null +++ b/skyrl-train/skyrl_train/metrics/__init__.py @@ -0,0 +1,21 @@ +"""Unified metrics for SkyRL training.""" + +from skyrl_train.metrics.reward_metrics import ( + compute_pass_at_n, + compute_reward_metrics, + compute_per_group_metrics, + compute_signal_ratio, + compute_variance_per_prompt, + flatten_rewards, + sanitize_metric_key, +) + +__all__ = [ + "compute_pass_at_n", + "compute_reward_metrics", + "compute_per_group_metrics", + "compute_signal_ratio", + "compute_variance_per_prompt", + "flatten_rewards", + "sanitize_metric_key", +] diff --git a/skyrl-train/skyrl_train/metrics/reward_metrics.py b/skyrl-train/skyrl_train/metrics/reward_metrics.py new file mode 100644 index 0000000000..3978ee1acd --- /dev/null +++ b/skyrl-train/skyrl_train/metrics/reward_metrics.py @@ -0,0 +1,253 @@ +"""Unified reward metrics for SkyRL and Tinker. + +This module provides shared metric calculation functions used by both: +- SkyRL trainer (skyrl_train/trainer.py, skyrl_train/utils/trainer_utils.py) +- Tinker integration (integrations/fleet/entrypoints/main_fleet_tinker.py) + +All metrics follow the same naming convention for WandB logging: +- reward/{group}/pass_at_{n} - Pass@n metric for group +- reward/{group}/variance_per_prompt - Mean within-prompt reward variance (GRPO learning signal) +- reward/{group}/signal_ratio - Fraction of prompts with non-zero variance (% with signal) +- reward/{group}/mean_positive_reward - Mean of positive rewards for group + +Rewards can be in two formats: +- Scalar rewards: List[float] - one reward per trajectory +- Token-level rewards: List[List[float]] - per-token rewards per trajectory (summed to scalar) +""" + +from collections import defaultdict +from typing import Any, Dict, List, Union + +import numpy as np + + +def flatten_rewards(rewards: Union[List[float], List[List[float]]]) -> List[float]: + """Flatten rewards to scalar format. + + Handles both scalar rewards (List[float]) and token-level rewards (List[List[float]]). + For token-level rewards, sums each trajectory's rewards into a single scalar. + + Args: + rewards: Either List[float] (scalar per trajectory) or + List[List[float]] (token-level per trajectory) + + Returns: + List[float]: Flattened scalar rewards, one per trajectory + """ + if not rewards: + return [] + + flat_rewards: List[float] = [] + for r in rewards: + if isinstance(r, list): + # Token-level rewards: sum to get trajectory reward + flat_rewards.append(float(sum(r))) + else: + flat_rewards.append(float(r)) + return flat_rewards + + +def sanitize_metric_key(key: str) -> str: + """Sanitize metric key for wandb (replace / with _). + + Args: + key: Raw metric key that may contain slashes + + Returns: + Sanitized key with slashes replaced by underscores + """ + return key.replace("/", "_") + + +def compute_pass_at_n( + rewards: Union[List[float], List[List[float]]], + uids: List[str], +) -> float: + """Compute pass@n: fraction of unique prompts with at least one fully successful rollout. + + For each unique prompt (identified by uid), if ANY of its rollouts achieves a + perfect reward (>= 1.0), that prompt counts as a "pass". This metric measures + how often the model can fully solve a task when given multiple attempts. + Partial rewards (e.g. 0.3 from partial_reward mode) do not count as a pass. + + Args: + rewards: List of rewards (one per rollout). Can be scalar (List[float]) + or token-level (List[List[float]]) - will be flattened. + uids: List of unique IDs (one per rollout, same uid = same prompt) + + Returns: + Float between 0.0 and 1.0 representing the fraction of prompts that passed + """ + flat_rewards = flatten_rewards(rewards) + uid_to_rewards: Dict[str, List[float]] = defaultdict(list) + for uid, reward in zip(uids, flat_rewards): + uid_to_rewards[uid].append(reward) + + if not uid_to_rewards: + return 0.0 + + passed = sum(1 for r_list in uid_to_rewards.values() if any(r >= 1.0 for r in r_list)) + return passed / len(uid_to_rewards) + + +def compute_variance_per_prompt( + rewards: Union[List[float], List[List[float]]], + uids: List[str], +) -> float: + """Compute mean within-prompt reward variance (GRPO learning signal). + + For GRPO to learn, there must be variance in rewards within each prompt's rollouts. + If all rollouts for a prompt get the same reward, there's no learning signal. + + This metric computes the variance of rewards for each prompt, then returns the + mean variance across all prompts. + + Args: + rewards: List of rewards (one per rollout). Can be scalar (List[float]) + or token-level (List[List[float]]) - will be flattened. + uids: List of unique IDs (one per rollout, same uid = same prompt) + + Returns: + Mean variance across prompts. Higher = more learning signal. + Returns 0.0 if no prompts or all prompts have single rollouts. + """ + flat_rewards = flatten_rewards(rewards) + uid_to_rewards: Dict[str, List[float]] = defaultdict(list) + for uid, reward in zip(uids, flat_rewards): + uid_to_rewards[uid].append(reward) + + if not uid_to_rewards: + return 0.0 + + # Compute variance for each prompt (need at least 2 samples for variance) + variances = [] + for r_list in uid_to_rewards.values(): + if len(r_list) >= 2: + variances.append(float(np.var(r_list))) + + return float(np.mean(variances)) if variances else 0.0 + + +def compute_signal_ratio( + rewards: Union[List[float], List[List[float]]], + uids: List[str], +) -> float: + """Compute fraction of prompts with non-zero variance (GRPO signal ratio). + + This metric shows what percentage of prompts have any learning signal at all. + A prompt has signal if at least one rollout differs from others (variance > 0). + + Unlike variance_per_prompt (which averages variance magnitudes), this metric + counts how many prompts contribute ANY signal, making it easier to interpret: + - 100% = every prompt has at least one differing rollout + - 0% = all prompts have identical rewards across rollouts (no learning possible) + + Args: + rewards: List of rewards (one per rollout). Can be scalar (List[float]) + or token-level (List[List[float]]) - will be flattened. + uids: List of unique IDs (one per rollout, same uid = same prompt) + + Returns: + Float between 0.0 and 1.0 representing fraction of prompts with signal. + Returns 0.0 if no prompts or all prompts have single rollouts. + """ + flat_rewards = flatten_rewards(rewards) + uid_to_rewards: Dict[str, List[float]] = defaultdict(list) + for uid, reward in zip(uids, flat_rewards): + uid_to_rewards[uid].append(reward) + + if not uid_to_rewards: + return 0.0 + + # Count prompts with variance > 0 (need at least 2 samples) + prompts_with_signal = 0 + prompts_total = 0 + for r_list in uid_to_rewards.values(): + if len(r_list) >= 2: + prompts_total += 1 + if np.var(r_list) > 0: + prompts_with_signal += 1 + + return prompts_with_signal / prompts_total if prompts_total > 0 else 0.0 + + +def compute_reward_metrics( + rewards: Union[List[float], List[List[float]]], + uids: List[str], + n_samples_per_prompt: int, +) -> Dict[str, float]: + """Compute core reward metrics. + + Args: + rewards: List of rewards (one per rollout). Can be scalar (List[float]) + or token-level (List[List[float]]) - will be flattened. + uids: List of unique IDs for pass@n grouping + n_samples_per_prompt: Number of samples per prompt (used in metric key name) + + Returns: + Dictionary with keys: + - "pass_at_{n}": Pass@n metric + - "variance_per_prompt": Mean within-prompt reward variance (GRPO learning signal) + - "signal_ratio": Fraction of prompts with non-zero variance (% with signal) + - "mean_positive_reward": Mean of positive rewards only + """ + # Flatten rewards once for efficiency (each sub-function would otherwise flatten again) + flat_rewards = flatten_rewards(rewards) + pass_at_n = compute_pass_at_n(flat_rewards, uids) + variance = compute_variance_per_prompt(flat_rewards, uids) + signal_ratio = compute_signal_ratio(flat_rewards, uids) + positive_rewards = [r for r in flat_rewards if r > 0] + mean_positive = float(np.mean(positive_rewards)) if positive_rewards else 0.0 + + return { + f"pass_at_{n_samples_per_prompt}": pass_at_n, + "variance_per_prompt": variance, + "signal_ratio": signal_ratio, + "mean_positive_reward": mean_positive, + } + + +def compute_per_group_metrics( + rewards: Union[List[float], List[List[float]]], + uids: List[str], + groups: List[str], + n_samples_per_prompt: int, + prefix: str = "reward", +) -> Dict[str, float]: + """Compute metrics grouped by a key (env_key, data_source, etc). + + This function computes reward metrics for each group separately, enabling + per-environment analysis in training and evaluation. + + Args: + rewards: List of rewards (one per rollout). Can be scalar (List[float]) + or token-level (List[List[float]]) - will be flattened. + uids: List of unique IDs for pass@n grouping within each group + groups: List of group keys (e.g., env_key or data_source per rollout) + n_samples_per_prompt: Number of samples per prompt (used in metric key name) + prefix: Metric prefix ("reward" for training, "eval" for evaluation) + + Returns: + Dictionary with keys like: + - "{prefix}/{group}/avg_score" + - "{prefix}/{group}/pass_at_{n}" + - "{prefix}/{group}/mean_positive_reward" + """ + # Flatten rewards once before grouping + flat_rewards = flatten_rewards(rewards) + + # Group data by group key + group_data: Dict[str, Dict[str, List[Any]]] = defaultdict(lambda: {"rewards": [], "uids": []}) + for reward, uid, group in zip(flat_rewards, uids, groups): + group_key = group if group is not None else "unknown" + group_data[group_key]["rewards"].append(reward) + group_data[group_key]["uids"].append(uid) + + metrics: Dict[str, float] = {} + for group_key, data in group_data.items(): + sanitized = sanitize_metric_key(group_key) + group_metrics = compute_reward_metrics(data["rewards"], data["uids"], n_samples_per_prompt) + for metric_name, value in group_metrics.items(): + metrics[f"{prefix}/{sanitized}/{metric_name}"] = value + + return metrics diff --git a/skyrl-train/skyrl_train/metrics/tests/__init__.py b/skyrl-train/skyrl_train/metrics/tests/__init__.py new file mode 100644 index 0000000000..9b6252fde0 --- /dev/null +++ b/skyrl-train/skyrl_train/metrics/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for skyrl_train.metrics package.""" diff --git a/skyrl-train/skyrl_train/metrics/tests/test_reward_metrics.py b/skyrl-train/skyrl_train/metrics/tests/test_reward_metrics.py new file mode 100644 index 0000000000..b8992991fd --- /dev/null +++ b/skyrl-train/skyrl_train/metrics/tests/test_reward_metrics.py @@ -0,0 +1,433 @@ +"""Tests for reward_metrics module.""" + +from skyrl_train.metrics.reward_metrics import ( + compute_pass_at_n, + compute_per_group_metrics, + compute_reward_metrics, + compute_variance_per_prompt, + flatten_rewards, + sanitize_metric_key, +) + + +class TestFlattenRewards: + """Tests for flatten_rewards function.""" + + def test_scalar_rewards(self): + """Scalar rewards remain unchanged.""" + rewards = [1.0, 0.5, 0.0, -0.5] + assert flatten_rewards(rewards) == [1.0, 0.5, 0.0, -0.5] + + def test_token_level_rewards(self): + """Token-level rewards are summed.""" + rewards = [[0.5, 0.3, 0.2], [0.0, 0.0, 1.0], [0.0, 0.0, 0.0]] + assert flatten_rewards(rewards) == [1.0, 1.0, 0.0] + + def test_mixed_length_token_rewards(self): + """Token-level rewards with varying lengths.""" + rewards = [[1.0], [0.5, 0.5], [0.1, 0.2, 0.3, 0.4]] + assert flatten_rewards(rewards) == [1.0, 1.0, 1.0] + + def test_empty_list(self): + """Empty list returns empty list.""" + assert flatten_rewards([]) == [] + + def test_empty_token_rewards(self): + """Empty token reward lists sum to 0.""" + rewards = [[], [], [1.0]] + assert flatten_rewards(rewards) == [0.0, 0.0, 1.0] + + def test_single_element(self): + """Single scalar reward.""" + assert flatten_rewards([0.5]) == [0.5] + + def test_single_token_list(self): + """Single token-level reward.""" + assert flatten_rewards([[0.3, 0.7]]) == [1.0] + + def test_negative_token_rewards(self): + """Negative token rewards sum correctly.""" + rewards = [[-0.5, 0.5], [0.0, -1.0]] + assert flatten_rewards(rewards) == [0.0, -1.0] + + +class TestSanitizeMetricKey: + def test_replaces_slash(self): + assert sanitize_metric_key("foo/bar") == "foo_bar" + + def test_multiple_slashes(self): + assert sanitize_metric_key("a/b/c") == "a_b_c" + + def test_no_slash(self): + assert sanitize_metric_key("foobar") == "foobar" + + def test_empty_string(self): + assert sanitize_metric_key("") == "" + + +class TestComputePassAtN: + def test_all_pass(self): + """All prompts have at least one positive reward.""" + rewards = [1.0, 1.0, 1.0, 1.0] + uids = ["a", "a", "b", "b"] + assert compute_pass_at_n(rewards, uids) == 1.0 + + def test_none_pass(self): + """No prompts have positive rewards.""" + rewards = [0.0, 0.0, 0.0, 0.0] + uids = ["a", "a", "b", "b"] + assert compute_pass_at_n(rewards, uids) == 0.0 + + def test_partial_pass(self): + """uid 'a' has one pass, uid 'b' has none -> 50% pass rate.""" + rewards = [1.0, 0.0, 0.0, 0.0] + uids = ["a", "a", "b", "b"] + assert compute_pass_at_n(rewards, uids) == 0.5 + + def test_any_positive_counts(self): + """uid 'a' has one pass out of 4 rollouts, still counts as pass.""" + rewards = [0.0, 0.0, 0.0, 1.0] + uids = ["a", "a", "a", "a"] + assert compute_pass_at_n(rewards, uids) == 1.0 + + def test_empty_input(self): + """Empty input returns 0.0.""" + assert compute_pass_at_n([], []) == 0.0 + + def test_single_prompt_pass(self): + """Single prompt with positive reward.""" + rewards = [1.0] + uids = ["a"] + assert compute_pass_at_n(rewards, uids) == 1.0 + + def test_single_prompt_fail(self): + """Single prompt with zero reward.""" + rewards = [0.0] + uids = ["a"] + assert compute_pass_at_n(rewards, uids) == 0.0 + + def test_fractional_positive_reward(self): + """Fractional positive reward counts as pass.""" + rewards = [0.5, 0.0] + uids = ["a", "a"] + assert compute_pass_at_n(rewards, uids) == 1.0 + + def test_negative_reward_not_pass(self): + """Negative rewards don't count as pass.""" + rewards = [-1.0, -0.5] + uids = ["a", "a"] + assert compute_pass_at_n(rewards, uids) == 0.0 + + def test_token_level_rewards_pass(self): + """Token-level rewards are summed - positive sum counts as pass.""" + # First trajectory: [0.5, 0.3, 0.2] -> 1.0 (pass) + # Second trajectory: [0.0, 0.0, 0.0] -> 0.0 (fail) + rewards = [[0.5, 0.3, 0.2], [0.0, 0.0, 0.0]] + uids = ["a", "a"] + assert compute_pass_at_n(rewards, uids) == 1.0 # uid "a" passes + + def test_token_level_rewards_all_zero(self): + """Token-level rewards that sum to zero don't pass.""" + rewards = [[0.0, 0.0], [0.0, 0.0, 0.0]] + uids = ["a", "b"] + assert compute_pass_at_n(rewards, uids) == 0.0 + + def test_token_level_rewards_partial(self): + """Some prompts pass with token-level rewards.""" + rewards = [[1.0, 0.0], [0.0, 0.0], [0.0, 0.5], [0.0, 0.0]] + uids = ["a", "a", "b", "b"] + # uid "a": has [1.0, 0.0] -> sum=1.0 (pass) + # uid "b": has [0.5, 0.0] -> sum=0.5 (pass) + assert compute_pass_at_n(rewards, uids) == 1.0 + + +class TestComputeVariancePerPrompt: + """Tests for within-prompt variance (GRPO learning signal).""" + + def test_high_variance(self): + """High variance within prompt = good learning signal.""" + rewards = [1.0, 0.0, 1.0, 0.0] # All for same prompt + uids = ["a", "a", "a", "a"] + variance = compute_variance_per_prompt(rewards, uids) + assert variance == 0.25 # np.var([1, 0, 1, 0]) = 0.25 + + def test_zero_variance(self): + """Zero variance = no learning signal.""" + rewards = [1.0, 1.0, 1.0, 1.0] # All same reward + uids = ["a", "a", "a", "a"] + variance = compute_variance_per_prompt(rewards, uids) + assert variance == 0.0 + + def test_multiple_prompts_average(self): + """Variance is averaged across prompts.""" + # Prompt "a": rewards [1.0, 0.0] -> variance = 0.25 + # Prompt "b": rewards [1.0, 1.0] -> variance = 0.0 + # Mean variance = (0.25 + 0.0) / 2 = 0.125 + rewards = [1.0, 0.0, 1.0, 1.0] + uids = ["a", "a", "b", "b"] + variance = compute_variance_per_prompt(rewards, uids) + assert variance == 0.125 + + def test_single_sample_per_prompt_excluded(self): + """Prompts with single sample are excluded (can't compute variance).""" + rewards = [1.0, 0.0, 0.5] # Three prompts with 1 sample each + uids = ["a", "b", "c"] + variance = compute_variance_per_prompt(rewards, uids) + assert variance == 0.0 # No prompts with 2+ samples + + def test_empty_input(self): + """Empty input returns 0.0.""" + assert compute_variance_per_prompt([], []) == 0.0 + + def test_saturated_environment(self): + """Saturated env (all 1.0) has zero variance = no learning signal.""" + rewards = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + uids = ["a", "a", "b", "b", "c", "c"] + variance = compute_variance_per_prompt(rewards, uids) + assert variance == 0.0 # All prompts have zero variance + + def test_mixed_variance_across_prompts(self): + """Some prompts have variance, others don't.""" + # Prompt "a": [1.0, 0.0] -> var = 0.25 + # Prompt "b": [0.5, 0.5] -> var = 0.0 + # Prompt "c": [1.0, 0.5] -> var = 0.0625 + # Mean = (0.25 + 0.0 + 0.0625) / 3 = 0.104166... + rewards = [1.0, 0.0, 0.5, 0.5, 1.0, 0.5] + uids = ["a", "a", "b", "b", "c", "c"] + variance = compute_variance_per_prompt(rewards, uids) + assert abs(variance - 0.10416666666666667) < 1e-10 + + def test_token_level_rewards_variance(self): + """Token-level rewards are summed before variance calculation.""" + # Trajectory 1: [1.0, 0.0] -> 1.0 + # Trajectory 2: [0.0, 0.0] -> 0.0 + # Both for uid "a", so variance = 0.25 + rewards = [[1.0, 0.0], [0.0, 0.0]] + uids = ["a", "a"] + variance = compute_variance_per_prompt(rewards, uids) + assert variance == 0.25 + + def test_token_level_rewards_no_variance(self): + """Token-level rewards that sum to same value have zero variance.""" + # Both sum to 1.0 + rewards = [[0.5, 0.5], [0.3, 0.7]] + uids = ["a", "a"] + variance = compute_variance_per_prompt(rewards, uids) + assert variance == 0.0 + + +class TestComputeRewardMetrics: + def test_basic_metrics(self): + """Test basic metric calculation with mixed rewards.""" + rewards = [1.0, 0.0, 0.5, 0.0] + uids = ["a", "a", "b", "b"] + metrics = compute_reward_metrics(rewards, uids, n_samples_per_prompt=2) + + assert metrics["pass_at_2"] == 1.0 # both uids have positive + assert metrics["mean_positive_reward"] == 0.75 # (1+0.5)/2 + # variance: uid "a" [1,0] var=0.25, uid "b" [0.5,0] var=0.0625 -> mean=0.15625 + assert abs(metrics["variance_per_prompt"] - 0.15625) < 1e-10 + + def test_no_positive_rewards(self): + """All zero rewards.""" + rewards = [0.0, 0.0] + uids = ["a", "b"] + metrics = compute_reward_metrics(rewards, uids, n_samples_per_prompt=1) + + assert metrics["pass_at_1"] == 0.0 + assert metrics["mean_positive_reward"] == 0.0 + assert metrics["variance_per_prompt"] == 0.0 # single sample per uid + + def test_all_positive_rewards(self): + """All positive rewards with variance.""" + rewards = [1.0, 0.5, 0.75, 0.25] + uids = ["a", "a", "b", "b"] + metrics = compute_reward_metrics(rewards, uids, n_samples_per_prompt=2) + + assert metrics["pass_at_2"] == 1.0 # all pass + assert metrics["mean_positive_reward"] == 0.625 # (1+0.5+0.75+0.25)/4 + # variance: uid "a" [1,0.5] var=0.0625, uid "b" [0.75,0.25] var=0.0625 -> mean=0.0625 + assert abs(metrics["variance_per_prompt"] - 0.0625) < 1e-10 + + def test_empty_input(self): + """Empty input returns zeros.""" + metrics = compute_reward_metrics([], [], n_samples_per_prompt=1) + + assert metrics["pass_at_1"] == 0.0 + assert metrics["mean_positive_reward"] == 0.0 + assert metrics["variance_per_prompt"] == 0.0 + + def test_different_n_samples(self): + """Test that n_samples_per_prompt is reflected in key name.""" + rewards = [1.0, 0.0] + uids = ["a", "a"] + + metrics_4 = compute_reward_metrics(rewards, uids, n_samples_per_prompt=4) + assert "pass_at_4" in metrics_4 + assert "variance_per_prompt" in metrics_4 + + metrics_8 = compute_reward_metrics(rewards, uids, n_samples_per_prompt=8) + assert "pass_at_8" in metrics_8 + assert "variance_per_prompt" in metrics_8 + + def test_token_level_rewards(self): + """Test with token-level rewards (List[List[float]]).""" + # Trajectory 1 for uid "a": [0.5, 0.5] -> 1.0 + # Trajectory 2 for uid "a": [0.0, 0.0] -> 0.0 + # Trajectory 3 for uid "b": [0.3, 0.2] -> 0.5 + # Trajectory 4 for uid "b": [0.0, 0.0] -> 0.0 + rewards = [[0.5, 0.5], [0.0, 0.0], [0.3, 0.2], [0.0, 0.0]] + uids = ["a", "a", "b", "b"] + metrics = compute_reward_metrics(rewards, uids, n_samples_per_prompt=2) + + # Both uids have at least one positive (1.0, 0.5) + assert metrics["pass_at_2"] == 1.0 + # Positive rewards: 1.0, 0.5 -> mean = 0.75 + assert metrics["mean_positive_reward"] == 0.75 + # Variance: uid "a" [1.0, 0.0] var=0.25, uid "b" [0.5, 0.0] var=0.0625 -> mean=0.15625 + assert abs(metrics["variance_per_prompt"] - 0.15625) < 1e-10 + + def test_token_level_rewards_all_zero(self): + """Test token-level rewards that all sum to zero.""" + rewards = [[0.0, 0.0], [0.0, 0.0, 0.0]] + uids = ["a", "b"] + metrics = compute_reward_metrics(rewards, uids, n_samples_per_prompt=1) + + assert metrics["pass_at_1"] == 0.0 + assert metrics["mean_positive_reward"] == 0.0 + assert metrics["variance_per_prompt"] == 0.0 + + +class TestComputePerGroupMetrics: + def test_per_env_grouping(self): + """Test grouping by environment.""" + rewards = [1.0, 0.0, 0.5, 0.0] + uids = ["task1", "task1", "task2", "task2"] + groups = ["github", "github", "booking", "booking"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="reward") + + # Check all expected keys exist + assert "reward/github/variance_per_prompt" in metrics + assert "reward/github/pass_at_2" in metrics + assert "reward/github/mean_positive_reward" in metrics + assert "reward/booking/variance_per_prompt" in metrics + assert "reward/booking/pass_at_2" in metrics + assert "reward/booking/mean_positive_reward" in metrics + + # Check values + assert metrics["reward/github/pass_at_2"] == 1.0 # task1 passes + assert metrics["reward/github/variance_per_prompt"] == 0.25 # [1,0] var=0.25 + assert metrics["reward/booking/pass_at_2"] == 1.0 # task2 passes + assert metrics["reward/booking/variance_per_prompt"] == 0.0625 # [0.5,0] var=0.0625 + + def test_eval_prefix(self): + """Test with eval prefix instead of reward.""" + rewards = [1.0, 0.0] + uids = ["a", "a"] + groups = ["env1", "env1"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="eval") + + assert "eval/env1/variance_per_prompt" in metrics + assert "eval/env1/pass_at_2" in metrics + assert "eval/env1/mean_positive_reward" in metrics + + def test_group_with_slash_sanitized(self): + """Test that group names with slashes are sanitized.""" + rewards = [1.0, 0.0] + uids = ["a", "a"] + groups = ["foo/bar", "foo/bar"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="reward") + + assert "reward/foo_bar/variance_per_prompt" in metrics + + def test_multiple_groups_independent(self): + """Each group's metrics are independent.""" + # github: all pass with variance, booking: none pass + rewards = [1.0, 0.5, 0.0, 0.0] + uids = ["t1", "t1", "t2", "t2"] + groups = ["github", "github", "booking", "booking"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="reward") + + assert metrics["reward/github/pass_at_2"] == 1.0 + assert metrics["reward/github/variance_per_prompt"] == 0.0625 # [1,0.5] var=0.0625 + assert metrics["reward/booking/pass_at_2"] == 0.0 + assert metrics["reward/booking/variance_per_prompt"] == 0.0 # [0,0] var=0 + + def test_none_group_becomes_unknown(self): + """None group values become 'unknown'.""" + rewards = [1.0, 0.0] + uids = ["a", "a"] + groups = [None, None] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="reward") + + assert "reward/unknown/variance_per_prompt" in metrics + + def test_empty_input(self): + """Empty input returns empty dict.""" + metrics = compute_per_group_metrics([], [], [], n_samples_per_prompt=1, prefix="reward") + assert metrics == {} + + def test_single_group_all_data(self): + """All data in single group.""" + rewards = [1.0, 0.0, 0.5, 0.5] + uids = ["a", "a", "b", "b"] + groups = ["env1", "env1", "env1", "env1"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="reward") + + assert len([k for k in metrics.keys() if k.startswith("reward/env1/")]) == 3 + # uid "a" [1,0] var=0.25, uid "b" [0.5,0.5] var=0 -> mean=0.125 + assert abs(metrics["reward/env1/variance_per_prompt"] - 0.125) < 1e-10 + + def test_token_level_rewards_per_group(self): + """Test with token-level rewards grouped by environment.""" + # github: [[1.0, 0.0], [0.0, 0.0]] -> [1.0, 0.0] + # booking: [[0.3, 0.2], [0.0, 0.0]] -> [0.5, 0.0] + rewards = [[1.0, 0.0], [0.0, 0.0], [0.3, 0.2], [0.0, 0.0]] + uids = ["task1", "task1", "task2", "task2"] + groups = ["github", "github", "booking", "booking"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="eval") + + # github: uid "task1" has [1.0, 0.0] -> pass, var=0.25 + assert metrics["eval/github/pass_at_2"] == 1.0 + assert metrics["eval/github/variance_per_prompt"] == 0.25 + + # booking: uid "task2" has [0.5, 0.0] -> pass, var=0.0625 + assert metrics["eval/booking/pass_at_2"] == 1.0 + assert metrics["eval/booking/variance_per_prompt"] == 0.0625 + + +class TestIntegrationWithSkyRLPatterns: + """Test that the functions produce output matching SkyRL's expected patterns.""" + + def test_training_metrics_pattern(self): + """Verify output matches reward/{env}/metric pattern for training.""" + rewards = [1.0, 0.0] + uids = ["task1", "task1"] + groups = ["github", "github"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=2, prefix="reward") + + # These are the exact keys SkyRL trainer expects + assert "reward/github/variance_per_prompt" in metrics + assert "reward/github/pass_at_2" in metrics + assert "reward/github/mean_positive_reward" in metrics + + def test_eval_metrics_pattern(self): + """Verify output matches eval/{env}/metric pattern for evaluation.""" + rewards = [1.0, 0.0] + uids = ["task1", "task1"] + groups = ["github", "github"] + + metrics = compute_per_group_metrics(rewards, uids, groups, n_samples_per_prompt=3, prefix="eval") + + # These are the exact keys SkyRL evaluate.py expects + assert "eval/github/variance_per_prompt" in metrics + assert "eval/github/pass_at_3" in metrics + assert "eval/github/mean_positive_reward" in metrics diff --git a/skyrl-train/skyrl_train/model_wrapper.py b/skyrl-train/skyrl_train/model_wrapper.py new file mode 100644 index 0000000000..1704cd3eee --- /dev/null +++ b/skyrl-train/skyrl_train/model_wrapper.py @@ -0,0 +1,797 @@ +# This code is adapted from OpenRLHF and OpenReasonerZero +# https://github.com/Open-Reasoner-Zero/Open-Reasoner-Zero/blob/main/orz/ppo/models.py +# https://github.com/OpenRLHF/OpenRLHF/blob/main/openrlhf/models/actor.py +# https://github.com/OpenRLHF/OpenRLHF/blob/main/openrlhf/models/model.py + +from typing import Any, Dict, Optional, Tuple, Union +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint as gradient_checkpoint +from loguru import logger +from peft import LoraConfig, TaskType, get_peft_model +from peft.tuners.lora import LoraLayer +import transformers +from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, BitsAndBytesConfig +import numpy as np +from skyrl_train.distributed.ulysses.utils import ulysses_pad_and_slice_inputs, gather_outputs_and_unpad +from skyrl_train.utils.torch_utils import chunked_entropy_from_logits, logprobs_from_logits + +# Import conditionally - only needed for sample packing with flash attention +try: + from flash_attn.bert_padding import pad_input, unpad_input +except ImportError: + pad_input = None + unpad_input = None +from packaging.version import Version + + +def _chunked_logprobs_only(hidden_chunk, labels_chunk, weight, bias, temperature): + """Compute logprobs for one chunk. Used inside gradient_checkpoint. + Uses F.linear with raw weight/bias to avoid DTensor issues with FSDP2. + """ + logits = F.linear(hidden_chunk, weight, bias) + logits = logits / temperature + return logprobs_from_logits(logits, labels_chunk, inplace_backward=False) + + +def _chunked_logprobs_and_entropy(hidden_chunk, labels_chunk, weight, bias, temperature): + """Compute logprobs and entropy for one chunk. Used inside gradient_checkpoint. + Uses F.linear with raw weight/bias to avoid DTensor issues with FSDP2. + """ + logits = F.linear(hidden_chunk, weight, bias) + logits = logits / temperature + lp = logprobs_from_logits(logits, labels_chunk, inplace_backward=False) + log_softmax_vals = F.log_softmax(logits, dim=-1) + entropy = -(log_softmax_vals.exp() * log_softmax_vals).sum(dim=-1) + return lp, entropy + + +class _IdentityLMHead(nn.Module): + """Dummy lm_head that passes hidden states through unchanged. + Used to prevent the HF model from materializing the full (B, S, vocab) logits tensor. + """ + + def forward(self, x): + return x + + +class HFModelWrapper(nn.Module): + """ + Base class for wrapped HF models in reinforcement learning. + + This class serves as a foundation for implementing various model roles. + + Args: + pretrain_or_model (nn.Module): A pretrained model or a new model instance to be used as the actor. + use_flash_attention_2 (bool, optional): Whether to utilize Flash Attention 2.0 for improved performance. Defaults to False. + bf16 (bool, optional): Enable bfloat16 precision for model computations. Defaults to True. + load_in_4bit (bool, optional): Load the model in 4-bit precision. Defaults to False. + lora_rank (int, optional): Rank for LoRA adaptation. Defaults to 0. + lora_alpha (int, optional): Alpha parameter for LoRA. Defaults to 16. + lora_dropout (float, optional): Dropout rate for LoRA layers. Defaults to 0. + lora_init_method (str, optional): Initialization method for LoRA layers. Defaults to "kaiming". + target_modules (list, optional): List of target modules for applying LoRA. Defaults to None. + exclude_modules (list, optional): List of modules to exclude from applying LoRA. Defaults to None. + device_map (dict, optional): Device mapping for loading the model onto specific devices. Defaults to None. + packing_samples (bool, optional): Whether to pack samples during training. Defaults to False. + temperature (float, optional): Temperature for action selection. Defaults to 1.0. + use_liger_kernel (bool, optional): Whether to use Liger Kernel for the model. Defaults to False. + """ + + def __init__( + self, + pretrain_or_model, + use_flash_attention_2=False, + bf16=True, + load_in_4bit=False, + # TODO(shu): combine all LoRA specific configs into one place? + lora_rank=0, + lora_alpha=16, + lora_dropout=0, + lora_init_method="kaiming", + target_modules=None, + exclude_modules=None, + device_map=None, + temperature=1.0, + use_liger_kernel=False, + sequence_parallel_size=1, + use_sample_packing: bool = False, + loss_chunk_size: int = 0, + use_torch_compile: bool = False, + rope_scaling: Dict[str, Any] = {}, + rope_theta: float | None = None, + model_config_kwargs: dict = {}, + **kwargs, + ) -> None: + super().__init__() + self.temperature = temperature + self.sequence_parallel_size = sequence_parallel_size + self.attn_implementation = "flash_attention_2" if use_flash_attention_2 else "sdpa" + self.use_sample_packing = use_sample_packing + self.loss_chunk_size = loss_chunk_size + # packing samples using Flash Attention 2 + if use_sample_packing: + assert ( + self.attn_implementation == "flash_attention_2" + ), "Flash attention 2 should be used for `use_sample_packing`" + + if isinstance(pretrain_or_model, str): + if load_in_4bit: + assert bf16, "we only support bnb_4bit_compute_dtype = bf16" + nf4_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=True, + bnb_4bit_compute_dtype=torch.bfloat16, + ) + else: + nf4_config = None + + model_config = AutoConfig.from_pretrained(pretrain_or_model, trust_remote_code=True, **model_config_kwargs) + + # Detect VL (Vision-Language) models by config class name + is_vl_model = "VL" in model_config.__class__.__name__ or hasattr(model_config, "vision_config") + + if use_liger_kernel: + from liger_kernel.transformers import AutoLigerKernelForCausalLM + + model_class = AutoLigerKernelForCausalLM + elif is_vl_model: + try: + from transformers import AutoModelForImageTextToText + + model_class = AutoModelForImageTextToText + except ImportError: + from transformers import AutoModelForVision2Seq + + model_class = AutoModelForVision2Seq + logger.info( + f"Detected VL model config: {model_config.__class__.__name__}, using {model_class.__name__}" + ) + else: + model_class = AutoModelForCausalLM + + # Set rope_scaling on config (not as kwarg to from_pretrained) + if rope_scaling: + model_config.rope_scaling = dict(rope_scaling) if hasattr(rope_scaling, "keys") else rope_scaling + if rope_theta: + model_config.rope_theta = rope_theta + + self.model = model_class.from_pretrained( + pretrain_or_model, + config=model_config, + trust_remote_code=True, + attn_implementation=self.attn_implementation, + quantization_config=nf4_config, + torch_dtype=torch.bfloat16 if bf16 else torch.float32, + device_map=device_map, + ) + + # gpt oss + if Version(transformers.__version__) >= Version("4.56.2"): + from transformers import GptOssConfig + + if isinstance(self.model.config, GptOssConfig): + # patch attention with Unsloth's flex attn + from skyrl_train.patches.gptoss.patch_transformers import ( + custom_attention, + custom_attention_mask, + patch_GptOssAttention, + ) + from transformers import AttentionInterface, AttentionMaskInterface + + AttentionInterface.register("custom_flex", custom_attention) + AttentionMaskInterface.register("custom_flex", custom_attention_mask) + # set attention implementation to be `custom_flex` + self.model.set_attn_implementation("custom_flex") + self.attn_implementation = "custom_flex" + # NOTE: Even though we set a custom attn implementation, we + # also patch the full attention function for GPT OSS + patch_GptOssAttention() + + # LoRA + if lora_rank > 0: + # https://github.com/huggingface/peft/issues/137 + self.model.enable_input_require_grads() + lora_config = LoraConfig( + task_type=TaskType.CAUSAL_LM, + r=lora_rank, + lora_alpha=lora_alpha, + target_modules=target_modules, + exclude_modules=exclude_modules, + lora_dropout=lora_dropout, + bias="none", + init_lora_weights=True if lora_init_method == "kaiming" else lora_init_method, + ) + self.model = get_peft_model(self.model, lora_config) + + if load_in_4bit: + for name, module in self.model.named_modules(): + if isinstance(module, LoraLayer): + module = module.to(torch.bfloat16) + if "norm" in name: + module = module.to(torch.float32) + if "lm_head" in name or "embed_tokens" in name: + if hasattr(module, "weight"): + module = module.to(torch.bfloat16) + + # MoE - balancing loss + model_config = self.model.config.to_dict() + if "output_router_logits" in model_config: + logger.info("[MoE] set output_router_logits as True") + self.model.config.output_router_logits = True + + # https://github.com/huggingface/transformers/issues/26877 + # Use `model.generate(use_cache=True)` instead.` + self.model.config.use_cache = False + else: + self.model = pretrain_or_model + + # TODO (sumanthrh): do the same for `logprobs_from_logits` and test. + # Credits: https://www.tylerromero.com/posts/2025-02-selective-log-softmax/#efficient-solution + self.chunked_entropy_from_logits_fn = ( + torch.compile(chunked_entropy_from_logits, dynamic=True) + if use_torch_compile + else chunked_entropy_from_logits + ) + + @torch.no_grad() + def generate(self, input_ids: torch.Tensor, **kwargs) -> Union[ + Tuple[torch.LongTensor, torch.LongTensor], + Tuple[torch.LongTensor, torch.LongTensor, torch.BoolTensor], + ]: + generate_args = { + "input_ids": input_ids, + "top_k": kwargs.get("top_k", None), + "top_p": kwargs.get("top_p", None), + "min_p": kwargs.get("min_p", None), + "do_sample": kwargs.get("do_sample", True), + "early_stopping": kwargs.get("num_beams", 1) > 1, + "temperature": kwargs.get("temperature", 1), + "use_cache": True, + "num_beams": kwargs.get("num_beams", 1), + "attention_mask": kwargs.get("attention_mask"), + "eos_token_id": kwargs.get("eos_token_id"), + "pad_token_id": kwargs.get("pad_token_id"), + "min_new_tokens": kwargs.get("min_new_tokens", 1), + } + + if kwargs.get("max_new_tokens", None): + generate_args["max_new_tokens"] = kwargs.get("max_new_tokens") + if kwargs.get("max_length", None): + generate_args["max_length"] = kwargs.get("max_length") + + # Call generate + sequences = self.model.generate(**generate_args) + + # Prepare mask tensor + eos_token_id = generate_args["eos_token_id"] + pad_token_id = generate_args["pad_token_id"] + + return self.process_sequences(sequences, input_ids.size(1), eos_token_id, pad_token_id) + + def process_sequences(self, sequences: torch.Tensor, input_len, eos_token_id, pad_token_id): + """ + Process generated sequences to create attention masks and action masks. + + Args: + sequences (torch.Tensor): Generated sequence tensor + input_len (int): Length of the input sequence + eos_token_id (int): Token ID for the end-of-sequence token + pad_token_id (int): Token ID for the padding token + + Returns: + tuple: A tuple containing three elements: + - sequences: Original sequence + - attention_mask: Attention mask indicating valid token positions + - action_mask: Action mask indicating valid action token positions + """ + # Create initial attention mask by marking positions that are neither EOS nor padding tokens + attention_mask = (sequences.ne(eos_token_id) & sequences.ne(pad_token_id)).to(dtype=torch.long) + seq_length = attention_mask.size(1) + + # Find the position of the last valid token in each sequence + eos_indices = seq_length - attention_mask.long().fliplr().argmax(dim=1, keepdim=True).clamp(min=1) + + # Handle cases where EOS tokens might appear in the middle of the prompt (for Llama3 and Qwen2 models) + # Find the position of the first valid token in each sequence + first_token_indices = attention_mask.long().argmax(dim=1, keepdim=True) + # Create position mask + mask = torch.arange(seq_length).unsqueeze(0).expand(sequences.size(0), -1).to(device=sequences.device) + # Generate final attention mask, keeping only positions between first and last valid tokens + attention_mask = (mask >= first_token_indices) & (mask <= eos_indices).to(dtype=torch.long) + + # In reinforcement learning, the state transition is represented as: + # state_i (current token) + action_i (next token) -> state_i+1 (next token) + # Generate state sequence from input_len-1 to second-to-last token + state_seq = sequences[:, input_len - 1 : -1] + # Generate action mask indicating valid action token positions + action_mask = state_seq.ne(eos_token_id) & state_seq.ne(pad_token_id) + action_mask[:, 0] = 1 + + return sequences, attention_mask, action_mask + + def forward( + self, + sequences: torch.LongTensor, + num_actions: Union[int, list[int]], + attention_mask: Optional[torch.Tensor] = None, + temperature: float = 1.0, + return_output=False, + compute_entropy=False, + entropy_requires_grad=True, + ) -> torch.Tensor: + """Returns action log probs""" + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + + sequences_fwd = sequences + position_ids_fwd = position_ids + attention_mask_fwd = attention_mask + if self.use_sample_packing: + with torch.no_grad(): + # Removes padding to get a packed tensor. `unpad_input` expects 3 dimensional tensor so we unsqueeze first + sequences_fwd, nnz_indices, _, _, _ = unpad_input( + sequences.unsqueeze(-1), attention_mask=attention_mask + ) + # (nnz, 1) -> (1, nnz) + sequences_fwd = sequences_fwd.transpose(0, 1) + position_ids_fwd, _, _, _, _ = unpad_input(position_ids.unsqueeze(-1), attention_mask) + # (nnz, 1) -> (1, nnz) + position_ids_fwd = position_ids_fwd.transpose(0, 1) + attention_mask_fwd = None # no attention mask with FA 2 + + sequences_rolled = torch.roll(sequences_fwd, shifts=-1, dims=1) + if self.sequence_parallel_size > 1: + # NOTE: don't pass any attn mask with sample packing + attention_mask_fwd = None if self.use_sample_packing else attention_mask_fwd + + # slice for sequence parallelism + # (bsz, seqlen) -> (bsz, seqlen//sp_size) + sequences_fwd, position_ids_fwd, attention_mask_fwd, pad_size = ulysses_pad_and_slice_inputs( + sequences_fwd, position_ids_fwd, attention_mask_fwd, self.sequence_parallel_size + ) + sequences_rolled, _, _, _ = ulysses_pad_and_slice_inputs( + sequences_rolled, None, None, self.sequence_parallel_size + ) + + # NOTE (sumanthrh): Once we have position_ids, we don't need attention mask with flash attention. + use_chunked = self.loss_chunk_size > 0 + + if use_chunked: + # Chunked lm_head: avoid materializing full (B, S, vocab_size) logits tensor. + # Replace lm_head with identity so the model returns hidden states instead. + lm_head = self.model.lm_head + self.model.lm_head = _IdentityLMHead() + + try: + if self.use_sample_packing and self.attn_implementation == "flash_attention_2": + output = self.model(sequences_fwd, attention_mask=None, position_ids=position_ids_fwd) + else: + output = self.model(sequences_fwd, attention_mask=attention_mask_fwd, position_ids=position_ids_fwd) + finally: + if use_chunked: + self.model.lm_head = lm_head + + if use_chunked: + # output["logits"] is actually hidden_states (B, S, hidden_dim) since lm_head was identity + hidden_states = output["logits"] + entropy_mask = None + if compute_entropy and not self.use_sample_packing: + entropy_mask = attention_mask_fwd + log_probs, entropy_BS = self._chunked_lm_head_forward( + hidden_states, + lm_head, + sequences_rolled, + temperature, + self.loss_chunk_size, + compute_entropy=compute_entropy, + entropy_requires_grad=entropy_requires_grad, + attention_mask=entropy_mask, + ) + # Replace hidden_states in output with None to free memory + output["logits"] = None + else: + logits_BSV = output["logits"] + logits_BSV.div_(temperature) + + # NOTE: this is slightly inaccurate with sample packing because last token from nth seq -> first token of n+1th seq loss is added. + log_probs = logprobs_from_logits( + logits_BSV, + sequences_rolled, + inplace_backward=True, + ) + + # gather output if sp > 1 + if self.sequence_parallel_size > 1: + dim = log_probs.ndim - 1 + log_probs = gather_outputs_and_unpad( + log_probs, gather_dim=dim, unpad_dim=dim, padding_size=pad_size + ) # shape can be (1, nnz) - with packing or (B, S) - without packing + + if self.use_sample_packing: + # add padding back - postprocess logprobs to be compatible with original tensor + batch_size, seqlen = attention_mask.shape + # (1, nnz-1) -> (batch_size, seqlen). Pad token ID used by flash attention is 0. + log_probs = pad_input( + log_probs.transpose(0, 1), indices=nnz_indices, batch=batch_size, seqlen=seqlen + ).squeeze(-1) + + if use_chunked: + # Entropy already computed in _chunked_lm_head_forward + if compute_entropy: + if self.sequence_parallel_size > 1: + dim = entropy_BS.ndim - 1 + entropy_BS = gather_outputs_and_unpad( + entropy_BS, gather_dim=dim, unpad_dim=dim, padding_size=pad_size + ) + if self.use_sample_packing: + entropy_BS = pad_input( + entropy_BS.transpose(0, 1), indices=nnz_indices, batch=batch_size, seqlen=seqlen + ).squeeze(-1) + output["entropy"] = entropy_BS + elif compute_entropy: + # For sample packing: entropy is calculated on unpacked data, so no attention mask needed + # For non-sample packing: pass the attention mask to exclude padding tokens + entropy_mask = None + if not self.use_sample_packing: + # Non-sample packing: pass attention mask to handle padding + # Use attention_mask_fwd which may be sliced (if sequence_parallel_size > 1) or full + entropy_mask = attention_mask_fwd + + entropy_BS = self.chunked_entropy_from_logits_fn( + logits_BSV, requires_grad=entropy_requires_grad, attention_mask=entropy_mask + ) + + if self.sequence_parallel_size > 1: + dim = entropy_BS.ndim - 1 + entropy_BS = gather_outputs_and_unpad( + entropy_BS, gather_dim=dim, unpad_dim=dim, padding_size=pad_size + ) # shape can be (1, nnz) - with packing or (B,S) - without packing + if self.use_sample_packing: + entropy_BS = pad_input( + entropy_BS.transpose(0, 1), indices=nnz_indices, batch=batch_size, seqlen=seqlen + ).squeeze( + -1 + ) # (1, nnz) -> (B, S) + + output["entropy"] = entropy_BS + + if isinstance(num_actions, list): + if len(num_actions) == 1: + num_actions = num_actions[0] + else: + num_actions = np.array(num_actions) + action_log_probs = log_probs[:, -num_actions - 1 : -1] + + if return_output: + return (action_log_probs, output) + else: + return action_log_probs + + def _chunked_lm_head_forward( + self, + hidden_states, + lm_head, + labels, + temperature, + chunk_size, + compute_entropy=False, + entropy_requires_grad=True, + attention_mask=None, + ): + """Compute log_probs (and optionally entropy) via chunked lm_head projection. + + Instead of materializing the full (B, S, vocab_size) logits tensor, this + computes lm_head in chunks of `chunk_size` tokens along the sequence dimension. + Each chunk uses gradient checkpointing so logits are recomputed during backward + rather than stored, keeping peak memory at (B, chunk_size, vocab_size). + + Ported from upstream SkyRL's JAX backend (PR #902, loss_chunk_size). + """ + B, S, H = hidden_states.shape + all_log_probs = [] + all_entropy = [] if compute_entropy else None + + # Extract weight/bias from lm_head module. With FSDP2, parameters are DTensors; + # calling the module inside gradient_checkpoint causes DTensor/Tensor mismatch. + # We all-gather DTensors to regular tensors via full_tensor() which is differentiable. + weight = lm_head.weight + bias = lm_head.bias + try: + from torch.distributed.tensor import DTensor + + if isinstance(weight, DTensor): + weight = weight.full_tensor() + if bias is not None and isinstance(bias, DTensor): + bias = bias.full_tensor() + except ImportError: + pass + + # When not computing gradients (ref model), skip gradient_checkpoint entirely — + # just compute each chunk directly with no_grad already active from caller. + use_checkpointing = torch.is_grad_enabled() + + for start in range(0, S, chunk_size): + end = min(start + chunk_size, S) + chunk_hidden = hidden_states[:, start:end] + chunk_labels = labels[:, start:end] + + if compute_entropy: + if use_checkpointing: + chunk_lp, chunk_ent = gradient_checkpoint( + _chunked_logprobs_and_entropy, + chunk_hidden, + chunk_labels, + weight, + bias, + temperature, + use_reentrant=False, + ) + else: + chunk_lp, chunk_ent = _chunked_logprobs_and_entropy( + chunk_hidden, chunk_labels, weight, bias, temperature + ) + if not entropy_requires_grad: + chunk_ent = chunk_ent.detach() + if attention_mask is not None: + chunk_mask = attention_mask[:, start:end] + chunk_ent = chunk_ent * chunk_mask + all_entropy.append(chunk_ent) + else: + if use_checkpointing: + chunk_lp = gradient_checkpoint( + _chunked_logprobs_only, + chunk_hidden, + chunk_labels, + weight, + bias, + temperature, + use_reentrant=False, + ) + else: + chunk_lp = _chunked_logprobs_only(chunk_hidden, chunk_labels, weight, bias, temperature) + + all_log_probs.append(chunk_lp) + + log_probs = torch.cat(all_log_probs, dim=1) + entropy = torch.cat(all_entropy, dim=1) if compute_entropy else None + return log_probs, entropy + + def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs={"use_reentrant": False}): + self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs=gradient_checkpointing_kwargs) + + def gradient_checkpointing_disable(self): + self.model.gradient_checkpointing_disable() + + def print_trainable_parameters(self): + self.model.print_trainable_parameters() + + +def reset_position_ids(attention_mask): + position_ids = torch.zeros_like(attention_mask, dtype=torch.long) + for i in range(attention_mask.size(0)): + mask = attention_mask[i] + seq_num = mask.max().item() + for index in range(1, seq_num + 1): + sample_mask = mask == index + sample_length = sample_mask.sum().item() + position_ids[i, sample_mask] = torch.arange(sample_length, device=mask.device) + return position_ids + + +def _get_critic_model( + base_pretrained_model, + base_llm_model, + value_head_prefix="value_head", + sequence_parallel_size=1, + use_sample_packing: bool = False, +): + class CriticModel(base_pretrained_model): + supports_gradient_checkpointing = True + + def __init__(self, config: AutoConfig): + super().__init__(config) + setattr(self, self.base_model_prefix, base_llm_model(config)) + + self.value_head_prefix = value_head_prefix + setattr(self, value_head_prefix, nn.Linear(config.hidden_size, 1, bias=False)) + + self.sequence_parallel_size = sequence_parallel_size + self.use_sample_packing = use_sample_packing + if use_sample_packing: + assert ( + config._attn_implementation == "flash_attention_2" + ), "Flash attention must be used with sample packing" + + if self.sequence_parallel_size > 1: + logger.info("Critic model using sequence parallelism with size: ", self.sequence_parallel_size) + + def forward( + self, + input_ids: torch.LongTensor = None, + num_actions: Optional[Union[int, list[int]]] = None, + attention_mask: Optional[torch.Tensor] = None, + return_output=False, + ) -> torch.Tensor: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + input_ids_fwd = input_ids + position_ids_fwd = position_ids + attention_mask_fwd = attention_mask + + if self.use_sample_packing: + with torch.no_grad(): + # remove padding. `unpad_input` expects 3 dimensional tensor + input_ids_fwd, nnz_indices, _, _, _ = unpad_input( + input_ids.unsqueeze(-1), attention_mask=attention_mask + ) + # (nnz, 1) -> (1, nnz) + input_ids_fwd = input_ids_fwd.transpose(0, 1) + position_ids_fwd, _, _, _, _ = unpad_input( + position_ids.unsqueeze(-1), attention_mask=attention_mask + ) + # (nnz, 1) -> (1, nnz) + position_ids_fwd = position_ids_fwd.transpose(0, 1) + # don't use attention mask with FA2 + attention_mask_fwd = None + + if self.sequence_parallel_size > 1: + assert self.use_sample_packing, "sample packing must be true for sequence parallelism" + # don't pass any attention mask for flash attention 2. this will save an all gather. + attention_mask_fwd = None if self.config._attn_implementation == "flash_attention_2" else attention_mask + # slice for sequence parallelism + # (bsz, seqlen) -> (bsz, seqlen//sp_size) + input_ids_fwd, position_ids_fwd, attention_mask_fwd, pad_size = ulysses_pad_and_slice_inputs( + input_ids_fwd, position_ids_fwd, attention_mask_fwd, self.sequence_parallel_size + ) + + if self.sequence_parallel_size > 1 and self.config._attn_implementation == "flash_attention_2": + outputs = getattr(self, self.base_model_prefix)(input_ids_fwd, position_ids=position_ids_fwd) + else: + outputs = getattr(self, self.base_model_prefix)( + input_ids_fwd, attention_mask=attention_mask_fwd, position_ids=position_ids_fwd + ) + last_hidden_states_BSH = outputs["last_hidden_state"] + + if self.sequence_parallel_size > 1: + last_hidden_states_SH = last_hidden_states_BSH.squeeze(0) + # (seqlen*bsz//sp_size, 1) -> (seqlen*bsz, 1) + last_hidden_states_SH = gather_outputs_and_unpad( + last_hidden_states_SH, gather_dim=0, unpad_dim=0, padding_size=pad_size + ) + last_hidden_states_BSH = last_hidden_states_SH.unsqueeze(0) + + values_BSH = getattr(self, self.value_head_prefix)(last_hidden_states_BSH) + + if self.use_sample_packing: + # add padding back - postprocess logits to be compatible with original tensors + batch_size, seqlen = attention_mask.shape + # (1, nnz, 1) -> (nnz, 1) -> (batch_size, seqlen, 1) + values_BSH = pad_input(values_BSH.squeeze(0), indices=nnz_indices, batch=batch_size, seqlen=seqlen) + + values = values_BSH.squeeze(-1)[:, :-1] + + if num_actions is None: + assert return_output + return outputs + + action_values = values[:, -num_actions:] + + if return_output: + return (action_values, outputs) + else: + return action_values + + return CriticModel + + +# Construct transformer with a value head for sequence classification. +# https://github.com/huggingface/transformers/blob/405b56269812056d9593869e22b7b264d806cb1e/src/transformers/models/llama/modeling_llama.py#L1254 +def get_llm_for_sequence_regression( + model_name_or_path: str, + model_type: str, + *, + bf16=True, + load_in_4bit=False, + lora_rank=0, + lora_alpha=16, + target_modules=None, + exclude_modules=None, + lora_dropout=0, + use_flash_attention_2=False, + init_value_head: bool = False, + value_head_prefix="value_head", + device_map=None, + sequence_parallel_size=1, + use_sample_packing: bool = False, + model_config_kwargs: dict = {}, + **kwargs, +) -> nn.Module: + """Get transformer with a sequence classification head on top (linear layer). + + Args: + model_name_or_path (str): Path to pretrained model. + model_type (str): Type of sequence classification model. Only `critic` is supported. + bf16 (bool, optional): Whether enable bfloat16. Defaults to True. + use_flash_attention_2 (bool, optional): Whether use Flash Attention 2.0. Defaults to False. + + Returns: + nn.Module: pretrained transformer model. + """ + assert model_type == "critic", f"Only model_type critic is supported, got: {model_type}." + + config = AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=True, **model_config_kwargs) + config._attn_implementation = "flash_attention_2" if use_flash_attention_2 else "eager" + + base_class = AutoModel._model_mapping[type(config)] + base_pretrained_class = base_class.__base__ + cls_class = _get_critic_model( + base_pretrained_class, + base_class, + value_head_prefix, + sequence_parallel_size=sequence_parallel_size, + use_sample_packing=use_sample_packing, + ) + + if load_in_4bit: + assert bf16, "we only support bnb_4bit_compute_dtype = bf16" + nf4_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=True, + bnb_4bit_compute_dtype=torch.bfloat16, + ) + else: + nf4_config = None + + model = cls_class.from_pretrained( + model_name_or_path, + config=config, + trust_remote_code=True, + torch_dtype=torch.bfloat16 if bf16 else torch.float32, + quantization_config=nf4_config, + device_map=device_map, + **kwargs, + ) + + # LoRA + if lora_rank > 0: + model.enable_input_require_grads() + lora_config = LoraConfig( + r=lora_rank, + lora_alpha=lora_alpha, + target_modules=target_modules, + exclude_modules=exclude_modules, + lora_dropout=lora_dropout, + bias="none", + ) + model = get_peft_model(model, lora_config) + + if load_in_4bit: + for name, module in model.named_modules(): + if isinstance(module, LoraLayer): + module = module.to(torch.bfloat16) + if "norm" in name: + module = module.to(torch.float32) + if value_head_prefix in name or "embed_tokens" in name: + if hasattr(module, "weight"): + module = module.to(torch.bfloat16) + + # MoE - balancing loss + model_config = model.config.to_dict() + if "output_router_logits" in model_config: + logger.info("[MoE] set output_router_logits as True") + model.config.output_router_logits = True + + # https://github.com/huggingface/transformers/issues/26877 + model.config.use_cache = False + + # NOTE: For reward model training only, intialize value_head manually. + # TODO: Find a better way to clarify reward model training. + if init_value_head: + value_head = getattr(model, value_head_prefix) + value_head.weight.data.normal_(mean=0.0, std=1 / (config.hidden_size + 1)) + + return model diff --git a/skyrl-train/skyrl_train/patches/gptoss/flex_attn_sink.py b/skyrl-train/skyrl_train/patches/gptoss/flex_attn_sink.py new file mode 100644 index 0000000000..ccba5cf4b2 --- /dev/null +++ b/skyrl-train/skyrl_train/patches/gptoss/flex_attn_sink.py @@ -0,0 +1,171 @@ +"""Flex attention implementation for attention sink + +Modified from Unsloth to support attention masks: https://github.com/unslothai/unsloth-zoo/blob/main/unsloth_zoo/flex_attention/attention_sink.py +""" + +# Unsloth Zoo - Utilities for Unsloth +# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +__all__ = [ + "old_flex_attention_with_sink", +] + +import torch +import functools +from torch.nn.attention.flex_attention import ( + create_block_mask as _create_block_mask, +) +from .flex_attn_utils import ( + _flex_attention as uncompiled_flex_attention, + flex_attention, +) + + +def causal_mask_with_sink(batch, head, q_idx, kv_idx): + """ + 0 1 2 3 0 1 2 3 + 0 X X 1 X + 1 X X X 2 X X + 2 X X X X 3 X X X + """ + # We add (q_idx + 1) since first column is sink token + causal_mask = (q_idx + 1) >= kv_idx + sink_first_column = kv_idx == 0 + return causal_mask | sink_first_column + + +@functools.lru_cache +def generate_sliding_window_with_sink(window_size: int): + def sliding_window(batch, head, q_idx, kv_idx): + causal_mask = (q_idx + 1) >= kv_idx + # Official PyTorch attends to 1 extra token + # windowed_mask = q_idx - kv_idx <= window_size + # HuggingFace and official GPT OSS attends to only 128 tokens not (128+1) + windowed_mask = (q_idx + 1) - kv_idx < window_size + sink_first_column = kv_idx == 0 + return (causal_mask & windowed_mask) | sink_first_column + + sliding_window.__name__ = sliding_window.__doc__ = f"sliding_window_{window_size}_sink" + return sliding_window + + +@functools.lru_cache +def generate_sink_score_mod( + sink_weights: torch.Tensor, +): + def sink_score_mod(score, batch, head, q_idx, kv_idx): + # Sink token is at the first location + return torch.where( + kv_idx == 0, + sink_weights[head].to(score.dtype) + 0.0, # Add +0 to allow gradients + score, + ) + + return sink_score_mod + + +@functools.lru_cache +def generate_padding_mask(attention_mask): + if attention_mask.ndim == 4: + assert attention_mask.shape[1] == 1, f"invalid shape {attention_mask.shape}" + B, _, Q, K = attention_mask.shape + attention_mask = attention_mask.squeeze(1).flatten() + + def padding_mask(batch, head, q_idx, kv_idx): + return ~attention_mask[batch * (Q * K) + K * q_idx + kv_idx].bool() + + else: + assert attention_mask.ndim == 2, f"Unexpected ndim: {attention_mask.ndim}" + B, K = attention_mask.shape + attention_mask = attention_mask.flatten() + + def padding_mask(batch, head, q_idx, kv_idx): + return attention_mask[batch * K + kv_idx].bool() + + return padding_mask + + +def old_flex_attention_with_sink( + query, + key, + value, + attention_mask=None, + scale=None, + sliding_window=None, + sinks=None, + num_key_value_groups=1, + compile=True, + **kwargs, +): + """ + Allows one sink token to be attended to for full/sliding window attention + Similar to Efficient Streaming Language Models with Attention Sinks + Primarily for GPT-OSS 2025 + + [WARNING] This only works for training. Inference fails since KV cache's + absolute positioning will fail. + """ + + enable_gqa = num_key_value_groups != 1 + bsz, heads_Q, qlen_Q, dim = query.shape + _, heads_KV, qlen_KV, _ = key.shape + + # Add K and V with a row of 0s to allow sinks to be placed there + key_padded = torch.cat([key.new_zeros(bsz, heads_KV, 1, dim), key], dim=2) + value_padded = torch.cat([value.new_zeros(bsz, heads_KV, 1, dim), value], dim=2) + + # Check for sliding window + mask_mod = ( + generate_sliding_window_with_sink(sliding_window) + if type(sliding_window) is int and sliding_window != 0 + else causal_mask_with_sink + ) + + if attention_mask is not None: + attn_size = attention_mask.size() + # In some cases, huggingface will preprocess this to be a 4d attention mask + # 0 -> token it can attend to + if attention_mask.ndim == 4: + attention_mask = torch.cat([attention_mask.new_zeros((*attn_size[:-1], 1)), attention_mask], dim=-1) + else: + assert attention_mask.ndim == 2, f"Unexpected ndim {attention_mask.ndim}" + attention_mask = torch.cat([attention_mask.new_ones((*attn_size[:-1], 1)), attention_mask], dim=-1) + _padding_mask = generate_padding_mask(attention_mask) + + def combine_masks(mask1, mask2): + def final_mask(b, h, q, k): + return mask1(b, h, q, k) & mask2(b, h, q, k) + + return final_mask + + mask_mod = combine_masks(_padding_mask, mask_mod) + + score_mod = generate_sink_score_mod( + sinks, + ) + # NOTE: if the block mask is not compiled, it leads to O(N^2) memory usage + block_mask = _create_block_mask(mask_mod, bsz, heads_Q, qlen_Q, qlen_KV + 1, device=key.device, _compile=True) + attn_output = (flex_attention if compile else uncompiled_flex_attention)( + query, + key_padded, + value_padded, + block_mask=block_mask, + score_mod=score_mod, + enable_gqa=enable_gqa, + scale=scale, + ) + attn_output = attn_output.transpose(1, 2).contiguous() + return attn_output diff --git a/skyrl-train/skyrl_train/patches/gptoss/flex_attn_utils.py b/skyrl-train/skyrl_train/patches/gptoss/flex_attn_utils.py new file mode 100644 index 0000000000..f9124baff6 --- /dev/null +++ b/skyrl-train/skyrl_train/patches/gptoss/flex_attn_utils.py @@ -0,0 +1,382 @@ +"""Flex attention utilities from Unsloth Zoo""" + +# Unsloth Zoo - Utilities for Unsloth +# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +__all__ = [ + "HAS_FLEX_ATTENTION", + "FLEX_ATTENTION_BLOCK_SIZE", + "_flex_attention", + "flex_attention", + "create_block_mask_cached", + "create_block_mask", + "compiled_create_block_mask", + "FlexAttentionCache", + "causal_mask", + "generate_causal_mask_with_padding", + "generate_decoding_causal_mask_with_padding", + "generate_sliding_window_mask", + "generate_sliding_window_mask_with_padding", + "generate_decoding_sliding_window_mask_with_padding", +] + +import torch +import functools + +FLEX_ATTENTION_KV_INCREMENT = 512 + +try: + from torch.nn.attention.flex_attention import _DEFAULT_SPARSE_BLOCK_SIZE as FLEX_ATTENTION_BLOCK_SIZE + from torch.nn.attention.flex_attention import ( + flex_attention as _flex_attention, + create_block_mask as _create_block_mask, + ) + + HAS_FLEX_ATTENTION = True + from torch.nn.attention.flex_attention import _score_mod_signature, _mask_mod_signature + + try: + # Determine kernel_options since low memory GPUs will go out of memory + # InductorError: RuntimeError: No valid triton configs. OutOfMemoryError: out of resource: triton_tem_fused_0 Required: 65536 Hardware limit:65536 Reducing block sizes or `num_stages` may help. + # See https://github.com/pytorch/pytorch/issues/133254#issuecomment-2408710459 + # https://github.com/pytorch/pytorch/issues/133254#issuecomment-2539969593 + vram_of_gpu = min( + torch.cuda.memory.mem_get_info(i)[-1] / 1024 / 1024 / 1024 for i in range(torch.cuda.device_count()) + ) + kernel_options = None + if vram_of_gpu <= 16: + kernel_options = { + "BLOCK_M": 32, + "BLOCK_N": 32, + "BLOCK_M1": 32, + "BLOCK_N1": 32, + "BLOCK_M2": 32, + "BLOCK_N2": 32, + } + _flex_attention = functools.partial(_flex_attention, kernel_options=kernel_options) + elif vram_of_gpu <= 24: + kernel_options = { + "BLOCK_M": 64, + "BLOCK_N": 64, + "BLOCK_M1": 32, + "BLOCK_N1": 64, + "BLOCK_M2": 64, + "BLOCK_N2": 32, + } + _flex_attention = functools.partial(_flex_attention, kernel_options=kernel_options) + except Exception: + pass + flex_attention = torch.compile(_flex_attention) + + @functools.lru_cache + def create_block_mask_cached(mask_mod, M, N, device="cuda"): + """Create block mask for Flex Attention. Assume bsz=any(None), head=any(None)""" + return _create_block_mask(mask_mod, None, None, M, N, device=device) + + @functools.lru_cache + def create_block_mask(mask_mod, bsz, head, M, N, device="cuda"): + """Create block mask for Flex Attention. Assume bsz=any(None), head=any(None)""" + return _create_block_mask(mask_mod, bsz, head, M, N, device=device) + + def compiled_create_block_mask_cached(mask_mod, M, N, device="cuda"): + """Create block mask for Flex Attention. Assume bsz=any(None), head=any(None)""" + # See https://github.com/meta-pytorch/attention-gym/issues/15#issuecomment-2284148665 + # _compile MUST be on to reduce VRAM otherwise O(N^2) usage + return _create_block_mask(mask_mod, None, None, M, N, device=device, _compile=True) + + def compiled_create_block_mask(mask_mod, bsz, head, M, N, device="cuda"): + """Create block mask for Flex Attention. Assume bsz=any(None), head=any(None)""" + # _compile MUST be on to reduce VRAM otherwise O(N^2) usage + return _create_block_mask(mask_mod, bsz, head, M, N, device=device, _compile=True) + + def causal_mask(batch_idx, head_idx, q_idx, kv_idx): + """Causal mask for Flex Attention""" + return q_idx >= kv_idx + + def generate_causal_mask_with_padding(padding_start_idx=None): + """ + Causal mask for Flex Attention with left padding support. + Normal causal mask: + k0 k1 k2 k3 k4 + q0 X + q1 X X + q2 X X X + q3 X X X X + q4 X X X X X + If we add 2 tokens as padded tokens, we get: + #0 #1 k2 k3 k4 + #0 + #1 + q2 X + q3 X X + q4 X X X + Assume padding_start_idx == [2] + """ + assert padding_start_idx is not None and type(padding_start_idx) is torch.Tensor + assert padding_start_idx.dim() == 1 + assert padding_start_idx.shape[0] >= 1 + + def causal_mask(batch_idx, head_idx, q_idx, kv_idx): + """Causal mask for Flex Attention""" + q_start = q_idx >= padding_start_idx[batch_idx] + k_start = kv_idx >= padding_start_idx[batch_idx] + return q_start & k_start & (q_idx >= kv_idx) + + causal_mask.__name__ = causal_mask.__doc__ = "causal_mask_with_left_padding" + return causal_mask + + def generate_decoding_causal_mask_with_padding(padding_start_idx=None): + """ + For decoding purposes only. We remove q_padded since decoding attends to 1 q + Assume padded tokens = 5 + #0 #1 #2 #3 #4 k5 k6 + #0 # + #1 # # + #2 # # # + #3 # # # # + #4 # # # # # + q5 # # # # # X + q6 # # # # # X X + """ + assert padding_start_idx is not None and type(padding_start_idx) is torch.Tensor + assert padding_start_idx.dim() == 1 + assert padding_start_idx.shape[0] >= 1 + + def causal_mask(batch_idx, head_idx, q_idx, kv_idx): + """Causal mask for Flex Attention""" + k_start = kv_idx >= padding_start_idx[batch_idx] + return k_start & (q_idx >= kv_idx) + + causal_mask.__name__ = causal_mask.__doc__ = "decoding_causal_mask_with_left_padding" + return causal_mask + + @functools.lru_cache + def generate_sliding_window_mask(window_size: int): + """Sliding window mask for Flex Attention""" + + def sliding_window(batch_idx, head_idx, q_idx, kv_idx): + causal_mask = q_idx >= kv_idx + """ + [NOTE] Official PyTorch uses <= window_size which is (window_size+1) tokens + Ie if window_size = 3, HuggingFace does the below: + X + X X + X X X + X X X + X X X + Whilst official PyTorch does (ie attends to the last 3 and itself) + X + X X + X X X + X X X X + X X X X + """ + windowed_mask = q_idx - kv_idx < window_size + # Official PyTorch attends to 1 extra token + # windowed_mask = q_idx - kv_idx <= window_size + return causal_mask & windowed_mask + + sliding_window.__name__ = sliding_window.__doc__ = f"sliding_window_{window_size}" + return sliding_window + + def generate_sliding_window_mask_with_padding(window_size: int, padding_start_idx=None): + assert padding_start_idx is not None and type(padding_start_idx) is torch.Tensor + assert padding_start_idx.dim() == 1 + assert padding_start_idx.shape[0] >= 1 + + def sliding_window(batch_idx, head_idx, q_idx, kv_idx): + causal_mask = q_idx >= kv_idx + windowed_mask = q_idx - kv_idx < window_size + q_padded = q_idx >= padding_start_idx[batch_idx] + k_padded = kv_idx >= padding_start_idx[batch_idx] + return q_padded & k_padded & causal_mask & windowed_mask + + sliding_window.__name__ = sliding_window.__doc__ = f"sliding_window_with_left_padding_{window_size}" + return sliding_window + + def generate_decoding_sliding_window_mask_with_padding(window_size: int, padding_start_idx=None): + """ + We cannot use padding_start_idx[batch_idx] for SWA decoding since + assume padding_start_idx=[3406, 4000, 0] and SW=128 then it'll always + be masked since the KV size=128. + + Since we set padded tokens = 0 always, we simply return the generic SWA. + """ + return generate_sliding_window_mask(window_size) + + # For inference see https://pytorch.org/blog/flexattention-for-inference + def get_score_mod_w_offset(score_mod: _score_mod_signature, _offset: torch.tensor): + def _score_mod(score, b, h, q, kv): + return score_mod(score, b, h, q + _offset, kv) + + return _score_mod + + def get_mask_mod_w_offset(mask_mod: _mask_mod_signature, _offset: torch.tensor): + def _mask_mod(b, h, q, kv): + return mask_mod(b, h, q + _offset, kv) + + return _mask_mod + + # Used for every attention layer + class FlexAttentionCache: + __slots__ = ( + "offset", + "offset_tensor", + "mask_mod_with_offset", + "block_mask", + "mask_mod", + "max_length", + "block_size", + "sliding_window", + "block_mask_slice", + ) + + def __init__(self, key, mask_mod, sliding_window): + bsz, heads_KV, qlen_KV, dim = key.shape + if sliding_window is None: + """ + Normal causal mask: + k0 k1 k2 k3 k4 + q0 X + q1 X X + q2 X X X + q3 X X X X + q4 X X X X X + During decoding: + k0 k1 k2 k3 k4 + q0 + q1 + q2 + q3 + q4 X X X X X + But q_index = 0, so we need an offset = 4 + If no offset, we get: + k0 k1 k2 k3 k4 + q0 + q1 + q2 + q3 + q4 X + Which is wrong. We need q_idx=0 + offset=4 = 0+4 = 4 + Note it's offset==index since it's indexed from 0 as seen in + https://pytorch.org/blog/flexattention-for-inference/ + https://github.com/meta-pytorch/gpt-fast/blob/6ecad9b5b6b987d17ac4303965545873d0192086/generate.py#L91 + """ + # Get next multiple of FLEX_ATTENTION_KV_INCREMENT + div, mod = divmod(qlen_KV, FLEX_ATTENTION_KV_INCREMENT) + n = FLEX_ATTENTION_KV_INCREMENT * div + (FLEX_ATTENTION_KV_INCREMENT if mod != 0 else 0) + self.offset = qlen_KV - 2 # Minus two since we need the block mask to use the saved offset_tensor + # Minus 2 and not -1 since we do pre-incrementing and not post-incrementing + # See self.offset += 1 + if self.offset <= -2: + # Minimum is -1 + self.offset = -1 + # During decoding we do self.offset += 1, so self.offset = 0 + self.sliding_window = None + else: + """ + Sliding window of 2 + causal mask: + k0 k1 k2 k3 k4 + q0 X + q1 X X + q2 X X + q3 X X + q4 X X + During decoding: + k0 k1 k2 k3 k4 + q0 + q1 + q2 + q3 + q4 X X + If we set cache_implementation = "static" which we assume, we don't use an offset + since the K is a rolling matrix of the past window size. + Ie if sliding_window = 2, K is shape (2, dim). So in actuality, we get: + -- -- -- k3 k4 + q0 + q1 + q2 + q3 + q4 X X + But since we use a rolling matrix, offset = sliding_window-1 always ie 2-1 = 1 + -- -- -- k0 k1 + q0 + q1 + q2 + q3(0) + q4(1) X X + Note it's offset==index since it's indexed from 0 as seen in + https://pytorch.org/blog/flexattention-for-inference/ + https://github.com/meta-pytorch/gpt-fast/blob/6ecad9b5b6b987d17ac4303965545873d0192086/generate.py#L91 + For sliding window, it's always sliding_window - 1 + since 128 means index 127 + """ + n = sliding_window + self.offset = min(sliding_window, qlen_KV) - 2 # Minus 2 since block mask is indexing + if self.offset <= -2: + # Minimum is -1 + self.offset = -1 + # During decoding we do self.offset += 1, so self.offset = 0 + self.sliding_window = sliding_window - 1 # Minus 1 since token 128 means index 127 + self.offset_tensor = torch.tensor(self.offset, device=key.device, dtype=torch.int32) + self.block_mask = compiled_create_block_mask(mask_mod, bsz, heads_KV, n, n, device=key.device) + self.mask_mod = mask_mod + self.max_length = n + self.block_size = self.block_mask.BLOCK_SIZE[0] + self.mask_mod_with_offset = get_mask_mod_w_offset(self.mask_mod, self.offset_tensor) + self.block_mask_slice = None + + def __call__(self, key): + bsz, heads_KV, qlen_KV, dim = key.shape + # We increment beforehand to get the correct index since offset_tensor is used + # Assume sliding_window=128-1 = 127 + # offset=126, so offset+1 = 127 + if (self.sliding_window is None) or (self.offset < self.sliding_window): + self.offset += 1 + self.offset_tensor.add_(1) + elif self.sliding_window is not None: + # Quick return since sliding window mask has the same block mask always + # Can only enter here if (self.offset < self.sliding_window) fails + # ie the maximum sliding window has been reached already + return self.block_mask_slice + if self.offset >= self.max_length: + # Must be >= since offset=127, max_length=128 means size=127+1=128 + # since we do zero indexing + self.max_length += FLEX_ATTENTION_KV_INCREMENT + self.block_mask = compiled_create_block_mask( + self.mask_mod, bsz, heads_KV, self.max_length, self.max_length, device=key.device + ) + self.block_size = self.block_mask.BLOCK_SIZE[0] + block_offset = self.offset // self.block_size + block_mask_slice = self.block_mask[:, :, block_offset] + block_mask_slice.mask_mod = self.mask_mod_with_offset + # Must set seq_lengths as seen in + # https://github.com/meta-pytorch/gpt-fast/blob/6ecad9b5b6b987d17ac4303965545873d0192086/generate.py#L80 + block_mask_slice.seq_lengths = (1, qlen_KV) + self.block_mask_slice = block_mask_slice + return block_mask_slice + + pass + +except Exception: + HAS_FLEX_ATTENTION = False + FLEX_ATTENTION_BLOCK_SIZE = None + flex_attention = None + create_block_mask_cached = None + causal_mask = None + generate_sliding_window_mask = None + FlexAttentionCache = None +pass diff --git a/skyrl-train/skyrl_train/patches/gptoss/patch_transformers.py b/skyrl-train/skyrl_train/patches/gptoss/patch_transformers.py new file mode 100644 index 0000000000..8adb8bbcc6 --- /dev/null +++ b/skyrl-train/skyrl_train/patches/gptoss/patch_transformers.py @@ -0,0 +1,301 @@ +"""Patches for GPTOSS's transformer layer + +Adapted from Unsloth's flex attention integration: https://github.com/unslothai/unsloth-zoo/blob/main/unsloth_zoo/temporary_patches/gpt_oss.py +""" + +from typing import Optional, Union, Callable, List, Any +import torch +import torch.nn as nn +from packaging.version import Version +import inspect + +from loguru import logger +from transformers.masking_utils import causal_mask_function + + +def patch_function_past_key_values( + target_obj: Any, + attr_name: str, + new_functions: Union[Callable, List[Callable]], +) -> bool: + """Patch either past_key_value or past_key_values""" + if not hasattr(target_obj, attr_name): + logger.error(f"Attribute '{attr_name}' not found on {target_obj.__name__}") + return False + + original_func = getattr(target_obj, attr_name) + try: + old_keys = inspect.signature(original_func).parameters.keys() + except Exception: + logger.error(f"Cannot inspect {target_obj.__name__}") + return False + success = False + error = "" + for func in new_functions: + try: + new_keys = inspect.signature(func).parameters.keys() + except Exception as e: + error = str(e) + continue + # Check if either is provided + for key in ( + "past_key_value", + "past_key_values", + ): + if key in new_keys and key in old_keys: + try: + orig_func = getattr(target_obj, attr_name) # noqa: F841 + setattr(target_obj, attr_name, func) + success = True + break + except Exception as e: + error = str(e) + continue + if not success: + print(f" Failed to patch {target_obj.__name__}.{attr_name}: {error}") + return success + + +pass + + +def patch_GptOssAttention(): + try: + from .flex_attn_sink import ( + old_flex_attention_with_sink, + ) + + assert old_flex_attention_with_sink is not None + except Exception: + raise + try: + import transformers.models.gpt_oss.modeling_gpt_oss + + transformers.models.gpt_oss.modeling_gpt_oss.GptOssAttention + from transformers.models.gpt_oss.modeling_gpt_oss import apply_rotary_pos_emb + except Exception: + raise + + torch._dynamo.config.cache_size_limit = 256 + + def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + F_softmax = torch.nn.functional.softmax + F_dropout = nn.functional.dropout + matmul = torch.matmul + + def inplace_eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs, + ): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + bsz, n_heads, qlen, _ = query.shape + bsz, n_heads, kvlen, _ = key_states.shape + combined_logits = key_states.new_empty((bsz, n_heads, qlen, kvlen + 1)) + + attn_weights = matmul(query, key_states.transpose(2, 3), out=combined_logits[:, :, :, :kvlen]) + attn_weights *= scaling + if attention_mask is not None: + causal_mask = attention_mask[..., key_states.shape[-2]] + attn_weights += causal_mask + + # sinks = module.sinks.reshape(1, -1, 1, 1).expand(query.shape[0], -1, query.shape[-2], -1) + # combined_logits = torch.cat([attn_weights, sinks], dim=-1) + combined_logits[:, :, :, -1] = module.sinks.reshape(1, -1, 1) + + # This was not in the original implementation and slightly affect results; it prevents overflow in BF16/FP16 + # when training with bsz>1 we clamp max values. + # combined_logits = combined_logits - combined_logits.max(dim=-1, keepdim=True).values + combined_logits[:] = F_softmax(combined_logits, dim=-1, dtype=torch.float32) + probs = combined_logits + scores = probs[..., :-1] # we drop the sink here + attn_weights = F_dropout(scores, p=dropout, training=module.training, inplace=True) + attn_output = matmul(attn_weights, value_states, out=query) + attn_output = attn_output.transpose(1, 2).contiguous() + return attn_output, None + + pass + + def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs, + ): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + attn_weights = matmul(query, key_states.transpose(2, 3)) + attn_weights *= scaling + if attention_mask is not None: + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights += causal_mask + + sinks = module.sinks.reshape(1, -1, 1, 1).expand(query.shape[0], -1, query.shape[-2], -1) + combined_logits = torch.cat([attn_weights, sinks], dim=-1) + + # This was not in the original implementation and slightly affect results; it prevents overflow in BF16/FP16 + # when training with bsz>1 we clamp max values. + # combined_logits = combined_logits - combined_logits.max(dim=-1, keepdim=True).values + combined_logits[:] = F_softmax(combined_logits, dim=-1, dtype=torch.float32) + probs = combined_logits + scores = probs[..., :-1] # we drop the sink here + attn_weights = F_dropout(scores, p=dropout, training=module.training, inplace=True) + attn_output = matmul(attn_weights, value_states, out=query) + attn_output = attn_output.transpose(1, 2).contiguous() + return attn_output, None + + pass + + apply_rotary_pos_emb = torch.compile(apply_rotary_pos_emb) + if Version(torch.__version__) >= Version("2.9.0"): + eager_attention_forward = torch.compile(eager_attention_forward, dynamic=None, fullgraph=True) + else: + # Too many recompilation failures on 2.8.0 + eager_attention_forward = inplace_eager_attention_forward + + def forward_function( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Any] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor]: + # print(f"ENTER CUSTOM ATTN: key value shape: {hidden_states.shape=} {self.head_dim=}") + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) # 2, 32, 2880/64, 64 + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) # 1, -1, 7, 64 + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + if past_key_value is not None: + cache_kwargs = {"cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + assert getattr(self, "sinks", None) is not None, "self_attn must have sinks" + sinks = self.sinks + num_key_value_groups = getattr(self, "num_key_value_groups", 1) + scale = getattr(self, "scaling", None) or getattr(self, "scale", None) + sliding_window = getattr(self, "sliding_window", None) + attn_output = old_flex_attention_with_sink( + query_states, + key_states, + value_states, + attention_mask=attention_mask, + scale=scale, + num_key_value_groups=num_key_value_groups, + sinks=sinks, + sliding_window=sliding_window, + ) + attn_weights = None + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + functions = [] + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Any] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + return forward_function( + self, hidden_states, position_embeddings, attention_mask, past_key_value, cache_position, **kwargs + ) + + functions.append(forward) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_values: Optional[Any] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + return forward_function( + self, hidden_states, position_embeddings, attention_mask, past_key_values, cache_position, **kwargs + ) + + functions.append(forward) + patch_function_past_key_values(transformers.models.gpt_oss.modeling_gpt_oss.GptOssAttention, "forward", functions) + + +def custom_attention( + module: torch.nn.Module, # required arg + query: torch.Tensor, # required arg + key: torch.Tensor, # required arg + value: torch.Tensor, # required arg + attention_mask: Optional[torch.Tensor], # required arg + **kwargs, +): + from .flex_attn_sink import ( + old_flex_attention_with_sink, + ) + + assert getattr(module, "sinks", None) is not None, "self_attn must have sinks" + sinks = module.sinks + num_key_value_groups = getattr(module, "num_key_value_groups", 1) + scale = getattr(module, "scaling", None) or getattr(module, "scale", None) + sliding_window = getattr(module, "sliding_window", None) + + attn_output = old_flex_attention_with_sink( + query, + key, + value, + attention_mask=attention_mask, + scale=scale, + num_key_value_groups=num_key_value_groups, + sinks=sinks, + sliding_window=sliding_window, + ) + return attn_output, None + + +def custom_attention_mask( + batch_size: int, + cache_position: torch.Tensor, + kv_length: int, + kv_offset: int = 0, + mask_function: Callable = causal_mask_function, + attention_mask: Optional[torch.Tensor] = None, + **kwargs, +): + """Custom attention mask for flex attention + + This is the same as the `flash_attention_mask` in transformers, but we simply propagate + the attention mask as-is without postprocessing + """ + if attention_mask is not None: + # Here we need to slice from the right if using sliding or chunked (for full attention, this is equivalent to doing nothing) + attention_mask = attention_mask[:, -kv_length:] + + return attention_mask diff --git a/skyrl-train/skyrl_train/trainer.py b/skyrl-train/skyrl_train/trainer.py new file mode 100644 index 0000000000..a672957821 --- /dev/null +++ b/skyrl-train/skyrl_train/trainer.py @@ -0,0 +1,1472 @@ +import copy +import math +import os +import shutil +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import ray +import torch +from jaxtyping import Float +from loguru import logger +from omegaconf import DictConfig +from ray import ObjectRef +from ray.util.placement_group import PlacementGroup, placement_group +from tqdm import tqdm +from transformers import AutoTokenizer + +from skyrl_train.dataset import PromptDataset +from skyrl_train.dataset.preprocess import ( + convert_prompts_responses_to_batch_tensors, +) +from skyrl_train.distributed.dispatch import ( + ActorInfo, + MeshRank, +) +from skyrl_train.evaluate import evaluate, evaluate_step_wise +from skyrl_train.generators.base import ( + GeneratorInput, + GeneratorInterface, + GeneratorOutput, +) +from skyrl_train.generators.utils import ( + get_metrics_from_generator_output, + prepare_generator_input, +) +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.training_batch import TrainingInputBatch +from skyrl_train.utils import ( + Timer, + get_ray_pg_ready_with_timeout, + ppo_utils, + trainer_utils, +) +from skyrl_train.utils.trainer_utils import calculate_per_source_reward_metrics +from skyrl_train.env_vars import SKYRL_RAY_PG_TIMEOUT_IN_S +from skyrl_train.utils.io import io +from skyrl_train.utils.logging_utils import log_example +from skyrl_train.utils.ppo_utils import ( + AdaptiveKLController, + FixedKLController, + compute_approx_kl, + get_kl_controller, + masked_mean, + normalize_advantages_dict, +) +from skyrl_train.utils.tracking import Tracking +from skyrl_train.utils.trainer_utils import ( + GLOBAL_STEP_PREFIX, + DynamicSamplingState, + ResumeMode, + build_dataloader, + cleanup_old_checkpoints, + extract_step_from_path, + run_on_each_node, + validate_consistency_for_latest_checkpoint, + validate_generator_output, + zero_variance_filter, +) +from skyrl_train.utils.alerts import check_tool_error_rate_alert +from skyrl_train.utils.utils import configure_ray_worker_logging +from skyrl_train.workers.worker import PPORayActorGroup +from skyrl_train.workers.worker_dispatch import WorkerDispatch +from skyrl_train.workers.worker_utils import reduce_metrics + + +class RayPPOTrainer: + def __init__( + self, + cfg: DictConfig, + tracker: Tracking, + tokenizer: AutoTokenizer, + train_dataset: Optional[PromptDataset], + inference_engine_client: InferenceEngineClient, + generator: GeneratorInterface, + colocate_pg: Optional[PlacementGroup] = None, + eval_dataset: Optional[PromptDataset] = None, + ): + self.cfg = cfg + self.colocate_all = cfg.trainer.placement.colocate_all + self.tracker = tracker + self.tokenizer = tokenizer + self.train_dataset = train_dataset + self.eval_dataset = eval_dataset + self.inference_engine_client = inference_engine_client + self.generator = generator + self.train_dataloader = None + self.total_training_steps = None + self._build_train_dataloader_and_compute_training_steps() + + self.eval_dataloader = ( + build_dataloader(self.cfg, eval_dataset, is_train=False) if eval_dataset is not None else None + ) + self.colocate_pg = colocate_pg + + self.resume_mode = ResumeMode(cfg.trainer.resume_mode) + + self.all_metrics = {} + self.all_timings = {} + self.global_step = 0 + + # initialized in `build_models` + self.policy_model: PPORayActorGroup = None + self.critic_model: Optional[PPORayActorGroup] = None + self.ref_model: Optional[PPORayActorGroup] = None + # used for checkpoint cleanup + self._node_ids: Optional[List[str]] = None + + self.dynamic_sampling_state: Optional[DynamicSamplingState] = None + + self.reward_kl_controller: Optional[Union[FixedKLController, AdaptiveKLController]] = None + self.dispatch: WorkerDispatch = None + configure_ray_worker_logging() + + @property + def has_critic(self) -> bool: + """Check if critic model is configured.""" + return self.cfg.trainer.critic.model.path is not None + + def _build_train_dataloader_and_compute_training_steps(self): + """ + Hook for constructing the training dataloader. Subclasses can override + this to customize dataloader behavior. For instance, fully async training + needs a batch size of 1, among other features. + Defaults to `trainer_utils.build_dataloader` with `is_train=True`. + """ + self.train_dataloader = build_dataloader(self.cfg, self.train_dataset, is_train=True) + self.total_training_steps = len(self.train_dataloader) * self.cfg.trainer.epochs + + @torch.no_grad() + async def eval(self) -> Dict[str, float]: + """ + Run generation and scoring on the evaluation dataset. + + The eval metrics are recorded after having finished training `self.global_step` steps. + Metrics recorded in global_step 0 corresponds to evaluations before training. + + Returns: + A dictionary of evaluation metrics. + """ + # Create Fleet trace job for eval traces (if FLEET_API_KEY is set) + trace_job_id = None + fleet_api_key = os.environ.get("FLEET_API_KEY") + if fleet_api_key: + try: + from envs.fleet_env.trace import create_trace_job + from integrations.fleet.env import FleetTaskEnv as FleetEnv + + job_name = f"{self.cfg.trainer.run_name}_step_{self.global_step}" + trace_job_id = await create_trace_job(fleet_api_key, job_name) + model_path = self.cfg.trainer.policy.model.path + FleetEnv.set_trace_config(job_id=trace_job_id, model=model_path) + logger.info(f"Created Fleet trace job: {trace_job_id}") + except Exception as e: + logger.warning(f"Failed to create Fleet trace job: {e}") + + try: + if self.cfg.generator.step_wise_trajectories: + eval_metrics = await evaluate_step_wise( + eval_dataloader=self.eval_dataloader, + generator=self.generator, + cfg=self.cfg, + global_step=self.global_step, + tokenizer=self.tokenizer, + ) + else: + eval_metrics = await evaluate( + eval_dataloader=self.eval_dataloader, + generator=self.generator, + cfg=self.cfg, + global_step=self.global_step, + tokenizer=self.tokenizer, + ) + finally: + if trace_job_id: + try: + from integrations.fleet.env import FleetTaskEnv as FleetEnv + + FleetEnv.clear_trace_config() + except Exception: + pass + + return eval_metrics + + async def train(self): + """ + Main training loop for PPO + """ + # Initialize weight sync state between policy model and inference engines. + with Timer("init_weight_sync_state"): + self.init_weight_sync_state() + + # Load checkpoint state if resumption is enabled. + if self.resume_mode != ResumeMode.NONE: + with Timer("load_checkpoints"): + self.global_step, _ = self.load_checkpoints() + + self.dispatch.prepare_for_weight_sync() + if self.colocate_all: + await self.inference_engine_client.wake_up(tags=["weights"]) + with Timer("sync_weights"): + self.dispatch.broadcast_to_inference_engines(self.inference_engine_client) + self.dispatch.finish_weight_sync() + if self.colocate_all: + await self.inference_engine_client.wake_up(tags=["kv_cache"]) + + # Eval before training - always log at step 0 regardless of resumed checkpoint + if self.cfg.trainer.eval_interval > 0 and self.cfg.trainer.eval_before_train: + with Timer("eval", self.all_timings): + eval_metrics = await self.eval() + self.tracker.log(eval_metrics, step=0, commit=True) + + # initialize kl controller + if self.cfg.trainer.algorithm.use_kl_in_reward: + self.reward_kl_controller = get_kl_controller(self.cfg.trainer.algorithm) + + # main training loop + pbar = tqdm(total=self.total_training_steps, initial=self.global_step, desc="Training Batches Processed") + start_epoch = self.global_step // len(self.train_dataloader) + self.global_step += 1 # start training at global_step 1 + for epoch in range(start_epoch, self.cfg.trainer.epochs): + for iter, rand_prompts in enumerate(self.train_dataloader): + with Timer("step", self.all_timings): + # for colocate_all=true, inference engine is always on GPU when starting the training step + + # 0. truncate data to have even shards + rand_prompts = self._remove_tail_data(rand_prompts) + generator_input, uids = prepare_generator_input( + rand_prompts, + self.cfg.generator.n_samples_per_prompt, + get_sampling_params_for_backend(self.cfg.generator.backend, self.cfg.generator.sampling_params), + self.cfg.environment.env_class, + "train", + self.global_step, + ) + + # 1.1 generation phase + with Timer("generate", self.all_timings): + generator_output: GeneratorOutput = await self.generate(generator_input) + + if self.cfg.generator.step_wise_trajectories or len(generator_output["response_ids"]) != len(uids): + # Re-derive uids from trajectory_ids when output size differs from input + # (step-wise training expands per-step, hint augmentation adds extra rollouts) + uids = [tid.instance_id for tid in generator_output["trajectory_ids"]] + + # dynamic sampling + if self.cfg.trainer.algorithm.dynamic_sampling.type is not None: + generator_output, uids, keep_sampling = self.handle_dynamic_sampling(generator_output, uids) + if keep_sampling: # continue sampling + # update progress bar for current batch (but not global step) + pbar.update(1) + continue + + if self.colocate_all: + # if we are not continuing sampling, we sleep the inference engine + await self.inference_engine_client.sleep() + + # 1.2 postprocess rewards + with Timer("postprocess_generator_output", self.all_timings): + # Extract data_sources for per-environment metrics + data_sources = [ + env_extra.get("data_source") for env_extra in generator_input.get("env_extras", []) + ] + generator_output = self.postprocess_generator_output(generator_output, uids, data_sources) + + # 2. print example just for debugging + vis = self.tokenizer.decode(generator_output["response_ids"][0]) + log_example( + logger, + prompt=generator_input["prompts"][0], + response=vis, + reward=generator_output["rewards"][0], + ) + + with Timer("convert_to_training_input", self.all_timings): + training_input: TrainingInputBatch = self.convert_to_training_input(generator_output, uids) + logger.info(f"Number of sequences: {len(training_input['sequences'])}") + + # 1.4 inference and calculate values, log probs, rewards, kl divergence + with Timer("fwd_logprobs_values_reward", self.all_timings): + training_input = self.fwd_logprobs_values_reward(training_input) + + # 1.5 apply kl divergence penalty to rewards + if self.cfg.trainer.algorithm.use_kl_in_reward: + with Timer("apply_reward_kl_penalty", self.all_timings): + training_input = self.apply_reward_kl_penalty(training_input) + + # 3. calculate advantages and returns + with Timer("compute_advantages_and_returns", self.all_timings): + training_input = self.compute_advantages_and_returns(training_input) + # remove some unwanted keys + for key in ["rewards"]: + training_input.pop(key) + training_input.metadata.pop("uids") + + if self.cfg.trainer.algorithm.advantage_batch_normalize: + training_input = normalize_advantages_dict(training_input) + + if self.cfg.trainer.dump_data_batch: + # dump data to file + with Timer("dump_data_batch"): + self.dump_data(training_input, file_name=f"global_step_{self.global_step}_training_input") + + # 4. train policy/critic model + # Policy model is backloaded to GPU during training + with Timer("train_critic_and_policy", self.all_timings): + status = self.train_critic_and_policy(training_input) + + # 5. conditionally save checkpoints and hf model + if self.cfg.trainer.ckpt_interval > 0 and self.global_step % self.cfg.trainer.ckpt_interval == 0: + with Timer("save_checkpoints", self.all_timings): + self.save_checkpoints() + if ( + self.cfg.trainer.hf_save_interval > 0 + and self.global_step % self.cfg.trainer.hf_save_interval == 0 + ): + with Timer("save_hf_model", self.all_timings): + self.save_models() + + # 6. conditionally sync policy and ref at the end of the epoch + if ( + self.cfg.trainer.update_ref_every_epoch + and self.ref_model is not None + and iter == len(self.train_dataloader) - 1 + and epoch != self.cfg.trainer.epochs - 1 # skip updating ref at the end of the last epoch + ): + with Timer("update_ref_with_policy", self.all_timings): + self.update_ref_with_policy() + + # 7. sync weights to inference engines + self.dispatch.prepare_for_weight_sync() + if self.colocate_all: + await self.inference_engine_client.wake_up(tags=["weights"]) + with Timer("sync_weights", self.all_timings): + self.dispatch.broadcast_to_inference_engines(self.inference_engine_client) + self.dispatch.finish_weight_sync() + if self.colocate_all: + await self.inference_engine_client.wake_up(tags=["kv_cache"]) + + # 8. set logs + logger.info(status) + # log epoch info + self.all_metrics.update({"trainer/epoch": epoch, "trainer/global_step": self.global_step}) + if self.cfg.trainer.eval_interval > 0 and ( + self.global_step % self.cfg.trainer.eval_interval == 0 + or self.global_step == self.total_training_steps + ): + with Timer("eval", self.all_timings): + eval_metrics = await self.eval() + self.all_metrics.update(eval_metrics) + + log_payload = { + **self.all_metrics, + **{f"timing/{k}": v for k, v in self.all_timings.items()}, + } + self.tracker.log(log_payload, step=self.global_step, commit=True) + + # Check for high tool error rate and alert via Slack + run_name = self.cfg.trainer.get("experiment_name", None) + check_tool_error_rate_alert(self.all_metrics, self.global_step, threshold=0.5, run_name=run_name) + + self.all_metrics = {} + self.all_timings = {} + + # update progress bar after logging + pbar.update(1) + + self.global_step += 1 + + del training_input, generator_output + + pbar.close() + if self.colocate_all: + await self.inference_engine_client.sleep() + if self.cfg.trainer.ckpt_interval > 0: + with Timer("save_checkpoints", self.all_timings): + self.save_checkpoints() + logger.info("Saved final checkpoint.") + if self.cfg.trainer.hf_save_interval > 0: + with Timer("save_hf_model", self.all_timings): + self.save_models() + logger.info("Saved final model.") + logger.info("Training done!") + + def _remove_tail_data(self, entries: List[Any]) -> List[Any]: + """Remove tail data to have even shards in terms of *effective* samples. + + Each prompt produces `n_samples_per_prompt` samples. For data-parallel + training we care that the total number of samples is nicely splittable + across the (combined) data-parallel size of all enabled models. + """ + lcm_dp_size = self.dispatch.get_lcm_dp_size() + + n_samples_per_prompt = self.cfg.generator.n_samples_per_prompt + + # We want the largest m <= len(entries) such that: + # (m * n_samples_per_prompt) % lcm_dp_size == 0 + # + # Let g = gcd(lcm_dp_size, n_samples_per_prompt). Then this is equivalent + # to requiring m to be a multiple of (lcm_dp_size / g). + stride = lcm_dp_size // math.gcd(lcm_dp_size, n_samples_per_prompt) + if stride <= 1: + # Every prompt count is valid, keep all entries. + return entries + + kept_prompts = (len(entries) // stride) * stride + return entries[:kept_prompts] + + def build_models(self, PolicyWorker, CriticWorker, RefWorker): + """ + Initialize the actors for training, and handle colocation logic + """ + cfg = self.cfg + pg = None + + use_ref_model = cfg.trainer.algorithm.use_kl_loss or cfg.trainer.algorithm.use_kl_in_reward + + if cfg.trainer.placement.colocate_all: + num_policy_gpus = cfg.trainer.placement.policy_num_gpus_per_node * cfg.trainer.placement.policy_num_nodes + num_critic_gpus = cfg.trainer.placement.critic_num_gpus_per_node * cfg.trainer.placement.critic_num_nodes + num_ref_gpus = cfg.trainer.placement.ref_num_gpus_per_node * cfg.trainer.placement.ref_num_nodes + num_rollout_gpus = ( + cfg.generator.num_inference_engines + * cfg.generator.inference_engine_tensor_parallel_size + * cfg.generator.inference_engine_pipeline_parallel_size + * cfg.generator.inference_engine_data_parallel_size + ) + assert ( + num_policy_gpus == num_rollout_gpus + ), "num_policy_gpus and num_rollout_gpus must be the same when colocating all models" + pg = self.colocate_pg + + policy_model = PPORayActorGroup( + cfg, + cfg.trainer.placement.policy_num_nodes, + cfg.trainer.placement.policy_num_gpus_per_node, + PolicyWorker, + pg=pg, + num_gpus_per_actor=0.2 if pg else 1, + colocate_all=True, + sequence_parallel_size=cfg.trainer.policy.sequence_parallel_size, + record_memory=cfg.trainer.policy.record_memory, + ) + if use_ref_model: + assert ( + num_policy_gpus == num_ref_gpus + ), "num_policy_gpus and num_ref_gpus must be the same when colocating policy and ref model" + ref_model = PPORayActorGroup( + cfg, + cfg.trainer.placement.ref_num_nodes, + cfg.trainer.placement.ref_num_gpus_per_node, + RefWorker, + pg=pg, + num_gpus_per_actor=0.2 if pg else 1, + colocate_all=True, + sequence_parallel_size=cfg.trainer.ref.sequence_parallel_size, + ) + else: + ref_model = None + + if cfg.trainer.critic.model.path: + assert ( + num_policy_gpus == num_critic_gpus + ), "num_policy_gpus and num_critic_gpus must be the same when colocating policy and critic model" + critic_model = PPORayActorGroup( + cfg, + cfg.trainer.placement.critic_num_nodes, + cfg.trainer.placement.critic_num_gpus_per_node, + CriticWorker, + pg=pg, + num_gpus_per_actor=0.2, + colocate_all=True, + sequence_parallel_size=cfg.trainer.critic.sequence_parallel_size, + ) + else: + critic_model = None + + else: + if cfg.trainer.placement.colocate_policy_ref and use_ref_model: + assert ( + cfg.trainer.placement.policy_num_nodes == cfg.trainer.placement.ref_num_nodes + and cfg.trainer.placement.policy_num_gpus_per_node == cfg.trainer.placement.ref_num_gpus_per_node + ), "num_nodes and num_gpus_per_node must be the same when colocate policy and ref model." + + bundles = [ + { + "GPU": cfg.trainer.placement.policy_num_gpus_per_node, + "CPU": cfg.trainer.placement.policy_num_gpus_per_node, + } + for _ in range(cfg.trainer.placement.policy_num_nodes) + ] + pg = placement_group(bundles, strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=SKYRL_RAY_PG_TIMEOUT_IN_S) + + policy_model = PPORayActorGroup( + cfg, + cfg.trainer.placement.policy_num_nodes, + cfg.trainer.placement.policy_num_gpus_per_node, + PolicyWorker, + pg=pg, + num_gpus_per_actor=0.75 if pg else 1, + colocate_all=False, + sequence_parallel_size=cfg.trainer.policy.sequence_parallel_size, + ) + if use_ref_model: + ref_model = PPORayActorGroup( + cfg, + cfg.trainer.placement.ref_num_nodes, + cfg.trainer.placement.ref_num_gpus_per_node, + RefWorker, + pg=pg, + num_gpus_per_actor=0.25 if pg else 1, + colocate_all=False, + sequence_parallel_size=cfg.trainer.ref.sequence_parallel_size, + ) + else: + ref_model = None + + if cfg.trainer.critic.model.path: + critic_model = PPORayActorGroup( + cfg, + cfg.trainer.placement.critic_num_nodes, + cfg.trainer.placement.critic_num_gpus_per_node, + CriticWorker, + num_gpus_per_actor=1, + colocate_all=False, + sequence_parallel_size=cfg.trainer.critic.sequence_parallel_size, + ) + else: + critic_model = None + + policy_steps_per_train_batch = ( + cfg.trainer.train_batch_size // cfg.trainer.policy_mini_batch_size * cfg.trainer.update_epochs_per_batch + ) + critic_steps_per_train_batch = 0 + if cfg.trainer.critic.model.path: + critic_steps_per_train_batch = ( + cfg.trainer.train_batch_size // cfg.trainer.critic_mini_batch_size * cfg.trainer.update_epochs_per_batch + ) + if not cfg.trainer.placement.colocate_all: + refs = [] + if ref_model is not None: + refs.extend(ref_model.async_init_model(cfg.trainer.ref.model.path)) + refs.extend( + policy_model.async_init_model( + cfg.trainer.policy.model.path, + num_training_steps=self.total_training_steps * policy_steps_per_train_batch, + ) + ) + if cfg.trainer.critic.model.path: + refs.extend( + critic_model.async_init_model( + cfg.trainer.critic.model.path, + num_training_steps=self.total_training_steps * critic_steps_per_train_batch, + ) + ) + ray.get(refs) + ray.get(policy_model.async_run_ray_method("pass_through", "_set_pad_token_id", self.tokenizer.pad_token_id)) + else: + if ref_model is not None: + ray.get(ref_model.async_init_model(cfg.trainer.ref.model.path)) + ref_model.offload_to_cpu() + ray.get( + policy_model.async_init_model( + cfg.trainer.policy.model.path, + num_training_steps=self.total_training_steps * policy_steps_per_train_batch, + ) + ) + ray.get(policy_model.async_run_ray_method("pass_through", "_set_pad_token_id", self.tokenizer.pad_token_id)) + policy_model.offload_to_cpu() + if cfg.trainer.critic.model.path: + ray.get( + critic_model.async_init_model( + cfg.trainer.critic.model.path, + num_training_steps=self.total_training_steps * critic_steps_per_train_batch, + ) + ) + critic_model.offload_to_cpu() + + self.policy_model: PPORayActorGroup = policy_model + self.critic_model: Optional[PPORayActorGroup] = critic_model + self.ref_model: Optional[PPORayActorGroup] = ref_model + + # Create unified dispatch that manages all actor groups + self.dispatch = WorkerDispatch( + cfg=self.cfg, + policy_actor_group=policy_model, + critic_actor_group=critic_model, + ref_actor_group=ref_model, + ) + + # Mark all models as offloaded if colocate_all (they were offloaded above) + if self.colocate_all: + self.dispatch.mark_all_offloaded() + + logger.info("init policy/ref/critic models done") + + def init_weight_sync_state(self): + """ + Setup the connection between policy model and inference engine for weight syncing. + """ + ray.get( + self.policy_model.async_run_ray_method( + "pass_through", "init_weight_sync_state", self.inference_engine_client + ) + ) + logger.info("Initialized weight sync state for policy model and inference engines.") + + def convert_to_training_input(self, generator_output: GeneratorOutput, uids: List[str]) -> TrainingInputBatch: + """Converts lists to a padded batch of tensors for training""" + prompt_ids: List[List[int]] = generator_output["prompt_token_ids"] + response_ids: List[List[int]] = generator_output["response_ids"] + rewards: List[List[float]] = generator_output["rewards"] + loss_masks: List[List[int]] = generator_output["loss_masks"] + + logprobs: Optional[List[List[float]]] = generator_output.get("rollout_logprobs", None) + + ( + sequences_tensor, + attention_masks_tensor, + response_masks_tensor, + rewards_tensor, + loss_masks_tensor, + rollout_logprobs_tensor, + ) = convert_prompts_responses_to_batch_tensors( + self.tokenizer, + prompt_ids, + response_ids, + rewards, + loss_masks, + logprobs, + ) + # sanity check for tis + if self.cfg.trainer.algorithm.use_tis: + assert ( + rollout_logprobs_tensor is not None + ), "expected non-null rollout logprobs tensor with `trainer.algorithm.use_tis` as `True`" + assert rollout_logprobs_tensor.shape == loss_masks_tensor.shape, "Logprobs should look like responses" + training_input = TrainingInputBatch( + { + "sequences": sequences_tensor, # Full trajectories (padded and concatenated prompts and responses) + "attention_mask": attention_masks_tensor, + "response_mask": response_masks_tensor, + "rewards": rewards_tensor, + "loss_mask": loss_masks_tensor, + "rollout_logprobs": rollout_logprobs_tensor, + "is_last_step": ( + torch.tensor(generator_output["is_last_step"], dtype=torch.bool) + if generator_output.get("is_last_step", None) is not None + else None + ), + }, + ) + training_input.metadata = {"uids": uids} + # Track which samples are hint-augmented for first-turn baseline + if generator_output.get("is_hinted") is not None: + training_input.metadata["is_hinted"] = generator_output["is_hinted"] + # padded response length + training_input.metadata["response_length"] = response_masks_tensor.shape[1] + if self.cfg.generator.step_wise_trajectories: + assert ( + "trajectory_ids" in generator_output + ), "Expected `trajectory_ids` in generator output for step wise training" + training_input.metadata["trajectory_ids"] = [ + trajectory_id.to_string() for trajectory_id in generator_output["trajectory_ids"] + ] + training_input.metadata["avg_response_length"] = sum( + len(sample_response_ids) + for sample_response_ids, is_last_step in zip(response_ids, generator_output["is_last_step"]) + if is_last_step + ) / len(response_ids) + else: + training_input.metadata["avg_response_length"] = sum( + len(sample_response_ids) for sample_response_ids in response_ids + ) / len(response_ids) + + logger.info(f"Number of sequences before padding: {len(training_input['sequences'])}") + training_input = self.pad_batch(training_input) + logger.info(f"Number of sequences after padding: {len(training_input['sequences'])}") + + return training_input + + @torch.no_grad() + async def generate( + self, + input_batch: GeneratorInput, + ) -> GeneratorOutput: + """ + Generate rollouts. + + If colocate_all is enabled: + - before calling this method, the policy model should be on CPU and inference engine should + be awake (i.e. on GPU). + - after calling this method, the same model placement still holds. + """ + # NOTE: we assume that .generate returns samples in the same order as passed in + generator_output: GeneratorOutput = await self.generator.generate(input_batch) + + # add rollout metrics to self.all_metrics + if generator_output["rollout_metrics"] is not None: + self.all_metrics.update(generator_output["rollout_metrics"]) + + if not self.cfg.generator.step_wise_trajectories: + validate_generator_output(len(input_batch["prompts"]), generator_output) + + return generator_output + + @torch.no_grad() + def postprocess_generator_output( + self, generator_output: GeneratorOutput, uids: List[str], data_sources: Optional[List[str]] = None + ) -> GeneratorOutput: + """ + Converts to per token rewards and computes pass@N. + + In the future algorithm specific reward or loss mask post processing should be done here. + """ + generator_output_for_metrics = generator_output + uids_for_metrics = uids + data_sources_for_metrics = data_sources + if self.cfg.generator.step_wise_trajectories: + generator_output_for_metrics = defaultdict(list) + for key in generator_output: + if isinstance(generator_output[key], list): + generator_output_for_metrics[key] = [ + generator_output[key][i] + for i in range(len(generator_output[key])) + if generator_output["is_last_step"][i] + ] + uids_for_metrics = [ + uid for uid, is_last_step in zip(uids, generator_output["is_last_step"]) if is_last_step + ] + if data_sources: + data_sources_for_metrics = [ + ds for ds, is_last_step in zip(data_sources, generator_output["is_last_step"]) if is_last_step + ] + + # only use `generator_output_for_metrics` for metrics calculation + # For step-wise training, we only calculate metrics for the last step of each trajectory + overall_metrics = get_metrics_from_generator_output( + generator_output_for_metrics, + uids_for_metrics, + ) + + # these use the full generator output + rewards: Union[List[float], List[List[float]]] = generator_output["rewards"] + responses: List[List[int]] = generator_output["response_ids"] + per_token_rewards: List[List[float]] = [] + + # Check if rewards are already token-level (List[List[float]]) or response-level (List[float]) + if rewards and isinstance(rewards[0], list): + # Token-level rewards: rewards is List[List[float]] + per_token_rewards = rewards + else: + if self.cfg.trainer.algorithm.zero_variance_filter: + kept_indices_set = set(zero_variance_filter(rewards, uids)) + generator_output["loss_masks"] = [ + [0] * len(mask) if i not in kept_indices_set else mask + for i, mask in enumerate(generator_output["loss_masks"]) + ] + # Response-level rewards: rewards is List[float], convert to per-token rewards + for reward, response in zip(rewards, responses): + per_token_reward = [0.0] * len(response) + per_token_reward[-1] = float(reward) + per_token_rewards.append(per_token_reward) + + n_samples_per_prompt = self.cfg.generator.n_samples_per_prompt + + reward_metrics = { + f"reward/avg_pass_at_{n_samples_per_prompt}": overall_metrics["pass_at_n"], + "reward/avg_raw_reward": overall_metrics["avg_score"], + "reward/mean_positive_reward": overall_metrics["mean_positive_reward"], + } + + # Calculate per-data-source metrics if data_sources provided + if data_sources_for_metrics: + per_source_metrics = calculate_per_source_reward_metrics( + generator_output_for_metrics, + uids_for_metrics, + data_sources_for_metrics, + n_samples_per_prompt, + ) + reward_metrics.update(per_source_metrics) + + self.all_metrics.update(reward_metrics) + logger.info( + f"reward/avg_pass_at_{n_samples_per_prompt}: {overall_metrics['pass_at_n']}, reward/avg_raw_reward: {overall_metrics['avg_score']}, reward/mean_positive_reward: {overall_metrics['mean_positive_reward']}" + ) + # re-assign reward but now it's per token rewards + generator_output["rewards"] = per_token_rewards + return generator_output + + @torch.no_grad() + def compute_advantages_and_returns(self, data: TrainingInputBatch) -> TrainingInputBatch: + """Calculate advantages and returns for the data batch. + + Expects: + - `["sequences"]`: Integer[torch.Tensor, "batch_size seqlen"] + - `["response_mask"]`: Integer[torch.Tensor, "batch_size seqlen"] + - `["loss_mask"]`: Integer[torch.Tensor, "batch_size seqlen"] + - `["values"]`: Float[torch.Tensor, "batch_size seqlen"] + - `["rewards"]`: Float[torch.Tensor, "batch_size seqlen"] + - `.metadata["uids"]`: List[str] + + Adds: + - `["advantages"]`: Float[torch.Tensor, "batch_size seqlen"] + - `["returns"]`: Float[torch.Tensor, "batch_size seqlen"] + """ + token_level_rewards = data["rewards"] + + # Convert is_hinted metadata to numpy array for advantage computation + is_hinted_list = data.metadata.get("is_hinted") + is_hinted = np.array(is_hinted_list) if is_hinted_list is not None else None + + if self.cfg.generator.step_wise_trajectories: + is_last_step = data["is_last_step"].bool() + response_mask = data["response_mask"] + index = np.array(data.metadata["uids"]) + adv_estimator = self.cfg.trainer.algorithm.advantage_estimator + config = self.cfg.trainer.algorithm + values = data["values"] + gamma = self.cfg.trainer.algorithm.gamma + lambd = self.cfg.trainer.algorithm.lambd + grpo_norm_by_std = self.cfg.trainer.algorithm.grpo_norm_by_std + last_step_rewards = token_level_rewards[is_last_step] + last_step_is_hinted = is_hinted[is_last_step.cpu().numpy()] if is_hinted is not None else None + # compatible with any advantage estimator + last_step_advantages, last_step_returns = ppo_utils.compute_advantages_and_returns( + token_level_rewards=last_step_rewards, + response_mask=response_mask[is_last_step], + index=index[is_last_step.cpu().numpy()], + adv_estimator=adv_estimator, + values=values[is_last_step] if values is not None else None, + config=config, + gamma=gamma, + lambd=lambd, + grpo_norm_by_std=grpo_norm_by_std, + is_hinted=last_step_is_hinted, + ) + traj_ids = ( + torch.cat([torch.tensor([False], device=is_last_step.device), is_last_step[:-1]]).int().cumsum(dim=0) + ) + num_groups = traj_ids[-1].item() + 1 + assert num_groups == len( + last_step_advantages + ), f"number of groups {num_groups} doesn't match the number of trajectories as given by `is_last_step` {len(last_step_advantages)}. The `is_last_step` tensor is likely malformed" + advantages = last_step_advantages[traj_ids] + returns = last_step_returns[traj_ids] + else: + advantages, returns = ppo_utils.compute_advantages_and_returns( + token_level_rewards=token_level_rewards, + response_mask=data["response_mask"], + index=data.metadata["uids"], + adv_estimator=self.cfg.trainer.algorithm.advantage_estimator, + config=self.cfg.trainer.algorithm, + values=data["values"], + gamma=self.cfg.trainer.algorithm.gamma, + lambd=self.cfg.trainer.algorithm.lambd, + grpo_norm_by_std=self.cfg.trainer.algorithm.grpo_norm_by_std, + is_hinted=is_hinted, + ) + data["returns"] = returns + data["advantages"] = advantages + + # remove padding while calculating metrics + pad_size = data.metadata.get("pad_size", 0) + num_samples = len(token_level_rewards) + + return_sums = token_level_rewards.sum(dim=-1)[: num_samples - pad_size] + if self.cfg.generator.step_wise_trajectories: + avg_rewards: float = return_sums[data["is_last_step"][: num_samples - pad_size]].mean().item() + else: + avg_rewards: float = return_sums.mean().item() + + avg_response_length = data.metadata["avg_response_length"] + data = data.to("cpu") + + valid_advantages = torch.masked_select( + data["advantages"][: num_samples - pad_size, ...], data["response_mask"][: num_samples - pad_size].bool() + ) + avg_advantages: float = valid_advantages.mean().item() + avg_advantages_abs: float = valid_advantages.abs().mean().item() + + if "metrics" not in data.metadata: + data.metadata["metrics"] = {} + data.metadata["metrics"].update( + { + "avg_final_rewards": avg_rewards, + "avg_response_length": avg_response_length, + "avg_advantages": avg_advantages, + "avg_advantages_abs": avg_advantages_abs, + } + ) + + logger.info(f"avg_final_rewards: {avg_rewards}, avg_response_length: {avg_response_length}") + self.all_metrics.update( + { + "loss/avg_final_rewards": avg_rewards, + "loss/avg_raw_advantages": avg_advantages, + "loss/avg_raw_advantages_abs": avg_advantages_abs, + } + ) + return data + + def dump_data(self, data: TrainingInputBatch, file_name: str): + """ + Dump data to pickle file + """ + data_save_dir = Path(self.cfg.trainer.export_path) / "dumped_data" + data_save_dir.mkdir(parents=True, exist_ok=True) + data.save(data_save_dir / f"{file_name}.pkl") + + def pad_batch(self, training_input: TrainingInputBatch) -> TrainingInputBatch: + """Pad the batch to be divisible by dp size""" + import math + + dp_size = self.dispatch.get_lcm_dp_size() + pad_size = math.ceil(training_input.batch_size / dp_size) * dp_size - training_input.batch_size + new_tensors = {} + training_input.metadata["pad_size"] = pad_size + if pad_size == 0: + return training_input + for key, tensor in training_input.items(): + if tensor is not None: + additional_dims = tuple(tensor.shape[1:]) if len(tensor.shape) > 1 else () + + if key == "is_last_step": + padding_tensor = torch.ones(pad_size, *additional_dims, dtype=tensor.dtype, device=tensor.device) + elif key == "loss_mask": + # ensures that padding tensors don't count towards the loss + padding_tensor = torch.zeros(pad_size, *additional_dims, dtype=tensor.dtype, device=tensor.device) + else: + # ensures all padding tensors are in a valid format by cloning `pad_size` from the original input + # `pad_size` is guaranteed to be smaller than batch_size + padding_tensor = tensor[:pad_size].clone() + new_tensors[key] = torch.cat([tensor, padding_tensor], dim=0) + + new_training_input = TrainingInputBatch(new_tensors) + new_training_input.metadata = {} + new_training_input.metadata["uids"] = training_input.metadata["uids"] + [f"pad{i}" for i in range(pad_size)] + if "trajectory_ids" in training_input.metadata: + new_training_input.metadata["trajectory_ids"] = training_input.metadata["trajectory_ids"] + [ + f"pad{i}" for i in range(pad_size) + ] + if "is_hinted" in training_input.metadata: + new_training_input.metadata["is_hinted"] = training_input.metadata["is_hinted"] + [False] * pad_size + for key, value in training_input.metadata.items(): + if key not in ["uids", "trajectory_ids", "is_hinted"]: + new_training_input.metadata[key] = copy.deepcopy(value) + return new_training_input + + @torch.no_grad() + def fwd_logprobs_values_reward( + self, + training_input: TrainingInputBatch, + ): + """ + Calculate values from the critic, log probs from the policy and ref model. + + Dispatch handles offload/backload automatically for all colocation configurations. + + Expects: + - `["sequences"]`: Integer[torch.Tensor, "batch_size seqlen"] + - `["attention_mask"]`: Integer[torch.Tensor, "batch_size seqlen"] + - `.metadata["response_length"]`: Int + + Adds: + - `["base_action_log_probs"]`: Float[torch.Tensor, "batch_size seqlen"] + - `["action_log_probs"]`: Float[torch.Tensor, "batch_size seqlen"] + - `["values"]`: Float[torch.Tensor, "batch_size seqlen"] + """ + data_fwd_pass = training_input.select(keys=["sequences", "attention_mask"], metadata_keys=["response_length"]) + + values = None + base_log_probs = None + action_log_probs = None + + # Critic forward (dispatch handles offload/backload automatically) + if self.has_critic: + critic_output = self.dispatch.forward("critic", data_fwd_pass) + values = critic_output["output"] + + # Ref forward + if self.ref_model is not None: + ref_output = self.dispatch.forward("ref", data_fwd_pass) + base_log_probs = ref_output["output"] + self.dispatch.empty_cache("ref") + + # Policy forward + policy_output = self.dispatch.forward("policy", data_fwd_pass) + action_log_probs = policy_output["output"] + + # Empty cache after all forward passes + self.dispatch.empty_cache() + + sequences_all: torch.Tensor = training_input["sequences"] + # NOTE (sumanthrh): The slicing is needed to make sure that the batch dimension doesn't change for the tensordict. + base_log_probs = base_log_probs[: len(sequences_all)] if base_log_probs is not None else None + action_log_probs = action_log_probs[: len(sequences_all)] + values = values[: len(sequences_all)] if values is not None else None + + training_input["base_action_log_probs"] = base_log_probs + training_input["action_log_probs"] = action_log_probs + training_input["values"] = values + + if self.cfg.generator.sampling_params.logprobs is not None: + # calculates the difference in probs between inference and trainer components + # only consider response tokens + logprobs_diff = ( + training_input["rollout_logprobs"][training_input["loss_mask"] > 0] + - action_log_probs[training_input["loss_mask"] > 0] + ) + prob_diff = logprobs_diff.exp().abs() + prob_diff_mean = prob_diff.mean().item() + prob_diff_std = prob_diff.std().item() + self.all_metrics.update( + { + "policy/rollout_train_prob_diff_mean": prob_diff_mean, + "policy/rollout_train_prob_diff_std": prob_diff_std, + } + ) + return training_input + + def apply_reward_kl_penalty( + self, + data: TrainingInputBatch, + ) -> TrainingInputBatch: + """Applies a penalty for KL divergence between the policy log probs and the base model log probs to the rewards.""" + loss_masks_all: torch.Tensor = data["loss_mask"] + rewards: torch.Tensor = data["rewards"] + base_action_log_probs: torch.Tensor = data["base_action_log_probs"] + action_log_probs: torch.Tensor = data["action_log_probs"] + + # single batched computation + kl: Float[torch.Tensor, "batch_size seqlen"] = compute_approx_kl( # type: ignore + action_log_probs, + base_action_log_probs, + loss_mask=loss_masks_all, + kl_estimator_type=self.cfg.trainer.algorithm.kl_estimator_type, + ) + kl_max: Float[torch.Tensor, "batch_size"] = torch.max(kl.abs(), dim=-1)[0] # noqa: F821 + kl_mean: Float[torch.Tensor, "batch_size"] = masked_mean(kl, loss_masks_all, dim=-1) # noqa: F821 + + # NOTE (erictang000): only supporting custom rewards currently + kl_loss_coef = ( + self.reward_kl_controller.value + if self.reward_kl_controller is not None + else self.cfg.trainer.algorithm.kl_loss_coef + ) + rewards = rewards - kl * max(0, kl_loss_coef) + data["rewards"] = rewards + + avg_kl: float = kl_mean.mean().item() + avg_kl_max: float = kl_max.mean().item() + + # update the kl controller + if self.reward_kl_controller is not None: + self.reward_kl_controller.update(current=avg_kl, n_steps=kl.shape[0]) # n_steps is just the batch size + if "metrics" not in data.metadata: + data.metadata["metrics"] = {} + + data.metadata["metrics"].update( + { + "avg_kl": avg_kl, + "avg_kl_max": avg_kl_max, + "kl_loss_coef": kl_loss_coef, + } + ) + + self.all_metrics.update( + { + "loss/avg_kl": avg_kl, + "loss/avg_kl_max": avg_kl_max, + "loss/kl_loss_coef": kl_loss_coef, + } + ) + + return data + + def sync_policy_weights_to_inference_engines(self) -> List[ObjectRef]: + return self.policy_model.async_run_ray_method( + "pass_through", "broadcast_to_inference_engines", self.inference_engine_client + ) + + def _execute_training_step(self, model: str, data: TrainingInputBatch) -> Dict[str, float]: + """ + Execute training step for FSDP strategy using forward_backward + optim_step. + + The trainer loops over epochs and mini-batches. Workers handle micro-batching + internally for gradient accumulation (memory efficiency). + + Args: + model: Model name ("policy" or "critic") + data: Training data batch + + Returns: + Dict of reduced metrics from training + """ + # Compute mini batch size from config (algorithm-level concept) + n_samples = self.cfg.generator.n_samples_per_prompt + if model == "policy": + mini_batch_size = self.cfg.trainer.policy_mini_batch_size * n_samples + else: + mini_batch_size = self.cfg.trainer.critic_mini_batch_size * n_samples + + all_metrics: Dict[str, List[float]] = defaultdict(list) + + # Training loop over epochs and mini-batches + for _epoch in range(self.cfg.trainer.update_epochs_per_batch): + num_mini_batches = len(data) // mini_batch_size + for local_step in range(num_mini_batches): + start_idx = local_step * mini_batch_size + end_idx = (local_step + 1) * mini_batch_size + mini_batch = data[start_idx:end_idx] + + status = self.dispatch.forward_backward(model, mini_batch) + for k, v in status.items(): + all_metrics[k].append(v) + + # Optimizer step after each mini batch + grad_norm = self.dispatch.optim_step(model) + if grad_norm is not None: + all_metrics["grad_norm"].append(grad_norm) + + # Compute gradient metrics once per training step (from accumulated stats) + # Only track gradient metrics for policy, not critic + if model == "policy" and self.cfg.trainer.policy.track_extra_gradient_metrics: + grad_metrics = self.dispatch.compute_gradient_metrics(model) + for k, v in grad_metrics.items(): + all_metrics[k].append(v) + # Free GPU tensors (mean/var_numerator) to reclaim memory for inference + self.dispatch.reset_gradient_stats(model) + + # Reduce metrics across all mini-batches and epochs + reduced_metrics = reduce_metrics(all_metrics, ignore_keys=["grad:update_diversity"]) + return reduced_metrics + + def train_critic_and_policy(self, data: TrainingInputBatch): + """ + Run the training step for the policy and critic models. + + For Megatron strategy: uses ppo_train (training loop inside worker) + For FSDP strategy: uses forward_backward + optim_step (training loop in trainer) + """ + data.metadata["global_step"] = self.global_step + critic_status = None + + if self.cfg.trainer.strategy == "megatron": + # Megatron: training loop inside worker via ppo_train + if self.has_critic: + with Timer("critic_train", self.all_timings): + critic_status = self.dispatch.ppo_train("critic", data) + with Timer("policy_train", self.all_timings): + policy_status = self.dispatch.ppo_train("policy", data) + else: + # FSDP: training loop in trainer via forward_backward + optim_step + if self.has_critic: + with Timer("critic_train", self.all_timings): + critic_status = self._execute_training_step("critic", data) + with Timer("policy_train", self.all_timings): + policy_status = self._execute_training_step("policy", data) + + # Update metrics + if critic_status is not None: + for k, v in critic_status.items(): + self.all_metrics.update({f"critic/{k}": v}) + + for k, v in policy_status.items(): + self.all_metrics.update({f"policy/{k}": v}) + + self.dispatch.empty_cache() + + return policy_status + + def handle_dynamic_sampling( + self, generator_output: GeneratorOutput, uids: List[str] + ) -> Tuple[GeneratorOutput, List[str], bool]: + """ + Handle dynamic sampling for the current batch. + + Accumulates the generator output and UIDs across batches if we are sampling repeatedly + and applies the dynamic sampling strategy (i.e. filter, replace) to the current batch. + If we hit the limit of max sample batches, we raise an error. + + Args: + generator_output: Current batch generator output + uids: Current batch UIDs + + Returns: + processed_output: Filtered generator output + processed_uids: Filtered UIDs + keep_sampling: Whether to keep sampling + """ + # Prepare sampling configuration + max_sample_batches = self.cfg.trainer.algorithm.dynamic_sampling.max_sample_batches + dynamic_sampling_config = { + "type": self.cfg.trainer.algorithm.dynamic_sampling.type, + "max_sample_batches": max_sample_batches, + "min_replace_ratio": self.cfg.trainer.algorithm.dynamic_sampling.min_replace_ratio, + "train_batch_size": self.cfg.trainer.train_batch_size, + "n_samples_per_prompt": self.cfg.generator.n_samples_per_prompt, + } + + if self.dynamic_sampling_state is None: + self.dynamic_sampling_state: DynamicSamplingState = { + "sample_batch_count": 1, + } + else: + self.dynamic_sampling_state["sample_batch_count"] += 1 + + # Handle dynamic sampling using utilities + processed_output, processed_uids, keep_sampling, updated_state = trainer_utils.handle_dynamic_sampling( + generator_output, uids, dynamic_sampling_config, self.dynamic_sampling_state + ) + + # Check max resample limit, and if we hit it, raise an error + if ( + keep_sampling + and max_sample_batches > 0 + and self.dynamic_sampling_state["sample_batch_count"] >= max_sample_batches + ): + raise RuntimeError( + f"Exiting training loop due to hitting dynamic sampling limit for " + f"{self.cfg.trainer.algorithm.dynamic_sampling.type} strategy with " + f"{self.cfg.trainer.algorithm.dynamic_sampling.max_sample_batches} max sample batches. " + f"Please check your data difficulty distribution." + ) + # Update state + self.dynamic_sampling_state = updated_state + + if not keep_sampling: + # Reset state when sampling is complete + self.dynamic_sampling_state = None + + return processed_output, processed_uids, keep_sampling + + def _get_dp_group_models(self, rank: int, model_type: str = ""): + model = getattr(self, model_type) + return model._actor_handlers[rank] + + def _get_mesh_rank(self, rank: int, model_type: str = "") -> MeshRank: + model: PPORayActorGroup = getattr(self, model_type) + actor_info: ActorInfo = model.actor_infos[rank] + return actor_info.rank + + def save_checkpoints(self): + """ + Save the model, optimizer, and training states to disk. + + Dispatch handles offload/backload automatically for all colocation configurations. + """ + # Create global step folder structure + global_step_folder = os.path.join(self.cfg.trainer.ckpt_path, f"global_step_{self.global_step}") + policy_save_dir = os.path.join(global_step_folder, "policy") + critic_save_dir = os.path.join(global_step_folder, "critic") + + io.makedirs(global_step_folder, exist_ok=True) + + # Save policy checkpoint (dispatch handles offload/backload) + self.dispatch.save_checkpoint("policy", policy_save_dir, self.tokenizer) + + # Save critic checkpoint (if it exists) + if self.has_critic: + self.dispatch.save_checkpoint("critic", critic_save_dir, self.tokenizer) + + # Save dataloader state + dataloader_save_path = os.path.join(global_step_folder, "data.pt") + try: + dataloader_state_dict = self.train_dataloader.state_dict() + with io.open_file(dataloader_save_path, "wb") as f: + torch.save(dataloader_state_dict, f) + logger.info(f"Saved dataloader state to {dataloader_save_path}") + except Exception as e: + logger.warning(f"Failed to save dataloader state: {e}") + + # Save additional trainer state + trainer_state = { + "global_step": self.global_step, + "config": self.cfg, + } + trainer_state_path = os.path.join(global_step_folder, "trainer_state.pt") + with io.open_file(trainer_state_path, "wb") as f: + torch.save(trainer_state, f) + logger.info(f"Saved trainer state to {trainer_state_path}") + + # Atomic tracking - write this last after all saves succeed + latest_checkpoint_file = os.path.join(self.cfg.trainer.ckpt_path, "latest_ckpt_global_step.txt") + with io.open_file(latest_checkpoint_file, "w") as f: + f.write(str(self.global_step)) + + logger.info(f"Successfully saved checkpoint for global_step_{self.global_step} to: {global_step_folder}") + + # Clean up old checkpoints after successful save + with Timer("cleanup_old_checkpoints", self.all_timings): + self._cleanup_old_checkpoints() + + def _cleanup_old_checkpoints(self): + if not self._node_ids: + self._node_ids = self.dispatch.get_node_ids() + run_on_each_node( + self._node_ids, + cleanup_old_checkpoints, + self.cfg.trainer.ckpt_path, + self.cfg.trainer.max_ckpts_to_keep, + ) + # run on driver as well + # NOTE (sumanthrh): the function will get called twice on the node with driver process, but it's ok because it's idempotent + cleanup_old_checkpoints(self.cfg.trainer.ckpt_path, self.cfg.trainer.max_ckpts_to_keep) + + def load_checkpoints(self) -> Tuple[int, str]: + """ + Load complete checkpoint state and return the global_step to resume from. + Returns 0 if no checkpoint is loaded. + + If colocate_all is True, assumes that the policy model is currently on GPU. + + Returns: + global_step: The global step to resume from. + checkpoint_path: The path to the checkpoint. + """ + checkpoint_path = None + # Check if resumption is enabled + if self.resume_mode == ResumeMode.NONE: + logger.info("Checkpoint resumption disabled, starting training from scratch") + return 0, None + # first, let's get resume_path + elif self.resume_mode == ResumeMode.LATEST: + latest_checkpoint_file = os.path.join(self.cfg.trainer.ckpt_path, "latest_ckpt_global_step.txt") + if not io.exists(latest_checkpoint_file): + logger.info("No checkpoint found, starting training from scratch") + return 0, None + with io.open_file(latest_checkpoint_file, "r") as f: + ckpt_iteration = int(f.read().strip()) + checkpoint_path = os.path.join(self.cfg.trainer.ckpt_path, f"{GLOBAL_STEP_PREFIX}{ckpt_iteration}") + # Run validation: Make sure ckpt folder is consistent with latest_ckpt_global_step.txt + validate_consistency_for_latest_checkpoint( + self.cfg.trainer.ckpt_path, + ckpt_iteration, + checkpoint_path, + latest_checkpoint_file, + self.cfg.trainer.ckpt_interval, + ) + else: + # Get and validate resume path + checkpoint_path = Path(self.cfg.trainer.resume_path) + if not checkpoint_path: + raise ValueError("`trainer.resume_path` must be specified when resume_mode is 'from_path'") + + # Validate that it's a global_step directory + if GLOBAL_STEP_PREFIX not in checkpoint_path.name: + raise ValueError( + f"`trainer.resume_path` must point to a directory whose name starting with {GLOBAL_STEP_PREFIX}, got: {checkpoint_path}" + ) + + # Validate that the path exists + if not io.exists(str(checkpoint_path)): + raise FileNotFoundError(f"Checkpoint path not found: {checkpoint_path}") + + logger.info(f"Loading checkpoint from: {checkpoint_path}") + + # Extract global step from checkpoint path + global_step = extract_step_from_path(Path(checkpoint_path)) + if global_step == -1: + raise ValueError(f"Checkpoint path {checkpoint_path} is not a valid checkpoint path") + logger.info(f"Resuming from global_step: {global_step}") + + # Define paths for different checkpoint components + policy_ckpt_dir = os.path.join(checkpoint_path, "policy") + critic_ckpt_dir = os.path.join(checkpoint_path, "critic") + trainer_state_path = os.path.join(checkpoint_path, "trainer_state.pt") + dataloader_state_path = os.path.join(checkpoint_path, "data.pt") + + # Validate that required checkpoint files exist + if not io.exists(trainer_state_path): + raise FileNotFoundError(f"Trainer state file not found: {trainer_state_path}") + + # 1. Load and validate trainer state + with io.open_file(trainer_state_path, "rb") as f: + trainer_state = torch.load(f, map_location="cpu", weights_only=False) + saved_global_step = trainer_state.get("global_step", global_step) + logger.info("Successfully loaded trainer state") + if saved_global_step != global_step: + logger.warning(f"Global step mismatch: path={global_step}, saved={saved_global_step}. Using path value.") + + # 2. Load dataloader state if available + if io.exists(dataloader_state_path): + try: + with io.open_file(dataloader_state_path, "rb") as f: + dataloader_state = torch.load(f, map_location="cpu", weights_only=False) + self.train_dataloader.load_state_dict(dataloader_state) + logger.info("Successfully loaded dataloader state") + except Exception as e: + logger.warning(f"Failed to load dataloader state: {e}. Dataloader will start from beginning.") + else: + logger.warning( + f"No dataloader state found at {dataloader_state_path}. Dataloader will start from beginning." + ) + + # 3. Load policy checkpoint (dispatch handles offload/backload) + logger.info(f"Loading policy checkpoint from {policy_ckpt_dir}") + self.dispatch.load_checkpoint( + "policy", + policy_ckpt_dir, + load_optimizer_states=True, + load_lr_scheduler_states=True, + ) + logger.info("Successfully loaded policy checkpoint") + + # 4. Load critic checkpoint if it exists and we have a critic model + if self.has_critic: + logger.info(f"Loading critic checkpoint from {critic_ckpt_dir}") + self.dispatch.load_checkpoint( + "critic", + critic_ckpt_dir, + load_optimizer_states=True, + load_lr_scheduler_states=True, + ) + logger.info("Successfully loaded critic checkpoint") + + logger.info(f"Successfully loaded complete checkpoint state from global_step_{global_step}") + return global_step, str(checkpoint_path) + + def save_models(self): + """ + Save the model parameters in HF format at `cfg.trainer.export_path`. + + Dispatch handles offload/backload automatically for all colocation configurations. + """ + policy_export_dir = os.path.join(self.cfg.trainer.export_path, f"global_step_{self.global_step}", "policy") + self.dispatch.save_hf_model("policy", policy_export_dir, self.tokenizer) + + if self.has_critic: + critic_export_dir = os.path.join(self.cfg.trainer.export_path, f"global_step_{self.global_step}", "critic") + self.dispatch.save_hf_model("critic", critic_export_dir, self.tokenizer) + + logger.info("Successfully saved model weights.") + + def update_ref_with_policy(self): + """ + Update the reference model with the policy model weights (required by some algorithms). + + Dispatch handles offload/backload automatically for all colocation configurations. + After this method, prepare_for_weight_sync() should be called to ensure policy is on GPU. + """ + # TODO(tgriggs): Make policy-to-ref sync faster. + policy_export_dir = os.path.join(self.cfg.trainer.export_path, f"global_step_{self.global_step}", "policy") + + # Save policy model (dispatch handles GPU state) + self.dispatch.save_hf_model("policy", policy_export_dir, self.tokenizer) + + # Re-initialize ref model from saved policy (dispatch handles offloading policy first) + self.dispatch.init_model("ref", policy_export_dir) + + # Clean up temporary saved model files + try: + shutil.rmtree(policy_export_dir) + logger.info(f"Cleaned up temporary policy export directory: {policy_export_dir}") + except Exception as e: + logger.warning(f"Failed to clean up temporary policy export directory {policy_export_dir}: {e}") + + logger.info("Successfully updated ref model with policy model, training continues.") diff --git a/skyrl-train/skyrl_train/training_batch.py b/skyrl-train/skyrl_train/training_batch.py new file mode 100644 index 0000000000..1c455db1c3 --- /dev/null +++ b/skyrl-train/skyrl_train/training_batch.py @@ -0,0 +1,347 @@ +"""Defines interfaces for training data.""" + +from typing import TypedDict, Dict, Any, List, Optional, Generic, TypeVar +import torch +from jaxtyping import Float, Integer +import pickle +import io + +DictType = TypeVar("DictType") + + +# NOTE (sumanthrh): This is inspired by `TensorDict` but is much simpler. +class TensorBatch(dict, Generic[DictType]): + """Base class for training batches + + This defines a generic container for a batch of training data (inputs or outputs). + Consists of a dictionary of tensors along with some metadata. + """ + + metadata: Optional[Dict[str, Any]] = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._batch_size = None + self._device = None + self._check_consistency() + + def select(self, keys: List[str], metadata_keys: Optional[List[str]] = None) -> "TensorBatch[DictType]": + """Select a subset of the data batch. + + Args: + keys: The keys to select + metadata_keys: The metadata keys to select + + Returns: + A new `TensorBatch` object with the selected keys and metadata + """ + selected_batch_data = {} + for key in keys: + selected_batch_data[key] = self[key] + selected_metadata = {} + if metadata_keys is None: + selected_metadata = self.metadata + else: + selected_metadata = {} + for key in metadata_keys: + selected_metadata[key] = self.metadata[key] + new_batch = self.__class__(selected_batch_data) + new_batch.metadata = selected_metadata + return new_batch + + def _check_consistency(self): + """Check consistency of all present fields""" + keys = list(self.keys()) + if len(keys) == 0: + return + + batch_size = len(self[keys[0]]) + self._batch_size = batch_size + for key in keys: + value = self[key] + if value is None: + continue + self._device = value.device if self._device is None else self._device + if not isinstance(value, torch.Tensor): + raise ValueError(f"Field {key} must be a tensor, got {type(value)}") + if len(value) != batch_size: + raise ValueError(f"Batch size mismatch in {key}") + if value.device != self._device: + raise ValueError(f"Device mismatch in {key}. Expected {self._device}, got {value.device}") + + def __getitem__(self, index) -> "TensorBatch[DictType]": + if isinstance(index, slice): + return self.slice(index.start, index.stop, index.step) + elif isinstance(index, int): + return self.slice(index, index + 1) + else: + return super().__getitem__(index) + + def __setitem__(self, key: str, value: Optional[torch.Tensor]) -> None: + if value is None: + super().__setitem__(key, value) + return + + if not isinstance(value, torch.Tensor): + raise ValueError(f"Field {key} must be a tensor, got {type(value)}") + + if hasattr(self, "_batch_size") and self._batch_size is not None and len(value) != self._batch_size: + raise ValueError( + f"Batch size mismatch in {key}. Expected tensor to be of size {self._batch_size}, got {len(value)}." + ) + + super().__setitem__(key, value) + + if hasattr(self, "_batch_size") and self._batch_size is None: + self._batch_size = len(value) + + def to( + self, device: torch.device = None, dtype: torch.dtype = None, *, non_blocking: bool = False + ) -> "TensorBatch": + """Move tensors to device and/or cast to dtype. + + Args: + device: The device to move the tensors to + dtype: The dtype to cast the tensors to + non_blocking: Whether the operation should be non-blocking + """ + for key, value in self.items(): + if value is None: + continue + assert isinstance(value, torch.Tensor), f"Field {key} must be a tensor, got {type(value)}" + self[key] = value.to(device, dtype, non_blocking=non_blocking) + return self + + def contiguous(self) -> "TensorBatch": + """Make the tensors contiguous""" + for key, value in self.items(): + if value is None: + continue + # some of these asserts are not needed, but it's kept for type safety + assert isinstance(value, torch.Tensor), f"Field {key} must be a tensor, got {type(value)}" + self[key] = value.contiguous() + return self + + @property + def batch_size(self) -> int: + """Batch size for the tensors""" + return self._batch_size + + @property + def device(self) -> torch.device: + """Get the device for the tensors""" + return self._device + + def __getstate__(self): + """Serialize the `TensorBatch` object for pickle protocol""" + self.contiguous() + if self._device is not None: + assert self._device == torch.device("cpu"), "Tensors must be on CPU before serialization" + batch_dict = {} + for key, value in self.items(): + buffer = io.BytesIO() + torch.save(value, buffer) + batch_dict[key] = buffer.getvalue() + + return { + "batch_dict": batch_dict, + "batch_size": self._batch_size, + "device": self._device, + "metadata": self.metadata, + } + + def __setstate__(self, state): + """Deserialize the `TensorBatch` object and load it into memory""" + for key, value in state["batch_dict"].items(): + buffer = io.BytesIO(value) + self[key] = torch.load(buffer) + + self._batch_size = state["batch_size"] + self._device = state["device"] + self.metadata = state["metadata"] + self._check_consistency() + return self + + def repeat(self, repeats: int): + """Repeat entries in the data batch a specified number of times. + + This is similar to `torch.repeat` (and `numpy.tile`). `metadata` is not repeated. + + Args: + repeats: The number of times to repeat the data batch + + Returns: + A new `TensorBatch` object with the data repeated + """ + new_batch = {} + for key, value in self.items(): + if value is None: + new_batch[key] = value + else: + assert isinstance(value, torch.Tensor), f"Field {key} must be a tensor, got {type(value)}" + new_batch[key] = value.repeat(repeats) + new_batch = self.__class__(new_batch) + new_batch.metadata = self.metadata + return new_batch + + def repeat_interleave(self, repeats: int): + """Repeat entries in the data batch a specified number of times. + + This is similar to `torch.repeat_interleave` (and `numpy.repeat`). `metadata` is not repeated. + + Args: + repeats: The number of times to repeat the data batch + + Returns: + A new `TensorBatch` object with the data repeated + """ + new_batch = {} + for key, value in self.items(): + if value is None: + new_batch[key] = value + else: + assert isinstance(value, torch.Tensor), f"Field {key} must be a tensor, got {type(value)}" + new_batch[key] = value.repeat_interleave(repeats) + new_batch = self.__class__(new_batch) + new_batch.metadata = self.metadata + return new_batch + + def chunk(self, chunk_size: int) -> List["TensorBatch[DictType]"]: + """Split into smaller chunks""" + chunks = [] + for i in range(0, self.batch_size, chunk_size): + chunk_data = {} + for key, value in self.items(): + if value is not None: + if isinstance(value, torch.Tensor): + chunk_data[key] = value[i : i + chunk_size] + else: + raise ValueError(f"Unsupported type {type(value)} for key {key}") + else: + # `None` values are not chunked + chunk_data[key] = value + chunk = self.__class__(chunk_data) + chunk.metadata = self.metadata + chunks.append(chunk) + return chunks + + def slice(self, start: int, end: int, step: int = 1) -> "TensorBatch[DictType]": + """Slice the data batch. + + Args: + start: The start index + end: The end index + step: The step size + + Returns: + A new `TensorBatch` object with the view of the specified slice. + """ + slice_obj = slice(start, end, step) + sliced_data = {} + for key, value in self.items(): + if value is not None: + if isinstance(value, torch.Tensor): + sliced_data[key] = value[slice_obj] + else: + raise ValueError(f"Unsupported type {type(value)} for key {key}") + else: + # `None` values are not sliced + sliced_data[key] = value + sliced_batch = self.__class__(sliced_data) + sliced_batch.metadata = self.metadata + return sliced_batch + + def save(self, path: str): + """Save the data to a pickle file""" + with open(path, "wb") as f: + pickle.dump(self, f) + + def load(self, path: str): + """Load the data from a pickle file""" + with open(path, "rb") as f: + return pickle.load(f) + + @classmethod + def cat(cls, shards: List["TensorBatch[DictType]"]) -> "TensorBatch[DictType]": + """Concatenate shards. + + Args: + shards: The list of `TensorBatch` objects to cat + + Returns: + A new `TensorBatch` object with the concatenated data + """ + cat_data = {} + assert len(shards) > 0, "Cannot cat an empty list of shards" + for key, value in shards[0].items(): + if value is not None: + if isinstance(value, torch.Tensor): + cat_data[key] = torch.cat([shard[key] for shard in shards]) + else: + raise ValueError(f"Unsupported type {type(value)} for key {key}") + else: + # `None` values are not cat'd + cat_data[key] = value + metadata = shards[0].metadata + cat_batch = cls(cat_data) + cat_batch.metadata = metadata + return cat_batch + + def __len__(self) -> int: + """Length of the batch. + + Note that this is the same as the batch size rather than the number of keys in the batch. + """ + return self._batch_size + + def __eq__(self, other: Any) -> bool: + """Check if two `TensorBatch` objects are equal""" + if not isinstance(other, TensorBatch): + return False + if self.metadata != other.metadata: + return False + if len(self) != len(other): + return False + if len(self.items()) != len(other.items()): + return False + for k, v in self.items(): + if k not in other or not torch.equal(v, other[k]): + return False + return True + + def __str__(self) -> str: + """String representation of the `TensorBatch` object""" + return f"TensorBatch(batch_size={self.batch_size}, device={self.device}, metadata={self.metadata}), items={self.items()}" + + def __repr__(self) -> str: + """String representation of the `TensorBatch` object""" + return self.__str__() + + +class TrainingInput(TypedDict, total=False): + """Schema for training input batch""" + + sequences: Integer[torch.Tensor, "batch_size seq_len"] + attention_mask: Integer[torch.Tensor, "batch_size seq_len"] + loss_mask: Integer[torch.Tensor, "batch_size seq_len"] + response_mask: Integer[torch.Tensor, "batch_size seq_len"] + action_log_probs: Float[torch.Tensor, "batch_size seq_len"] + base_action_log_probs: Float[torch.Tensor, "batch_size seq_len"] + values: Optional[Float[torch.Tensor, "batch_size seq_len"]] + returns: Float[torch.Tensor, "batch_size seq_len"] + advantages: Float[torch.Tensor, "batch_size seq_len"] + kl: Float[torch.Tensor, "batch_size seq_len"] + rewards: Optional[Float[torch.Tensor, "batch_size seq_len"]] + rollout_logprobs: Optional[Float[torch.Tensor, "batch_size seq_len"]] + + +class TrainingInputBatch(TensorBatch[TrainingInput]): + """Training input data""" + + pass + + +class TrainingOutputBatch(TensorBatch[Dict[str, torch.Tensor]]): + """Training output data""" + + pass diff --git a/skyrl-train/skyrl_train/utils/__init__.py b/skyrl-train/skyrl_train/utils/__init__.py new file mode 100644 index 0000000000..f06e76a37d --- /dev/null +++ b/skyrl-train/skyrl_train/utils/__init__.py @@ -0,0 +1,2 @@ +from .utils import * # noqa: F401, F403 +from . import io as io diff --git a/skyrl-train/skyrl_train/utils/alerts.py b/skyrl-train/skyrl_train/utils/alerts.py new file mode 100644 index 0000000000..03cfd59fa7 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/alerts.py @@ -0,0 +1,123 @@ +""" +Slack alerting utilities for training metric thresholds. + +Sends alerts to Slack channels when training metrics exceed configured thresholds. +Requires SLACK_BOT_TOKEN environment variable with chat:write scope. + +All functions are fault-tolerant: they catch all exceptions internally and will +never raise, so they are safe to call in the training loop without risk of +crashing a training run. +""" + +import logging +import os +from typing import Any, Dict, Optional +from urllib import request +import json + +logger = logging.getLogger(__name__) + +# Default channel for training alerts +DEFAULT_CHANNEL = "#fleet-training-runs" + +# Track whether we've already alerted for a given metric+step to avoid spam +_alerted_steps: Dict[str, int] = {} + + +def send_slack_alert( + message: str, + channel: str = DEFAULT_CHANNEL, + bot_token: Optional[str] = None, +) -> bool: + """Post an alert message to a Slack channel. + + This function is fault-tolerant and will never raise an exception. + + Args: + message: The message text to send. + channel: Slack channel name or ID (default: #fleet-training-runs). + bot_token: Slack bot token. Falls back to SLACK_BOT_TOKEN env var. + + Returns: + True if the message was sent successfully, False otherwise. + """ + try: + token = bot_token or os.environ.get("SLACK_BOT_TOKEN") + if not token: + logger.warning("SLACK_BOT_TOKEN not set, skipping Slack alert") + return False + + payload = json.dumps({"channel": channel, "text": message}).encode("utf-8") + req = request.Request( + "https://slack.com/api/chat.postMessage", + data=payload, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + ) + + with request.urlopen(req, timeout=10) as resp: + body = json.loads(resp.read().decode("utf-8")) + if not body.get("ok"): + logger.warning(f"Slack API error: {body.get('error', 'unknown')}") + return False + return True + except Exception as e: + logger.warning(f"Failed to send Slack alert: {e}") + return False + + +def check_tool_error_rate_alert( + metrics: Dict[str, Any], + global_step: int, + threshold: float = 0.5, + run_name: Optional[str] = None, + channel: str = DEFAULT_CHANNEL, +) -> None: + """Check if tool_error_rate exceeds threshold and send a Slack alert. + + Checks both per-environment and overall tool_error_rate metrics. + Only alerts once per metric per step to avoid spam. + + This function is fault-tolerant and will never raise an exception. + + Args: + metrics: Dictionary of training metrics (from self.all_metrics). + global_step: Current training step. + threshold: Alert if tool_error_rate exceeds this value (default 0.5 = 50%). + run_name: Optional run name for the alert message. + channel: Slack channel to alert. + """ + try: + alerts = [] + + for key, value in metrics.items(): + if not key.endswith("tool_error_rate"): + continue + if not isinstance(value, (int, float)) or value <= threshold: + continue + + # Deduplicate: only alert once per metric per step + alert_key = f"{key}:{global_step}" + if alert_key in _alerted_steps: + continue + _alerted_steps[alert_key] = global_step + + # Extract env name from key like "environment/outlook/tool_error_rate" + parts = key.rsplit("/tool_error_rate", 1) + env_name = parts[0] if parts[0] != key else "overall" + + alerts.append((env_name, value)) + + if not alerts: + return + + run_label = f" (run: `{run_name}`)" if run_name else "" + lines = [f":warning: *Tool Error Rate Alert* — step {global_step}{run_label}"] + for env_name, rate in alerts: + lines.append(f" • `{env_name}`: {rate:.1%} tool error rate (threshold: {threshold:.0%})") + + send_slack_alert("\n".join(lines), channel=channel) + except Exception as e: + logger.warning(f"Tool error rate alert check failed (non-fatal): {e}") diff --git a/skyrl-train/skyrl_train/utils/io/io.py b/skyrl-train/skyrl_train/utils/io/io.py new file mode 100644 index 0000000000..62877735ce --- /dev/null +++ b/skyrl-train/skyrl_train/utils/io/io.py @@ -0,0 +1,192 @@ +""" +File I/O utilities for handling both local filesystem and cloud storage (S3/GCS). + +This module provides a unified interface for file operations that works with: +- Local filesystem paths +- S3 paths (s3://bucket/path) +- Google Cloud Storage paths (gs://bucket/path or gcs://bucket/path) + +Uses fsspec for cloud storage abstraction. +""" + +import os +import tempfile +from contextlib import contextmanager +import fsspec +from loguru import logger +from .s3fs import get_s3_fs, s3_refresh_if_expiring, call_with_s3_retry, ClientError + + +def is_cloud_path(path: str) -> bool: + """Check if the given path is a cloud storage path.""" + return path.startswith(("s3://", "gs://", "gcs://")) + + +def _get_filesystem(path: str): + """Get the appropriate filesystem for the given path.""" + if not is_cloud_path(path): + return fsspec.filesystem("file") + + proto = path.split("://", 1)[0] + if proto == "s3": + fs = get_s3_fs() + s3_refresh_if_expiring(fs) + return fs + return fsspec.filesystem(proto) + + +def open_file(path: str, mode: str = "rb"): + """Open a file using fsspec, works with both local and cloud paths.""" + if not is_cloud_path(path): + return fsspec.open(path, mode) + + fs = _get_filesystem(path) + norm = fs._strip_protocol(path) + try: + return fs.open(norm, mode) + except ClientError as e: + code = getattr(e, "response", {}).get("Error", {}).get("Code") + if code in {"ExpiredToken", "ExpiredTokenException", "RequestExpired"} and hasattr(fs, "connect"): + try: + fs.connect(refresh=True) + except Exception: + pass + return fs.open(norm, mode) + raise + + +def makedirs(path: str, exist_ok: bool = True) -> None: + """Create directories. Only applies to local filesystem paths.""" + if not is_cloud_path(path): + os.makedirs(path, exist_ok=exist_ok) + + +def exists(path: str) -> bool: + """Check if a file or directory exists.""" + fs = _get_filesystem(path) + if is_cloud_path(path) and path.startswith("s3://"): + return call_with_s3_retry(fs, fs.exists, path) + return fs.exists(path) + + +def isdir(path: str) -> bool: + """Check if path is a directory.""" + fs = _get_filesystem(path) + if is_cloud_path(path) and path.startswith("s3://"): + return call_with_s3_retry(fs, fs.isdir, path) + return fs.isdir(path) + + +def list_dir(path: str) -> list[str]: + """List contents of a directory.""" + fs = _get_filesystem(path) + if is_cloud_path(path) and path.startswith("s3://"): + return call_with_s3_retry(fs, fs.ls, path, detail=False) + return fs.ls(path, detail=False) + + +def remove(path: str) -> None: + """Remove a file or directory.""" + fs = _get_filesystem(path) + if is_cloud_path(path) and path.startswith("s3://"): + if call_with_s3_retry(fs, fs.isdir, path): + call_with_s3_retry(fs, fs.rm, path, recursive=True) + else: + call_with_s3_retry(fs, fs.rm, path) + return + if fs.isdir(path): + fs.rm(path, recursive=True) + else: + fs.rm(path) + + +def upload_directory(local_path: str, cloud_path: str) -> None: + """Upload a local directory to cloud storage.""" + if not is_cloud_path(cloud_path): + raise ValueError(f"Destination must be a cloud path, got: {cloud_path}") + + fs = _get_filesystem(cloud_path) + if cloud_path.startswith("s3://"): + call_with_s3_retry(fs, fs.put, local_path, fs._strip_protocol(cloud_path), recursive=True) + else: + fs.put(local_path, cloud_path, recursive=True) + logger.info(f"Uploaded {local_path} to {cloud_path}") + + +def download_directory(cloud_path: str, local_path: str) -> None: + """Download a cloud directory to local storage.""" + if not is_cloud_path(cloud_path): + raise ValueError(f"Source must be a cloud path, got: {cloud_path}") + + fs = _get_filesystem(cloud_path) + if cloud_path.startswith("s3://"): + call_with_s3_retry(fs, fs.get, fs._strip_protocol(cloud_path), local_path, recursive=True) + else: + fs.get(cloud_path, local_path, recursive=True) + logger.info(f"Downloaded {cloud_path} to {local_path}") + + +@contextmanager +def local_work_dir(output_path: str): + """ + Context manager that provides a local working directory. + + For local paths, returns the path directly. + For cloud paths, creates a temporary directory and uploads content at the end. + + Args: + output_path: The final destination path (local or cloud) + + Yields: + str: Local directory path to work with + + Example: + with local_work_dir("s3://bucket/model") as work_dir: + # Save files to work_dir + model.save_pretrained(work_dir) + # Files are automatically uploaded to s3://bucket/model at context exit + """ + if is_cloud_path(output_path): + with tempfile.TemporaryDirectory() as temp_dir: + try: + yield temp_dir + finally: + # Upload everything from temp_dir to cloud path + upload_directory(temp_dir, output_path) + logger.info(f"Uploaded directory contents to {output_path}") + else: + # For local paths, ensure directory exists and use it directly + makedirs(output_path, exist_ok=True) + yield output_path + + +@contextmanager +def local_read_dir(input_path: str): + """ + Context manager that provides a local directory with content from input_path. + + For local paths, returns the path directly. + For cloud paths, downloads content to a temporary directory. + + Args: + input_path: The source path (local or cloud) + + Yields: + str: Local directory path containing the content + + Example: + with local_read_dir("s3://bucket/model") as read_dir: + # Load files from read_dir + model = AutoModel.from_pretrained(read_dir) + """ + if is_cloud_path(input_path): + with tempfile.TemporaryDirectory() as temp_dir: + # Download everything from cloud path to temp_dir + download_directory(input_path, temp_dir) + logger.info(f"Downloaded directory contents from {input_path}") + yield temp_dir + else: + # For local paths, use directly (but check it exists) + if not exists(input_path): + raise FileNotFoundError(f"Path does not exist: {input_path}") + yield input_path diff --git a/skyrl-train/skyrl_train/utils/io/s3fs.py b/skyrl-train/skyrl_train/utils/io/s3fs.py new file mode 100644 index 0000000000..70caa92a28 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/io/s3fs.py @@ -0,0 +1,73 @@ +from datetime import datetime, timezone, timedelta +import fsspec + +# Optional AWS deps (present when s3fs is installed) +try: + import botocore.session as _botocore_session + from botocore.exceptions import ClientError + + _HAS_BOTOCORE = True +except Exception: + _HAS_BOTOCORE = False + + class ClientError(Exception): # fallback type + pass + + +_S3_FS = None # type: ignore + + +def get_s3_fs(): + """Return a cached S3 filesystem instance, creating it once.""" + global _S3_FS + if _S3_FS is None: + _S3_FS = fsspec.filesystem("s3") + return _S3_FS + + +def s3_expiry_time(): + """Return botocore credential expiry (datetime in UTC) or None.""" + if not _HAS_BOTOCORE: + return None + try: + sess = _botocore_session.get_session() + creds = sess.get_credentials() + if not creds: + return None + return getattr(creds, "expiry_time", None) or getattr(creds, "_expiry_time", None) + except Exception: + return None + + +def s3_refresh_if_expiring(fs) -> None: + """ + Simple refresh: + - If expiry exists and is within 300s (or past), refresh with fs.connect(refresh=True). + - Otherwise, do nothing. + """ + exp = s3_expiry_time() + if not exp: + return + now = datetime.now(timezone.utc) + if now >= exp - timedelta(seconds=300): + try: + fs.connect(refresh=True) # rebuild session + except Exception: + pass + + +def call_with_s3_retry(fs, fn, *args, **kwargs): + """ + Wrapper for calling an S3 method. If it fails with ExpiredToken, force refresh once and retry. + """ + try: + return fn(*args, **kwargs) + except ClientError as e: + code = getattr(e, "response", {}).get("Error", {}).get("Code") + if code in {"ExpiredToken", "ExpiredTokenException", "RequestExpired"} and hasattr(fs, "connect"): + try: + fs.connect(refresh=True) + except Exception: + pass + return fn(*args, **kwargs) + raise diff --git a/skyrl-train/skyrl_train/utils/logging_utils.py b/skyrl-train/skyrl_train/utils/logging_utils.py new file mode 100644 index 0000000000..863d31d788 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/logging_utils.py @@ -0,0 +1,84 @@ +from typing import Any, Dict, List, Optional, Union + +POSITIVE_RESPONSE_COLOR = "green" +NEGATIVE_RESPONSE_COLOR = "yellow" +BASE_PROMPT_COLOR = "cyan" + + +def _color_block_format_and_kwargs( + text: str, + color: str, + field_prefix: str, +) -> tuple[str, dict]: + """Build a format string and kwargs for a multi-line colored block. + + The format string will look like: + "{p0}\n{p1}\n..." + + where "p0", "p1", ... are placeholder names starting with `field_prefix`. + """ + # Ensure at least one line + lines = text.splitlines() or [""] + + fmt_lines = [] + kwargs: dict[str, str] = {} + + for i, line in enumerate(lines): + key = f"{field_prefix}{i}" + # NOTE: double braces {{ }} so that {key} survives into str.format + fmt_lines.append(f"<{color}>{{{key}}}") + kwargs[key] = line + + fmt = "\n".join(fmt_lines) + return fmt, kwargs + + +def log_example( + logger: Any, + prompt: List[Dict[str, Any]], + response: str, + reward: Optional[Union[float, List[float]]] = None, +) -> None: + """ + Log a single example prompt and response with formatting and colors. + + Args: + logger: The logger instance to use (expected to be loguru logger or compatible). + prompt: The input prompt in OpenAI message format. + response: The output response string. + reward: The reward value(s) associated with the response. + """ + reward_val = 0.0 + reward_str = "N/A" + try: + prompt_str = str(prompt) + response_str = str(response) + # --- Reward handling --- + if reward is not None: + if isinstance(reward, list): + reward_val = float(sum(reward)) + else: + reward_val = float(reward) + reward_str = f"{reward_val:.4f}" + + # --- Color selection --- + if reward is not None and reward_val > 0: + response_color = POSITIVE_RESPONSE_COLOR + else: + response_color = NEGATIVE_RESPONSE_COLOR + + # --- Build per-line colored blocks in the *format string* --- + prompt_fmt, prompt_kwargs = _color_block_format_and_kwargs(prompt_str, BASE_PROMPT_COLOR, "p") + response_fmt, response_kwargs = _color_block_format_and_kwargs(response_str, response_color, "r") + + # Single format string with only our own markup and placeholders + log_format = "Example:\n" f" Input: {prompt_fmt}\n" " Output (Total Reward: {reward}):\n" f"{response_fmt}" + + # Merge all args for str.format + format_kwargs = {**prompt_kwargs, **response_kwargs, "reward": reward_str} + + # Let Loguru parse tags in log_format and then substitute arguments. + logger.opt(colors=True).info(log_format, **format_kwargs) + except Exception as e: + print(f"Error pretty printing example, debug printing instead: {e}") + print(f"Example:\n Input: {prompt}\n Output (Total Reward: {reward_str}):\n{response}") diff --git a/skyrl-train/skyrl_train/utils/ppo_utils.py b/skyrl-train/skyrl_train/utils/ppo_utils.py new file mode 100644 index 0000000000..9ca65a898a --- /dev/null +++ b/skyrl-train/skyrl_train/utils/ppo_utils.py @@ -0,0 +1,1145 @@ +# This code is adapted from VERL +# https://github.com/volcengine/verl/blob/main/verl/trainer/ppo/core_algos.py +# The original copyright is reproduced below: +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict +from enum import StrEnum +from functools import wraps +from typing import Callable, List, Literal, Optional, Tuple, Union + +import numpy as np +import ray +import torch +from jaxtyping import Float +from loguru import logger +from omegaconf import DictConfig + +from skyrl_train.training_batch import TrainingInputBatch + +# Import cloudpickle for function serialization +try: + import cloudpickle +except ImportError: + # Fallback to pickle if cloudpickle is not available + import pickle as cloudpickle + + +# TODO (erictang000): unused right now, but will be useful as we add more algorithm support +class AdaptiveKLController: + """ + Adaptive KL controller described in the paper: + https://arxiv.org/pdf/1909.08593.pdf + """ + + def __init__(self, init_kl_coef, target, horizon): + self.value = init_kl_coef + self.target = target + self.horizon = horizon + + def update(self, current, n_steps): + target = self.target + proportional_error = np.clip(current / target - 1, -0.2, 0.2) + mult = 1 + proportional_error * n_steps / self.horizon + self.value *= mult + + +class FixedKLController: + """Fixed KL controller.""" + + def __init__(self, kl_coef): + self.value = kl_coef + + def update(self, current, n_steps): + pass + + +def get_kl_controller(algorithm_cfg: DictConfig): + if algorithm_cfg.kl_ctrl.type == "fixed": + return FixedKLController(kl_coef=algorithm_cfg.kl_loss_coef) + elif algorithm_cfg.kl_ctrl.type == "adaptive": + if algorithm_cfg.kl_ctrl.horizon <= 0: + raise ValueError(f"horizon must be larger than 0. Got {algorithm_cfg.kl_ctrl.horizon}") + return AdaptiveKLController( + init_kl_coef=algorithm_cfg.kl_loss_coef, + target=algorithm_cfg.kl_ctrl.kl_target, + horizon=algorithm_cfg.kl_ctrl.horizon, + ) + else: + raise ValueError(f"Invalid KL controller type: {algorithm_cfg.kl_ctrl.type}") + + +def masked_mean(tensor: torch.Tensor, mask: Optional[torch.Tensor], dim: Optional[int] = None) -> torch.Tensor: + if mask is None: + return tensor.mean(axis=dim) + return (tensor * mask).sum(axis=dim) / mask.sum(axis=dim).clamp(min=1.0) + + +@torch.no_grad() +def compute_approx_kl( + log_probs: torch.Tensor, + log_probs_base: torch.Tensor, + loss_mask: Optional[torch.Tensor] = None, + kl_estimator_type: str = "k3", +) -> torch.Tensor: + """ + Compute the approximate KL divergence between two distributions. + Schulman blog: http://joschu.net/blog/kl-approx.html + + Args: + log_probs: Log probabilities of the new distribution. + log_probs_base: Log probabilities of the base distribution. + action_mask: Mask for actions. + """ + if kl_estimator_type == "k1": + kld = log_probs - log_probs_base + elif kl_estimator_type == "abs": + kld = (log_probs - log_probs_base).abs() + elif kl_estimator_type == "k2": + kld = 0.5 * (log_probs - log_probs_base).square() + # J. Schulman. Approximating kl divergence, 2020. + # URL http://joschu.net/blog/kl-approx.html. + elif kl_estimator_type == "k3": + kl = log_probs_base - log_probs + # For numerical stability + kl = torch.clamp(kl, min=-20, max=20) + ratio = torch.exp(kl) + kld = (ratio - kl - 1).contiguous() + kld = torch.clamp(kld, min=-10, max=10) + else: + raise ValueError(f"Invalid KL estimator type: {kl_estimator_type}") + + if loss_mask is not None: + kld = kld * loss_mask + return kld + + +@torch.no_grad() +def normalize_advantages_dict(data: TrainingInputBatch) -> TrainingInputBatch: + """Normalizes the advantages in the data batch. + + Expects: + - `["advantages"]`: Float[torch.Tensor, "batch_size seqlen"] + - `["response_mask"]`: Float[torch.Tensor, "batch_size seqlen"] + """ + advantages: Float[torch.Tensor, "batch_size seqlen"] = data["advantages"] + response_masks: Float[torch.Tensor, "batch_size seqlen"] = data["response_mask"] + num_actions: float = response_masks.sum() + # mean + mean: float = advantages.mean() + # std + std: float = ((advantages - mean).pow(2) * response_masks).sum() + rstd: float = (std / num_actions).clamp(min=1e-8).rsqrt() + + data["advantages"] = (advantages - mean) * rstd + return data + + +def masked_var(values, mask, unbiased=True): + """Compute variance of tensor with masked values.""" + mean = masked_mean(values, mask) + centered_values = values - mean + variance = masked_mean(centered_values**2, mask) + if unbiased: + mask_sum = mask.sum() + if mask_sum == 0: + raise ValueError("At least one element in the mask has to be 1.") + # note that if mask_sum == 1, then there is a division by zero issue + # to avoid it you just need to use a larger minibatch_size + if mask_sum == 1: + raise ValueError("The sum of the mask is one, which can cause a division by zero.") + bessel_correction = mask_sum / (mask_sum - 1) + variance = variance * bessel_correction + return variance + + +def masked_whiten(values, mask, shift_mean=True): + """Whiten values with masked values.""" + mean, var = masked_mean(values, mask), masked_var(values, mask) + whitened = (values - mean) * torch.rsqrt(var + 1e-8) + if not shift_mean: + whitened += mean + return whitened + + +def ppo_critic_loss( + values: torch.Tensor, + old_values: torch.Tensor, + returns: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, Optional[float]]: + if config.value_clip is not None: + values_clipped = old_values + (values - old_values).clamp(-config.value_clip, config.value_clip) + surr1 = (values_clipped - returns) ** 2 + surr2 = (values - returns) ** 2 + loss = torch.max(surr1, surr2) + clipfrac = masked_mean((surr1 > surr2).float(), loss_mask).mean().detach().item() + else: + clipfrac = None + loss = (values - returns) ** 2 + + loss = masked_mean(loss, loss_mask, dim=-1).mean() + return 0.5 * loss, clipfrac + + +# Shared registry actor class for both policy loss and advantage estimator registries +@ray.remote +class RegistryActor: + """Shared Ray actor for managing function registries across processes.""" + + def __init__(self): + self.registry = {} + + def register(self, name: str, func_serialized: bytes): + """Register a serialized function.""" + self.registry[name] = func_serialized + + def get(self, name: str): + """Get a serialized function by name.""" + return self.registry.get(name) + + def list_available(self): + """List all available function names.""" + return list(self.registry.keys()) + + def unregister(self, name: str): + """Unregister a function by name.""" + return self.registry.pop(name, None) + + +class BaseFunctionRegistry: + """Base class for function registries with Ray actor synchronization.""" + + # Subclasses should override these class attributes + _actor_name = None + _function_type = "Function" + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls._functions = {} + cls._ray_actor = None + cls._synced_to_actor = False + + @classmethod + def _get_or_create_actor(cls): + """Get or create the Ray actor for managing the registry using get_if_exists.""" + if not ray.is_initialized(): + raise Exception("Ray is not initialized, cannot create registry actor") + + if cls._ray_actor is None: + # Use get_if_exists to create actor only if it doesn't exist + cls._ray_actor = RegistryActor.options(name=cls._actor_name, get_if_exists=True).remote() + return cls._ray_actor + + @classmethod + def _sync_local_to_actor(cls): + """Sync all local functions to Ray actor.""" + if cls._synced_to_actor: + return + if not ray.is_initialized(): + raise Exception("Ray is not initialized, cannot sync with actor") + + try: + actor = cls._get_or_create_actor() + if actor is not None: + for name, func in cls._functions.items(): + func_serialized = cloudpickle.dumps(func) + ray.get(actor.register.remote(name, func_serialized)) + cls._synced_to_actor = True + except Exception as e: + logger.error(f"Error syncing {cls._function_type} to actor: {e}") + raise e + + @classmethod + def sync_with_actor(cls): + """Sync local registry with Ray actor if Ray is available.""" + # Only try if Ray is initialized + if not ray.is_initialized(): + raise Exception("Ray is not initialized, cannot sync with actor") + + # First check if the actor is still alive + # NOTE(Charlie): This is mainly for unit tests, where we run multiple unit tests in the + # same Python process, and each unit test has ray init/shutdown. This makes cls's attributes + # outdated (e.g. the _ray_actor points to a stale actor in the previous ray session). + try: + _ = ray.get_actor(cls._actor_name) # this raises exception if the actor is stale + except ValueError: + cls._ray_actor = None + cls._synced_to_actor = False + + # First, sync our local functions to the actor + cls._sync_local_to_actor() + + actor = cls._get_or_create_actor() + if actor is None: + return + + available = ray.get(actor.list_available.remote()) + + # Sync any new functions from actor to local registry + for name in available: + if name not in cls._functions: + func_serialized = ray.get(actor.get.remote(name)) + if func_serialized is not None: + # Deserialize the function + try: + func = cloudpickle.loads(func_serialized) + cls._functions[name] = func + except Exception as e: + # If deserialization fails, skip this function + logger.error(f"Error deserializing {name} from actor: {e}") + raise e + + @classmethod + def register(cls, name: Union[str, StrEnum], func: Callable): + """Register a function. + + If ray is initialized, this function will get or create a named ray actor (RegistryActor) + for the registry, and sync the registry to the actor. + + If ray is not initalized, the function will be stored in the local registry only. + + To make sure all locally registered functions are available to all ray processes, + call sync_with_actor() after ray.init(). + + Args: + name: Name of the function to register. Can be a string or a StrEnum. + func: Function to register. + + Raises: + ValueError: If the function is already registered. + """ + # Convert enum to string if needed + # note: StrEnum is not cloudpickleable: https://github.com/cloudpipe/cloudpickle/issues/558 + if isinstance(name, StrEnum): + name = name.value + + if name in cls._functions: + raise ValueError(f"{cls._function_type} '{name}' already registered") + + # Always store in local registry first + cls._functions[name] = func + + # Try to sync with Ray actor if Ray is initialized + if ray.is_initialized(): + actor = cls._get_or_create_actor() + if actor is not None: + # Serialize the function using cloudpickle + func_serialized = cloudpickle.dumps(func) + ray.get(actor.register.remote(name, func_serialized)) + + @classmethod + def get(cls, name: str) -> Callable: + """Get a function by name. + + If ray is initialized, this function will first sync the local registry with the RegistryActor. + Then it will return the function if it is found in the registry. + + Args: + name: Name of the function to get. Can be a string or a StrEnum. + + Returns: + The function if it is found in the registry. + """ + # Try to sync with actor first if Ray is available + if ray.is_initialized(): + cls.sync_with_actor() + + if name not in cls._functions: + available = list(cls._functions.keys()) + raise ValueError(f"Unknown {cls._function_type.lower()} '{name}'. Available: {available}") + return cls._functions[name] + + @classmethod + def list_available(cls) -> List[str]: + """List all registered functions.""" + # Try to sync with actor first if Ray is available + if ray.is_initialized(): + cls.sync_with_actor() + return list(cls._functions.keys()) + + @classmethod + def unregister(cls, name: Union[str, StrEnum]): + """Unregister a function. Useful for testing.""" + # Convert enum to string if needed + if isinstance(name, StrEnum): + name = name.value + + # Try to sync with actor first to get any functions that might be in the actor but not local + if ray.is_initialized(): + cls.sync_with_actor() + + # Track if we found the function anywhere + found_locally = name in cls._functions + found_in_actor = False + + # Remove from local registry if it exists + if found_locally: + del cls._functions[name] + + # Try to remove from Ray actor if Ray is available + if ray.is_initialized(): + actor = cls._get_or_create_actor() + if actor is not None: + # Check if it exists in actor first + available_in_actor = ray.get(actor.list_available.remote()) + if name in available_in_actor: + found_in_actor = True + ray.get(actor.unregister.remote(name)) + + # Only raise error if the function wasn't found anywhere + if not found_locally and not found_in_actor: + raise ValueError(f"{cls._function_type} '{name}' not registered") + + @classmethod + def reset(cls): + """Resets the registry (useful for testing purposes).""" + if ray.is_initialized() and cls._ray_actor is not None: + try: + actor = ray.get_actor(cls._actor_name) # this raises exception if the actor is stale + ray.kill(actor) + except ValueError: + pass # Actor may already be gone + cls._functions.clear() + cls._ray_actor = None + cls._synced_to_actor = False + + @classmethod + def repopulate(cls): + """Repopulate the registry with the default functions.""" + cls.reset() + cls.register(cls._function_type, cls._function_type) + + +class AdvantageEstimator(StrEnum): + GAE = "gae" + GRPO = "grpo" + RLOO = "rloo" + REINFORCE_PP = "reinforce++" + + +class AdvantageEstimatorRegistry(BaseFunctionRegistry): + """ + Registry for advantage estimator functions. + + This registry allows users to register custom advantage estimators without modifying + the skyrl_train package. Custom estimators can be registered by calling + AdvantageEstimatorRegistry.register() directly or by using the @register_advantage_estimator + decorator. + + See examples/algorithms/custom_advantage_estimator for a simple example of how to + register and use custom advantage estimators. + """ + + _actor_name = "advantage_estimator_registry" + _function_type = "advantage estimator" + + @classmethod + def repopulate_registry(cls): + ae_avail = set(cls.list_available()) + ae_types = { + "grpo": [AdvantageEstimator.GRPO, compute_grpo_outcome_advantage], + "gae": [AdvantageEstimator.GAE, compute_gae_advantage_return], + "rloo": [AdvantageEstimator.RLOO, compute_rloo_outcome_advantage], + "reinforce++": [AdvantageEstimator.REINFORCE_PP, compute_reinforce_plus_plus_outcome_advantage], + } + + for ae_name, (ae_type, ae_func) in ae_types.items(): + if ae_name not in ae_avail: + cls.register(ae_type, ae_func) + + +class PolicyLossType(StrEnum): + REGULAR = "regular" + DUAL_CLIP = "dual_clip" + GSPO = "gspo" + CISPO = "cispo" + CLIP_COV = "clip_cov" + KL_COV = "kl_cov" + SAPO = "sapo" + + +class PolicyLossRegistry(BaseFunctionRegistry): + """ + Registry for policy loss functions. + + This registry allows users to register custom policy loss functions without modifying + the skyrl_train package. Custom functions can be registered by calling + PolicyLossRegistry.register() directly or by using the @register_policy_loss + decorator. + + See examples/algorithms/custom_policy_loss for a simple example of how to + register and use custom policy loss functions. + """ + + _actor_name = "policy_loss_registry" + _function_type = "policy loss" + + @classmethod + def repopulate_registry(cls): + """Repopulate the registry with default policy loss functions.""" + pl_avail = set(cls.list_available()) + pl_types = { + "regular": [PolicyLossType.REGULAR, ppo_policy_loss], + "dual_clip": [PolicyLossType.DUAL_CLIP, ppo_policy_loss], + "gspo": [PolicyLossType.GSPO, gspo_policy_loss], + "clip_cov": [PolicyLossType.CLIP_COV, compute_policy_loss_clip_cov], + "kl_cov": [PolicyLossType.KL_COV, compute_policy_loss_kl_cov], + "sapo": [PolicyLossType.SAPO, sapo_policy_loss], + } + + for pl_name, (pl_type, pl_func) in pl_types.items(): + if pl_name not in pl_avail: + cls.register(pl_type, pl_func) + + +def register_advantage_estimator(name: Union[str, AdvantageEstimator]): + """Decorator to register an advantage estimator function.""" + + def decorator(func: Callable): + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + AdvantageEstimatorRegistry.register(name, wrapper) + return wrapper + + return decorator + + +def register_policy_loss(name: Union[str, PolicyLossType]): + """Decorator to register a policy loss function.""" + + def decorator(func: Callable): + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + PolicyLossRegistry.register(name, wrapper) + return wrapper + + return decorator + + +def sync_registries(): + """Sync the registries with the ray actor once ray is initialized""" + if not ray.is_initialized(): + raise ValueError("Ray is not initialized, cannot sync registries") + PolicyLossRegistry.sync_with_actor() + AdvantageEstimatorRegistry.sync_with_actor() + logger.info("Synced registries to ray actor") + + +def _safe_exp_delta(delta: torch.Tensor, clip: float = 20.0, out_dtype=None) -> torch.Tensor: + """ + Clamp the delta before exponentiating to avoid potential overflow. + """ + y = torch.clamp(delta.to(torch.float32), -clip, clip).exp() + return y.to(out_dtype or delta.dtype) + + +@register_policy_loss(PolicyLossType.REGULAR) +@register_policy_loss(PolicyLossType.DUAL_CLIP) +def ppo_policy_loss( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, + rollout_logprobs: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, float]: + assert config.policy_loss_type in ["regular", "dual_clip"], "loss_type must be either 'regular' or 'dual_clip'" + loss_reduction = config.loss_reduction + assert loss_reduction in [ + "token_mean", + "sequence_mean", + "seq_mean_token_sum_norm", + ], "loss_reduction must be either 'token_mean', 'sequence_mean', or 'seq_mean_token_sum_norm'" + + ratio = _safe_exp_delta(log_probs - old_log_probs, clip=20.0, out_dtype=log_probs.dtype) + surr1 = ratio * advantages + surr2 = ratio.clamp(1 - config.eps_clip_low, 1 + config.eps_clip_high) * advantages + loss = -torch.min(surr1, surr2) + clip_ratio = masked_mean((-surr2 > -surr1).float(), loss_mask).mean().detach().item() + clip_pg_losses1 = loss + if config.policy_loss_type == "dual_clip": + pg_losses3 = -advantages * config.clip_ratio_c + clip_pg_losses2 = torch.min(pg_losses3, clip_pg_losses1) + loss = torch.where(advantages < 0, clip_pg_losses2, clip_pg_losses1) + + if config.use_tis: + from loguru import logger as logger_ + + logger_.debug(f"Using TIS with dtype: {rollout_logprobs.dtype}") + # Apply truncated importance sampling -> https://fengyao.notion.site/off-policy-rl + tis_imp_ratio = _safe_exp_delta(old_log_probs - rollout_logprobs, clip=20.0, out_dtype=log_probs.dtype) + tis_imp_ratio = torch.clamp(tis_imp_ratio, max=config.tis_imp_ratio_cap) + loss = loss * tis_imp_ratio + + loss = reduce_loss(loss, loss_mask, loss_reduction, config.max_seq_len) + return loss, clip_ratio + + +@register_policy_loss(PolicyLossType.SAPO) +def sapo_policy_loss( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, + rollout_logprobs: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, float]: + """ + SAPO (Soft Adaptive Policy Optimization) policy loss function. + + Compute the smoothed policy objective and related metrics for SAPO. + + See https://arxiv.org/pdf/2511.20347 for more details. + + """ + # SAPO must use sequence_mean reduction + loss_reduction = config.loss_reduction + if loss_reduction != "sequence_mean": + # The SAPO paper uses sequence_mean reduction; there's no reason + # why a user couldn't use token_mean reduction, but + # it's not clear whether it would be stable or not. + from loguru import logger as logger_ # have to do lazy import to avoid pickling error + + logger_.warning(f"With SAPO it's recommended to use 'sequence_mean' loss reduction; got {loss_reduction}") + + # temperature for positive and negative token updates + tau_pos = torch.as_tensor(config.sapo.tau_pos, dtype=advantages.dtype, device=advantages.device) + tau_neg = torch.as_tensor(config.sapo.tau_neg, dtype=advantages.dtype, device=advantages.device) + + def gate_function(x, tau): + """The gating function used in SAPO""" + return torch.sigmoid(tau * (x - 1.0)) * (4.0 / tau) + + # compute IS at token level: + # r_{i,t}(θ) = π_θ(y_{i,t}|x, y_{i, avoid overflow in exp() + log_ratio = torch.clamp(log_ratio, min=-20.0, max=20.0) + + # finally exp() to remove log and get r_{i,t}(θ) + ratio = torch.exp(log_ratio) + + # tau_{i,t} is tau_pos if adv > 0 else tau_neg + taus = torch.where( + condition=advantages > 0, + input=tau_pos, # if A_{i,t} > 0 we set to tau_pos + other=tau_neg, # if A_{i,t} <= 0 we set to tau_neg + ) + + # compute the gates f_{i,t}(r_{i,t}(θ)) at token level + gates = gate_function(ratio, taus) + + # compute policy gradient loss + loss = -gates * advantages + + # for SAPO, we need to aggregate the loss at the sequence level (seq-mean-token-mean) + loss = reduce_loss(loss, loss_mask, loss_reduction, config.max_seq_len) + + # SAPO does not use clipping, so we set clip_ratio to 0.0 for compatibility + clip_ratio = 0.0 + + return loss, clip_ratio + + +@register_policy_loss(PolicyLossType.GSPO) +def gspo_policy_loss( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, + rollout_logprobs: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, float]: + """ + GSPO (Group Sequence Policy Optimization) policy loss function, + as proposed in https://arxiv.org/abs/2507.18071. + + This implements sequence-level importance sampling instead of token-level importance sampling. + The key difference is that importance weights are computed at the sequence level and then + applied uniformly across all tokens in the sequence. This can lead to more stable training + dynamics by reducing the variance in clipping behavior within sequences. + + The variant of GSPO used here is GSPO-token, a generalization which allows for token-level + advantages [equations 14 and 15 in the paper]. + """ + # GSPO must use sequence_mean reduction + loss_reduction = config.loss_reduction + if loss_reduction != "sequence_mean": + # The GSPO paper uses sequence_mean reduction; there's no reason + # why a user couldn't use token_mean reduction, but + # it's not clear whether it would be stable or not. + from loguru import logger as logger_ # have to do lazy import to avoid pickling error + + logger_.warning(f"With GSPO it's recommended to use 'sequence_mean' loss reduction; got {loss_reduction}") + + # Compute log ratios + log_ratio = log_probs - old_log_probs + + # Key GSPO innovation: sequence-level importance sampling + # Instead of using per-token ratios, compute sequence-averaged ratios + log_importance_weights = masked_mean(log_ratio, loss_mask, dim=-1).unsqueeze(-1) + + # s_i,t(θ) = sg[s_i(θ)] · π_θ(y_i,t|x, y_i, -surr1).float(), loss_mask).mean().detach().item() + + loss = reduce_loss(loss, loss_mask, loss_reduction, config.max_seq_len) + + return loss, clip_ratio + + +@register_policy_loss(PolicyLossType.CISPO) +def compute_policy_loss_cispo( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, + rollout_logprobs: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, float]: + """Implementation of CISPO (Clipped IS-weight Policy Optimization) loss function, + as proposed in https://arxiv.org/abs/2506.13585. + + Instead of clipping the importance sampling ratio in the loss directly, as done + in PPO loss, CISPO clips the importance sampling ratio in the policy gradient + update. This means the model can still learn from samples whose importance sampling + ratio is clipped in CISPO, as opposed to PPO where these samples have zero + gradient and are essentially ignored. + """ + ratio = _safe_exp_delta(log_probs - old_log_probs, clip=20.0, out_dtype=log_probs.dtype) + clamped_ratio = torch.clamp(ratio, 1 - config.cispo.cispo_eps_clip_low, 1 + config.cispo.cispo_eps_clip_high) + loss = -advantages * clamped_ratio.detach() * log_probs + + is_clipped = (ratio < 1 - config.cispo.cispo_eps_clip_low) | (ratio > 1 + config.cispo.cispo_eps_clip_high) + clip_ratio = masked_mean(is_clipped.float(), loss_mask).mean().detach().item() + + loss = reduce_loss(loss, loss_mask, config.loss_reduction, config.max_seq_len) + return loss, clip_ratio + + +@register_policy_loss(PolicyLossType.CLIP_COV) +def compute_policy_loss_clip_cov( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, + rollout_logprobs: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, float]: + """Clip-Cov policy loss function implementation. + + Adapted from https://github.com/PRIME-RL/Entropy-Mechanism-of-RL/blob/main/verl/trainer/ppo/core_algos.py + + This method combines standard PPO clipping with covariance-based clipping + to provide more stable training dynamics. + """ + # Extract config parameters with defaults + clip_cov_ratio = config.clip_cov.clip_ratio + clip_cov_lb = config.clip_cov.clip_cov_lb + clip_cov_ub = config.clip_cov.clip_cov_ub + + negative_approx_kl = log_probs - old_log_probs + ratio = torch.exp(negative_approx_kl) + + pg_losses1 = -advantages * ratio + + pg_losses2 = -advantages * torch.clamp(ratio, 1 - config.eps_clip_low, 1 + config.eps_clip_high) + clip_by_origin = (pg_losses2 > pg_losses1) & (loss_mask > 0) + + # Compute covariance for clipping decision + cov_all = (advantages - masked_mean(advantages, loss_mask)) * ( + log_probs - masked_mean(log_probs.detach(), loss_mask) + ) + cov_all[loss_mask == 0] = -torch.inf + cov_all[clip_by_origin] = -torch.inf + + # Determine number of tokens to clip based on clip_ratio + clip_num = max(int(clip_cov_ratio * loss_mask.sum().item()), 1) + top_k_idx = (cov_all < clip_cov_ub) & (cov_all > clip_cov_lb) & (loss_mask > 0) + top_k_idx = torch.nonzero(top_k_idx) + + if len(top_k_idx) > 0: + perm = torch.randperm(len(top_k_idx)) + top_k_idx = top_k_idx[perm[: min(clip_num, len(top_k_idx))]] + else: + top_k_idx = torch.empty((0, 2), device=cov_all.device, dtype=torch.long) + + # Create correction mask + corr = torch.ones_like(advantages) + if len(top_k_idx) > 0: + corr[top_k_idx[:, 0], top_k_idx[:, 1]] = 0 + + # Compute clip fraction for monitoring + clip_frac = masked_mean((corr == 0).float(), loss_mask) + + # Apply correction mask to losses + pg_losses = torch.maximum(pg_losses1, pg_losses2) * corr + pg_loss = reduce_loss( + loss=pg_losses, + loss_mask=loss_mask, + loss_reduction=config.loss_reduction, + max_seq_len=config.max_seq_len, + ) + + return pg_loss, clip_frac.item() + + +@register_policy_loss(PolicyLossType.KL_COV) +def compute_policy_loss_kl_cov( + log_probs: torch.Tensor, + old_log_probs: torch.Tensor, + advantages: torch.Tensor, + config: DictConfig, + loss_mask: Optional[torch.Tensor] = None, + rollout_logprobs: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, float]: + """KL-Cov policy loss function implementation. + + Adapted from https://github.com/PRIME-RL/Entropy-Mechanism-of-RL/blob/main/verl/trainer/ppo/core_algos.py + + Uses covariance-based selection to apply KL regularization to a subset of tokens. + """ + kl_cov_frac = config.kl_cov.kl_cov_frac # This should be a percentage (e.g., 0.2 for 20%) + ppo_kl_coef = config.kl_cov.ppo_kl_coef + + negative_approx_kl = log_probs - old_log_probs + abs_kl = negative_approx_kl.abs() + ratio = torch.exp(negative_approx_kl) + + pg_losses1 = -advantages * ratio + pg_losses_kl = -advantages * ratio + ppo_kl_coef * abs_kl + pg_losses = pg_losses1.clone() + + all_valid = loss_mask > 0 + all_valid_idx = torch.nonzero(all_valid.reshape(-1), as_tuple=True)[0] + all_valid_adv = advantages[all_valid].detach().reshape(-1).cpu() + all_valid_logp = log_probs[all_valid].detach().reshape(-1).cpu() + + if len(all_valid_adv) > 0: + cov_lst_all = (all_valid_adv - all_valid_adv.mean()) * (all_valid_logp - all_valid_logp.mean()) + # Use percentage-based selection like the reference implementation + k_percent_nums = max(1, int(len(cov_lst_all) * kl_cov_frac)) + + if k_percent_nums > 0: + large_cov_idxs = torch.topk(cov_lst_all, min(k_percent_nums, len(cov_lst_all)), largest=True).indices + + if len(large_cov_idxs) > 0: + large_cov_idxs = all_valid_idx[large_cov_idxs] + pg_losses[ + large_cov_idxs // advantages.shape[1], + large_cov_idxs % advantages.shape[1], + ] = pg_losses_kl[ + large_cov_idxs // advantages.shape[1], + large_cov_idxs % advantages.shape[1], + ] + + pg_loss = reduce_loss( + loss=pg_losses, + loss_mask=loss_mask, + loss_reduction=config.loss_reduction, + max_seq_len=config.max_seq_len, + ) + + # NOTE (sumanthrh): Since the pg clip ratio is not applicable for KL-COV so we just use 0.0 + return pg_loss, 0.0 + + +def reduce_loss( + loss: torch.Tensor, + loss_mask: Optional[torch.Tensor], + loss_reduction: Literal["token_mean", "sequence_mean", "seq_mean_token_sum_norm"], + max_seq_len: Optional[int] = None, +) -> torch.Tensor: + if loss_reduction == "token_mean": + # sum over *all* valid tokens, divide by total valid-token count + loss = masked_mean(loss, loss_mask) + elif loss_reduction == "sequence_mean": + # per-sequence token-mean (dim=-1), then batch-mean + loss = masked_mean(loss, loss_mask, dim=-1).mean() + elif loss_reduction == "seq_mean_token_sum_norm": + # per-sequence token-sum, normalized by the max sequence length, then batch mean + # this is the Dr. GRPO loss reduction to avoid length bias by normalizing by a constant + assert max_seq_len is not None, "max_seq_len must be provided for seq_mean_token_sum_norm loss reduction" + # NOTE: max_seq_len is computed as cfg.generator.max_input_length + cfg.generator.sampling_params.max_generate_length by default + if loss_mask is not None: + seq_losses = torch.sum(loss * loss_mask, dim=-1) / max_seq_len + else: + # If no mask, assume all tokens are valid + seq_losses = torch.sum(loss, dim=-1) / max_seq_len + loss = torch.mean(seq_losses) + else: + raise ValueError(f"Invalid loss reduction type: {loss_reduction}") + return loss + + +# NOTE (erictang000): below ported from verl +@register_advantage_estimator(AdvantageEstimator.REINFORCE_PP) +def compute_reinforce_plus_plus_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + gamma: float, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for REINFORCE++. + This implementation is based on the paper: https://arxiv.org/abs/2501.03262 + + Args: + - token_level_rewards: Float[torch.Tensor, "batch_size seqlen"] + - response_mask: Float[torch.Tensor, "batch_size seqlen"] + + Returns: + - advantages: Float[torch.Tensor, "batch_size seqlen"] + - returns: Float[torch.Tensor, "batch_size seqlen"] + """ + with torch.no_grad(): + returns = torch.zeros_like(token_level_rewards) + running_return = 0 + + for t in reversed(range(token_level_rewards.shape[1])): + running_return = token_level_rewards[:, t] + gamma * running_return + returns[:, t] = running_return + # Reset after EOS + running_return = running_return * response_mask[:, t] + + advantages = masked_whiten(returns, response_mask) + advantages = advantages * response_mask + + return advantages, returns + + +@register_advantage_estimator(AdvantageEstimator.RLOO) +def compute_rloo_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for RLOO based on https://arxiv.org/abs/2402.14740 + + This advantage estimator is also used in LOOP (https://arxiv.org/pdf/2502.01600), + and was originally introduced in "Buy 4 REINFORCE Samples, Get a Baseline for Free!" + (https://openreview.net/pdf?id=r1lgTGL5DE). + + Args: + - token_level_rewards: Float[torch.Tensor, "batch_size seqlen"] + - response_mask: Float[torch.Tensor, "batch_size seqlen"] + - index: np.ndarray (batch_size) + + Returns: + - advantages: Float[torch.Tensor, "batch_size seqlen"] + - returns: Float[torch.Tensor, "batch_size seqlen"] + """ + from loguru import logger as logger_ + + scores = token_level_rewards.sum(dim=-1) + + id2score = defaultdict(list) + id2mean = {} + + with torch.no_grad(): + bsz = scores.shape[0] + for i in range(bsz): + id2score[index[i]].append(scores[i]) + for idx in id2score: + if len(id2score[idx]) == 1: + id2mean[idx] = torch.tensor(0.0, device=scores.device) + elif len(id2score[idx]) > 1: + id2mean[idx] = torch.mean(torch.stack(id2score[idx])) + for i in range(bsz): + response_num = len(id2score[index[i]]) + if response_num > 1: + factor = response_num / (response_num - 1) + scores[i] = (scores[i] - id2mean[index[i]]) * factor + else: + # if there's only one response, set the advantage to 0 + logger_.warning(f"Only one response for prompt index {index[i]}, setting advantage to 0") + scores[i] = 0.0 + scores = scores.unsqueeze(-1) * response_mask + + return scores, scores + + +@register_advantage_estimator(AdvantageEstimator.GAE) +def compute_gae_advantage_return( + token_level_rewards: Float[torch.Tensor, "batch_size seqlen"], + values: Float[torch.Tensor, "batch_size seqlen"], + response_mask: Float[torch.Tensor, "batch_size seqlen"], + gamma: float, + lambd: float, + **kwargs, +) -> Tuple[Float[torch.Tensor, "batch_size seqlen"], Float[torch.Tensor, "batch_size seqlen"]]: + """ + Compute advantage and return for GAE. + + Adapted from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py + """ + with torch.no_grad(): + lastgaelam = 0 + advantages_reversed = [] + gen_len = token_level_rewards.shape[-1] + + for t in reversed(range(gen_len)): + nextvalues = values[:, t + 1] if t < gen_len - 1 else 0.0 + delta = token_level_rewards[:, t] + gamma * nextvalues - values[:, t] + lastgaelam = delta + gamma * lambd * lastgaelam + advantages_reversed.append(lastgaelam) + advantages = torch.stack(advantages_reversed[::-1], dim=1) + + returns = advantages + values + advantages = masked_whiten(advantages, response_mask) + return advantages, returns + + +@register_advantage_estimator(AdvantageEstimator.GRPO) +def compute_grpo_outcome_advantage( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + epsilon: float = 1e-6, + grpo_norm_by_std: bool = True, + is_hinted: Optional[np.ndarray] = None, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Compute advantage for GRPO, operating only on Outcome reward (with only one scalar reward for each response). + + When ``is_hinted`` is provided, uses a **first-turn baseline**: the group mean and std + are computed from raw (unhinted) samples only. All samples (raw + hinted) are then + centered using this raw-only baseline. This prevents hinted samples from contaminating + the baseline for raw samples (RLTF-SD paper, Section 3.2). + + Expects: + - token_level_rewards: Float[torch.Tensor, "batch_size seqlen"] + - response_mask: Float[torch.Tensor, "batch_size seqlen"] + - index: np.ndarray (batch_size) + - epsilon: float + - grpo_norm_by_std: bool + - is_hinted: Optional[np.ndarray] bool array (batch_size), True for hinted samples + + Returns: + - advantages: Float[torch.Tensor, "batch_size seqlen"] + - returns: Float[torch.Tensor, "batch_size seqlen"] + """ + # this assumes response-level rewards + scores = token_level_rewards.sum(dim=-1) + + id2mean = {} + id2std = {} + + use_first_turn_baseline = is_hinted is not None and np.any(is_hinted) + + with torch.no_grad(): + bsz = scores.shape[0] + + if use_first_turn_baseline: + # First-turn baseline: compute mean/std from raw (unhinted) samples only + id2raw_scores = defaultdict(list) + for i in range(bsz): + if not is_hinted[i]: + id2raw_scores[index[i]].append(scores[i]) + + for idx in id2raw_scores: + raw = id2raw_scores[idx] + if len(raw) == 1: + id2mean[idx] = torch.tensor(0.0) + id2std[idx] = torch.tensor(1.0) + else: + id2mean[idx] = torch.mean(torch.tensor(raw)) + id2std[idx] = torch.std(torch.tensor([raw])) + + # For groups with only hinted samples (no raw), use 0 mean / 1 std + for i in range(bsz): + if index[i] not in id2mean: + id2mean[index[i]] = torch.tensor(0.0) + id2std[index[i]] = torch.tensor(1.0) + else: + # Standard GRPO baseline: compute mean/std from all samples + id2score = defaultdict(list) + for i in range(bsz): + id2score[index[i]].append(scores[i]) + for idx in id2score: + if len(id2score[idx]) == 1: + id2mean[idx] = torch.tensor(0.0) + id2std[idx] = torch.tensor(1.0) + elif len(id2score[idx]) > 1: + id2mean[idx] = torch.mean(torch.tensor(id2score[idx])) + id2std[idx] = torch.std(torch.tensor([id2score[idx]])) + else: + raise ValueError(f"no score in prompt index: {idx}") + + for i in range(bsz): + if grpo_norm_by_std: + scores[i] = (scores[i] - id2mean[index[i]]) / (id2std[index[i]] + epsilon) + else: + scores[i] = scores[i] - id2mean[index[i]] + scores = scores.unsqueeze(-1) * response_mask + + return scores, scores + + +def repopulate_all_registries(): + PolicyLossRegistry.repopulate_registry() + AdvantageEstimatorRegistry.repopulate_registry() + + +def compute_advantages_and_returns( + token_level_rewards: torch.Tensor, + response_mask: torch.Tensor, + index: np.ndarray, + adv_estimator: AdvantageEstimator, + config: DictConfig, + values: Optional[torch.Tensor] = None, + grpo_norm_by_std: bool = True, + gamma=1.0, + lambd=1.0, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + estimator_func = AdvantageEstimatorRegistry.get(adv_estimator) + + return estimator_func( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + values=values, + grpo_norm_by_std=grpo_norm_by_std, + gamma=gamma, + lambd=lambd, + config=config, + **kwargs, + ) diff --git a/skyrl-train/skyrl_train/utils/profiler.py b/skyrl-train/skyrl_train/utils/profiler.py new file mode 100644 index 0000000000..efd2ec4d66 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/profiler.py @@ -0,0 +1,98 @@ +import os +from loguru import logger +import torch +import torch.distributed + + +class Profiler: + """ + A PyTorch profiler wrapper class for collecting performance metrics. + """ + + def __init__(self, config): + """ + config contains: + - enable: bool + - ranks: list[int] + - save_path: str + """ + self.enable = config.enable + if not config.enable: + return + self.config = config + self.save_path = config.save_path + self.ranks = config.ranks + self.saved = False + self.prof = None + self.rank = torch.distributed.get_rank() + if self.rank in self.ranks: + logger.info(f"[Profiler] Profiler init for rank {self.rank}") + + self.prof = torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + schedule=torch.profiler.schedule( + wait=0, + warmup=0, + active=1, + repeat=1, + ), + record_shapes=True, + with_stack=True, + ) + + def check(self): + return self.prof is not None and self.enable + + def start(self): + if self.check(): + logger.info(f"[Profiler] started for rank {self.rank}") + self.prof.start() + + def step(self): + if self.check(): + self.prof.step() + + def stop(self): + if self.check(): + logger.info(f"[Profiler] stopped for rank {self.rank}") + self.prof.stop() + + def save(self): + if self.prof is not None and not self.saved: + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + save_file_name = f"/prof_rank_{self.rank}.json" + logger.info(f"[Profiler] Saving trace to {self.save_path + save_file_name}") + self.prof.export_chrome_trace(self.save_path + save_file_name) + self.enable = False + self.saved = True + + def stop_and_save(self): + if self.check(): + self.stop() + self.save() + + def stop_trace(self): + if self.check(): + logger.info(f"[Profiler] Trace stopped for rank {self.rank}") + self.enable = False + + +class CudaTimer: + def __init__(self, device): + self.device = device + + self.start_event = torch.cuda.Event(enable_timing=True) + self.end_event = torch.cuda.Event(enable_timing=True) + + def __enter__(self): + self.start_event.record() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.end_event.record() + torch.cuda.synchronize(self.device) + self.elapsed_time = self.start_event.elapsed_time(self.end_event) # Calculate the elapsed time diff --git a/skyrl-train/skyrl_train/utils/templates/README.md b/skyrl-train/skyrl_train/utils/templates/README.md new file mode 100644 index 0000000000..71618acf06 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/templates/README.md @@ -0,0 +1,50 @@ +This folder contains customized chat templates. + +### `qwen3_acc_thinking.jinja2` + +Exactly the same as [the official Qwen3 jinja template](https://huggingface.co/Qwen/Qwen3-8B/blob/895c8d171bc03c30e113cd7a28c02494b5e068b7/tokenizer_config.json#L230) +except that we do not read `reasoning_content` at all and always do `{{- '<|im_start|>' + message.role + '\n' + content }}` for assistant messages. Therefore, you +should not parse the thinking content and pass in all generated text as part of `content`. + +The motivation is to not strip thinking tokens and keep the chat history in multi-turn training strictly appending, making the training on-policy without performing step-wise training. + +Specifically, we change from + +```jinja2 + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + ... +``` + +to + +```jinja2 + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + ... +``` + +When using `/chat/completions` with vllm via the HTTP endpoint for rollout, you can pass this in with + +```bash ++generator.engine_init_kwargs.chat_template=/path/to/templates/qwen3_acc_thinking.jinja2 +``` diff --git a/skyrl-train/skyrl_train/utils/templates/qwen3_acc_thinking.jinja2 b/skyrl-train/skyrl_train/utils/templates/qwen3_acc_thinking.jinja2 new file mode 100644 index 0000000000..1643aa5fbb --- /dev/null +++ b/skyrl-train/skyrl_train/utils/templates/qwen3_acc_thinking.jinja2 @@ -0,0 +1,72 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} diff --git a/skyrl-train/skyrl_train/utils/torch_utils.py b/skyrl-train/skyrl_train/utils/torch_utils.py new file mode 100644 index 0000000000..ac091e09db --- /dev/null +++ b/skyrl-train/skyrl_train/utils/torch_utils.py @@ -0,0 +1,191 @@ +# This code is adapted from VERL +# https://github.com/volcengine/verl/blob/main/verl/utils/torch_functional.py +# The original copyright is reproduced below: +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import nullcontext +import torch +import torch.nn.functional as F +from jaxtyping import Float, Integer + +try: + from flash_attn.ops.triton.cross_entropy import cross_entropy_loss + + FLASH_ATTN_CROSS_ENTROPY_LOSS_AVAILABLE = True +except ImportError: + FLASH_ATTN_CROSS_ENTROPY_LOSS_AVAILABLE = False + + +CHUNK_SIZE = 1024 + + +def chunked_cross_entropy_from_log_probs( + logprobs: Float[torch.Tensor, "batch_size seqlen vocab_size"], requires_grad: bool = False +) -> Float[torch.Tensor, "batch_size seqlen"]: + cm = nullcontext() if requires_grad else torch.no_grad() + with cm: + # Calculate entropy in chunks to avoid OOM + chunk_size = CHUNK_SIZE + num_chunks = (logprobs.size(1) + chunk_size - 1) // chunk_size + entropy_tensor = torch.zeros( + (logprobs.shape[0], logprobs.shape[1]), dtype=logprobs.dtype, device=logprobs.device + ) + + for i in range(num_chunks): + start_idx = i * chunk_size + end_idx = min((i + 1) * chunk_size, logprobs.size(1)) + # (bsz, seq, vocab_size) + chunk = logprobs[:, start_idx:end_idx] + + # Calculate entropy for this chunk + chunk_probs = chunk.exp() + chunk_entropy = -(chunk_probs * chunk).sum(-1) + entropy_tensor[:, start_idx:end_idx] = chunk_entropy + return entropy_tensor + + +# NOTE: we don't actually use jaxtype for runtime type checking since it doesn't play well with torch compile +def chunked_entropy_from_logits( + logits: Float[torch.Tensor, "batch_size seqlen vocab"], + requires_grad: bool = False, + attention_mask: Float[torch.Tensor, "batch_size seqlen"] = None, +) -> Float[torch.Tensor, "batch_size seqlen"]: + """Chunked entropy calculation from logits. + + Avoids allocating a full log probabilities tensor to save memory. For models like Qwen with large vocab sizes, this can reduce gpu memory significantly (~O(10GB)) + + Args: + logits: Input logits of shape (batch_size, seqlen, vocab_size) + requires_grad: Whether to enable gradient computation + attention_mask: Optional attention mask of shape (batch_size, seqlen). When provided, + entropy values for padded positions (mask=0) will be zeroed out. + + Returns: + Entropy tensor of shape (batch_size, seqlen). If attention_mask is provided, + positions with mask=0 will have entropy=0. + """ + # Validate attention mask shape if provided + if attention_mask is not None: + if attention_mask.shape != (logits.shape[0], logits.shape[1]): + raise ValueError( + f"attention_mask shape {attention_mask.shape} does not match logits shape " + f"(batch_size={logits.shape[0]}, seqlen={logits.shape[1]}). " + f"Expected attention_mask shape: ({logits.shape[0]}, {logits.shape[1]})" + ) + + cm = nullcontext() if requires_grad else torch.no_grad() + with cm: + # Calculate entropy in chunks to avoid OOM + chunk_size = CHUNK_SIZE + num_chunks = (logits.size(1) + chunk_size - 1) // chunk_size + entropy_tensor = torch.zeros((logits.shape[0], logits.shape[1]), dtype=logits.dtype, device=logits.device) + + for i in range(num_chunks): + start_idx = i * chunk_size + end_idx = min((i + 1) * chunk_size, logits.size(1)) + # (bsz, seq, vocab_size) + chunk = logits[:, start_idx:end_idx] + chunk_logprob = F.log_softmax(chunk, dim=-1) + + # Calculate entropy for this chunk + chunk_probs = chunk_logprob.exp() + chunk_entropy = -(chunk_probs * chunk_logprob).sum(-1) + + # Apply attention mask if provided + if attention_mask is not None: + chunk_mask = attention_mask[:, start_idx:end_idx] + chunk_entropy = chunk_entropy * chunk_mask + + entropy_tensor[:, start_idx:end_idx] = chunk_entropy + return entropy_tensor + + +# Adapt from VERL +def logprobs_from_logits( + logits: Float[torch.Tensor, "batch_size seqlen vocab_size"], + labels: Integer[torch.Tensor, "batch_size seqlen"], + inplace_backward=True, +) -> Float[torch.Tensor, "batch_size seqlen"]: + """ + Compute per-token log-probabilities for the given labels. + + Uses a Flash-Attention-based cross-entropy (if available) for efficient backward, + otherwise falls back to a standard log-softmax+gather approach. + + See: https://github.com/pytorch/pytorch/issues/563#issuecomment-330103591 + + Args: + logits (Tensor): Model outputs of shape (..., vocab_size). + labels (LongTensor): True class indices of shape matching logits[..., :-1]. + inplace_backward (bool): If True and Flash-Attn is available, perform backward in-place. + + Returns: + Tensor: Log-probabilities of the target labels, shape logits.shape[:-1]. + """ + if FLASH_ATTN_CROSS_ENTROPY_LOSS_AVAILABLE: + batch_dim = logits.shape[:-1] + last_dim = logits.shape[-1] + logits = logits.reshape(-1, last_dim) + labels = labels.reshape(-1) + output = logprobs_from_logits_flash_attn(logits, labels, inplace_backward=inplace_backward) + output = output.view(*batch_dim) + else: + output = logprobs_from_logits_v2(logits, labels) + return output + + +def logprobs_from_logits_flash_attn(logits, labels, inplace_backward=True): + output = cross_entropy_loss(logits, labels, inplace_backward=inplace_backward) + assert isinstance( + output, tuple + ), "please make sure flash-attn>=2.4.3 where cross_entropy_loss returns Tuple[losses, z_losses]." + return -output[0] + + +# Credits: https://www.tylerromero.com/posts/2025-02-selective-log-softmax/ +# https://github.com/volcengine/verl/pull/220 +def logprobs_from_logits_v2( + logits: Float[torch.Tensor, "batch_size seqlen vocab_size"], labels: Integer[torch.Tensor, "batch_size seqlen"] +) -> Float[torch.Tensor, "batch_size seqlen"]: + """ + A memory efficient implementation of logprobs_from_logits + """ + if logits.dtype in [torch.float32, torch.float64]: + logits_labels = torch.gather(logits, dim=-1, index=labels.unsqueeze(-1)).squeeze(-1) + # loop to reduce peak mem consumption + logsumexp_values = torch.stack([torch.logsumexp(logit, dim=-1) for logit in logits]) + logprobs_labels = logits_labels - logsumexp_values # log_softmax(x_i) = x_i - logsumexp(x) + else: + # Chunked gather + logsumexp in float32 to avoid materializing full (seqlen, vocab) tensors. + # F.log_softmax allocates a (seqlen, vocab) float32 output which OOMs for long sequences + # with large vocabs (e.g. 64K tokens * 151K vocab * 4 bytes = 36 GiB). + # Instead, chunk along the sequence dim and use gather + logsumexp per chunk. + # Use adaptive chunk size: each chunk is upcasted to float32, plus logsumexp needs + # an intermediate of the same size, so peak memory per chunk ≈ 2 * chunk_size * vocab * 4 bytes. + # Target ~64 MiB per chunk to leave headroom during training backward pass. + vocab_size = logits.shape[-1] + logprobs_chunk_size = max(1, min(CHUNK_SIZE, (64 * 1024 * 1024) // (vocab_size * 4))) + logprobs_labels = [] + for row_logits, row_labels in zip(logits, labels): # loop over batch dim + row_results = [] + for start in range(0, row_logits.shape[0], logprobs_chunk_size): + chunk = row_logits[start : start + logprobs_chunk_size].float() # (chunk, vocab) in float32 + chunk_labels = row_labels[start : start + logprobs_chunk_size] + label_logits = chunk.gather(dim=-1, index=chunk_labels.unsqueeze(-1)).squeeze(-1) + lse = torch.logsumexp(chunk, dim=-1) + row_results.append(label_logits - lse) # log_softmax(x_i) = x_i - logsumexp(x) + del chunk + logprobs_labels.append(torch.cat(row_results)) + logprobs_labels = torch.stack(logprobs_labels) + return logprobs_labels diff --git a/skyrl-train/skyrl_train/utils/tracking.py b/skyrl-train/skyrl_train/utils/tracking.py new file mode 100644 index 0000000000..f05dcab223 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/tracking.py @@ -0,0 +1,316 @@ +# This code is adapted from VERL +# https://github.com/volcengine/verl/blob/main/verl/utils/tracking.py +# The original copyright is reproduced below: +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +from enum import Enum +from functools import partial +from pathlib import Path +from typing import Any, Dict, List, Union +from loguru import logger +from omegaconf import DictConfig, OmegaConf +import pprint + +import numpy as np + + +# TODO(tgriggs): Test all backends. +class Tracking: + supported_backends = ["wandb", "mlflow", "swanlab", "tensorboard", "console"] + + def __init__( + self, + project_name, + experiment_name, + backends: Union[str, List[str]] = "console", + config=None, + ): + if isinstance(backends, str): + backends = [backends] + for backend in backends: + assert backend in self.supported_backends, f"{backend} is not supported" + + self.logger = {} + + if "wandb" in backends: + import wandb + from omegaconf import OmegaConf + + wandb.init(project=project_name, name=experiment_name, config=OmegaConf.to_container(config, resolve=True)) + self.logger["wandb"] = wandb + + if "mlflow" in backends: + self.logger["mlflow"] = _MlflowLoggingAdapter(project_name, experiment_name, config) + + if "swanlab" in backends: + import os + + import swanlab + + SWANLAB_API_KEY = os.environ.get("SWANLAB_API_KEY", None) + SWANLAB_LOG_DIR = os.environ.get("SWANLAB_LOG_DIR", "swanlog") + SWANLAB_MODE = os.environ.get("SWANLAB_MODE", "cloud") + if SWANLAB_API_KEY: + swanlab.login(SWANLAB_API_KEY) # NOTE: previous login information will be overwritten + swanlab.init( + project=project_name, + experiment_name=experiment_name, + config=config, + logdir=SWANLAB_LOG_DIR, + mode=SWANLAB_MODE, + ) + self.logger["swanlab"] = swanlab + + if "tensorboard" in backends: + self.logger["tensorboard"] = _TensorboardAdapter() + + if "console" in backends: + self.console_logger = ConsoleLogger() + self.logger["console"] = self.console_logger + + def log(self, data, step, commit=False): + for logger_name, logger_instance in self.logger.items(): + if logger_name == "wandb": + # Convert list metrics to wandb.Histogram for visualization + wandb_data = {} + for k, v in data.items(): + if isinstance(v, list) and len(v) > 0 and all(isinstance(x, (int, float)) for x in v): + # List of numbers → histogram + wandb_data[k] = logger_instance.Histogram(v) + else: + wandb_data[k] = v + logger_instance.log(data=wandb_data, step=step, commit=commit) + else: + logger_instance.log(data=data, step=step) + + def __del__(self): + # NOTE (sumanthrh): We use a try-except block here while finishing tracking. + # This is because wandb often errors out with a BrokenPipeError when closing. + # https://github.com/wandb/wandb/issues/6449 + # TODO (sumanthrh): Check if this is really needed. Trackers like wandb will automatically finish at program exit. + try: + if "wandb" in self.logger: + self.logger["wandb"].finish(exit_code=0) + if "swanlab" in self.logger: + self.logger["swanlab"].finish() + if "tensorboard" in self.logger: + self.logger["tensorboard"].finish() + if "mlflow" in self.logger: + self.logger["mlflow"].finish() + except Exception as e: + logger.warning(f"Attempted to finish tracking but got error {e}") + + +class ConsoleLogger: + def __init__(self): + pass + + def log(self, data: Dict[str, Any], step: int): + # pprint the data and log with logger + data_as_str = pprint.pformat(ConsoleLogger.stringify_floats(data)) + logger.info(f"Step {step}: \n{data_as_str}") + + def finish(self): + pass + + @staticmethod + def stringify_floats(obj: Any) -> Any: + if isinstance(obj, float): + return f"{obj:.4f}" + elif isinstance(obj, dict): + return {k: ConsoleLogger.stringify_floats(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [ConsoleLogger.stringify_floats(v) for v in obj] + elif isinstance(obj, tuple): + return tuple(ConsoleLogger.stringify_floats(v) for v in obj) + return obj + + +class _TensorboardAdapter: + def __init__(self): + import os + + from torch.utils.tensorboard import SummaryWriter + + tensorboard_dir = os.environ.get("TENSORBOARD_DIR", "tensorboard_log") + os.makedirs(tensorboard_dir, exist_ok=True) + print(f"Saving tensorboard log to {tensorboard_dir}.") + self.writer = SummaryWriter(tensorboard_dir) + + def log(self, data, step): + for key in data: + value = data[key] + if isinstance(value, list): + # Log histogram distribution for list values + self.writer.add_histogram(key, np.array(value), step) + else: + self.writer.add_scalar(key, value, step) + + def finish(self): + self.writer.close() + + +class _MlflowLoggingAdapter: + def __init__(self, project_name, experiment_name, config): + import os + + import mlflow + + if mlflow.active_run() is None: + self.we_created_mlflow = True + if mlflow_tracking_uri := os.environ.get("MLFLOW_TRACKING_URI", None): + mlflow.set_tracking_uri(mlflow_tracking_uri) + + # Project_name is actually experiment_name in MLFlow + # If experiment does not exist, will create a new experiment + experiment = mlflow.set_experiment(project_name) + mlflow.start_run(experiment_id=experiment.experiment_id, run_name=experiment_name) + + else: + self.we_created_mlflow = False + + mlflow.log_params(_compute_mlflow_params_from_objects(config)) + self.mlflow = mlflow + + def log(self, data, step): + results = {k.replace("@", "_at_"): v for k, v in data.items()} + self.mlflow.log_metrics(metrics=results, step=step) + + def finish(self): + if self.we_created_mlflow: + self.mlflow.end_run() + + +def _compute_mlflow_params_from_objects(params) -> Dict[str, Any]: + if params is None: + return {} + + if isinstance(params, DictConfig): + params = OmegaConf.to_container(params, resolve=True) + + return _flatten_dict(_transform_params_to_json_serializable(params, convert_list_to_dict=True), sep="/") + + +def _transform_params_to_json_serializable(x, convert_list_to_dict: bool): + _transform = partial(_transform_params_to_json_serializable, convert_list_to_dict=convert_list_to_dict) + + if dataclasses.is_dataclass(x): + return _transform(dataclasses.asdict(x)) + if isinstance(x, dict): + return {k: _transform(v) for k, v in x.items()} + if isinstance(x, list): + if convert_list_to_dict: + return {"list_len": len(x)} | {f"{i}": _transform(v) for i, v in enumerate(x)} + else: + return [_transform(v) for v in x] + if isinstance(x, Path): + return str(x) + if isinstance(x, Enum): + return x.value + + return x + + +def _flatten_dict(raw: Dict[str, Any], *, sep: str) -> Dict[str, Any]: + import pandas as pd + + ans = pd.json_normalize(raw, sep=sep).to_dict(orient="records")[0] + assert isinstance(ans, dict) + return ans + + +@dataclasses.dataclass +class ValidationGenerationsLogger: + def log(self, loggers, samples, step): + if "wandb" in loggers: + self.log_generations_to_wandb(samples, step) + if "swanlab" in loggers: + self.log_generations_to_swanlab(samples, step) + if "mlflow" in loggers: + self.log_generations_to_mlflow(samples, step) + + def log_generations_to_wandb(self, samples, step): + """Log samples to wandb as a table""" + import wandb + + # Create column names for all samples + columns = ["step"] + sum( + [[f"input_{i + 1}", f"output_{i + 1}", f"score_{i + 1}"] for i in range(len(samples))], [] + ) + + if not hasattr(self, "validation_table"): + # Initialize the table on first call + self.validation_table = wandb.Table(columns=columns) + + # Create a new table with same columns and existing data + # Workaround for https://github.com/wandb/wandb/issues/2981#issuecomment-1997445737 + new_table = wandb.Table(columns=columns, data=self.validation_table.data) + + # Add new row with all data + row_data = [] + row_data.append(step) + for sample in samples: + row_data.extend(sample) + + new_table.add_data(*row_data) + + # Update reference and log + wandb.log({"val/generations": new_table}, step=step) + self.validation_table = new_table + + def log_generations_to_swanlab(self, samples, step): + """Log samples to swanlab as text""" + import swanlab + + swanlab_text_list = [] + for i, sample in enumerate(samples): + row_text = f""" + input: {sample[0]} + + --- + + output: {sample[1]} + + --- + + score: {sample[2]} + """ + swanlab_text_list.append(swanlab.Text(row_text, caption=f"sample {i + 1}")) + + # Log to swanlab + swanlab.log({"val/generations": swanlab_text_list}, step=step) + + def log_generations_to_mlflow(self, samples, step): + """Log validation generation to mlflow as artifacts""" + # https://mlflow.org/docs/latest/api_reference/python_api/mlflow.html?highlight=log_artifact#mlflow.log_artifact + + import json + import tempfile + + import mlflow + + try: + with tempfile.TemporaryDirectory() as tmp_dir: + validation_gen_step_file = Path(tmp_dir, f"val_step{step}.json") + row_data = [] + for sample in samples: + data = {"input": sample[0], "output": sample[1], "score": sample[2]} + row_data.append(data) + with open(validation_gen_step_file, "w") as file: + json.dump(row_data, file) + mlflow.log_artifact(validation_gen_step_file) + except Exception as e: + logger.warning(f"save validation generation file to mlflow failed with error {e}") diff --git a/skyrl-train/skyrl_train/utils/trainer_utils.py b/skyrl-train/skyrl_train/utils/trainer_utils.py new file mode 100644 index 0000000000..d0d6b8b992 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/trainer_utils.py @@ -0,0 +1,929 @@ +from typing import List, Dict, Any, Union, Callable, Optional, Tuple, TypedDict +from omegaconf import OmegaConf, DictConfig +from enum import Enum +import ray +from skyrl_train.workers.worker import PPORayActorGroup +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy +import os +from loguru import logger +import json +import torch +import numpy as np +from collections import defaultdict +from skyrl_train.generators.utils import concatenate_generator_outputs +from skyrl_train.generators.base import GeneratorOutput +from skyrl_train.metrics.reward_metrics import compute_per_group_metrics, sanitize_metric_key +from transformers import AutoTokenizer +from pathlib import Path +from skyrl_train.utils.io import io +from skyrl_train.dataset import PromptDataset +from torchdata.stateful_dataloader import StatefulDataLoader +from torch.utils.data import DataLoader + +BasicType = Union[int, float, str, bool, type(None)] + +GLOBAL_STEP_PREFIX = "global_step_" + + +class ResumeMode(Enum): + NONE = "none" + LATEST = "latest" + FROM_PATH = "from_path" + + @classmethod + def _missing_(cls, value): + if value is None: + return cls.NONE + return super()._missing_(value) + + +def get_node_ids( + policy_model: PPORayActorGroup, critic_model: Optional[PPORayActorGroup], ref_model: Optional[PPORayActorGroup] +) -> List[str]: + """Get the node ids of the policy, critic, and ref models. + + Args: + policy_model: Policy model actor group + critic_model: Critic model actor group (Optional) + ref_model: Ref model actor group (Optional) + """ + policy_node_ids: List[str] = ray.get(policy_model.async_run_ray_method("pass_through", "get_ray_node_id")) + if critic_model is not None: + critic_node_ids: List[str] = ray.get(critic_model.async_run_ray_method("pass_through", "get_ray_node_id")) + else: + critic_node_ids = [] + if ref_model is not None: + ref_node_ids: List[str] = ray.get(ref_model.async_run_ray_method("pass_through", "get_ray_node_id")) + else: + ref_node_ids = [] + + unique_node_ids = list(set(policy_node_ids + critic_node_ids + ref_node_ids)) + return unique_node_ids + + +def run_on_each_node(node_ids: List[str], fn: Callable, *args, **kwargs): + """Simple helper to run a function on each node. + + Args: + node_ids: List of node ids to run the function on + fn: Function to run + *args: Arguments to pass to the function + **kwargs: Keyword arguments to pass to the function + """ + node_ids = list(set(node_ids)) + task = ray.remote(num_cpus=0.25)(fn) + refs = [] + + for node_id in node_ids: + node_task = task.options( + scheduling_strategy=NodeAffinitySchedulingStrategy( + node_id=node_id, + soft=False, + ) + ) + refs.append(node_task.remote(*args, **kwargs)) + + return ray.get(refs) + + +def extract_step_from_path(path: str) -> int: + basename = os.path.basename(path) + if basename.startswith(GLOBAL_STEP_PREFIX): + return int(basename.split(GLOBAL_STEP_PREFIX)[1]) + return -1 + + +def list_checkpoint_dirs(checkpoint_base_path: str) -> list[str]: + """ + List all checkpoint directories in the base path. + + Args: + checkpoint_base_path: Base path where checkpoints are stored + + Returns: + list[str]: List of checkpoint directory names + """ + if not io.exists(checkpoint_base_path): + return [] + + try: + all_items = io.list_dir(checkpoint_base_path) + + # Filter for directories that match the global_step_* pattern + checkpoint_dirs = [] + for item in all_items: + # Get just the basename for pattern matching + basename = os.path.basename(item) + if basename.startswith("global_step_") and io.isdir(os.path.join(checkpoint_base_path, basename)): + checkpoint_dirs.append(basename) + + return sorted(checkpoint_dirs) + except Exception as e: + logger.warning(f"Failed to list checkpoint directories from {checkpoint_base_path}: {e}") + return [] + + +def cleanup_old_checkpoints(checkpoint_base_path: str, max_checkpoints: int) -> None: + """ + Clean up old checkpoints, keeping only the most recent `max_checkpoints` checkpoints. + + Args: + checkpoint_base_path: Base path where checkpoints are stored + max_checkpoints: Maximum number of checkpoints to keep + """ + if max_checkpoints < 0: + return + + checkpoint_dirs = list_checkpoint_dirs(checkpoint_base_path) + + if len(checkpoint_dirs) <= max_checkpoints: + return + + # Sort by step number (extract number from global_step_N) + def extract_step(dirname): + try: + return int(dirname.split("global_step_")[1]) + except (IndexError, ValueError): + return 0 + + checkpoint_dirs.sort(key=extract_step) + + # Remove oldest checkpoints + dirs_to_remove = checkpoint_dirs[:-max_checkpoints] if max_checkpoints > 0 else checkpoint_dirs + + for dir_name in dirs_to_remove: + full_path = os.path.join(checkpoint_base_path, dir_name) + try: + io.remove(full_path) + step_num = extract_step(dir_name) + logger.info(f"Cleaned up old checkpoint: global_step_{step_num} at {full_path}") + except Exception as e: + logger.warning(f"Failed to remove old checkpoint {full_path}: {e}") + + +def validate_consistency_for_latest_checkpoint( + root_ckpt_folder: str, ckpt_iteration: int, checkpoint_path: str, latest_checkpoint_file: str, save_interval: int +): + """Validate that the checkpoint folder is consistent with the latest checkpoint file. + + Asserts that the folder with the highest global step is the latest checkpoint tracked by `latest_checkpoint_file`. + Otherwise, the folder state is inconsistent and the user should delete other checkpoints. + """ + if io.exists(root_ckpt_folder): + checkpoint_dirs = list_checkpoint_dirs(root_ckpt_folder) + if checkpoint_dirs: + global_step_values = [extract_step_from_path(d) for d in checkpoint_dirs] + max_global_step_in_folder = max(global_step_values) + # NOTE (sumanthrh): We allow a checkpoint folder to be `save_interval` steps ahead of the latest checkpoint + # in `latest_checkpoint_file`. This is because the last checkpoint can be an incomplete checkpoint. + if max_global_step_in_folder - ckpt_iteration > save_interval: + max_global_step_in_folder_path = os.path.join( + root_ckpt_folder, f"{GLOBAL_STEP_PREFIX}{max_global_step_in_folder}" + ) + raise ValueError( + f"Inconsistent checkpoint folder. Latest checkpoint file {latest_checkpoint_file} points to " + f"{ckpt_iteration}, but the folder has checkpoints with higher global step - Found global steps " + f"{max_global_step_in_folder_path}. This is likely because checkpoint " + f"{max_global_step_in_folder_path} was created in a previous run while the latest run is at " + f"{checkpoint_path}. Please delete/move checkpoints from older runs and try again." + ) + + +def sanitize_data_source(data_source: str) -> str: + """Sanitize data source name for use in file paths. + + Note: This function is kept for backward compatibility. + For new code, use skyrl_train.metrics.sanitize_metric_key instead. + """ + if data_source is None: + return "unknown" + return sanitize_metric_key(data_source) + + +def _extract_trajectory_rewards(generator_output: GeneratorOutput) -> List[float]: + """Extract trajectory-level rewards from generator output. + + Handles both token-level rewards (List[List[float]]) and trajectory-level rewards (List[float]). + For token-level rewards, sums the rewards to get a single trajectory reward. + + Args: + generator_output: Generator output containing rewards + + Returns: + List of trajectory-level rewards + """ + rewards = generator_output["rewards"] + if not rewards: + return [] + + if isinstance(rewards[0], list): + # Token-level rewards: sum to get trajectory reward + return [float(sum(r)) for r in rewards] + else: + # Already trajectory-level + return [float(r) for r in rewards] + + +def calculate_per_dataset_metrics( + concat_generator_outputs: GeneratorOutput, + concat_uids: List[str], + concat_data_sources: List[str], + n_samples_per_prompt: int, +) -> Dict[str, float]: + """Calculate metrics per data source for evaluation. + + Uses the shared compute_per_group_metrics function for consistent metric calculation + across SkyRL trainer and Tinker integration. + """ + rewards = _extract_trajectory_rewards(concat_generator_outputs) + return compute_per_group_metrics( + rewards=rewards, + uids=concat_uids, + groups=concat_data_sources, + n_samples_per_prompt=n_samples_per_prompt, + prefix="eval", + ) + + +def calculate_per_source_reward_metrics( + generator_outputs: GeneratorOutput, + uids: List[str], + data_sources: List[str], + n_samples_per_prompt: int, +) -> Dict[str, float]: + """Calculate reward metrics per data source for training. + + Uses the shared compute_per_group_metrics function for consistent metric calculation + across SkyRL trainer and Tinker integration. + """ + rewards = _extract_trajectory_rewards(generator_outputs) + return compute_per_group_metrics( + rewards=rewards, + uids=uids, + groups=data_sources, + n_samples_per_prompt=n_samples_per_prompt, + prefix="reward", + ) + + +def dump_per_dataset_eval_results( + dump_dir_path: Path, + tokenizer: AutoTokenizer, + concat_generator_outputs: GeneratorOutput, + concat_data_sources: List[str], + concat_all_envs: List[str], + concat_env_extras: List[Dict[str, Any]], + eval_metrics: Dict[str, float], +): + """Dump evaluation results per dataset and overall aggregated results.""" + + # Prepare common data + input_prompts = [tokenizer.decode(prompt) for prompt in concat_generator_outputs["prompt_token_ids"]] + output_responses = [tokenizer.decode(response) for response in concat_generator_outputs["response_ids"]] + + # Group indices by data source + data_source_indices = {} + for i, data_source in enumerate(concat_data_sources): + if data_source is None: + data_source = "unknown" + if data_source not in data_source_indices: + data_source_indices[data_source] = [] + data_source_indices[data_source].append(i) + + # Dump per-dataset files + for data_source, indices in data_source_indices.items(): + sanitized_data_source = sanitize_data_source(data_source) + filename = dump_dir_path / f"{sanitized_data_source}.jsonl" + + with open(filename, "w") as f: + for i in indices: + entry = { + "input_prompt": input_prompts[i], + "output_response": output_responses[i], + "score": concat_generator_outputs["rewards"][i], + "stop_reason": concat_generator_outputs.get("stop_reasons", [None] * len(input_prompts))[i], + "env_class": concat_all_envs[i], + "env_extras": concat_env_extras[i], + "data_source": data_source, + } + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + logger.info(f"Dumped eval data for {data_source} to {filename}") + + # Dump aggregated results file + aggregated_filename = dump_dir_path / "aggregated_results.jsonl" + with open(aggregated_filename, "w") as f: + f.write(json.dumps(eval_metrics, ensure_ascii=False) + "\n") + + logger.info(f"Dumped aggregated eval metrics to {aggregated_filename}") + + +class DynamicSamplingState(TypedDict, total=False): + """Schema for dynamic sampling state dictionary. + + Fields: + sample_batch_count: Counter for the number of sample batches processed + collected_generator_output: Accumulated generator output (filter strategy only) + collected_uids: Accumulated UIDs (filter strategy only) + num_prompts_in_batch: Number of prompts collected so far (filter strategy only) + """ + + sample_batch_count: int + collected_generator_output: Optional[GeneratorOutput] + collected_uids: Optional[List[str]] + num_prompts_in_batch: Optional[int] + + +def handle_dynamic_sampling( + generator_output: GeneratorOutput, + uids: List[str], + sampling_config: Dict[str, Any], + collected_state: Optional[DynamicSamplingState] = None, +) -> Tuple[GeneratorOutput, List[str], bool, Optional[DynamicSamplingState]]: + """ + Handle dynamic sampling with different strategies (filter, replace). + + filter (used in DAPO) - filter out groups with std == 0 and group size > 1 and resample until we have enough prompts + replace (used in POLARIS, WebSailor) - replace bad (std == 0) samples with good (std > 0) samples + + Args: + generator_output: Current batch generator output + uids: Current batch UIDs + sampling_config: Configuration dict with sampling parameters + collected_state: State for accumulating data across batches (for filter strategy) + + Returns: + Tuple of (processed_generator_output, processed_uids, keep_sampling, updated_state) + """ + sampling_type = sampling_config.get("type", None) + + if sampling_type is None: + return generator_output, uids, False, None + + if sampling_type == "replace": + # For "replace" strategy, the collected state is not used. + processed_output, processed_uids, keep_sampling = handle_replace_sampling( + generator_output, uids, sampling_config + ) + return processed_output, processed_uids, keep_sampling, collected_state + elif sampling_type == "filter": + # For filter strategies, accumulate the generator output and UIDs + # across batches in collected_state if we are sampling repeatedly. + return handle_filter_sampling(generator_output, uids, sampling_config, collected_state) + else: + raise ValueError(f"Invalid dynamic sampling type: {sampling_type}") + + +def handle_replace_sampling( + generator_output: GeneratorOutput, uids: List[str], sampling_config: Dict[str, Any] +) -> Tuple[GeneratorOutput, List[str], bool]: + """ + Handle replace sampling strategy based on POLARIS implementation + + Reference: https://github.com/ChenxinAn-fdu/POLARIS/blob/8c82adb16b8e45c1a34f6d0e23e35deb66dd1ae7/verl/verl/trainer/ppo/ray_trainer.py#L995-L1022. + + Args: + generator_output: Current batch generator output + uids: Current batch UIDs + sampling_config: Configuration dict with sampling parameters + Returns: + Tuple of (processed_generator_output, processed_uids, keep_sampling) + """ + n_samples_per_prompt = sampling_config["n_samples_per_prompt"] + min_replace_ratio = sampling_config["min_replace_ratio"] + + # Extract rewards and convert to sequence-level if needed + rewards_list = generator_output["rewards"] + if rewards_list and isinstance(rewards_list[0], list): + # Token-level rewards: sum to get sequence rewards + rewards = np.array([sum(r) for r in rewards_list]) + else: + rewards = np.array(rewards_list) + + # get mapping of uids to list of indices and metrics + uid2indices = defaultdict(list) + uid2metric_vals = defaultdict(list) + for idx, uid in enumerate(uids): + uid2indices[uid].append(idx) + uid2metric_vals[uid].append(rewards[idx]) + + # Group by UID and calculate metrics + uid2metric_std = {} + for uid, metric_vals in uid2metric_vals.items(): + uid2metric_std[uid] = np.std(metric_vals) + + # Determine good UIDs: those with std > 0 (or group size == 1) + good_uids = set([uid for uid, std in uid2metric_std.items() if std > 0 or n_samples_per_prompt == 1]) + bad_uids = set([uid for uid, std in uid2metric_std.items() if std == 0 and n_samples_per_prompt > 1]) + + logger.info(f"Replace sampling: {len(good_uids)} good UIDs out of {len(uid2metric_vals)} total prompts") + + # Check if we have enough good UIDs (more than min_replace_ratio of the batch) + if len(good_uids) > len(uid2metric_vals) * min_replace_ratio: + logger.info("============= Dynamic sampling replace ===========") + logger.info(f"Number of good prompts: {len(good_uids)}") + logger.info(f"Number of bad prompts: {len(bad_uids)}") + + # Get good uids to replace the bad uids (length of bad uids) + replacement_uids = get_bad_sample_replacements(good_uids, bad_uids) # uids to replace the bad uids + # get replacement indices + replacement_indices = [] + for uid in replacement_uids: + replacement_indices.extend(uid2indices[uid]) + # get bad indices + bad_indices = [] + for uid in bad_uids: + bad_indices.extend(uid2indices[uid]) + + # Replace bad samples with good ones (modify in place because replacement_idx and bad_idx should not overlap) + for bad_idx, replacement_idx in zip(bad_indices, replacement_indices): + generator_output["prompt_token_ids"][bad_idx] = generator_output["prompt_token_ids"][replacement_idx].copy() + generator_output["response_ids"][bad_idx] = generator_output["response_ids"][replacement_idx].copy() + replacement_reward = generator_output["rewards"][replacement_idx] + generator_output["rewards"][bad_idx] = ( + replacement_reward.copy() if isinstance(replacement_reward, list) else replacement_reward + ) + generator_output["loss_masks"][bad_idx] = generator_output["loss_masks"][replacement_idx].copy() + if generator_output["stop_reasons"]: + generator_output["stop_reasons"][bad_idx] = generator_output["stop_reasons"][replacement_idx] + + if generator_output["rollout_logprobs"]: + generator_output["rollout_logprobs"][bad_idx] = generator_output["rollout_logprobs"][replacement_idx] + + # Update UIDs accordingly + replaced_uids = uids.copy() + for bad_idx, replacement_idx in zip(bad_indices, replacement_indices): + replaced_uids[bad_idx] = uids[replacement_idx] + + logger.info(f"After replacement - Replaced {len(bad_indices) // n_samples_per_prompt} bad prompts") + logger.info("==================================================") + + return generator_output, replaced_uids, False + else: + logger.warning("===================== Warning (Dynamic sampling replace) ====================") + logger.warning("In this mini-batch, most training samples receive low variance rewards.") + logger.warning("If you continue to see this warning, please check your data difficulty distribution.") + logger.warning("==================================================") + + return generator_output, uids, True + + +def handle_filter_sampling( + generator_output: GeneratorOutput, + uids: List[str], + sampling_config: Dict[str, Any], + collected_state: DynamicSamplingState, +) -> Tuple[GeneratorOutput, List[str], bool, DynamicSamplingState]: + """ + Handle filter-based sampling strategy (like DAPO). + + Args: + generator_output: Current batch generator output + uids: Current batch UIDs + sampling_config: Configuration dict with sampling parameters + collected_state: State for accumulating data across batches + + Returns: + Tuple of (processed_generator_output, processed_uids, keep_sampling, updated_state) + """ + target_batch_size = sampling_config["train_batch_size"] + n_samples_per_prompt = sampling_config["n_samples_per_prompt"] + + # Extract rewards from collected output + rewards_list = generator_output["rewards"] + if rewards_list and isinstance(rewards_list[0], list): + # Token-level rewards: sum to get sequence rewards + rewards = np.array([sum(r) for r in rewards_list]) + else: + rewards = np.array(rewards_list) + + # Group by UID and calculate standard deviation + uid2metric_vals = defaultdict(list) + for uid, reward in zip(uids, rewards): + uid2metric_vals[uid].append(reward) + + uid2metric_std = {} + for uid, metric_vals in uid2metric_vals.items(): + uid2metric_std[uid] = np.std(metric_vals) + + # Filter out groups with std == 0 and group size > 1 + kept_uids = [uid for uid, std in uid2metric_std.items() if std > 0 or n_samples_per_prompt == 1] + kept_uids_set = set(kept_uids) + + # Filter trajectories based on kept UIDs + kept_traj_idxs = [] + for idx, traj_uid in enumerate(uids): + if traj_uid in kept_uids_set: + kept_traj_idxs.append(idx) + + # Apply filtering to generator output + filtered_output = filter_generator_output(generator_output, kept_traj_idxs) + filtered_uids = [uids[idx] for idx in kept_traj_idxs] + + if "collected_generator_output" not in collected_state: + collected_state.update( + { + "collected_generator_output": filtered_output, + "collected_uids": filtered_uids.copy(), + "num_prompts_in_batch": len(kept_uids), + } + ) + else: + collected_state["collected_generator_output"] = concatenate_generator_outputs( + [collected_state["collected_generator_output"], filtered_output] + ) + collected_state["collected_uids"].extend(filtered_uids) + collected_state["num_prompts_in_batch"] += len(kept_uids) + + # Check if we have enough prompts + if collected_state["num_prompts_in_batch"] < target_batch_size: + logger.info("============= Dynamic sampling filter =============") + logger.info(f"Dynamic sampling: {collected_state['num_prompts_in_batch']} < {target_batch_size} prompts") + logger.info(f"Resample batch {collected_state['sample_batch_count']}, continue sampling...") + logger.info("==================================================") + return generator_output, uids, True, collected_state + else: + logger.info("============= Dynamic sampling filter =============") + logger.info( + f"Dynamic sampling: collected {collected_state['num_prompts_in_batch']} >= {target_batch_size} prompts" + ) + logger.info("==================================================") + # Truncate to exact batch size if needed + n_samples_per_prompt = sampling_config.get("n_samples_per_prompt", 1) + max_trajectories = target_batch_size * n_samples_per_prompt + final_output = collected_state["collected_generator_output"] + final_uids = collected_state["collected_uids"] + + if len(final_uids) > max_trajectories: + final_output = filter_generator_output(final_output, list(range(max_trajectories))) + final_uids = final_uids[:max_trajectories] + + return final_output, final_uids, False, None + + +def get_bad_sample_replacements(good_uids: List[str], bad_uids: List[str]) -> List[str]: + num_replacements = len(bad_uids) + num_candidates = len(good_uids) + + if num_candidates >= num_replacements: + perm = np.random.permutation(num_candidates) + chosen_replacement_uids = np.array(list(good_uids))[perm[:num_replacements]] + else: + indices = np.random.randint(low=0, high=num_candidates, size=(num_replacements,)) + chosen_replacement_uids = np.array(list(good_uids))[indices] + + return chosen_replacement_uids + + +def filter_generator_output(output: GeneratorOutput, kept_indices: List[int]) -> GeneratorOutput: + """Filter GeneratorOutput based on kept indices.""" + filtered = { + "prompt_token_ids": [output["prompt_token_ids"][i] for i in kept_indices], + "response_ids": [output["response_ids"][i] for i in kept_indices], + "rewards": [output["rewards"][i] for i in kept_indices], + "loss_masks": [output["loss_masks"][i] for i in kept_indices], + "stop_reasons": None, + "rollout_metrics": output.get("rollout_metrics"), + "rollout_logprobs": ( + [output["rollout_logprobs"][i] for i in kept_indices] if output["rollout_logprobs"] else None + ), + } + + if output.get("stop_reasons"): + filtered["stop_reasons"] = [output["stop_reasons"][i] for i in kept_indices] + + return filtered + + +def zero_variance_filter(rewards: List[float], uids: List[str]) -> List[int]: + """ + Given a list of trajectory level rewards and uids, return the indices of the trajectories with non-zero variance rewards. + + Args: + rewards: List[float] + uids: List[str] + + Returns: + List[int] + """ + # Group by UID and calculate standard deviation + uid2metric_vals = defaultdict(list) + for uid, reward in zip(uids, rewards): + uid2metric_vals[uid].append(reward) + + # Identify UIDs to keep: non-zero variance or singletons + kept_uids_set = { + uid for uid, metric_vals in uid2metric_vals.items() if np.std(metric_vals) > 0 or len(metric_vals) == 1 + } + + # Return indices of trajectories with kept UIDs + return [i for i, uid in enumerate(uids) if uid in kept_uids_set] + + +def validate_generator_output(num_prompts: int, generator_output: GeneratorOutput): + """Validate the generator output. + + Args: + num_prompts: Number of input prompts used to produce this output. + generator_output: The generated output batch to validate. + """ + if len(generator_output["response_ids"]) <= 0: + raise RuntimeError("No outputs generated") + + # check that input prompts, response ids, and prompt token ids are all the same length + num_responses = len(generator_output["response_ids"]) + num_prompt_tokens = len(generator_output["prompt_token_ids"]) + assert num_prompts == num_responses, f"Mismatch between prompts ({num_prompts}) and responses ({num_responses})" + assert ( + num_responses == num_prompt_tokens + ), f"Mismatch between responses ({num_responses}) and prompt_token_ids ({num_prompt_tokens})" + + # make sure all batch elements have the same length as response_ids (which should be non-zero) + for key in generator_output: + if isinstance(generator_output[key], list) and key in [ + "response_ids", + "loss_masks", + "rewards", + "rollout_logprobs", + ]: + assert len(generator_output[key]) == len(generator_output["response_ids"]), ( + f"Generator output {key} length must be equal to response_ids length, " + f"got {len(generator_output[key])} and {len(generator_output['response_ids'])}" + ) + + # make sure that each element of response ids and loss masks are all the same length + # (and token level rewards if used) + for i, (response_ids, loss_masks, rewards) in enumerate( + zip(generator_output["response_ids"], generator_output["loss_masks"], generator_output["rewards"]) + ): + assert len(response_ids) == len(loss_masks), ( + f"Response ids and loss masks must have the same length, " + f"for sample {i} got {len(response_ids)} and {len(loss_masks)}" + ) + if isinstance(rewards, list): + assert len(rewards) == len(response_ids), ( + f"Token rewards and response ids must have the same length, " + f"for sample {i} got {len(rewards)} and {len(response_ids)}" + ) + + if generator_output["rollout_logprobs"]: + assert len(response_ids) == len(generator_output["rollout_logprobs"][i]), ( + f"Response ids and rollout logprobs must have the same length, " + f"for sample {i} got {len(response_ids)} and {len(generator_output['rollout_logprobs'][i])}" + ) + + # loss masks should be non-zero for at least one element for trainer + if np.concatenate(generator_output["loss_masks"]).sum() == 0: + logger.warning("All outputs are loss masked, which may lead to NaN loss, please check your generation logic!!") + + # check that the rewards are either List[float-like] or List[List[float-like]] + rewards = generator_output["rewards"] + if isinstance(rewards[0], list): + assert all( + isinstance(reward, list) for reward in rewards + ), "rewards must be `List[float]` or `List[List[float]]`" + else: + assert all( + not isinstance(reward, list) for reward in rewards + ), "rewards must be `List[float]` or `List[List[float]]`" + + +class HybridEnvSampler(torch.utils.data.Sampler): + """ + Hybrid environment sampler that ensures minimum representation from each environment + while filling remaining slots proportionally from larger environments. + + This prevents "unlucky" batches where all samples come from hard environments + with low success rates, ensuring learning signal in every batch. + + Args: + dataset: PromptDataset with env_class information + batch_size: Number of samples per batch + min_samples_per_env: Minimum samples from each environment per batch (default: 1) + generator: Random number generator for reproducibility + drop_last: Whether to drop the last incomplete batch + """ + + def __init__( + self, + dataset: PromptDataset, + batch_size: int, + min_samples_per_env: int = 1, + generator: torch.Generator = None, + drop_last: bool = True, + ): + self.dataset = dataset + self.batch_size = batch_size + self.min_samples_per_env = min_samples_per_env + self.generator = generator + self.drop_last = drop_last + + # Build index mapping: env_class -> list of sample indices + self.env_to_indices: Dict[str, List[int]] = defaultdict(list) + for idx in range(len(dataset)): + # Access the underlying dataframe to get env_class without full __getitem__ + env_class = dataset.dataframe[idx].get(dataset.env_class_key, "unknown") + self.env_to_indices[env_class].append(idx) + + self.env_classes = list(self.env_to_indices.keys()) + self.num_envs = len(self.env_classes) + + # Validate that we can fit minimum samples + min_required = self.num_envs * min_samples_per_env + if min_required > batch_size: + logger.warning( + f"HybridEnvSampler: {self.num_envs} envs × {min_samples_per_env} min samples = {min_required} " + f"> batch_size {batch_size}. Reducing min_samples_per_env to fit." + ) + self.min_samples_per_env = max(1, batch_size // self.num_envs) + + # Calculate proportional weights for remaining slots + total_samples = len(dataset) + self.env_weights = {env: len(indices) / total_samples for env, indices in self.env_to_indices.items()} + + logger.info( + f"HybridEnvSampler initialized: {self.num_envs} envs, " + f"batch_size={batch_size}, min_per_env={self.min_samples_per_env}" + ) + for env, indices in sorted(self.env_to_indices.items()): + logger.info(f" {env}: {len(indices)} samples ({self.env_weights[env]*100:.1f}%)") + + def __iter__(self): + # Create shuffled copies of indices for each environment + env_indices_shuffled = {} + for env, indices in self.env_to_indices.items(): + shuffled = indices.copy() + if self.generator is not None: + # Use generator for reproducibility + perm = torch.randperm(len(shuffled), generator=self.generator).tolist() + else: + perm = torch.randperm(len(shuffled)).tolist() + env_indices_shuffled[env] = [shuffled[i] for i in perm] + + # Track position in each env's shuffled list + env_positions = {env: 0 for env in self.env_classes} + + # Calculate number of complete batches we can form + # Each batch consumes batch_size samples total. We need at least min_samples_per_env from each env. + # The limiting factor is the smallest env - it determines how many batches we can form. + # We use a conservative estimate: total_samples / batch_size, but limited by smallest env. + min_batches_per_env = [len(indices) // self.min_samples_per_env for indices in self.env_to_indices.values()] + num_batches = min(min_batches_per_env) + + # Further limit by total available samples / batch_size to avoid over-consuming + total_samples = sum(len(indices) for indices in self.env_to_indices.values()) + num_batches = min(num_batches, total_samples // self.batch_size) + + for batch_idx in range(num_batches): + batch_indices = [] + + # Step 1: Add minimum samples from each environment + for env in self.env_classes: + available = len(env_indices_shuffled[env]) - env_positions[env] + samples_to_take = min(self.min_samples_per_env, available) + for _ in range(samples_to_take): + idx = env_indices_shuffled[env][env_positions[env]] + batch_indices.append(idx) + env_positions[env] += 1 + + # Step 2: Fill remaining slots proportionally + remaining_slots = self.batch_size - len(batch_indices) + if remaining_slots > 0: + # Weighted sampling for remaining slots + # Collect available indices from all envs (those not yet used in this epoch) + available_by_env = {env: env_indices_shuffled[env][env_positions[env] :] for env in self.env_classes} + + # Sample proportionally + for _ in range(remaining_slots): + # Find envs that still have samples + envs_with_samples = [env for env, avail in available_by_env.items() if avail] + if not envs_with_samples: + break + + # Weight by original proportions (normalized to available envs) + weights = [self.env_weights[env] for env in envs_with_samples] + total_weight = sum(weights) + weights = [w / total_weight for w in weights] + + # Sample an env + if self.generator is not None: + rand_val = torch.rand(1, generator=self.generator).item() + else: + rand_val = torch.rand(1).item() + + cumsum = 0 + chosen_env = envs_with_samples[-1] + for env, w in zip(envs_with_samples, weights): + cumsum += w + if rand_val < cumsum: + chosen_env = env + break + + # Take next sample from chosen env + idx = available_by_env[chosen_env].pop(0) + batch_indices.append(idx) + env_positions[chosen_env] += 1 + + # Shuffle the batch to avoid predictable ordering + if self.generator is not None: + perm = torch.randperm(len(batch_indices), generator=self.generator).tolist() + else: + perm = torch.randperm(len(batch_indices)).tolist() + batch_indices = [batch_indices[i] for i in perm] + + # Yield the complete batch (for use with batch_sampler=) + yield batch_indices + + def __len__(self): + # Number of batches per epoch (for batch_sampler, __len__ returns batch count) + min_batches_per_env = [len(indices) // self.min_samples_per_env for indices in self.env_to_indices.values()] + num_batches = min(min_batches_per_env) + # Also limit by total samples / batch_size + total_samples = sum(len(indices) for indices in self.env_to_indices.values()) + num_batches = min(num_batches, total_samples // self.batch_size) + return num_batches + + +def build_dataloader( + cfg: DictConfig, dataset: PromptDataset, is_train=True, is_fully_async=False +) -> StatefulDataLoader: + """ + Build the dataloader for the training or evaluation dataset. + + Args: + cfg: Config object + dataset: Dataset object + is_train: Whether to build the dataloader for training or evaluation + is_fully_async: If is_train, whether to build the dataloader for fully async training, which + mainly makes the batch size 1. + """ + # prepare dataloader + batch_size = cfg.trainer.train_batch_size if is_train else cfg.trainer.eval_batch_size + + # Seed the dataloader for reproducibility. + seeded_generator = torch.Generator() + seeded_generator.manual_seed(cfg.trainer.seed) + + # Check if we should use hybrid env sampling for training + # Requires dataset to have dataframe and env_class_key attributes (like PromptDataset) + use_hybrid_sampling = ( + is_train + and not is_fully_async + and getattr(cfg.trainer, "use_hybrid_env_sampling", False) + and hasattr(dataset, "dataframe") + and hasattr(dataset, "env_class_key") + ) + + if use_hybrid_sampling: + min_samples_per_env = getattr(cfg.trainer, "min_samples_per_env", 1) + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=seeded_generator, + drop_last=True, + ) + # Use standard DataLoader instead of StatefulDataLoader for hybrid sampling + # StatefulDataLoader ignores batch_sampler and falls back to index-based fetching + dataloader = DataLoader( + dataset, + batch_sampler=sampler, # HybridEnvSampler yields batches of indices + collate_fn=dataset.collate_fn, + num_workers=0 if cfg.generator.enable_http_endpoint else 8, + ) + logger.info(f"Using HybridEnvSampler for training with min_samples_per_env={min_samples_per_env}") + else: + dataloader = StatefulDataLoader( + dataset, + batch_size=batch_size if not is_fully_async else 1, + shuffle=True if is_train else False, + collate_fn=dataset.collate_fn, + # TODO(Charlie): debug why inference http endpoint is slow when num_workers is 8 + num_workers=0 if cfg.generator.enable_http_endpoint else 8, + drop_last=True if is_train else False, + generator=seeded_generator, + ) + + if is_train: + if not is_fully_async: + logger.info(f"Total steps: {len(dataloader) * cfg.trainer.epochs}") + else: + logger.info(f"Total steps: {len(dataloader) // cfg.trainer.train_batch_size * cfg.trainer.epochs}") + else: + logger.info(f"Validation set size: {len(dataloader)}") + + return dataloader + + +def get_rope_scaling_config(trainer_cfg: DictConfig) -> dict[str, Any]: + if "rope_scaling" not in trainer_cfg: + return {} + if trainer_cfg.rope_scaling is None: + return None + return OmegaConf.to_container(trainer_cfg.rope_scaling) + + +def get_rope_theta_config(trainer_cfg: DictConfig) -> int | None: + if "rope_theta" not in trainer_cfg: + return None + return trainer_cfg.rope_theta diff --git a/skyrl-train/skyrl_train/utils/utils.py b/skyrl-train/skyrl_train/utils/utils.py new file mode 100644 index 0000000000..2f0918e3d6 --- /dev/null +++ b/skyrl-train/skyrl_train/utils/utils.py @@ -0,0 +1,868 @@ +import ipaddress +import os +import time +import sys +import logging +import math +import socket + +import ray +import torch +from loguru import logger +from omegaconf import DictConfig, OmegaConf +from ray.util.placement_group import ( + placement_group, + PlacementGroupSchedulingStrategy, + PlacementGroup, + placement_group_table, +) + +from skyrl_train.env_vars import SKYRL_LD_LIBRARY_PATH_EXPORT, SKYRL_RAY_PG_TIMEOUT_IN_S, SKYRL_PYTHONPATH_EXPORT + + +class Timer: + def __init__(self, message, update_dict=None): + self.message = message + self.update_dict = update_dict + + def __enter__(self): + self.start_time = time.time() + logger.opt(depth=1).info(f"Started: '{self.message}'") + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + logger.opt(depth=1).info(f"Finished: '{self.message}', time cost: {time.time() - self.start_time:.2f}s") + if self.update_dict is not None: + self.update_dict[self.message] = self.update_dict.get(self.message, 0.0) + time.time() - self.start_time + + async def __aenter__(self): + self.start_time = time.time() + logger.opt(depth=1).info(f"Started: '{self.message}'") + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + logger.opt(depth=1).info(f"Finished: '{self.message}', time cost: {time.time() - self.start_time:.2f}s") + if self.update_dict is not None: + self.update_dict[self.message] = self.update_dict.get(self.message, 0.0) + time.time() - self.start_time + + +def validate_batch_sizes(cfg: DictConfig): + """ + Validate configured batch sizes. + + Explanation of how batching operates: + 1. Each prompt in train_batch_size creates `n_samples_per_prompt` total samples. + 2. During training, these samples are split across data parallel (DP) workers, making the effective per-GPU + batch size: `train_batch_size * n_samples_per_prompt / dp_size`. + 3. Mini batches are similarly normalized to per-gpu mini batches with size: + `mini_batch_size * n_samples_per_prompt / dp_size`. + 4. Per-gpu train batch size must be divisible by per-gpu mini batch size, otherwise the last mini batch will + be incomplete. + 5. Per-gpu mini batch size must be divisible by per-gpu micro batch size, otherwise the last micro batch will + be incomplete. + """ + assert cfg.trainer.train_batch_size >= cfg.trainer.policy_mini_batch_size + assert cfg.trainer.policy_mini_batch_size > 0, "policy_mini_batch_size must be greater than 0" + if cfg.trainer.critic.model.path is not None: + assert cfg.trainer.train_batch_size >= cfg.trainer.critic_mini_batch_size + assert cfg.trainer.critic_mini_batch_size > 0, "critic_mini_batch_size must be greater than 0" + assert cfg.trainer.micro_train_batch_size_per_gpu > 0, "micro_train_batch_size_per_gpu must be greater than 0" + assert cfg.trainer.micro_forward_batch_size_per_gpu > 0, "micro_forward_batch_size_per_gpu must be greater than 0" + + # Validate policy mini batch size + policy_world_size = cfg.trainer.placement.policy_num_nodes * cfg.trainer.placement.policy_num_gpus_per_node + + if cfg.trainer.strategy == "megatron": + pp = cfg.trainer.policy.megatron_config.pipeline_model_parallel_size + cp = cfg.trainer.policy.megatron_config.context_parallel_size + tp = cfg.trainer.policy.megatron_config.tensor_model_parallel_size + assert policy_world_size % (pp * cp * tp) == 0, ( + f"policy_world_size {policy_world_size} should be divisible by (pp * cp * tp) {pp * cp * tp}. " + "This ensures that the data parallel size is an integer." + ) + policy_dp_size = policy_world_size // (pp * cp * tp) + else: + policy_dp_size = policy_world_size // cfg.trainer.policy.sequence_parallel_size + + assert cfg.trainer.train_batch_size % cfg.trainer.policy_mini_batch_size == 0, ( + f"train_batch_size {cfg.trainer.train_batch_size} should be divisible by " + f"policy_mini_batch_size {cfg.trainer.policy_mini_batch_size}" + ) + policy_mini_batch_size_per_gpu = ( + cfg.trainer.policy_mini_batch_size * cfg.generator.n_samples_per_prompt // policy_dp_size + ) + assert policy_mini_batch_size_per_gpu > 0, ( + f"Invalid policy_mini_batch_size_per_gpu: {policy_mini_batch_size_per_gpu}. " + f"mini_batch_size={cfg.trainer.policy_mini_batch_size}, " + f"n_samples_per_prompt={cfg.generator.n_samples_per_prompt}, " + f"dp_size={policy_dp_size}" + ) + assert policy_mini_batch_size_per_gpu % cfg.trainer.micro_train_batch_size_per_gpu == 0, ( + f"normalized policy_mini_batch_size_per_gpu {policy_mini_batch_size_per_gpu} should be divisible " + f"by micro_train_batch_size_per_gpu {cfg.trainer.micro_train_batch_size_per_gpu}" + ) + assert policy_mini_batch_size_per_gpu // cfg.trainer.micro_train_batch_size_per_gpu > 0, ( + f"normalized policy_mini_batch_size_per_gpu {policy_mini_batch_size_per_gpu} should be larger than " + f"micro_train_batch_size_per_gpu {cfg.trainer.micro_train_batch_size_per_gpu}" + ) + policy_train_batch_size_per_gpu = ( + cfg.trainer.train_batch_size * cfg.generator.n_samples_per_prompt // policy_dp_size + ) + + # `train_batch_size_per_gpu` should be divisible by `policy_mini_batch_size_per_gpu` + assert policy_train_batch_size_per_gpu % policy_mini_batch_size_per_gpu == 0, ( + f"normalized policy_train_batch_size_per_gpu (train_batch_size * n_samples_per_prompt // policy_dp_size) " + f"{policy_train_batch_size_per_gpu} should be divisible by policy_mini_batch_size_per_gpu " + f"(policy_mini_batch_size * n_samples_per_prompt // policy_dp_size) {policy_mini_batch_size_per_gpu}" + ) + + # Validate critic mini batch size + critic_world_size = cfg.trainer.placement.critic_num_nodes * cfg.trainer.placement.critic_num_gpus_per_node + critic_dp_size = critic_world_size // cfg.trainer.critic.sequence_parallel_size + + if cfg.trainer.critic.model.path is not None: + assert cfg.trainer.train_batch_size % cfg.trainer.critic_mini_batch_size == 0, ( + f"train_batch_size {cfg.trainer.train_batch_size} should be divisible by " + f"critic_mini_batch_size {cfg.trainer.critic_mini_batch_size}" + ) + critic_mini_batch_size_per_gpu = ( + cfg.trainer.critic_mini_batch_size * cfg.generator.n_samples_per_prompt // critic_dp_size + ) + assert critic_mini_batch_size_per_gpu > 0, ( + f"Invalid critic_mini_batch_size_per_gpu: {critic_mini_batch_size_per_gpu}. " + f"mini_batch_size={cfg.trainer.critic_mini_batch_size}, " + f"n_samples_per_prompt={cfg.generator.n_samples_per_prompt}, " + f"dp_size={critic_dp_size}" + ) + assert critic_mini_batch_size_per_gpu % cfg.trainer.micro_train_batch_size_per_gpu == 0, ( + f"normalized critic_mini_batch_size_per_gpu {critic_mini_batch_size_per_gpu} should be divisible by " + f"micro_train_batch_size_per_gpu {cfg.trainer.micro_train_batch_size_per_gpu}" + ) + assert critic_mini_batch_size_per_gpu // cfg.trainer.micro_train_batch_size_per_gpu > 0, ( + f"normalized critic_mini_batch_size_per_gpu {critic_mini_batch_size_per_gpu} should be larger than " + f"micro_train_batch_size_per_gpu {cfg.trainer.micro_train_batch_size_per_gpu}" + ) + critic_train_batch_size_per_gpu = ( + cfg.trainer.train_batch_size * cfg.generator.n_samples_per_prompt // critic_dp_size + ) + assert critic_train_batch_size_per_gpu % critic_mini_batch_size_per_gpu == 0, ( + f"normalized critic_train_batch_size_per_gpu (train_batch_size * n_samples_per_prompt // critic_dp_size) " + f"{critic_train_batch_size_per_gpu} should be divisible by critic_mini_batch_size_per_gpu " + f"(critic_mini_batch_size * n_samples_per_prompt // critic_dp_size) {critic_mini_batch_size_per_gpu}" + ) + + # Validate training batch size is larger than the least common multiple of the DP sizes of policy (and ref if used). + lcm_dp_size = policy_dp_size + + use_ref_model = cfg.trainer.algorithm.use_kl_loss or cfg.trainer.algorithm.use_kl_in_reward + if use_ref_model: + ref_world_size = cfg.trainer.placement.ref_num_nodes * cfg.trainer.placement.ref_num_gpus_per_node + if cfg.trainer.strategy == "megatron": + pp = cfg.trainer.ref.megatron_config.pipeline_model_parallel_size + cp = cfg.trainer.ref.megatron_config.context_parallel_size + tp = cfg.trainer.ref.megatron_config.tensor_model_parallel_size + assert ref_world_size % (pp * cp * tp) == 0, ( + f"ref_world_size {ref_world_size} should be divisible by (pp * cp * tp) {pp * cp * tp}. " + "This ensures that the data parallel size is an integer." + ) + ref_dp_size = ref_world_size // (pp * cp * tp) + else: + ref_dp_size = ref_world_size // cfg.trainer.ref.sequence_parallel_size + lcm_dp_size = math.lcm(lcm_dp_size, ref_dp_size) + + assert cfg.trainer.train_batch_size * cfg.generator.n_samples_per_prompt >= lcm_dp_size, ( + f"train_batch_size * n_samples_per_prompt ({cfg.trainer.train_batch_size * cfg.generator.n_samples_per_prompt}) " + f"should be larger than or equal to the least common multiple of the data parallel sizes of the enabled models: " + f"policy_dp_size={policy_dp_size}, " + f"ref_dp_size={ref_dp_size if use_ref_model else 'None'}, " + f"lcm_dp_size={lcm_dp_size}" + ) + + +def validate_megatron_cfg(cfg: DictConfig): + # not yet supported + tested features + assert cfg.generator.weight_sync_backend == "nccl", "only nccl is supported for megatron weight sync" + assert cfg.generator.backend == "vllm", "only vllm is supported for with megatron" + assert cfg.trainer.critic.model.path is None, "only GRPO training is currently supported for megatron" + + if cfg.trainer.flash_attn: + import flash_attn + + version = flash_attn.__version__ + if version > "2.8.1": + logger.warning("flash_attn > 2.8.1 is not supported for using the megatron backend with flash_attn") + + worker_configs = [(cfg.trainer.policy, "policy"), (cfg.trainer.ref, "ref")] + for config, worker_type in worker_configs: + # context, expert, and expert tensor parallel are not yet supported for megatron + if config.megatron_config.context_parallel_size > 1: + assert cfg.trainer.use_sample_packing, "context parallel is only supported with sample packing" + # check that sequence parallel is not configured outside of megatron + assert config.sequence_parallel_size == 1, ( + f"found {worker_type}.sequence_parallel_size={config.sequence_parallel_size}, ulysses style sequence " + f"parallel is not supported for megatron" + ) + + +def validate_cfg(cfg: DictConfig): + # Validate generation config separately + validate_generator_cfg(cfg) + from .ppo_utils import AdvantageEstimatorRegistry, PolicyLossRegistry, repopulate_all_registries + + assert ( + cfg.trainer.sequence_parallel_backend == "ulysses" + ), f"only ulysses is supported as of now, got {cfg.trainer.sequence_parallel_backend}" + + # if advantage estimator is GAE, then critic path should be provided + if cfg.trainer.algorithm.advantage_estimator == "gae": + assert ( + cfg.trainer.critic.model.path + ), "`trainer.critic.model.path` should be provided for PPO training, got `None`" + + assert not ( + cfg.trainer.algorithm.use_kl_in_reward and cfg.trainer.algorithm.use_kl_loss + ), "use_kl_in_reward and use_kl_loss should be mutually exclusive" + + if cfg.trainer.strategy in ("fsdp", "fsdp2"): + assert not ( + cfg.trainer.policy.fsdp_config.cpu_offload and cfg.trainer.strategy == "fsdp" + ), "fwd pass cpu offloading is not supported for FSDP1 policy worker, use FSDP2 instead" + assert not ( + cfg.trainer.critic.fsdp_config.cpu_offload and cfg.trainer.strategy == "fsdp" + ), "fwd pass cpu offloading is not supported for FSDP1 critic worker, use FSDP2 instead" + + validate_batch_sizes(cfg) + + if cfg.trainer.max_ckpts_to_keep == 0: + raise ValueError( + "`max_ckpts_to_keep` must be greater than 0 to keep the last N checkpoints " + "or negative to keep all checkpoints" + ) + + # TODO (devpatel): move to initializing ray and syncing registries codepath at startup + repopulate_all_registries() + available_policy_losses = PolicyLossRegistry.list_available() + assert available_policy_losses != [], "Policy loss registry is not populated." + + assert ( + cfg.trainer.algorithm.policy_loss_type in available_policy_losses + ), f"invalid policy_loss_type: {cfg.trainer.algorithm.policy_loss_type}. Must be one of {available_policy_losses}" + + available_advantage_estimators = AdvantageEstimatorRegistry.list_available() + assert cfg.trainer.algorithm.advantage_estimator in available_advantage_estimators, ( + f"invalid advantage_estimator: {cfg.trainer.algorithm.advantage_estimator}. " + f"Must be one of {available_advantage_estimators}" + ) + + assert cfg.trainer.algorithm.loss_reduction in ( + "token_mean", + "sequence_mean", + "seq_mean_token_sum_norm", + ), ( + f"invalid loss_reduction: {cfg.trainer.algorithm.loss_reduction}. " + f"Must be one of `['token_mean', 'sequence_mean', 'seq_mean_token_sum_norm']`" + ) + + # add field to algorithm config needed for loss functions + # create a new config to make it modifiable + algorithm_config = OmegaConf.create(cfg.trainer.algorithm) + # NOTE (erictang000): this is the max sequence length including the prompt, since max response length + # per batch can be variable based on the prompt length. This is used to normalize the loss for + # seq_mean_token_sum_norm loss reduction. Potentially revisit this if we update to use a + # fixed max response budget. + algorithm_config.max_seq_len = cfg.generator.max_input_length + cfg.generator.sampling_params.max_generate_length + + # TODO (erictang000): remove these after deprecation period + if algorithm_config.use_abs_kl: + logger.warning("`use_abs_kl` will be deprecated, overriding to use `kl_estimator_type='abs'` instead") + algorithm_config.kl_estimator_type = "abs" + elif algorithm_config.use_kl_estimator_k3: + logger.warning("`use_kl_estimator_k3` will be deprecated, overriding to use `kl_estimator_type='k3'` instead") + algorithm_config.kl_estimator_type = "k3" + cfg.trainer.algorithm = algorithm_config + + if cfg.trainer.algorithm.use_tis: + if cfg.trainer.algorithm.tis_imp_ratio_cap <= 0: + raise ValueError( + f"If `trainer.algorithm.use_tis` is `True` then `cfg.trainer.algorithm.tis_imp_ratio_cap` " + f"should be > 0, got {cfg.trainer.algorithm.tis_imp_ratio_cap }" + ) + if cfg.generator.sampling_params.logprobs is None: + logger.warning( + "`generator.sampling_params.logprobs` is `None` but `trainer.algorithm.use_tis` is `True`." + " Setting `logprobs` to `True`." + ) + # just set to 0 for better user exp + cfg.generator.sampling_params.logprobs = 0 + + if cfg.generator.backend == "sglang": + raise NotImplementedError("`trainer.algorithm.use_tis` doesn't support Sglang backend, please use vLLM") + assert cfg.trainer.algorithm.policy_loss_type in [ + "regular", + "dual_clip", + ], "TIS is only implemented for regular and dual_clip policy loss types" + + if cfg.trainer.policy.model.lora.rank > 0: + # LoRA enabled + # Right now: assert generator backend must be vllm, training backend must be fsdp/fsdp2 + assert cfg.generator.backend == "vllm", "LoRA enabled requires vLLM backend" + assert cfg.trainer.strategy in ( + "fsdp", + "fsdp2", + "megatron", + ), "LoRA enabled requires fsdp/fsdp2/megatron training backend" + + if cfg.trainer.target_modules is not None: + logger.warning( + "`trainer.target_modules` is deprecated, use `trainer.policy.model.lora.target_modules` or `trainer.critic.model.lora.target_modules` instead" + ) + if cfg.trainer.exclude_modules is not None: + logger.warning( + "`trainer.exclude_modules` is deprecated, use `trainer.policy.model.lora.exclude_modules` or `trainer.critic.model.lora.exclude_modules` instead" + ) + + # Validate placement + if cfg.trainer.placement.colocate_all: + num_policy_gpus = cfg.trainer.placement.policy_num_gpus_per_node * cfg.trainer.placement.policy_num_nodes + num_rollout_gpus = ( + cfg.generator.num_inference_engines + * cfg.generator.inference_engine_tensor_parallel_size + * cfg.generator.inference_engine_pipeline_parallel_size + * cfg.generator.inference_engine_data_parallel_size + ) + assert num_policy_gpus == num_rollout_gpus, ( + f"num_policy_gpus ({num_policy_gpus}) and num_rollout_gpus ({num_rollout_gpus}) " + "must be the same when colocating all models" + ) + else: + use_ref_model = cfg.trainer.algorithm.use_kl_loss or cfg.trainer.algorithm.use_kl_in_reward + if cfg.trainer.placement.colocate_policy_ref and use_ref_model: + assert cfg.trainer.placement.policy_num_nodes == cfg.trainer.placement.ref_num_nodes, ( + f"policy_num_nodes ({cfg.trainer.placement.policy_num_nodes}) and ref_num_nodes " + f"({cfg.trainer.placement.ref_num_nodes}) must be the same when colocate policy and ref model." + ) + assert cfg.trainer.placement.policy_num_gpus_per_node == cfg.trainer.placement.ref_num_gpus_per_node, ( + f"policy_num_gpus_per_node ({cfg.trainer.placement.policy_num_gpus_per_node}) and " + f"ref_num_gpus_per_node ({cfg.trainer.placement.ref_num_gpus_per_node}) must be the same " + f"when colocate policy and ref model." + ) + + +def validate_generator_cfg(cfg: DictConfig): + """Validates the correctness of generator-related config. + + Args: + cfg (DictConfig): config to validate + + Raises: + NotImplementedError: if feature is not supported, such as sglang for multiturn generation + ValueError: when cfg.generator.sampling_params.logprobs > 0 + """ + + if cfg.generator.max_turns == 1: + assert ( + cfg.generator.max_input_length == cfg.trainer.max_prompt_length + ), "generator.max_input_length should be set equal to trainer.max_prompt_length for single-turn generation" + else: + assert cfg.generator.max_input_length >= cfg.trainer.max_prompt_length, ( + "generator.max_input_length should be set greater than or equal to trainer.max_prompt_length " + "for multi-turn generation" + ) + + if not cfg.generator.run_engines_locally: + assert cfg.generator.num_inference_engines == len( + cfg.generator.remote_inference_engine_urls + ), "num_inference_engines should be equal to the number of remote_inference_engine_urls" + + if not cfg.generator.async_engine and cfg.generator.backend == "vllm": + assert ( + cfg.generator.batched + ), "if we are using the offline vLLM engine, we need to put generator in batched mode for faster generation" + + # TODO(tgriggs): use a more modular config validation + if cfg.trainer.logger == "wandb": + assert os.environ.get("WANDB_API_KEY"), "`WANDB_API_KEY` is required for `wandb` logger" + + if cfg.generator.override_existing_update_group == "auto": + if cfg.generator.backend == "vllm" and not cfg.generator.run_engines_locally: + # remote engines can be launched separately so we `enable` by default + cfg.generator.override_existing_update_group = "enable" + else: + # for local engines or sglang, we disable + cfg.generator.override_existing_update_group = "disable" + + # TODO: fix once we support these features with SGLang + if cfg.generator.backend == "sglang" and cfg.generator.run_engines_locally: + assert cfg.generator.inference_engine_tensor_parallel_size == 1, ( + "As of now, We do not support tensor parallel inference engine with SGLang when running engines locally. " + "Please set `inference_engine_tensor_parallel_size` to 1." + ) + + if cfg.generator.backend == "sglang" and not cfg.generator.use_conversation_multi_turn: + raise NotImplementedError("`use_conversation_multi_turn=False` is not supported for SGLang backend") + + if cfg.generator.sampling_params.logprobs is not None: + assert isinstance(cfg.generator.sampling_params.logprobs, int) + if cfg.generator.sampling_params.logprobs > 0: + raise ValueError( + f"`logprobs` if set should be 0 i.e only for the chosen token, " + f"got {cfg.generator.sampling_params.logprobs}" + ) + if not cfg.generator.run_engines_locally: + raise NotImplementedError("Remote inference mode doesn't support `sampling_params.logprobs`") + + if cfg.trainer.strategy == "megatron": + validate_megatron_cfg(cfg) + if cfg.generator.backend == "sglang": + # Some sampling parameters are not supported in SGLang when `skip_tokenizer_init` is True. + if cfg.generator.sampling_params.stop is not None or cfg.generator.eval_sampling_params.stop is not None: + raise ValueError( + "`sampling_params.stop` and `eval_sampling_params.stop` are not supported for SGLang backend " + "since we always set `skip_tokenizer_init` to True. If you have to use these parameters, you can switch " + "to vLLM. " + "See this issue for more: https://github.com/sgl-project/sglang/issues/9039#issuecomment-3218331087" + ) + if "min_new_tokens" in cfg.generator.sampling_params or "min_new_tokens" in cfg.generator.eval_sampling_params: + raise ValueError( + "`sampling_params.min_new_tokens` and `eval_sampling_params.min_new_tokens` are not " + "supported for SGLang backend since we always set `skip_tokenizer_init` to True. " + "If you have to use these parameters, you can switch to vLLM. " + "See this issue for more: https://github.com/sgl-project/sglang/issues/9039#issuecomment-3218331087" + ) + + if cfg.generator.use_conversation_multi_turn: + if ( + cfg.generator.sampling_params.stop is not None or cfg.generator.eval_sampling_params.stop is not None + ) and not cfg.generator.append_eos_token_after_stop_str_in_multi_turn: + logger.warning( + "WARNING: `sampling_params.stop` and `eval_sampling_params.stop` are specified and we " + "are using multi-turn generation. You might want to set `append_eos_token_after_stop_str_in_multi_turn`" + " to `True` to append tokenizer.eos_token_id to the assistant-generated response " + "to match the chat template." + ) + + if cfg.generator.enable_http_endpoint: + if cfg.generator.backend == "sglang": + # TODO(Charlie): sglang_server.py not supported for /chat/completion yet because we have + # skip_tokenizer_init=True in engine creation. Fix by getting tokens via return logprobs + # instead. sglang_engine.py not supported yet because we still need to figure out how + # to make SGLang Python engine take OAI request. + raise ValueError( + "generator.enable_http_endpoint is not supported for SGLang backend yet. " + 'Please set generator.backend="vllm".' + ) + if not cfg.generator.async_engine: + raise ValueError("generator.async_engine must be True when generator.enable_http_endpoint==True.") + + # Validate inference engine parallelism. + ep_size = cfg.generator.inference_engine_expert_parallel_size + dp_size = cfg.generator.inference_engine_data_parallel_size + tp_size = cfg.generator.inference_engine_tensor_parallel_size + if cfg.generator.backend == "sglang": + assert dp_size == 1, "Inference data parallelism is not yet supported for SGLang backend." + assert ep_size == 1, "Inference expert parallelism is not yet supported for SGLang backend." + if ep_size > 1: + assert dp_size * tp_size == ep_size, ( + f"If inference expert parallel is enabled, data parallel size * tensor parallel size must equal expert " + f"parallel size. " + f"Got dp_size={dp_size}, tp_size={tp_size}, ep_size={ep_size}" + ) + + +@ray.remote +def get_all_env_variables(): + import os + + return os.environ + + +def ray_noset_visible_devices(env_vars=os.environ): + # Refer to + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/nvidia_gpu.py#L95-L96 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/amd_gpu.py#L102-L103 + # https://github.com/ray-project/ray/blob/3b9e729f6a669ffd85190f901f5e262af79771b0/python/ray/_private/accelerators/amd_gpu.py#L114-L115 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/npu.py#L94-L95 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/hpu.py#L116-L117 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/neuron.py#L108-L109 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/tpu.py#L171-L172 + # https://github.com/ray-project/ray/blob/161849364a784442cc659fb9780f1a6adee85fce/python/ray/_private/accelerators/intel_gpu.py#L97-L98 + NOSET_VISIBLE_DEVICES_ENV_VARS_LIST = [ + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES", + "RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES", + "RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES", + "RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS", + "RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR", + ] + return any(env_vars.get(env_var) for env_var in NOSET_VISIBLE_DEVICES_ENV_VARS_LIST) + + +def get_physical_gpu_id(): + import torch + + device = torch.cuda.current_device() + props = torch.cuda.get_device_properties(device) + return str(props.uuid) + + +def prepare_runtime_environment(cfg: DictConfig) -> dict[str, str]: + """ + Prepare environment variables for Ray runtime environment. + + Args: + cfg: Training config + + Returns: + Dict[str, str]: Environment variables to be used in Ray runtime environment + """ + # TODO(sumanthrh): introduce a debug mode and add debugging flags like `CUDA_LAUNCH_BLOCKING` here + env_vars = {} + + # NOTE (charlie): See https://github.com/vllm-project/vllm/blob/c6b0a7d3ba03ca414be1174e9bd86a97191b7090/vllm/worker/worker_base.py#L445 + # and https://docs.vllm.ai/en/v0.9.2/usage/troubleshooting.html?h=nccl_cumem_enable#known-issues + # Same for SGLang as we set `NCCL_CUMEM_ENABLE` to 0 in `sglang_engine.py`'s _patched_set_envs_and_config + # + # Skip on GCP: GCP provides a custom NCCL shim + gIB plugin that manages all + # NCCL configuration. The shim's "Guest Config Checker" expects NCCL_CUMEM_ENABLE, + # NCCL_P2P_DISABLE, and NCCL_NVLS_ENABLE to be UNSET. Setting any of these + # conflicts with the shim and causes FSDP workers to crash during dist.broadcast(). + # On GCP, NVSwitch is managed at the host level (FM reports "nothing to do"), + # and NVLink P2P works through the host-managed fabric without Fabric Manager. + _on_gcp = os.path.exists("/usr/local/gib") + if not _on_gcp: + try: + _dmi_product = open("/sys/class/dmi/id/product_name").read().strip() + _on_gcp = "Google" in _dmi_product + except (FileNotFoundError, PermissionError): + pass + if cfg.generator.weight_sync_backend == "nccl" and not _on_gcp: + env_vars["NCCL_CUMEM_ENABLE"] = "0" + logger.info("Setting NCCL_CUMEM_ENABLE=0 (not on GCP)") + elif cfg.generator.weight_sync_backend == "nccl" and _on_gcp: + # GCP deep learning images auto-load gIB (NCCL_NET=gIB) via /etc/profile.d/nccl_env.sh. + # gIB requires RDMA hardware (ConnectX NICs + GPUDirect VPC networks). + # SkyPilot provisions VMs with a single management NIC — no RDMA networking. + # Without RDMA devices (/sys/class/infiniband), gIB can't init and SIGKILL's workers. + # The shell script (fleet-common-run.sh) detects RDMA absence and strips gIB vars + + # LD_LIBRARY_PATH before starting Ray, so workers inherit the correct env. + # NCCL falls back to NVLink P2P (intra-node) + Socket/TCP (inter-node). + _has_rdma = os.path.isdir("/sys/class/infiniband") and bool(os.listdir("/sys/class/infiniband")) + _num_nodes = getattr(getattr(cfg.trainer, "placement", None), "policy_num_nodes", 1) + if _has_rdma: + logger.info(f"On GCP ({_num_nodes} node(s)): RDMA devices found, gIB enabled for GPUDirect RDMA") + else: + logger.info( + f"On GCP ({_num_nodes} node(s)): no RDMA — gIB stripped by shell script, using NVLink P2P + Socket" + ) + + if cfg.trainer.strategy == "megatron": + # this is needed for megatron-core >= 0.15.0, which requires devices to be visible while importing megatron.core + env_vars["RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO"] = "0" + # useful when tp > 1 (and thus megatron sequence_parallel is enabled) + # see: https://github.com/NVIDIA/Megatron-LM/issues/533#issuecomment-1760193239 + env_vars["CUDA_DEVICE_MAX_CONNECTIONS"] = "1" + if cfg.trainer.flash_attn: + # disable fused attention for megatron with flash_attn + # (otherwise flash_attn choice is overridden in TransformerEngine for Hopper+ devices) + # https://github.com/NVIDIA/TransformerEngine/blob/release_v2.5/transformer_engine/pytorch/attention/dot_product_attention/utils.py#L916 + env_vars["NVTE_FUSED_ATTN"] = "0" + + if cfg.generator.backend == "vllm": + env_vars["VLLM_ALLOW_RUNTIME_LORA_UPDATING"] = "true" + + # NOTE (sumanthrh): In vllm >= 0.9.0, we need to explicitly allow for serialization via pickle + # for collective RPCs. During weight transfer, we use IPC handles, which contains a `function` + # object and requires pickling. + env_vars["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" + + # NOTE (sumanthrh): In vLLM >= 0.9.0, we've observed compilatiion failures with torch compile. + # removing the compilation directory and trying again does not fix the issue. Temporarily we disable + # compilation cache, which seems to fix the issue. This should not have any effect on performance - + # compilation will still happen, it's just not cached + # TODO (sumanthrh): remove this once vLLM fixes the issue + env_vars["VLLM_DISABLE_COMPILE_CACHE"] = "1" + + if not os.environ.get("VLLM_USE_V1", False): + logger.info( + "`VLLM_USE_V1` is not specified, setting `VLLM_USE_V1` to 1. To override, set `VLLM_USE_V1` explicitly" + ) + env_vars["VLLM_USE_V1"] = "1" + env_vars["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" + + # Use max of available GPU counts, defaulting to 1 if none found + gpu_counts = [] + if hasattr(cfg.generator, "inference_engine_tensor_parallel_size"): + gpu_counts.append(cfg.generator.inference_engine_tensor_parallel_size) + if hasattr(cfg, "trainer") and hasattr(cfg.trainer, "placement"): + placement = cfg.trainer.placement + gpu_counts.extend( + [ + placement.policy_num_gpus_per_node, + placement.critic_num_gpus_per_node, + placement.ref_num_gpus_per_node, + ] + ) + max_num_gpus_per_node = max(gpu_counts) if gpu_counts else 1 + if not peer_access_supported(max_num_gpus_per_node=max_num_gpus_per_node): + logger.info("Peer access is not supported on this node type, disabling NCCL P2P and SHM") + env_vars["NCCL_P2P_DISABLE"] = "1" + env_vars["NCCL_SHM_DISABLE"] = "1" + + # TODO: this can be removed if we standardize on env files. + # But it's helpful for a quickstart + if os.environ.get("WANDB_API_KEY"): + logger.info("Exporting wandb api key to ray runtime env") + env_vars["WANDB_API_KEY"] = os.environ["WANDB_API_KEY"] + + if os.environ.get("MLFLOW_TRACKING_URI"): + logger.info("Exporting mlflow tracking uri to ray runtime env") + env_vars["MLFLOW_TRACKING_URI"] = os.environ["MLFLOW_TRACKING_URI"] + + if os.environ.get("MLFLOW_TRACKING_TOKEN"): + logger.info("Exporting mlflow tracking token to ray runtime env") + env_vars["MLFLOW_TRACKING_TOKEN"] = os.environ["MLFLOW_TRACKING_TOKEN"] + + if SKYRL_LD_LIBRARY_PATH_EXPORT: + # export `LD_LIBRARY_PATH` to ray runtime env. + # For some reason the `LD_LIBRARY_PATH` is not exported to the worker with .env file. + logger.info(f"Exporting `LD_LIBRARY_PATH` to ray runtime env: {os.environ['LD_LIBRARY_PATH']}") + env_vars["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + + if SKYRL_PYTHONPATH_EXPORT: + # allow pythonpath to be updated as a fall back for deps that are not shipped with UV + # not recommended since it can cause unexpected conflicts with UV packages, + # but keeping for backwards compatibility + logger.info(f"Exporting `PYTHONPATH` to ray runtime env: {os.environ['PYTHONPATH']}") + env_vars["PYTHONPATH"] = os.environ["PYTHONPATH"] + + return env_vars + + +def configure_ray_worker_logging() -> None: + """ + In Ray workers, stderr/stdout are not TTYs, so Loguru disables color. + This method forces color and formatting (e.g., bold) and routes stdlib `logging` + through Loguru so third-party logs match formatting + """ + level_name = os.getenv("LOG_LEVEL", "INFO").upper() + + # 1) Loguru formatting (force colors) + logger.remove() + logger.level("INFO", color="") + logger.add( + sys.stderr, + colorize=True, # keep ANSI even without a TTY + level=level_name, # ensure Loguru filters below this level + enqueue=True, + backtrace=False, + diagnose=False, + format="{time:YYYY-MM-DD HH:mm:ss.SSS} | " + "{level: <8} | " + "{name}:{function}:{line} - " + "{message}", + ) + + # 2) Route stdlib logging -> Loguru (so vLLM/transformers/etc. are formatted) + class _InterceptHandler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + try: + level = logger.level(record.levelname).name + except ValueError: + level = record.levelno + logger.opt(depth=6, exception=record.exc_info).log(level, record.getMessage()) + + logging.root.handlers = [_InterceptHandler()] + level = getattr(logging, level_name, logging.INFO) + logging.root.setLevel(level) + + +def initialize_ray(cfg: DictConfig): + """ + Initialize Ray cluster with prepared runtime environment. + + Args: + cfg: Training config + """ + from .ppo_utils import sync_registries + + env_vars = prepare_runtime_environment(cfg) + ray.init(runtime_env={"env_vars": env_vars}) + + # create the named ray actors for the registries to make available to all workers + sync_registries() + + +def get_ray_pg_ready_with_timeout(pg: PlacementGroup, timeout: int = 60): + try: + ray.get(pg.ready(), timeout=timeout) + except Exception as e: + # Extract resource demands from the placement group + bundles = pg.bundle_specs + total_gpus = sum(bundle.get("GPU", 0) for bundle in bundles) + total_cpus = sum(bundle.get("CPU", 0) for bundle in bundles) + + raise RuntimeError( + f"Failed to create placement group with {len(bundles)} bundles " + f"(requiring {total_gpus} GPUs, {total_cpus} CPUs total) in {timeout} seconds. " + f"This might indicate insufficient GPU resources.\n" + f"Error: {e}" + ) + + +@ray.remote(num_gpus=1) +class InfoActor: + def get_gpu_id(self): + return ray.get_gpu_ids()[0] + + +def get_reordered_bundle_indices(pg: PlacementGroup): + pg_data = placement_group_table(pg) + num_bundles = len(pg_data["bundles"]) + bundle_to_node_ids = pg_data["bundles_to_node_id"] + + # use info actor to get the GPU id + info_actors = [] + for i in range(num_bundles): + info_actors.append( + InfoActor.options( + # set both num_cpus and num_gpus to be small values to enable assignment in colocated case + num_cpus=0.01, + num_gpus=0.01, + resources=None, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, + placement_group_bundle_index=i, + ), + ).remote() + ) + + gpu_ids = ray.get([actor.get_gpu_id.remote() for actor in info_actors]) + for actor in info_actors: + ray.kill(actor) + + # original index, node_id, gpu_id + bundle_infos = [(i, bundle_to_node_ids[i], gpu_ids[i]) for i in range(num_bundles)] + pg_reordered_bundle_indices = [ + bundle_info[0] for bundle_info in sorted(bundle_infos, key=lambda x: (x[1], x[2])) + ] # sort by node_id, then gpu_id + return pg_reordered_bundle_indices + + +# NOTE (sumanthrh): For SGLang, the string representations here should also match those used by +# (and supported by) SGLang. This is because we do not control the update weight implementation +# with SGLang backend. With VLLM, we use a custom Worker extension to have a custom update weight implementation. +def torch_dtype_to_str(dtype: torch.dtype) -> str: + if dtype == torch.bfloat16: + return "bfloat16" + elif dtype == torch.float16: + return "float16" + elif dtype == torch.float32: + return "float32" + else: + return str(dtype) + + +def str_to_torch_dtype(dtype: str) -> torch.dtype: + if dtype == "bfloat16": + return torch.bfloat16 + elif dtype == "float16": + return torch.float16 + elif dtype == "float32": + return torch.float32 + else: + return torch.dtype(dtype) + + +def format_gib(mem_bytes: int) -> str: + return f"{mem_bytes / (1024 ** 3):.2f} GiB" + + +def print_mem(tag: str, mem: dict): + logger.info( + f"{tag} - Allocated: {format_gib(mem['allocated'])}, " + f"Reserved: {format_gib(mem['reserved'])}, " + f"Free: {format_gib(mem['free'])}, " + f"Total: {format_gib(mem['total'])}" + ) + + +def run_p2p_access_check(): + device_count = torch.cuda.device_count() + if device_count < 2: + return False + + # Check P2P access between all GPU pairs + for i in range(device_count): + for j in range(device_count): + if i != j: + # This checks if device i can access device j's memory + can_access = torch.cuda.can_device_access_peer(i, j) + if not can_access: + return False + + return True + + +def peer_access_supported(max_num_gpus_per_node: int): + # whatever the max num gpus per node is, we can check p2p access if there are at least 2 GPUs + # if max is 1, p2p access is not supported + if max_num_gpus_per_node <= 1: + return False + + if not torch.cuda.is_available(): + # we are on cpu head node, so we need to check P2P access on a node with 2 GPUs + ray.init() + pg = placement_group([{"CPU": 1, "GPU": 2}], strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=SKYRL_RAY_PG_TIMEOUT_IN_S) + result = ray.get( + ray.remote(num_gpus=2, scheduling_strategy=PlacementGroupSchedulingStrategy(pg))( + run_p2p_access_check + ).remote() + ) + ray.shutdown() + return result + else: + return run_p2p_access_check() + + +def update_model_config(module_config, override_config_kwargs): + """Update the module config with the override_config_kwargs. + + Args: + module_config: The module config from Huggingface Transformers. + override_config_kwargs: The kwargs to override the module config. + """ + for key, val in override_config_kwargs.items(): + if isinstance(val, dict): + update_model_config(getattr(module_config, key), val) + else: + setattr(module_config, key, val) + + +def get_tcp_url(host: str, port: int) -> str: + """ + Formats the TCP URL for the given host and port, handling IPv6 addresses correctly. + + Args: + host (str): The hostname or IP address. + port (int): The port number. + Returns: + str: The formatted TCP URL. + """ + try: + if isinstance(ipaddress.ip_address(host), ipaddress.IPv6Address): + return f"tcp://[{host}]:{port}" + except ValueError: + # not a literal IP, probably a hostname + pass + return f"tcp://{host}:{port}" + + +def get_free_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", 0)) + port = s.getsockname()[1] + s.close() + return port diff --git a/skyrl-train/skyrl_train/weight_sync/__init__.py b/skyrl-train/skyrl_train/weight_sync/__init__.py new file mode 100644 index 0000000000..e1390bcb41 --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/__init__.py @@ -0,0 +1,74 @@ +"""Weight synchronization abstractions for distributed RL training.""" + +from typing import Type, TYPE_CHECKING + +if TYPE_CHECKING: + from omegaconf import DictConfig + +from .base import WeightChunk, WeightUpdateRequest, LoraLoadRequest +from .weight_extractor import WeightExtractor +from .weight_loader import WeightLoader +from .transfer_strategy import ( + WeightTransferStrategy, + WeightTransferSender, + WeightTransferReceiver, + WeightSyncInitInfo, +) +from .broadcast_strategy import ( + BroadcastInitInfo, + BroadcastTransferStrategy, + BroadcastWeightTransferSender, + BroadcastWeightTransferReceiver, + BroadcastWeightUpdateRequest, +) +from .cuda_ipc_strategy import ( + CudaIpcInitInfo, + CudaIpcTransferStrategy, + CudaIpcWeightTransferSender, + CudaIpcWeightTransferReceiver, + CudaIpcWeightUpdateRequest, +) + + +def get_transfer_strategy_cls(cfg: "DictConfig") -> Type[WeightTransferStrategy]: + """Get the appropriate transfer strategy class based on config. + + Uses CUDA IPC when: + - weight_sync_backend is "nccl" + - colocate_all is True (training and inference on same nodes) + + Otherwise uses broadcast. + + Args: + cfg: Configuration object containing generator and trainer settings. + + Returns: + The strategy class (CudaIpcTransferStrategy or BroadcastTransferStrategy). + """ + if cfg.generator.weight_sync_backend == "nccl" and cfg.trainer.placement.colocate_all: + return CudaIpcTransferStrategy + return BroadcastTransferStrategy + + +__all__ = [ + "WeightChunk", + "WeightExtractor", + "WeightLoader", + "WeightUpdateRequest", + "LoraLoadRequest", + "BroadcastWeightUpdateRequest", + "CudaIpcWeightUpdateRequest", + "WeightTransferStrategy", + "WeightTransferSender", + "WeightTransferReceiver", + "WeightSyncInitInfo", + "BroadcastInitInfo", + "CudaIpcInitInfo", + "BroadcastTransferStrategy", + "BroadcastWeightTransferSender", + "BroadcastWeightTransferReceiver", + "CudaIpcTransferStrategy", + "CudaIpcWeightTransferSender", + "CudaIpcWeightTransferReceiver", + "get_transfer_strategy_cls", +] diff --git a/skyrl-train/skyrl_train/weight_sync/base.py b/skyrl-train/skyrl_train/weight_sync/base.py new file mode 100644 index 0000000000..d25f2682e5 --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/base.py @@ -0,0 +1,89 @@ +"""Base data structures for weight synchronization.""" + +from dataclasses import dataclass +from functools import cached_property +from typing import List + +import torch + + +@dataclass +class WeightUpdateRequest: + """Base class for weight update requests. + + Each transfer strategy has its own request type with strategy-specific fields. + """ + + names: List[str] + dtypes: List[str] + shapes: List[List[int]] + + def __post_init__(self): + lengths = [len(self.names), len(self.dtypes), len(self.shapes)] + if len(set(lengths)) != 1: + raise ValueError( + f"names, dtypes, shapes must have the same length. " + f"Got names={len(self.names)}, dtypes={len(self.dtypes)}, shapes={len(self.shapes)}" + ) + + def __len__(self) -> int: + return len(self.names) + + +class LoraLoadRequest(WeightUpdateRequest): + """Request to load LoRA weights from disk. + + This is a special request type used for loading pre-trained LoRA adapters + from disk rather than transferring weights from training. Unlike other + WeightUpdateRequest subclasses, this doesn't transfer weights - it tells + the inference engine to load LoRA from a path. + """ + + def __init__(self, lora_path: str): + super().__init__(names=[], dtypes=[], shapes=[]) + self.lora_path = lora_path + + +@dataclass +class WeightChunk: + """Represents one or more model parameters to be transferred. + + A WeightChunk can contain multiple parameters grouped together for efficient + transfer (e.g., Q/K/V projections for FlashRL fusion). + + Attributes: + names: List of parameter names (e.g., ["model.layer.0.weight"]) + dtypes: List of dtype strings (e.g., ["torch.bfloat16"]) + shapes: List of tensor shapes (e.g., [[4096, 4096]]) + tensors: List of actual tensor data (populated during extraction) + total_numel: Total number of elements (cached property, auto-calculated) + total_size_bytes: Total memory footprint (cached property, auto-calculated) + """ + + names: List[str] + dtypes: List[str] + shapes: List[List[int]] + tensors: List[torch.Tensor] + + def __post_init__(self): + """Validate that all input lists have the same length.""" + lengths = [len(self.names), len(self.dtypes), len(self.shapes), len(self.tensors)] + if len(set(lengths)) != 1: + raise ValueError( + f"All lists must have the same length. Got names={len(self.names)}, " + f"dtypes={len(self.dtypes)}, shapes={len(self.shapes)}, tensors={len(self.tensors)}" + ) + + def __len__(self) -> int: + """Return the number of parameters in this chunk.""" + return len(self.names) + + @cached_property + def total_numel(self) -> int: + """Calculate total number of elements across all tensors.""" + return sum(t.numel() for t in self.tensors) + + @cached_property + def total_size_bytes(self) -> int: + """Calculate total memory footprint in bytes.""" + return sum(t.numel() * t.element_size() for t in self.tensors) diff --git a/skyrl-train/skyrl_train/weight_sync/broadcast_strategy.py b/skyrl-train/skyrl_train/weight_sync/broadcast_strategy.py new file mode 100644 index 0000000000..180a1b46db --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/broadcast_strategy.py @@ -0,0 +1,300 @@ +"""Broadcast-based weight transfer strategy using torch.distributed. + +This module implements the broadcast transfer strategy for synchronizing model weights +from training workers to inference engines using NCCL/Gloo broadcast operations. +""" + +import asyncio +import socket +from dataclasses import dataclass, replace +from typing import Iterable, Iterator, Optional, Tuple, TYPE_CHECKING + +if TYPE_CHECKING: + from omegaconf import DictConfig + +import ray +import torch + +from skyrl_train.distributed.utils import init_custom_process_group +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.utils import get_tcp_url +from skyrl_train.weight_sync.base import WeightChunk, WeightUpdateRequest +from skyrl_train.weight_sync.transfer_strategy import ( + WeightSyncInitInfo, + WeightTransferStrategy, + WeightTransferSender, + WeightTransferReceiver, +) + + +@dataclass +class BroadcastInitInfo(WeightSyncInitInfo): + """Initialization info for broadcast-based weight transfer.""" + + master_addr: str + master_port: int + rank_offset: int + world_size: int + group_name: str + backend: str + model_dtype_str: str + + @staticmethod + def strategy_type() -> type: + """Return the strategy class for this init info type.""" + return BroadcastTransferStrategy + + def for_engine(self, engine_index: int, tp_size: int, pp_size: int) -> "BroadcastInitInfo": + """Return init_info with rank_offset adjusted for this engine. + + Args: + engine_index: Index of the engine (0-based). + tp_size: Tensor parallel size of the engine. + pp_size: Pipeline parallel size of the engine. + + Returns: + BroadcastInitInfo with adjusted rank_offset. + """ + cumulative_offset = engine_index * tp_size * pp_size + return replace(self, rank_offset=self.rank_offset + cumulative_offset) + + +@dataclass +class BroadcastWeightUpdateRequest(WeightUpdateRequest): + """Request for broadcast-based weight transfer. + + Contains only metadata - actual tensor data is sent via torch.distributed.broadcast. + """ + + pass + + +class BroadcastWeightTransferSender(WeightTransferSender): + """Sends weights via torch.distributed.broadcast. + + The sender broadcasts tensors from rank 0 to all other ranks in the + model update group, while coordinating with inference engines via RPC. + """ + + def __init__( + self, + init_info: BroadcastInitInfo, + model_update_group: Optional[torch.distributed.ProcessGroup], + inference_client: InferenceEngineClient, + ) -> None: + """Initialize the broadcast sender. + + Args: + init_info: BroadcastInitInfo containing all config-derived args. + model_update_group: Process group for broadcast operations (None on non-rank-0 training workers). + inference_client: Client for coordinating with inference engines. + """ + self._init_info = init_info + self._model_update_group = model_update_group + self._inference_client = inference_client + + async def send_chunks(self, chunks: Iterable[WeightChunk]) -> None: + """Send chunks via broadcast. + + Each chunk should contain exactly one parameter for broadcast strategy. + All training ranks iterate through chunks (weight extraction may involve + collective ops), but only rank 0 broadcasts to inference engines via the + model_update_group. + + Args: + chunks: Iterable of WeightChunk objects to send. + """ + rank = torch.distributed.get_rank() + + # Rank 0 must have a process group to broadcast to inference engines + if rank == 0: + assert self._model_update_group is not None, "Rank 0 must have model_update_group" + + # All ranks iterate through chunks (weight extraction may involve collective ops) + for chunk in chunks: + assert len(chunk) == 1, f"Broadcast strategy expects single-parameter chunks, got {len(chunk)}" + + name = chunk.names[0] + tensor = chunk.tensors[0] + shape = chunk.shapes[0] + + # Only rank 0 sends request to inference engines + if rank == 0: + request = BroadcastWeightUpdateRequest( + names=[name], + dtypes=[self._init_info.model_dtype_str], + shapes=[shape], + ) + update_weight_task = asyncio.create_task(self._inference_client.update_named_weights(request)) + + # Broadcast tensor from rank 0 to inference engines (no-op on other training ranks) + def broadcast_tensor(t: torch.Tensor) -> None: + if rank == 0 and self._model_update_group is not None: + torch.distributed.broadcast(t.data, 0, group=self._model_update_group) + + await asyncio.to_thread(broadcast_tensor, tensor) + + if rank == 0: + await update_weight_task + + torch.distributed.barrier() + + def teardown(self) -> None: + """Destroy the process group used for weight transfer. + + TODO: Integrate with training workers to call this during shutdown. + """ + if self._model_update_group is not None: + torch.distributed.destroy_process_group(self._model_update_group) + + +class BroadcastWeightTransferReceiver(WeightTransferReceiver): + """Receives weights via torch.distributed.broadcast. + + Allocates tensors locally and receives data via broadcast from training workers. + """ + + def __init__( + self, + model_dtype: torch.dtype, + model_update_group: torch.distributed.ProcessGroup, + ) -> None: + """Initialize the broadcast receiver. + + Args: + model_dtype: Expected dtype for received tensors. + model_update_group: Process group for broadcast operations. + """ + self._model_dtype = model_dtype + self._model_update_group = model_update_group + + def receive_weights(self, request: BroadcastWeightUpdateRequest) -> Iterator[Tuple[str, torch.Tensor]]: + """Receive weights via broadcast. + + Args: + request: Broadcast weight update request with names, dtypes, shapes. + + Yields: + Tuples of (parameter_name, tensor) for each weight. + """ + from skyrl_train.utils import str_to_torch_dtype + + for name, dtype_str, shape in zip(request.names, request.dtypes, request.shapes): + dtype = str_to_torch_dtype(dtype_str) + assert dtype == self._model_dtype, f"dtype mismatch: request {dtype}, model {self._model_dtype}" + + # Allocate tensor and receive via broadcast + weight = torch.empty(shape, dtype=dtype, device="cuda") + torch.distributed.broadcast(weight, 0, group=self._model_update_group) + yield name, weight + + def teardown(self) -> None: + """Destroy the process group used for weight transfer.""" + torch.distributed.destroy_process_group(self._model_update_group) + + +class BroadcastTransferStrategy(WeightTransferStrategy): + """Factory for broadcast-based weight transfer. + + This strategy uses NCCL/Gloo broadcast operations to transfer weights from + training workers to inference engines. + + All methods are static - no instance state needed. + """ + + @staticmethod + def create_init_info(cfg: "DictConfig") -> BroadcastInitInfo: + """Create init info with all config-derived args. + + Args: + cfg: Configuration object containing generator settings. + + Returns: + BroadcastInitInfo containing all args needed for sender/receiver creation. + """ + + num_inference_engines = cfg.generator.num_inference_engines + tensor_parallel_size = cfg.generator.inference_engine_tensor_parallel_size + pipeline_parallel_size = cfg.generator.inference_engine_pipeline_parallel_size + data_parallel_size = cfg.generator.inference_engine_data_parallel_size + world_size = num_inference_engines * tensor_parallel_size * pipeline_parallel_size * data_parallel_size + 1 + + master_addr = ray._private.services.get_node_ip_address() + with socket.socket() as sock: + sock.bind(("", 0)) + master_port = sock.getsockname()[1] + + return BroadcastInitInfo( + master_addr=master_addr, + master_port=master_port, + rank_offset=1, + world_size=world_size, + group_name="skyrl", + backend=cfg.generator.weight_sync_backend, + model_dtype_str=cfg.generator.model_dtype, + override_existing_receiver=cfg.generator.override_existing_update_group == "enable", + ) + + @staticmethod + def create_sender( + init_info: BroadcastInitInfo, + inference_client: InferenceEngineClient, + ) -> BroadcastWeightTransferSender: + """Create a broadcast sender. + + Sets up the process group on rank 0 only (other training ranks don't join). + + Args: + init_info: BroadcastInitInfo containing config-derived args. + inference_client: Client for coordinating with inference engines. + + Returns: + A configured BroadcastWeightTransferSender instance. + """ + # Only rank 0 joins the model_update_group (with inference engines) + # Other training ranks don't participate in the process group + model_update_group = None + if torch.distributed.get_rank() == 0: + model_update_group = init_custom_process_group( + backend=init_info.backend, + init_method=get_tcp_url(init_info.master_addr, init_info.master_port), + world_size=init_info.world_size, + rank=0, + group_name=init_info.group_name, + ) + + return BroadcastWeightTransferSender( + init_info=init_info, + model_update_group=model_update_group, + inference_client=inference_client, + ) + + @staticmethod + def create_receiver(init_info: BroadcastInitInfo) -> BroadcastWeightTransferReceiver: + """Create a broadcast receiver. + + Sets up the process group and returns a configured receiver. + + Args: + init_info: BroadcastInitInfo from the sender. + + Returns: + A configured BroadcastWeightTransferReceiver instance. + """ + from skyrl_train.utils import str_to_torch_dtype + + # Setup process group (receiver rank = local rank + rank_offset) + rank = torch.distributed.get_rank() + init_info.rank_offset + model_update_group = init_custom_process_group( + backend=init_info.backend, + init_method=get_tcp_url(init_info.master_addr, init_info.master_port), + world_size=init_info.world_size, + rank=rank, + group_name=init_info.group_name, + ) + + model_dtype = str_to_torch_dtype(init_info.model_dtype_str) + return BroadcastWeightTransferReceiver( + model_dtype=model_dtype, + model_update_group=model_update_group, + ) diff --git a/skyrl-train/skyrl_train/weight_sync/cuda_ipc_strategy.py b/skyrl-train/skyrl_train/weight_sync/cuda_ipc_strategy.py new file mode 100644 index 0000000000..4c2f3b6708 --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/cuda_ipc_strategy.py @@ -0,0 +1,293 @@ +"""CUDA IPC-based weight transfer strategy. + +This module implements the CUDA IPC transfer strategy for synchronizing model weights +from training workers to inference engines using CUDA IPC handles. +""" + +from dataclasses import dataclass +from typing import Any, Callable, Dict, Iterable, Iterator, List, Tuple, TYPE_CHECKING + +if TYPE_CHECKING: + from omegaconf import DictConfig + +import torch + +from torch.multiprocessing.reductions import reduce_tensor + +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.utils import get_physical_gpu_id, str_to_torch_dtype +from skyrl_train.weight_sync.base import WeightChunk, WeightUpdateRequest +from skyrl_train.weight_sync.transfer_strategy import ( + WeightSyncInitInfo, + WeightTransferStrategy, + WeightTransferSender, + WeightTransferReceiver, +) + +# IPC handle type: (rebuild_func, args) returned by reduce_tensor +IpcHandle = Tuple[Callable[..., torch.Tensor], Tuple[Any, ...]] + + +@dataclass +class CudaIpcInitInfo(WeightSyncInitInfo): + """Initialization info for CUDA IPC-based weight transfer.""" + + model_dtype_str: str + + @staticmethod + def strategy_type() -> type: + """Return the strategy class for this init info type.""" + return CudaIpcTransferStrategy + + +_IPC_REQUEST_END_MARKER = b"__END_OF_REQUEST__" + + +@dataclass +class CudaIpcWeightUpdateRequest(WeightUpdateRequest): + """Request for CUDA IPC-based weight transfer. + + Contains IPC handles for direct GPU memory access. Tensors are packed into + a contiguous buffer to reduce the number of IPC handles. + """ + + sizes: List[int] # Size in elements per parameter (for unpacking) + ipc_handles: Dict[str, IpcHandle] # Physical GPU UUID -> IPC handle for the packed buffer + + def serialize(self) -> bytes: + """Serialize the request to bytes.""" + import pickle + import base64 + + request_data = pickle.dumps(self) + request_data_encoded = base64.b64encode(request_data) + data_with_marker = request_data_encoded + _IPC_REQUEST_END_MARKER + + # Pad for 4-byte alignment + data_size = len(data_with_marker) + padded_size = ((data_size + 3) // 4) * 4 + result = bytearray(data_with_marker) + result.extend(b"\x00" * (padded_size - data_size)) + return bytes(result) + + @classmethod + def deserialize(cls, data: bytes) -> "CudaIpcWeightUpdateRequest": + """Deserialize the request from bytes.""" + import pickle + import base64 + + end_index = data.find(_IPC_REQUEST_END_MARKER) + if end_index == -1: + raise ValueError("End marker not found in serialized data") + request_data = data[:end_index] + try: + request_data_decoded = base64.b64decode(request_data) + return pickle.loads(request_data_decoded) + except Exception as e: + raise ValueError("Failed to deserialize request") from e + + +class CudaIpcWeightTransferSender(WeightTransferSender): + """Sends weights via CUDA IPC handles. + + Creates IPC handles for tensors, gathers them across ranks, and sends + the handle metadata to inference engines. + """ + + def __init__( + self, + init_info: CudaIpcInitInfo, + inference_client: InferenceEngineClient, + ) -> None: + """Initialize the CUDA IPC sender. + + Args: + init_info: CudaIpcInitInfo containing config-derived args. + inference_client: Client for coordinating with inference engines. + """ + self._init_info = init_info + self._inference_client = inference_client + + async def send_chunks(self, chunks: Iterable[WeightChunk]) -> None: + """Send chunks via CUDA IPC with packed tensors. + + Each chunk can contain multiple parameters. All tensors in a chunk are + packed into a single contiguous buffer, and one IPC handle is created + for the packed buffer. This reduces the number of IPC handles and file + descriptors. + + Args: + chunks: Iterable of WeightChunk objects to send. + """ + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + device = torch.cuda.current_device() + dtype = str_to_torch_dtype(self._init_info.model_dtype_str) + + for chunk in chunks: + # Collect metadata + names = [] + dtypes = [] + shapes = [] + sizes = [] + + # Pack all tensors in this chunk into a single contiguous buffer + total_numel = sum(t.numel() for t in chunk.tensors) + packed_tensor = torch.empty( + total_numel, + device=device, + dtype=dtype, + requires_grad=False, + ) + + offset = 0 + for name, tensor, shape in zip(chunk.names, chunk.tensors, chunk.shapes): + size = tensor.numel() + packed_tensor[offset : offset + size].copy_(tensor.detach().view(-1)) + offset += size + names.append(name) + dtypes.append(self._init_info.model_dtype_str) + shapes.append(shape) + sizes.append(size) + + # Create single IPC handle for the packed buffer + ipc_handle: IpcHandle = reduce_tensor(packed_tensor) + ipc_handle_dict: Dict[str, IpcHandle] = {get_physical_gpu_id(): ipc_handle} + ipc_handle_list: List[Dict[str, IpcHandle] | None] = [None] * world_size + torch.distributed.all_gather_object(ipc_handle_list, ipc_handle_dict) + + ipc_handles: Dict[str, IpcHandle] = {} + if rank == 0: + for d in ipc_handle_list: + if d is not None: + ipc_handles.update(d) + + torch.distributed.barrier() + torch.cuda.synchronize() + + # Send the packed chunk + if rank == 0: + request = CudaIpcWeightUpdateRequest( + names=names, + dtypes=dtypes, + shapes=shapes, + sizes=sizes, + ipc_handles=ipc_handles, + ) + await self._inference_client.update_named_weights(request) + + torch.cuda.ipc_collect() + torch.distributed.barrier() + torch.cuda.synchronize() + + def teardown(self) -> None: + """No-op for CUDA IPC sender (no custom process group to clean up).""" + pass + + +class CudaIpcWeightTransferReceiver(WeightTransferReceiver): + """Receives weights via CUDA IPC handles. + + Opens IPC handles to access tensors shared from training workers. + """ + + def __init__(self, model_dtype: torch.dtype) -> None: + """Initialize the CUDA IPC receiver. + + Args: + model_dtype: Expected dtype for received tensors. + """ + self._model_dtype = model_dtype + + def receive_weights(self, request: CudaIpcWeightUpdateRequest) -> Iterator[Tuple[str, torch.Tensor]]: + """Receive weights via CUDA IPC handles. + + Args: + request: CUDA IPC weight update request with names, dtypes, shapes, sizes, and IPC handles. + + Yields: + Tuples of (parameter_name, tensor) for each weight. + """ + assert len(set(request.dtypes)) == 1, "packed weight update should have all tensors with the same dtype" + assert ( + str_to_torch_dtype(request.dtypes[0]) == self._model_dtype + ), f"mismatch dtype: src {request.dtypes[0]}, dst {self._model_dtype}" + assert len(request.sizes) == len(request), "sizes must be provided for packed weight update" + assert all(isinstance(size, int) for size in request.sizes), "sizes should be a list of integers" + + device_id = torch.cuda.current_device() + physical_gpu_id = get_physical_gpu_id() + + handle = request.ipc_handles[physical_gpu_id] + func, args = handle + list_args = list(args) + list_args[6] = device_id + packed_tensor = func(*list_args) + + offset = 0 + for name, shape, size in zip(request.names, request.shapes, request.sizes): + yield name, packed_tensor[offset : offset + size].view(*shape) + offset += size + + def teardown(self) -> None: + """No-op for CUDA IPC receiver (no custom process group to clean up).""" + pass + + +class CudaIpcTransferStrategy(WeightTransferStrategy): + """Factory for CUDA IPC-based weight transfer. + + This strategy uses CUDA IPC handles to share GPU memory between training + workers and inference engines on the same machine. + + All methods are static - no instance state needed. + """ + + @staticmethod + def create_init_info(cfg: "DictConfig") -> CudaIpcInitInfo: + """Create init info with all config-derived args. + + Args: + cfg: Configuration object containing generator settings. + + Returns: + CudaIpcInitInfo containing all args needed for sender/receiver creation. + """ + return CudaIpcInitInfo( + model_dtype_str=cfg.generator.model_dtype, + override_existing_receiver=cfg.generator.override_existing_update_group == "enable", + ) + + @staticmethod + def create_sender( + init_info: CudaIpcInitInfo, + inference_client: InferenceEngineClient, + ) -> CudaIpcWeightTransferSender: + """Create a CUDA IPC sender. + + Args: + init_info: CudaIpcInitInfo containing config-derived args. + inference_client: Client for coordinating with inference engines. + + Returns: + A configured CudaIpcWeightTransferSender instance. + """ + return CudaIpcWeightTransferSender( + init_info=init_info, + inference_client=inference_client, + ) + + @staticmethod + def create_receiver(init_info: CudaIpcInitInfo) -> CudaIpcWeightTransferReceiver: + """Create a CUDA IPC receiver. + + Args: + init_info: CudaIpcInitInfo from the sender. + + Returns: + A configured CudaIpcWeightTransferReceiver instance. + """ + from skyrl_train.utils import str_to_torch_dtype + + model_dtype = str_to_torch_dtype(init_info.model_dtype_str) + return CudaIpcWeightTransferReceiver(model_dtype=model_dtype) diff --git a/skyrl-train/skyrl_train/weight_sync/transfer_strategy.py b/skyrl-train/skyrl_train/weight_sync/transfer_strategy.py new file mode 100644 index 0000000000..be12efc353 --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/transfer_strategy.py @@ -0,0 +1,170 @@ +"""Weight transfer strategy abstractions for distributed RL training. + +This module defines the abstract interfaces for transferring model weights +from training workers to inference engines. The strategy pattern allows different +transfer mechanisms (broadcast, CUDA IPC) to be used interchangeably. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, Iterable, Iterator, Tuple + +import torch + +from skyrl_train.weight_sync.base import WeightChunk + +if TYPE_CHECKING: + from omegaconf import DictConfig + from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient + from skyrl_train.weight_sync.base import WeightUpdateRequest + + +@dataclass +class WeightSyncInitInfo(ABC): + """Base class for weight sync initialization info.""" + + override_existing_receiver: bool + """Whether to override an existing weight receiver. If False and a receiver exists, init is skipped.""" + + @staticmethod + @abstractmethod + def strategy_type() -> type: + """Return the strategy class for this init info type.""" + ... + + def for_engine(self, engine_index: int, tp_size: int, pp_size: int) -> "WeightSyncInitInfo": + """Return init_info adjusted for a specific engine. + + Override in subclasses that need per-engine adjustments (e.g., rank offset). + Default implementation returns self unchanged. + + Args: + engine_index: Index of the engine (0-based). + tp_size: Tensor parallel size of the engine. + pp_size: Pipeline parallel size of the engine. + + Returns: + Adjusted init_info for the specific engine. + """ + return self + + +class WeightTransferSender(ABC): + """Strategy-specific component that sends WeightChunk data to inference actors. + + Implementations handle the transfer primitive (broadcast, CUDA IPC) and coordinate + with inference actors. + """ + + @abstractmethod + async def send_chunks(self, chunks: Iterable[WeightChunk]) -> None: + """Send chunks using this transfer strategy. + + This method must be called on all training ranks. Implementations may have + different behavior for different ranks. + + Args: + chunks: Iterable of WeightChunk objects to send. + """ + ... + + @abstractmethod + def teardown(self) -> None: + """Clean up resources used by the sender (e.g., destroy process groups).""" + ... + + +class WeightTransferReceiver(ABC): + """Strategy-specific component that receives weights from training workers. + + Implementations handle receiving data via the transfer primitive (broadcast, CUDA IPC) + and yielding tensors for the loader to apply to the model. + """ + + @abstractmethod + def receive_weights(self, request: "WeightUpdateRequest") -> Iterator[Tuple[str, torch.Tensor]]: + """Yield (name, tensor) tuples by pulling data from transfer channel. + + This method must be called on all inference engine ranks. Implementations may have + different behavior for different ranks. + + Args: + request: Weight update request. + + Yields: + Tuples of (parameter_name, tensor) for each weight in the request. + """ + ... + + @abstractmethod + def teardown(self) -> None: + """Clean up resources used by the receiver (e.g., destroy process groups).""" + ... + + +class WeightTransferStrategy(ABC): + """Stateless factory for creating init info, senders and receivers. + + Each strategy implementation provides static methods to create: + - init_info: Contains all config-derived args + - sender: Uses init_info + inference_client + - receiver: Uses init_info + strategy-specific args + + Usage on sender side: + init_info = Strategy.create_init_info(cfg) + sender = Strategy.create_sender(init_info, inference_client) + + Usage on receiver side (for each engine): + engine_init_info = init_info.for_engine(engine_index, tp_size, pp_size) + receiver = engine_init_info.strategy_type().create_receiver(engine_init_info) + """ + + @staticmethod + @abstractmethod + def create_init_info(cfg: "DictConfig") -> WeightSyncInitInfo: + """Create init info with all config-derived args. + + Args: + cfg: Configuration object containing generator settings. + + Returns: + WeightSyncInitInfo containing all args needed for sender/receiver creation. + """ + ... + + @staticmethod + @abstractmethod + def create_sender( + init_info: WeightSyncInitInfo, + inference_client: "InferenceEngineClient", + ) -> WeightTransferSender: + """Create a sender for the training worker side. + + This method must be called on all training ranks. Implementations may + have different initialization logic for different ranks (e.g., only rank 0 + joins a process group for broadcast, while all ranks participate for IPC). + + Args: + init_info: WeightSyncInitInfo containing config-derived args. + inference_client: Client for coordinating with inference engines. + + Returns: + A configured WeightTransferSender instance. + """ + ... + + @staticmethod + @abstractmethod + def create_receiver(init_info: WeightSyncInitInfo) -> WeightTransferReceiver: + """Create a receiver for the inference engine side. + + This method must be called on all inference engine ranks. Implementations may + have different initialization logic for different ranks. + + Args: + init_info: WeightSyncInitInfo from the sender. + + Returns: + A configured WeightTransferReceiver instance. + """ + ... diff --git a/skyrl-train/skyrl_train/weight_sync/weight_extractor.py b/skyrl-train/skyrl_train/weight_sync/weight_extractor.py new file mode 100644 index 0000000000..323a029d08 --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/weight_extractor.py @@ -0,0 +1,34 @@ +"""Weight extractor interface for extracting weights from training backends.""" + +from abc import ABC, abstractmethod +from typing import Iterator + +import torch + +from .base import WeightChunk + + +class WeightExtractor(ABC): + """Extracts weights from training backend models. + + Subclasses implement backend-specific logic to extract model weights, + handle sharding, and prepare them for transfer to inference engines. + """ + + @abstractmethod + def extract_weights(self, dtype: torch.dtype) -> Iterator[WeightChunk]: + """Extract weights from the model as WeightChunk objects. + + Implementations should: + - Gather sharded weights into full tensors + - Convert tensors to the specified dtype for inference + - Ensure tensors are contiguous in memory + - Optionally group related parameters (e.g., QKV for efficiency) + + Args: + dtype: Target dtype for inference (e.g., torch.bfloat16, torch.float16) + + Yields: + WeightChunk objects containing model parameters ready for transfer + """ + ... diff --git a/skyrl-train/skyrl_train/weight_sync/weight_extractor_utils.py b/skyrl-train/skyrl_train/weight_sync/weight_extractor_utils.py new file mode 100644 index 0000000000..91acc27f01 --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/weight_extractor_utils.py @@ -0,0 +1,107 @@ +"""Utility functions for weight extraction.""" + +from collections import defaultdict +from typing import Dict, List, Callable, Iterator, Any +import torch + +from skyrl_train.weight_sync import WeightChunk + + +def yield_module_grouped_chunks( + params: Dict[str, Any], + dtype: torch.dtype, + gather_tensor_fn: Callable[[Any], torch.Tensor], + get_shape_fn: Callable[[str, Any, torch.Tensor], List[int]], + batch_size_threshold_gb: float = 0.0, +) -> Iterator[WeightChunk]: + """Yield WeightChunk objects grouped by module. + + This helper function eliminates duplication between different weight extractors + that need to group parameters by module (e.g., for FlashRL QKV fusion). + + Groups parameters by their parent module by removing the last two components + from the parameter name. For example: + "model.layers.0.self_attn.q_proj.weight" -> "model.layers.0.self_attn" + + Args: + params: Dictionary mapping parameter names to parameter objects + dtype: Target dtype for inference + gather_tensor_fn: Backend-specific function to gather sharded tensors into full tensors + get_shape_fn: Function to extract shape from param_name, param, and prepared tensor + batch_size_threshold_gb: If > 0, batch complete modules together until threshold is reached + + Yields: + WeightChunk objects containing all parameters for each module (or batched modules if threshold set) + """ + # Group parameters by module for FlashRL + # NOTE (sumanthrh): We sync weights module by module. Ex: weights for self attn together, weights for mlp together + # For FlashRL integration, we allocate new storage for each param. Since q, k and v layer weights are fused internally by vllm, + # we need to pass the weights for all of these together. + # Overall, this doesn't hurt perf even in the general case + module_to_params: Dict[str, List[str]] = defaultdict(list) + for param_name in params.keys(): + # Extract module name (e.g., "model.layers.0.self_attn" from "model.layers.0.self_attn.q_proj.weight") + # TODO (sumanthrh): When would this fail? Works for many AutoModelForCausalLM models for now + module_name = ".".join(param_name.split(".")[:-2]) + module_to_params[module_name].append(param_name) + + # Accumulate complete modules until threshold reached + batch_tensors = [] + batch_names = [] + batch_shapes = [] + batch_dtypes = [] + current_size = 0 + threshold_bytes = batch_size_threshold_gb * 1024**3 + + for module_name, param_names in module_to_params.items(): + module_tensors = [] + module_names = [] + module_shapes = [] + module_dtypes = [] + module_size = 0 + + # Prepare all tensors for this module + # TODO: Allow gather_tensor_fn to accept a list of params for batched gathering + # to improve efficiency for sharded backends that support multi-parameter collects. + for param_name in param_names: + param = params[param_name] + tensor = gather_tensor_fn(param) + tensor = tensor.to(dtype).detach().contiguous() + shape = get_shape_fn(param_name, param, tensor) + module_tensors.append(tensor) + module_names.append(param_name) + module_shapes.append(shape) + module_dtypes.append(str(dtype)) + module_size += tensor.nbytes + + # Check if adding this module would exceed threshold + if current_size > 0 and current_size + module_size > threshold_bytes: + # Yield current batch before adding this module + yield WeightChunk( + names=batch_names, + dtypes=batch_dtypes, + shapes=batch_shapes, + tensors=batch_tensors, + ) + # Start new batch + batch_tensors = [] + batch_names = [] + batch_shapes = [] + batch_dtypes = [] + current_size = 0 + + # Add module to current batch + batch_tensors.extend(module_tensors) + batch_names.extend(module_names) + batch_shapes.extend(module_shapes) + batch_dtypes.extend(module_dtypes) + current_size += module_size + + # Yield final batch if non-empty + if batch_tensors: + yield WeightChunk( + names=batch_names, + dtypes=batch_dtypes, + shapes=batch_shapes, + tensors=batch_tensors, + ) diff --git a/skyrl-train/skyrl_train/weight_sync/weight_loader.py b/skyrl-train/skyrl_train/weight_sync/weight_loader.py new file mode 100644 index 0000000000..df97682297 --- /dev/null +++ b/skyrl-train/skyrl_train/weight_sync/weight_loader.py @@ -0,0 +1,25 @@ +"""Weight loader interface for inference engines.""" + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from skyrl_train.weight_sync.base import WeightUpdateRequest + + +class WeightLoader(ABC): + """Loads received weights into inference engine. + + Implementations are engine-specific (vLLM, SGLang, etc.) and handle + the mechanics of coordinating weight transfer and applying weights + to the inference model. + """ + + @abstractmethod + async def load_weights(self, request: "WeightUpdateRequest") -> None: + """Load weights into the inference engine. + + Coordinates with the receiver to fetch weights and applies them + to the model. Handles RPC coordination for distributed engines. + """ + ... diff --git a/skyrl-train/skyrl_train/workers/__init__.py b/skyrl-train/skyrl_train/workers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/skyrl_train/workers/fsdp/fsdp_worker.py b/skyrl-train/skyrl_train/workers/fsdp/fsdp_worker.py new file mode 100644 index 0000000000..d3107cde54 --- /dev/null +++ b/skyrl-train/skyrl_train/workers/fsdp/fsdp_worker.py @@ -0,0 +1,395 @@ +from skyrl_train.utils.trainer_utils import get_rope_scaling_config, get_rope_theta_config +import ray +import torch +import torch.distributed +from transformers import AutoConfig +from torch.distributed.fsdp.api import ShardedStateDictConfig, StateDictType +from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP +import io + +try: + # for torch 2.5+ + from torch.distributed.tensor import DTensor +except ImportError: + from torch.distributed._tensor import DTensor + +from skyrl_train.model_wrapper import HFModelWrapper, get_llm_for_sequence_regression +from skyrl_train.distributed.fsdp_strategy import FSDPStrategy +from skyrl_train.utils import str_to_torch_dtype +from skyrl_train.training_batch import TrainingInputBatch, TrainingOutputBatch +from skyrl_train.distributed.fsdp_utils import fsdp_version, get_init_weight_context_manager +from skyrl_train.workers.worker import ( + PolicyWorkerBase, + CriticWorkerBase, + RefWorkerBase, +) +from skyrl_train.weight_sync import WeightExtractor, WeightChunk, LoraLoadRequest +from skyrl_train.weight_sync.weight_extractor_utils import yield_module_grouped_chunks + + +class FSDPWeightExtractor(WeightExtractor): + """Extracts weights from FSDP-sharded models. + + Args: + model: FSDP model to extract weights from + group_by_module: If True, group parameters by module (e.g., for FlashRL QKV fusion) + batch_size_threshold_gb: If > 0, batch complete modules together until threshold is reached + """ + + def __init__(self, model: torch.nn.Module, group_by_module: bool = False, batch_size_threshold_gb: float = 0.0): + self.model = model + self.group_by_module = group_by_module + self.batch_size_threshold_gb = batch_size_threshold_gb + + def extract_weights(self, dtype: torch.dtype): + """Extract weights from FSDP model. + + Args: + dtype: Target dtype for inference + + Yields: + WeightChunk objects (one per parameter, or grouped by module) + """ + # Configure state_dict type for FSDP v1 + if fsdp_version(self.model) == 1: + FSDP.set_state_dict_type( + self.model, + state_dict_type=StateDictType.SHARDED_STATE_DICT, + state_dict_config=ShardedStateDictConfig(), + ) + + # Get state dict (handles FSDP sharding) + params = self.model.state_dict() + + if not self.group_by_module: + # Simple path: yield one chunk per parameter + for name, param in params.items(): + tensor = self._gather_tensor(param).to(dtype).detach().contiguous() + yield WeightChunk( + names=[name], + dtypes=[str(dtype)], + shapes=[list(tensor.shape)], + tensors=[tensor], + ) + else: + for chunk in yield_module_grouped_chunks( + params=params, + dtype=dtype, + gather_tensor_fn=self._gather_tensor, + get_shape_fn=lambda name, param, tensor: list(tensor.shape), + batch_size_threshold_gb=self.batch_size_threshold_gb, + ): + yield chunk + + def _gather_tensor(self, param: torch.Tensor) -> torch.Tensor: + """Gather sharded tensor into full tensor.""" + device = torch.cuda.current_device() + return param.to(device, non_blocking=True).full_tensor() if isinstance(param, DTensor) else param + + +class FSDPPolicyWorkerBase(PolicyWorkerBase): + def offload_to_cpu(self, pin_memory=True, non_blocking=True, offload_optimizer=True, offload_model=True): + self._set_numa_affinity(torch.distributed.get_rank() % torch.cuda.device_count()) + self.strategy.offload_to_cpu( + self.model, self.optimizer, pin_memory, non_blocking, offload_optimizer, offload_model + ) + + def backload_to_gpu(self, non_blocking=True, backload_optimizer=True, backload_model=True): + self.strategy.backload_to_gpu(self.model, self.optimizer, non_blocking, backload_optimizer, backload_model) + + def init_model(self, model_path, num_training_steps: int = None): + assert self.cfg.trainer.strategy in ("fsdp", "fsdp2") + strategy = FSDPStrategy( + fsdp_config=self.cfg.trainer.policy.fsdp_config, + optimizer_config=self.cfg.trainer.policy.optimizer_config, + model_config=self.cfg.trainer.policy.model, + fsdp_strategy=self.cfg.trainer.strategy, + seed=self.cfg.trainer.seed, + micro_train_batch_size_per_gpu=self.cfg.trainer.micro_train_batch_size_per_gpu, + num_training_steps=num_training_steps, + ) + strategy.setup_distributed() + self.strategy = strategy + + self._is_lora = self.cfg.trainer.policy.model.lora.rank > 0 + + # Update per-gpu mini batch size based on device mesh + self._normalize_mini_batch_size() + + model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + init_context = get_init_weight_context_manager( + use_meta_tensor=not model_config.tie_word_embeddings, mesh=self.strategy.device_mesh + ) + with init_context(): + + wrapped_model = HFModelWrapper( + model_path, + use_flash_attention_2=self.cfg.trainer.flash_attn, + # NOTE (sumanthrh): Model initialization should always be in fp32 + # during training + bf16=False, + lora_rank=self.cfg.trainer.policy.model.lora.rank, + lora_alpha=self.cfg.trainer.policy.model.lora.alpha, + lora_dropout=self.cfg.trainer.policy.model.lora.dropout, + lora_init_method=self.cfg.trainer.policy.model.lora.init_method, + target_modules=self.cfg.trainer.policy.model.lora.target_modules, + exclude_modules=self.cfg.trainer.policy.model.lora.exclude_modules, + sequence_parallel_size=self.cfg.trainer.policy.sequence_parallel_size, + use_sample_packing=self.cfg.trainer.use_sample_packing, + use_torch_compile=self.cfg.trainer.policy.use_torch_compile, + rope_scaling=get_rope_scaling_config(self.cfg.trainer), + rope_theta=get_rope_theta_config(self.cfg.trainer), + model_config_kwargs=self.cfg.trainer.policy.model_config_kwargs, + loss_chunk_size=self.cfg.trainer.loss_chunk_size, + ) + # in-place patch + self._seq_parallel_monkey_patch(model=wrapped_model.model) + + if self.cfg.trainer.gradient_checkpointing: + wrapped_model.gradient_checkpointing_enable( + gradient_checkpointing_kwargs={ + "use_reentrant": self.cfg.trainer.gradient_checkpointing_use_reentrant + } + ) + + self.model, self.optimizer, self.scheduler = strategy.prepare( + (wrapped_model, None, None), + ) + assert ( + self.optimizer is not None and self.scheduler is not None + ), "FSDP preparation should create optimizer and scheduler" + + # Initialize weight extractor + # TODO(haochen): Now module grouping (in order to support FlashRL) is only enabled for the CUDA IPC + # transfer strategy, we can enable it for other strategies as well. + from skyrl_train.weight_sync import CudaIpcTransferStrategy + + group_by_module = self._transfer_strategy_cls is CudaIpcTransferStrategy + self.weight_extractor = FSDPWeightExtractor( + self.model.model, + group_by_module=group_by_module, + batch_size_threshold_gb=( + self.cfg.generator.weight_transfer_threshold_cuda_ipc_GB if group_by_module else 0.0 + ), + ) + + async def _save_lora_adapters_and_sync(self, peft_model, lora_sync_path, inference_engine_client): + """Collect LoRA parameters, save and call inference engine to load.""" + import os + import json + from dataclasses import asdict + from safetensors.torch import save_file + from skyrl_train.distributed.fsdp_utils import collect_lora_params + + lora_params = collect_lora_params(module=self.model.model) + + if torch.distributed.get_rank() == 0: + os.makedirs(lora_sync_path, exist_ok=True) + + peft_config = asdict(peft_model.peft_config.get("default", {})) + peft_config["task_type"] = peft_config["task_type"].value + peft_config["peft_type"] = peft_config["peft_type"].value + peft_config["target_modules"] = list(peft_config["target_modules"]) + + # Save LoRA parameters and config + save_file(lora_params, os.path.join(lora_sync_path, "adapter_model.safetensors")) + with io.open(os.path.join(lora_sync_path, "adapter_config.json"), "w", encoding="utf-8") as f: + json.dump(peft_config, f, ensure_ascii=False, indent=4) + + # Send LoRA disk loading request to inference engine + lora_request = LoraLoadRequest(lora_path=lora_sync_path) + await inference_engine_client.update_named_weights(lora_request) + + torch.distributed.barrier() + + async def broadcast_to_inference_engines(self, inference_engine_client): + use_prefix_cache = self.cfg.generator.enable_prefix_caching + generator_dtype = str_to_torch_dtype(self.cfg.generator.model_dtype) + cache_reset_task = None + if use_prefix_cache and torch.distributed.get_rank() == 0: + # clear prefix cache + cache_reset_task = inference_engine_client.reset_prefix_cache() + + torch.cuda.empty_cache() + + # Check if this is a LoRA model + peft_model = getattr(self.model.model, "_fsdp_wrapped_module", self.model.model) + + if self._is_lora: + assert hasattr(peft_model, "peft_config"), "LoRA model should have peft_config" + + # assume base model is already synced, sync LoRA adapters + lora_sync_path = self.cfg.trainer.policy.model.lora.lora_sync_path + await self._save_lora_adapters_and_sync(peft_model, lora_sync_path, inference_engine_client) + return + + # Extract and send weights using the sender created at init time + await self._weight_transfer_sender.send_chunks(self.weight_extractor.extract_weights(generator_dtype)) + + if cache_reset_task is not None: + await cache_reset_task + torch.cuda.empty_cache() + torch.distributed.barrier() + + def get_weight_statistics(self): + """Compute lightweight statistics for model weights""" + raise NotImplementedError() + + def _set_pad_token_id(self, pad_token_id): + # NOTE (sumanthrh): self.model -> HFModelWrapper; self.model.model -> AutoModelForCausalLM + self.model.model.config.pad_token_id = pad_token_id + + def forward( + self, + data: TrainingInputBatch, + ) -> TrainingOutputBatch: + """Run forward pass on data in inference mode. + + Reshard the model after forward pass to redistribute memory and allow for offloading to cpu. + """ + output = super().forward(data) + # unshard the root FSDP module (https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes) + if self._world_size > 1 and fsdp_version(self.model.model) == 1: + self.model.model._handle.reshard(True) + return output + + +class FSDPCriticWorkerBase(CriticWorkerBase): + def offload_to_cpu(self, pin_memory=True, non_blocking=True, offload_optimizer=True, offload_model=True): + self._set_numa_affinity(torch.distributed.get_rank() % torch.cuda.device_count()) + self.strategy.offload_to_cpu( + self.model, self.optimizer, pin_memory, non_blocking, offload_optimizer, offload_model + ) + + def backload_to_gpu(self, non_blocking=True, backload_optimizer=True, backload_model=True): + self.strategy.backload_to_gpu(self.model, self.optimizer, non_blocking, backload_optimizer, backload_model) + + def init_model(self, model_path, num_training_steps: int = None): + assert self.cfg.trainer.strategy in ("fsdp", "fsdp2") + strategy = FSDPStrategy( + fsdp_config=self.cfg.trainer.critic.fsdp_config, + optimizer_config=self.cfg.trainer.critic.optimizer_config, + fsdp_strategy=self.cfg.trainer.strategy, + seed=self.cfg.trainer.seed, + micro_train_batch_size_per_gpu=self.cfg.trainer.micro_train_batch_size_per_gpu, + num_training_steps=num_training_steps, + ) + strategy.setup_distributed() + self.strategy = strategy + + # Update per-gpu mini batch size based on device mesh + self._normalize_mini_batch_size() + + model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + init_context = get_init_weight_context_manager( + use_meta_tensor=not model_config.tie_word_embeddings, mesh=self.strategy.device_mesh + ) + with init_context(): + critic = get_llm_for_sequence_regression( + model_path, + "critic", + use_flash_attention_2=self.cfg.trainer.flash_attn, + # NOTE (sumanthrh): Model initialization should always be in fp32 + # during training + bf16=False, + lora_rank=self.cfg.trainer.critic.model.lora.rank, + lora_alpha=self.cfg.trainer.critic.model.lora.alpha, + lora_dropout=self.cfg.trainer.critic.model.lora.dropout, + target_modules=self.cfg.trainer.critic.model.lora.target_modules, + exclude_modules=self.cfg.trainer.critic.model.lora.exclude_modules, + value_head_prefix=self.cfg.trainer.algorithm.value_head_prefix, + init_value_head=self.cfg.trainer.policy.model.path == self.cfg.trainer.critic.model.path, + sequence_parallel_size=self.cfg.trainer.critic.sequence_parallel_size, + use_sample_packing=self.cfg.trainer.use_sample_packing, + model_config_kwargs=self.cfg.trainer.critic.model_config_kwargs, + ) + self._seq_parallel_monkey_patch(model=critic, use_parent_class=True) + + if self.cfg.trainer.gradient_checkpointing: + critic.gradient_checkpointing_enable( + gradient_checkpointing_kwargs={ + "use_reentrant": self.cfg.trainer.gradient_checkpointing_use_reentrant + } + ) + + # prepare models/optimizers... + self.model, self.optimizer, self.scheduler = strategy.prepare( + (critic, None, None), + ) + assert self.optimizer is not None + + def forward( + self, + data: TrainingInputBatch, + ) -> TrainingOutputBatch: + """Run forward pass on data in inference mode. + + Reshard the model after forward pass to redistribute memory and allow for offloading to cpu. + """ + output = super().forward(data) + # unshard the root FSDP module (https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes) + if self._world_size > 1 and fsdp_version(self.model.model) == 1: + self.model.model._handle.reshard(True) + return output + + +class FSDPRefWorkerBase(RefWorkerBase): + def offload_to_cpu(self, pin_memory=True, non_blocking=True, **kwargs): + self._set_numa_affinity(torch.distributed.get_rank() % torch.cuda.device_count()) + self.strategy.offload_to_cpu(self.model, None, pin_memory, non_blocking) + + def backload_to_gpu(self, non_blocking=True, **kwargs): + self.strategy.backload_to_gpu(self.model, None, non_blocking) + + def init_model(self, model_path): + assert self.cfg.trainer.strategy in ("fsdp", "fsdp2") + strategy = FSDPStrategy( + fsdp_config=self.cfg.trainer.ref.fsdp_config, + fsdp_strategy=self.cfg.trainer.strategy, + seed=self.cfg.trainer.seed, + micro_train_batch_size_per_gpu=self.cfg.trainer.micro_train_batch_size_per_gpu, + ) + strategy.setup_distributed() + self.strategy = strategy + + model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + init_context = get_init_weight_context_manager( + use_meta_tensor=not model_config.tie_word_embeddings, mesh=self.strategy.device_mesh + ) + + with init_context(): + wrapped_model = HFModelWrapper( + model_path, + use_flash_attention_2=self.cfg.trainer.flash_attn, + bf16=self.cfg.trainer.bf16, + sequence_parallel_size=self.cfg.trainer.ref.sequence_parallel_size, + use_sample_packing=self.cfg.trainer.use_sample_packing, + rope_scaling=get_rope_scaling_config(self.cfg.trainer), + rope_theta=get_rope_theta_config(self.cfg.trainer), + model_config_kwargs=self.cfg.trainer.ref.model_config_kwargs, + loss_chunk_size=self.cfg.trainer.loss_chunk_size, + ) + self._seq_parallel_monkey_patch(model=wrapped_model.model) + + self.model = strategy.prepare(wrapped_model) + self.model.eval() + + def forward( + self, + data: TrainingInputBatch, + ) -> TrainingOutputBatch: + """Run forward pass on data in inference mode. + + Reshard the model after forward pass to redistribute memory and allow for offloading to cpu. + """ + output = super().forward(data) + # unshard the root FSDP module (https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes) + if self._world_size > 1 and fsdp_version(self.model.model) == 1: + self.model.model._handle.reshard(True) + return output + + +# Ray remote actors +PolicyWorker = ray.remote(num_gpus=1)(FSDPPolicyWorkerBase) +CriticWorker = ray.remote(num_gpus=1)(FSDPCriticWorkerBase) +RefWorker = ray.remote(num_gpus=1)(FSDPRefWorkerBase) diff --git a/skyrl-train/skyrl_train/workers/megatron/megatron_model_wrapper.py b/skyrl-train/skyrl_train/workers/megatron/megatron_model_wrapper.py new file mode 100644 index 0000000000..d84673e662 --- /dev/null +++ b/skyrl-train/skyrl_train/workers/megatron/megatron_model_wrapper.py @@ -0,0 +1,338 @@ +from typing import Optional, Callable, List +from functools import partial +import torch +import torch.nn as nn + +from megatron.core.pipeline_parallel import get_forward_backward_func +import megatron.core.parallel_state as mpu +from megatron.core.distributed import finalize_model_grads + +from skyrl_train.distributed.megatron.model_utils import from_parallel_logits_to_logprobs, vocab_parallel_entropy +from skyrl_train.distributed.megatron.megatron_utils import get_model_config +from skyrl_train.utils.ppo_utils import compute_approx_kl, masked_mean + +from skyrl_train.distributed.megatron.megatron_utils import ( + make_batch_generator, + preprocess_packed_seqs, + postprocess_packed_seqs, + remove_left_padding, + recover_left_padding, +) + + +class MegatronModelWrapper: + def __init__( + self, + config, + actor_module: List[nn.Module], + actor_optimizer: Optional[torch.optim.Optimizer] = None, + policy_loss_fn: Optional[Callable] = None, + ): + self.cfg = config + self.actor_module = actor_module + self.actor_optimizer = actor_optimizer + self.policy_loss_fn = policy_loss_fn + self.use_sample_packing = self.cfg.trainer.use_sample_packing + + config = get_model_config(self.actor_module[0]) + # This is set to None by default: https://github.com/NVIDIA/Megatron-LM/blob/07b22a05136a3cb08ece05f7de38cf6aeeb165fb/megatron/core/model_parallel_config.py#L95 + # use the build in finalize_model_grads function to all reduce gradients across parallelism dimensions + config.finalize_model_grads_func = finalize_model_grads + + def train(self): + [module.train() for module in self.actor_module] + + def eval(self): + [module.eval() for module in self.actor_module] + + def __call__(self, *args, **kwargs): + return self.forward(*args, **kwargs) + + def forward( + self, + micro_batches: List[dict], + seq_len: int, + micro_batch_size: int, + temperature: float = 1.0, + ) -> torch.Tensor: + """ + Forward-only inference to compute log-probs over a full mini-batch consisting of multiple micro-batches. + + Args: + micro_batches: List of micro-batch dicts with keys: "sequences", "attention_mask", "position_ids", + and "num_actions". + seq_len: Padded sequence length per sample. + micro_batch_size: Per-micro-batch size. + temperature: Optional temperature scaling for logits. + + Returns: + torch.Tensor of concatenated log-probs across micro-batches (valid on pipeline last stage only). + """ + forward_backward_func = get_forward_backward_func() + + def collection_func(logits, data): + sequences = data["sequences"] + tp_grp = mpu.get_tensor_model_parallel_group() + tp_rank = mpu.get_tensor_model_parallel_rank() + + if temperature != 1.0: + logits.div_(temperature) + + token_logprobs = from_parallel_logits_to_logprobs( + logits, + sequences, + vocab_start_index=tp_rank * logits.shape[-1], + vocab_end_index=(tp_rank + 1) * logits.shape[-1], + tp_group=tp_grp, + inference_only=True, + cp_group=None, # we handle cp gathering in `postprocess_packed_seqs` + chunk_size=None, + ) + return torch.tensor(0.0, device=token_logprobs.device), {"log_probs": token_logprobs} + + def forward_step(batch_iter, model): + batch = next(batch_iter) + sequences = batch["sequences"] + attention_mask = batch["attention_mask"].to(bool) + position_ids = batch["position_ids"] + + if self.use_sample_packing: + new_sequences, packed_seq_params = preprocess_packed_seqs( + sequences, + attention_mask, + pre_process=mpu.is_pipeline_first_stage(ignore_virtual=True), + ) + new_attention_mask = None + new_position_ids = None + else: + new_sequences, new_attention_mask, new_position_ids = remove_left_padding( + sequences, + attention_mask, + position_ids, + pre_process=mpu.is_pipeline_first_stage(ignore_virtual=True), + ) + packed_seq_params = None + + outputs = model( + new_sequences, + new_position_ids, + new_attention_mask, + packed_seq_params=packed_seq_params, + ) + + if self.use_sample_packing: + outputs = postprocess_packed_seqs( + outputs, + packed_seq_params, + attention_mask, + micro_batch_size, + seq_len, + post_process=mpu.is_pipeline_last_stage(ignore_virtual=True), + ) + else: + outputs = recover_left_padding( + outputs, + new_attention_mask, + attention_mask, + seq_len, + post_process=mpu.is_pipeline_last_stage(ignore_virtual=True), + ) + + return outputs, partial(collection_func, data=batch) + + batch_generator = make_batch_generator(micro_batches, vpp_size=len(self.actor_module)) + + output = forward_backward_func( + forward_step_func=forward_step, + data_iterator=batch_generator, + model=self.actor_module, + num_microbatches=len(micro_batches), + seq_length=seq_len, + micro_batch_size=micro_batch_size, + forward_only=True, + ) + + if mpu.is_pipeline_last_stage(ignore_virtual=True): + log_probs = [o["log_probs"] for o in output] + log_probs = torch.cat(log_probs, dim=0) + # take last num_actions tokens per micro; concatenate later + # Assume all micros have same num_actions + num_actions = micro_batches[0]["num_actions"] + log_probs = log_probs[:, -num_actions:] + else: + # return dummy tensor for non-last pp stages + device = micro_batches[0]["sequences"].device + log_probs = torch.zeros(size=(1, 1), dtype=torch.bfloat16, device=device) + return log_probs + + def forward_backward_mini_batch( + self, + micro_batches: List[dict], + seq_len: int, + micro_batch_size: int, + temperature: float = 1.0, + ) -> List[dict]: + """ + Run forward-backward over a full mini-batch consisting of multiple micro-batches. + + Args: + micro_batches: A list of micro-batch dicts. Each dict must contain keys: + "sequences", "attention_mask", "position_ids", "num_actions", + "old_action_log_probs", "base_action_log_probs", "advantages", + "loss_mask", "rollout_action_logprobs". + seq_len: Sequence length (tokens) per sample (assumed same across micros after padding). + micro_batch_size: Micro-batch size per forward pass. + temperature: Optional temperature for logits scaling. + + Returns: + List[dict]: one metrics dict per micro-batch in order. + """ + forward_backward_func = get_forward_backward_func() + + def loss_func(logits, data): + sequences = data["sequences"] + num_actions = data["num_actions"] + old_action_log_probs = data["old_action_log_probs"] + base_action_log_probs = data["base_action_log_probs"] + advantages = data["advantages"] + loss_mask = data["loss_mask"] + rollout_action_logprobs = data["rollout_action_logprobs"] + + tp_grp = mpu.get_tensor_model_parallel_group() + tp_rank = mpu.get_tensor_model_parallel_rank() + + # temperature normalization + if temperature != 1.0: + logits.div_(temperature) + + token_logprobs = from_parallel_logits_to_logprobs( + logits, + sequences, + vocab_start_index=tp_rank * logits.shape[-1], + vocab_end_index=(tp_rank + 1) * logits.shape[-1], + tp_group=tp_grp, + inference_only=False, + cp_group=None, # we handle cp gathering in `postprocess_packed_seqs` + chunk_size=None, + ) + + action_log_probs = token_logprobs[:, -num_actions:] + + # policy loss should be calculated based on the selected token logprobs + policy_loss, clip_ratio = self.policy_loss_fn( + action_log_probs, + old_action_log_probs, + advantages, + config=self.cfg.trainer.algorithm, + loss_mask=loss_mask, + rollout_logprobs=rollout_action_logprobs, + ) + + with torch.set_grad_enabled(self.cfg.trainer.algorithm.use_entropy_loss): + action_logits = logits[:, -num_actions - 1 : -1, :] + entropy_BS = vocab_parallel_entropy(action_logits) + entropy = masked_mean(entropy_BS, loss_mask) + + if self.cfg.trainer.algorithm.use_entropy_loss: + entropy_loss_term = entropy * self.cfg.trainer.algorithm.entropy_loss_coef + else: + entropy_loss_term = torch.tensor(0.0) + + if self.cfg.trainer.algorithm.use_kl_loss: + kl_loss = compute_approx_kl( + action_log_probs, + base_action_log_probs, + loss_mask=loss_mask, + kl_estimator_type=self.cfg.trainer.algorithm.kl_estimator_type, + ) + kl_loss = masked_mean(kl_loss, loss_mask, dim=-1).mean() + else: + kl_loss = torch.tensor(0.0) + kl_loss_term = kl_loss * self.cfg.trainer.algorithm.kl_loss_coef + + loss = policy_loss + kl_loss_term - entropy_loss_term + + metrics = { + "final_loss": loss.detach().item(), + "policy_loss": policy_loss.detach().item(), + "policy_entropy": entropy.detach().item(), + "ppo_clip_ratio": clip_ratio, + "policy_kl": kl_loss.detach().item(), + } + return loss, metrics + + def forward_step(batch_iter, model): + batch = next(batch_iter) + + sequences = batch["sequences"] + attention_mask = batch["attention_mask"].to(bool) + position_ids = batch["position_ids"] + + if self.use_sample_packing: + new_sequences, packed_seq_params = preprocess_packed_seqs( + sequences, + attention_mask, + pre_process=mpu.is_pipeline_first_stage(ignore_virtual=True), + ) + new_attention_mask = None + new_position_ids = None + else: + new_sequences, new_attention_mask, new_position_ids = remove_left_padding( + sequences, + attention_mask, + position_ids, + pre_process=mpu.is_pipeline_first_stage(ignore_virtual=True), + ) + packed_seq_params = None + + outputs = model( + new_sequences, + new_position_ids, + new_attention_mask, + packed_seq_params=packed_seq_params, + ) + + if self.use_sample_packing: + outputs = postprocess_packed_seqs( + outputs, + packed_seq_params, + attention_mask, + micro_batch_size, + seq_len, + post_process=mpu.is_pipeline_last_stage(ignore_virtual=True), + ) + else: + outputs = recover_left_padding( + outputs, + new_attention_mask, + attention_mask, + seq_len, + post_process=mpu.is_pipeline_last_stage(ignore_virtual=True), + ) + + return outputs, partial(loss_func, data=batch) + + # batch should be a list of micro-batches + batch_generator = make_batch_generator(micro_batches, vpp_size=len(self.actor_module)) + + metrics_list = forward_backward_func( + forward_step_func=forward_step, + data_iterator=batch_generator, + model=self.actor_module, + num_microbatches=len(micro_batches), + seq_length=seq_len, + micro_batch_size=micro_batch_size, + forward_only=False, + ) + + # broadcast metrics to all pp ranks + if not mpu.is_pipeline_last_stage(ignore_virtual=True): + metrics_list = [None] * len(micro_batches) + with torch.no_grad(): + torch.distributed.broadcast_object_list( + metrics_list, + src=mpu.get_pipeline_model_parallel_last_rank(), + group=mpu.get_pipeline_model_parallel_group(), + ) + + return metrics_list diff --git a/skyrl-train/skyrl_train/workers/megatron/megatron_worker.py b/skyrl-train/skyrl_train/workers/megatron/megatron_worker.py new file mode 100644 index 0000000000..198905939c --- /dev/null +++ b/skyrl-train/skyrl_train/workers/megatron/megatron_worker.py @@ -0,0 +1,770 @@ +import torch +import torch.nn as nn +import torch.distributed +import ray +from transformers import AutoTokenizer, AutoConfig +from huggingface_hub import snapshot_download + +import os +from datetime import timedelta +from typing import List, Dict, Any, Optional +from collections import defaultdict +from tqdm import tqdm +from omegaconf import OmegaConf + +from megatron.bridge import AutoBridge +from megatron.bridge.peft.lora import LoRA +from megatron.bridge.peft.canonical_lora import CanonicalLoRA +import megatron.core.parallel_state as mpu +from megatron.core.optimizer import DistributedOptimizer +from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler + +from skyrl_train.distributed.megatron.optimizer import ( + init_megatron_optim_config, + get_megatron_optimizer, + get_megatron_optimizer_param_scheduler, +) +from skyrl_train.distributed.dispatch import MeshRank +from skyrl_train.distributed.megatron.megatron_strategy import MegatronStrategy +from skyrl_train.distributed.megatron.megatron_utils import print_model_size, broadcast_object_across_pp_ranks +from skyrl_train.utils.utils import update_model_config, str_to_torch_dtype +from skyrl_train.env_vars import SKYRL_WORKER_NCCL_TIMEOUT_IN_S +from skyrl_train.training_batch import TrainingOutputBatch +from skyrl_train.workers.worker_utils import BatchIterator, reduce_metrics +from skyrl_train.workers.worker import ( + PolicyWorkerBase, + RefWorkerBase, + CriticWorkerBase, +) +from skyrl_train.workers.megatron.megatron_model_wrapper import MegatronModelWrapper +from skyrl_train.utils.profiler import Profiler +from skyrl_train.weight_sync import WeightExtractor, WeightChunk + + +class MegatronWeightExtractor(WeightExtractor): + """Extracts weights from Megatron model-parallel models. + + Uses Megatron's bridge to export weights in HuggingFace format. + + Args: + bridge: Megatron AutoBridge instance for weight conversion + actor_module: The actor module to extract weights from + enable_bucketing: If True, group parameters into size-based buckets for packing + bucket_size_threshold_GB: Size threshold in GB for bucketing (only used if enable_bucketing=True) + training_dtype: Training dtype for size calculation (only used if enable_bucketing=True) + """ + + def __init__( + self, + bridge, + actor_module, + enable_bucketing: bool = False, + bucket_size_threshold_GB: float = 1.0, + training_dtype: torch.dtype = torch.bfloat16, + ): + self.bridge = bridge + self.actor_module = actor_module + self.enable_bucketing = enable_bucketing + self.bucket_size_threshold_GB = bucket_size_threshold_GB + self.training_dtype = training_dtype + + # Initialize bucketing if enabled + if enable_bucketing: + self._init_param_buckets() + else: + self.param_buckets = None + + def _init_param_buckets(self): + """Initialize parameter buckets for packing.""" + # Get conversion tasks from bridge + weight_conversion_tasks = self.bridge.get_conversion_tasks(self.actor_module) + + # Calculate size for each parameter + param_info = [] + + def calculate_size_in_bytes(param, tp_size, ep_size): + if param is None: + # need to broadcast for other pp ranks + size_in_bytes = None + else: + # Calculate size for this parameter + prec_to_bytes = { + torch.bfloat16: 2, + torch.float32: 4, + } + scale = prec_to_bytes[self.training_dtype] / prec_to_bytes[param.dtype] + size_in_bytes = param.element_size() * param.numel() * tp_size * ep_size * scale + + # Broadcast size_in_bytes across pipeline parallel ranks + return broadcast_object_across_pp_ranks(size_in_bytes) + + for task in weight_conversion_tasks: + param_info.append( + ( + task, + calculate_size_in_bytes( + task.param_weight, + task.mapping.tp_size, + task.mapping.ep_size if task.mapping.is_expert else 1, + ), + ) + ) + + # Group parameters into buckets based on size threshold + self.param_buckets = [[]] + curr_size = 0 + for task, size in param_info: + if curr_size + size > self.bucket_size_threshold_GB * 1024**3: + self.param_buckets.append([]) + curr_size = 0 + self.param_buckets[-1].append(task) + curr_size += size + + def extract_weights(self, dtype: torch.dtype): + """Extract weights from Megatron model. + + Args: + dtype: Target dtype for inference + + Yields: + WeightChunk objects (one per parameter, or one per bucket if bucketing enabled) + """ + device = torch.cuda.current_device() + + if not self.enable_bucketing: + # No bucketing: yield one chunk per parameter + hf_params_generator = self.bridge.export_hf_weights( + self.actor_module, + show_progress=False, + conversion_tasks=None, + ) + + for name, tensor in hf_params_generator: + # Move to device and convert dtype + tensor = tensor.to(device=device, dtype=dtype, non_blocking=True) + + yield WeightChunk( + names=[name], + dtypes=[str(dtype)], + shapes=[list(tensor.shape)], + tensors=[tensor], + ) + else: + # Bucketing mode: iterate over buckets, yield one chunk per bucket + for bucket in self.param_buckets: + hf_params_generator = self.bridge.export_hf_weights( + self.actor_module, + show_progress=False, + conversion_tasks=bucket, + ) + + # Collect all parameters in this bucket into one chunk + names = [] + dtypes_list = [] + shapes = [] + tensors = [] + + for name, tensor in hf_params_generator: + # Move to device and convert dtype + tensor = tensor.to(device=device, dtype=dtype, non_blocking=True) + + names.append(name) + dtypes_list.append(str(dtype)) + shapes.append(list(tensor.shape)) + tensors.append(tensor) + + # Yield one chunk containing all parameters in this bucket + if tensors: + yield WeightChunk( + names=names, + dtypes=dtypes_list, + shapes=shapes, + tensors=tensors, + ) + + +class MegatronWorker: + def init_configs( + self, + model_path, + megatron_config, + model_config_kwargs, + transformer_config_kwargs, + bf16=True, + flash_attn=False, + lora_config=None, + ): + """ + Initialize the Megatron-Bridge bridge and provider objects + hf_config and tokenizer + """ + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + hf_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + + override_config_kwargs = { + "bos_token_id": tokenizer.bos_token_id, + "eos_token_id": tokenizer.eos_token_id, + "pad_token_id": tokenizer.pad_token_id, + } + override_config_kwargs.update(model_config_kwargs.get("model_config", {})) + update_model_config(hf_config, override_config_kwargs=override_config_kwargs) + + # if flash_attn is enabled, we use flash attention backend, otherwise fall back to fused attention backend + transformer_config_kwargs = OmegaConf.to_container(transformer_config_kwargs, resolve=True) + transformer_config_kwargs["attention_backend"] = "flash" if flash_attn else "fused" + + if not self.cfg.trainer.gradient_checkpointing: + for key in ("recompute_granularity", "recompute_method", "recompute_num_layers"): + transformer_config_kwargs[key] = None + + bridge = AutoBridge.from_hf_pretrained(model_path, trust_remote_code=True) + provider = bridge.to_megatron_provider() + provider.tensor_model_parallel_size = megatron_config.tensor_model_parallel_size + provider.pipeline_model_parallel_size = megatron_config.pipeline_model_parallel_size + provider.pipeline_dtype = torch.bfloat16 if bf16 else torch.float32 + provider.context_parallel_size = megatron_config.context_parallel_size + provider.expert_model_parallel_size = megatron_config.expert_model_parallel_size + provider.expert_tensor_parallel_size = megatron_config.expert_tensor_parallel_size + provider.sequence_parallel = megatron_config.tensor_model_parallel_size > 1 + provider.attention_backend = "flash" if flash_attn else "fused" + provider.variable_seq_lengths = True + provider.masked_softmax_fusion = True + provider.moe_token_dispatcher_type = "alltoall" + provider.moe_router_load_balancing_type = "none" + + for k, v in transformer_config_kwargs.items(): + setattr(provider, k, v) + provider.finalize() + + self.provider = provider + self.bridge = bridge + + self.strategy.hf_config = hf_config + self.tokenizer = tokenizer + + def configure_lora(self, lora_config, lora_type: Optional[str] = "lora"): + if lora_type == "lora": + self.lora_cls = LoRA( + target_modules=( + ["linear_qkv", "linear_proj", "linear_fc1", "linear_fc2"] + if lora_config.target_modules == "all-linear" + else lora_config.target_modules + ), + dim=lora_config.rank, + alpha=lora_config.alpha, + dropout=lora_config.dropout, + lora_A_init_method=lora_config.init_method, + lora_B_init_method="zero", + exclude_modules=[] if lora_config.exclude_modules is None else lora_config.exclude_modules, + lora_dtype=torch.bfloat16 if self.cfg.trainer.bf16 else torch.float32, + ) + elif lora_type == "canonical_lora": + self.lora_cls = CanonicalLoRA( + target_modules=( + [ + "linear_q", + "linear_k", + "linear_v", + "linear_proj", + "linear_fc1_up", + "linear_fc1_gate", + "linear_fc2", + ] + if lora_config.target_modules == "all-linear" + else lora_config.target_modules + ), + dim=lora_config.rank, + alpha=lora_config.alpha, + dropout=lora_config.dropout, + lora_A_init_method=lora_config.init_method, + lora_B_init_method="zero", + exclude_modules=[] if lora_config.exclude_modules is None else lora_config.exclude_modules, + ) + + def make_megatron_module( + self, + wrap_with_ddp: bool = True, + ddp_config: Optional[Dict[str, Any]] = None, + lora_config: Optional[Dict[str, Any]] = None, + lora_type: Optional[str] = "lora", + bf16: bool = True, + ) -> List[nn.Module]: + """ + Creates a megatron GPTModel (optionally DDP wrapped) using the bridge. + """ + from megatron.core.distributed.distributed_data_parallel_config import DistributedDataParallelConfig + + if lora_config is not None: + self.configure_lora(lora_config, lora_type) + + def lora_pre_wrap_hook(model): + lora_model = self.lora_cls(model, training=True) + self.lora_cls.set_params_to_save(lora_model) + + return lora_model + + self.provider.register_pre_wrap_hook(lora_pre_wrap_hook) + + default_ddp_config = DistributedDataParallelConfig() + if wrap_with_ddp: + default_ddp_config.use_distributed_optimizer = True + if ddp_config is not None: + for k, v in ddp_config.items(): + setattr(default_ddp_config, k, v) + model = self.provider.provide_distributed_model( + ddp_config=default_ddp_config, wrap_with_ddp=wrap_with_ddp, bf16=bf16 + ) + return model + + def forward(self, data): + """ + Override `Worker.forward` to support passing the full mini batch to the MegatronModelWrapper.forward method. + """ + # Run in micro batches grouped into a single mini-batch + micro_bsz = self.cfg.trainer.micro_forward_batch_size_per_gpu + micro_batches = data.chunk(micro_bsz) + + # Build micro-batch dicts expected by policy.forward_mini_batch + micro_dicts = [] + device = torch.cuda.current_device() + for micro in micro_batches: + micro.to(device) + sequences = micro["sequences"] + attention_mask = micro["attention_mask"] + num_actions = micro.metadata["response_length"] + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 0) + micro_dicts.append( + { + "sequences": sequences, + "attention_mask": attention_mask, + "position_ids": position_ids, + "num_actions": num_actions, + } + ) + + self.model.eval() + seq_len = micro_dicts[0]["sequences"].shape[1] + mbs = micro_dicts[0]["sequences"].shape[0] + with torch.no_grad(): + log_probs = self.model.forward( + micro_batches=micro_dicts, + seq_len=seq_len, + micro_batch_size=mbs, + temperature=self.cfg.generator.sampling_params.temperature, + ) + + log_probs = log_probs.to("cpu") + output = TrainingOutputBatch({"output": log_probs}) + output.metadata = data.metadata + return output + + def save_hf_model(self, export_dir: str, tokenizer): + # Save model in HuggingFace safetensors format + self.strategy.save_hf_model( + self.bridge, + self.model, + export_dir, + tokenizer=tokenizer, + ) + + +class MegatronPolicyWorkerBase(MegatronWorker, PolicyWorkerBase): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.model: MegatronModelWrapper = None + self.actor_module: List[nn.Module] = None + self.scheduler: OptimizerParamScheduler = None + self.optimizer: DistributedOptimizer = None + self.profiler: Profiler = None + self._is_lora = self.cfg.trainer.policy.model.lora.rank > 0 + + def offload_to_cpu(self, pin_memory=True, non_blocking=True, offload_optimizer=True, offload_model=True): + self._set_numa_affinity(torch.distributed.get_rank() % torch.cuda.device_count()) + self.strategy.offload_to_cpu( + self.actor_module, self.optimizer, pin_memory, non_blocking, offload_optimizer, offload_model + ) + + def backload_to_gpu(self, non_blocking=True, backload_optimizer=True, backload_model=True): + self.strategy.backload_to_gpu( + self.actor_module, self.optimizer, non_blocking, backload_optimizer, backload_model + ) + + def init_worker_process_group(self): + """ + Override DistributedTorchRayActor.init_worker_process_group to use megatron distributed setup to create the mesh. + """ + if not torch.distributed.is_initialized(): + # Default torch dist pg init timeout is 10 minutes (600 seconds) + torch.distributed.init_process_group( + backend="nccl", timeout=timedelta(seconds=SKYRL_WORKER_NCCL_TIMEOUT_IN_S) + ) + + # Explicitly wrap torch.distributed.broadcast in torch.no_grad() to avoid a warning in Megatron training where the + # autograd engine tries to track gradients through the default Torch kernel. This fixes a deprecated behaviour in + # PyTorch, preventing potential silent errors in future versions. + + if not getattr(torch.distributed, "_skyrl_broadcast_no_grad_patched", False): + _orig_broadcast = torch.distributed.broadcast + + def _broadcast_no_grad(*args, **kwargs): + with torch.no_grad(): + return _orig_broadcast(*args, **kwargs) + + torch.distributed.broadcast = _broadcast_no_grad + torch.distributed._skyrl_broadcast_no_grad_patched = True + + self.strategy = MegatronStrategy( + megatron_config=self.cfg.trainer.policy.megatron_config, + optimizer_config=self.cfg.trainer.policy.optimizer_config, + seed=self.cfg.trainer.seed, + is_lora=self._is_lora, + ) + self.strategy.setup_distributed() + + self.mesh_rank = MeshRank( + dp=mpu.get_data_parallel_rank(), + sp=mpu.get_context_parallel_rank(), + tp=mpu.get_tensor_model_parallel_rank(), + pp=mpu.get_pipeline_model_parallel_rank(), + world_size=self._world_size, + dp_size=mpu.get_data_parallel_world_size(), + pp_size=mpu.get_pipeline_model_parallel_world_size(), + ) + + def _normalize_mini_batch_size(self): + """ + Override to set Megatron-specific batch size attributes. + + Megatron's ppo_train method needs policy_mini_batch_size_per_gpu to compute + how many micro batches fit in a mini batch for gradient accumulation. + """ + super()._normalize_mini_batch_size() # Sets _micro_batches_accumulated + + # Megatron-specific: compute mini batch size per GPU for ppo_train + n_samples = self.cfg.generator.n_samples_per_prompt + dp_size = self.mesh_rank.dp_size + self.policy_mini_batch_size_per_gpu = (self.cfg.trainer.policy_mini_batch_size * n_samples) // dp_size + + def init_model(self, model_path, num_training_steps: int = 1e9): + """ + Initialize the model, optimizer, and scheduler for the policy worker. + """ + # initialize the bridge and provider objects + self.init_configs( + model_path, + self.cfg.trainer.policy.megatron_config, + self.cfg.trainer.policy.megatron_config.model_config_kwargs, + self.cfg.trainer.policy.megatron_config.transformer_config_kwargs, + bf16=self.cfg.trainer.bf16, + flash_attn=self.cfg.trainer.flash_attn, + ) + + # wrap with DDP for training + self.actor_module = self.make_megatron_module( + wrap_with_ddp=True, + ddp_config=self.cfg.trainer.policy.megatron_config.ddp_config, + lora_config=self.cfg.trainer.policy.model.lora if self._is_lora else None, + lora_type=self.cfg.trainer.policy.megatron_config.lora_config.lora_type, + bf16=self.cfg.trainer.bf16, + ) + + if self._local_rank == 0 and not os.path.exists( + model_path + ): # if not local path, try downloading model weights from huggingface + snapshot_download(model_path) # will be no-op if already downloaded + torch.distributed.barrier() + + if self._rank == 0: + print_model_size(self.actor_module[0]) + + # create profiler + if self.cfg.trainer.policy.megatron_config.torch_profiler_config.enable: + self.profiler = Profiler(self.cfg.trainer.policy.megatron_config.torch_profiler_config) + + # create optimizer + optim_config = init_megatron_optim_config( + self.cfg.trainer.policy.optimizer_config, self.cfg.trainer.policy.megatron_config.optimizer_config_kwargs + ) + self.optimizer = get_megatron_optimizer(self.actor_module, optim_config) + + self._normalize_mini_batch_size() + + # create scheduler + self.scheduler = get_megatron_optimizer_param_scheduler( + optimizer=self.optimizer, + config=self.cfg.trainer.policy.optimizer_config, + num_training_steps=num_training_steps, + ) + + # create worker model + self.model = MegatronModelWrapper( + config=self.cfg, + actor_module=self.actor_module, + actor_optimizer=self.optimizer, + policy_loss_fn=self.policy_loss_fn, + ) + + # Initialize weight extractor + # TODO(haochen): Now bucketing is only enabled for the CUDA IPC + # transfer strategy, we can enable it for other strategies as well. + from skyrl_train.weight_sync import CudaIpcTransferStrategy + + self.weight_extractor = MegatronWeightExtractor( + bridge=self.bridge, + actor_module=self.actor_module, + enable_bucketing=self._transfer_strategy_cls is CudaIpcTransferStrategy, + bucket_size_threshold_GB=self.cfg.generator.weight_transfer_threshold_cuda_ipc_GB, + training_dtype=torch.bfloat16 if self.cfg.trainer.bf16 else torch.float32, + ) + + self.empty_cuda_cache = self.cfg.trainer.policy.megatron_config.empty_cuda_cache + + def ppo_train(self, train_data) -> "TrainingOutputBatch": + """ + Overrides `PolicyWorkerBase.ppo_train` for megatron. + + Since we want megatron to handle gradient accumulation over micro batches, we directly pass mini batches into the + worker MegatronModelWrapper.forward_backward_mini_batch method. + """ + dataloader = BatchIterator( + train_data, sample_batch_size=self.cfg.trainer.micro_train_batch_size_per_gpu, drop_last=False + ) + + micro_batches_per_mini_batch = ( + self.policy_mini_batch_size_per_gpu // self.cfg.trainer.micro_train_batch_size_per_gpu + ) + + status_list = [] + all_metrics = defaultdict(list) + policy_update_steps = 0 + + if self.profiler is not None: + self.profiler.start() + + for epoch in range(self.cfg.trainer.update_epochs_per_batch): + self.optimizer.zero_grad() + pbar = tqdm( + dataloader, + desc=f"Policy Train epoch [{epoch + 1}/{self.cfg.trainer.update_epochs_per_batch}]", + disable=not self.strategy.is_rank_0(), + ) + + # TODO: Convert this into 2 loops for minibatches and microbatches. + micro_buffer = [] + for local_step, experience in enumerate(pbar): + # BatchIterator now yields Experience objects directly + experience.to_device(torch.cuda.current_device()) + sequences = experience.sequences + attention_mask = experience.attention_mask + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 0) + + micro_buffer.append( + { + "sequences": sequences, + "attention_mask": attention_mask, + "position_ids": position_ids, + "num_actions": experience.num_actions, + "old_action_log_probs": experience.action_log_probs, + "base_action_log_probs": experience.base_action_log_probs, + "advantages": experience.advantages, + "loss_mask": experience.loss_mask, + "rollout_action_logprobs": experience.rollout_logprobs, + } + ) + + if len(micro_buffer) == micro_batches_per_mini_batch: + # run mini-batch forward-backward and then one optimizer step + self.model.train() + for chunk in self.actor_module: + # if use distributed optimizer, zero grad buffer will be handled by optimizer + chunk.zero_grad_buffer() + seq_len = micro_buffer[0]["sequences"].shape[1] + micro_bsz = micro_buffer[0]["sequences"].shape[0] + + metrics_list = self.model.forward_backward_mini_batch( + micro_batches=micro_buffer, + seq_len=seq_len, + micro_batch_size=micro_bsz, + temperature=self.cfg.generator.sampling_params.temperature, + ) + + if self.empty_cuda_cache: + torch.cuda.empty_cache() + + grad_norm = self.strategy.optimizer_step(self.optimizer, self.model, self.scheduler, name="actor") + + # within a DP group, metrics are already the same across all workers - we then just all reduce across + # the whole world size to get the metrics for the global micro batch + for i, metrics in enumerate(metrics_list): + status = { + "final_loss": metrics["final_loss"], + "policy_loss": metrics["policy_loss"], + "policy_lr": self.optimizer.param_groups[0]["lr"], + "ppo_clip_ratio": metrics["ppo_clip_ratio"], + "policy_entropy": metrics["policy_entropy"], + } + if self.cfg.trainer.algorithm.use_kl_loss: + status["policy_kl"] = metrics["policy_kl"] + + # Attach grad norm only for the last micro in the mini-batch + if i == len(metrics_list) - 1 and grad_norm is not None: + status["raw_grad_norm"] = grad_norm + + # attach response_length + status["response_length"] = micro_buffer[i]["num_actions"] + + status = self.strategy.all_reduce(status) + status_list.append(status) + for k, v in status.items(): + all_metrics[k].append(v) + + short_status = { + "pg": status_list[-1]["policy_loss"], + "glen": status_list[-1]["response_length"], + "policy_lr": status_list[-1]["policy_lr"], + "ent": status_list[-1]["policy_entropy"], + } + if "raw_grad_norm" in status_list[-1]: + short_status["grad_norm"] = status_list[-1]["raw_grad_norm"] + pbar.set_postfix(short_status) + + policy_update_steps += 1 + micro_buffer = [] + + # drop any trailing micros that don't fill a mini-batch (keep behavior consistent) + micro_buffer = [] + + torch.distributed.barrier() + if self.profiler is not None: + self.profiler.stop_and_save() + self.profiler.stop_trace() + + # not needed beyond status logging + all_metrics.pop("response_length", None) + + status_mean = reduce_metrics(all_metrics, ignore_keys=["grad:update_diversity"]) + status_mean["policy_update_steps"] = policy_update_steps + + output = TrainingOutputBatch() + output.metadata = {"train_status": status_mean} + return output + + async def broadcast_to_inference_engines(self, inference_engine_client): + use_prefix_cache = self.cfg.generator.enable_prefix_caching + generator_dtype = str_to_torch_dtype(self.cfg.generator.model_dtype) + cache_reset_task = None + if use_prefix_cache and torch.distributed.get_rank() == 0: + # clear prefix cache + cache_reset_task = inference_engine_client.reset_prefix_cache() + + torch.cuda.empty_cache() + + # Extract and send weights using the sender created at init time + await self._weight_transfer_sender.send_chunks(self.weight_extractor.extract_weights(generator_dtype)) + + if cache_reset_task is not None: + await cache_reset_task + torch.cuda.empty_cache() + torch.distributed.barrier() + + def get_weight_statistics(self): + """Compute lightweight statistics for model weights""" + raise NotImplementedError() + + def _set_pad_token_id(self, pad_token_id): + # this already gets set in the init_model method + pass + + +class MegatronRefWorkerBase(MegatronWorker, RefWorkerBase): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.model: MegatronModelWrapper = None + self.actor_module: List[nn.Module] = None + + def offload_to_cpu(self, pin_memory=True, non_blocking=True, **kwargs): + self._set_numa_affinity(torch.distributed.get_rank() % torch.cuda.device_count()) + self.strategy.offload_to_cpu(self.actor_module, None, pin_memory, non_blocking) + + def backload_to_gpu(self, non_blocking=True, **kwargs): + self.strategy.backload_to_gpu(self.actor_module, None, non_blocking) + + def init_worker_process_group(self): + """ + Override DistributedTorchRayActor.init_worker_process_group to use megatron distributed setup to create the mesh. + """ + if not torch.distributed.is_initialized(): + # Default torch dist pg init timeout is 10 minutes (600 seconds) + torch.distributed.init_process_group( + backend="nccl", timeout=timedelta(seconds=SKYRL_WORKER_NCCL_TIMEOUT_IN_S) + ) + + self.strategy = MegatronStrategy( + megatron_config=self.cfg.trainer.ref.megatron_config, + optimizer_config=None, + seed=self.cfg.trainer.seed, + ) + self.strategy.setup_distributed() + + self.mesh_rank = MeshRank( + dp=mpu.get_data_parallel_rank(), + sp=mpu.get_context_parallel_rank(), + tp=mpu.get_tensor_model_parallel_rank(), + pp=mpu.get_pipeline_model_parallel_rank(), + world_size=self._world_size, + dp_size=mpu.get_data_parallel_world_size(), + pp_size=mpu.get_pipeline_model_parallel_world_size(), + ) + + def init_model(self, model_path, num_training_steps: int = 1e9): + """ + Initialize the model for the ref worker. + """ + # initialize the bridge and provider objects + self.init_configs( + model_path, + self.cfg.trainer.ref.megatron_config, + self.cfg.trainer.ref.megatron_config.model_config_kwargs, + self.cfg.trainer.ref.megatron_config.transformer_config_kwargs, + bf16=self.cfg.trainer.bf16, + flash_attn=self.cfg.trainer.flash_attn, + ) + + self.actor_module = self.make_megatron_module( + wrap_with_ddp=False, + ddp_config=None, + bf16=self.cfg.trainer.bf16, + ) + + # download model weights from huggingface (need to be done for ref worker as well, else errors when colocate_all=False) + if self._local_rank == 0 and not os.path.exists( + model_path + ): # if not local path, try downloading model weights from huggingface + snapshot_download(model_path) # will be no-op if already downloaded + torch.distributed.barrier() + + # load weights + if self._rank == 0: + print_model_size(self.actor_module[0]) + + # create worker model + self.model = MegatronModelWrapper(config=self.cfg, actor_module=self.actor_module) + + def get_weight_statistics(self): + """Compute lightweight statistics for model weights""" + raise NotImplementedError() + + def _set_pad_token_id(self, pad_token_id): + # this already gets set in the init_model method + pass + + +class MegatronCriticWorkerBase(MegatronWorker, CriticWorkerBase): + def __init__(self, **kwargs): + raise NotImplementedError() + + +PolicyWorker = ray.remote(num_gpus=1)(MegatronPolicyWorkerBase) +RefWorker = ray.remote(num_gpus=1)(MegatronRefWorkerBase) +CriticWorker = ray.remote(num_gpus=1)(MegatronCriticWorkerBase) diff --git a/skyrl-train/skyrl_train/workers/worker.py b/skyrl-train/skyrl_train/workers/worker.py new file mode 100644 index 0000000000..4adfb03bba --- /dev/null +++ b/skyrl-train/skyrl_train/workers/worker.py @@ -0,0 +1,1313 @@ +import asyncio +import logging +import os +import socket +from collections import defaultdict +from ctypes import CDLL, POINTER, Structure, c_char_p, c_int, c_ulong, c_void_p +from datetime import timedelta +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Type + +import ray +import torch +import torch.distributed +import torch.nn as nn +from loguru import logger +from omegaconf import DictConfig +from ray import ObjectRef +from ray.util.placement_group import ( + PlacementGroup, + PlacementGroupSchedulingStrategy, + placement_group, + placement_group_table, +) +from torch.optim import Optimizer +from torch.optim.lr_scheduler import LRScheduler +from transformers import PreTrainedModel + +from skyrl_train.dataset.replay_buffer import Experience +from skyrl_train.distributed.dispatch import ( + ActorInfo, + Dispatch, + DispatchRegistry, + MeshRank, +) +from skyrl_train.distributed.strategy import DistributedStrategy +from skyrl_train.distributed.ulysses import ( + apply_monkey_patch, + set_ulysses_sequence_parallel_group, +) +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.training_batch import TrainingInputBatch, TrainingOutputBatch +from skyrl_train.utils import ( + get_ray_pg_ready_with_timeout, + get_reordered_bundle_indices, + ray_noset_visible_devices, +) +from skyrl_train.env_vars import ( + SKYRL_RAY_PG_TIMEOUT_IN_S, + SKYRL_WORKER_NCCL_TIMEOUT_IN_S, +) +from skyrl_train.utils.io import io +from skyrl_train.utils.ppo_utils import ( + PolicyLossRegistry, + compute_approx_kl, + masked_mean, + ppo_critic_loss, +) +from skyrl_train.utils.utils import configure_ray_worker_logging +from skyrl_train.workers.worker_utils import BatchIterator, reduce_metrics + +_SET_AFFINITY = False + + +# Adapted from OpenRLHF: https://github.com/OpenRLHF/OpenRLHF/blob/main/openrlhf/trainer/ray/launcher.py#L17 +class DistributedTorchRayActor: + def __init__( + self, world_size, rank, local_rank, master_addr, master_port, sequence_parallel_size, record_memory=False + ): + logging.basicConfig( + format="%(asctime)s %(levelname)-8s %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", + ) + self._world_size = world_size + self._rank = rank + self._local_rank = local_rank + self._master_addr = master_addr if master_addr else self._get_current_node_ip() + self._master_port = master_port if master_port else self._get_free_port() + os.environ["MASTER_ADDR"] = self._master_addr + os.environ["MASTER_PORT"] = str(self._master_port) + os.environ["WORLD_SIZE"] = str(self._world_size) + os.environ["RANK"] = str(self._rank) + # NOTE: Ray will automatically set the CUDA_VISIBLE_DEVICES + # environment variable for each actor, so always set device to 0 + # os.environ["LOCAL_RANK"] = str(self._local_rank) + os.environ["LOCAL_RANK"] = str(ray.get_gpu_ids()[0]) if ray_noset_visible_devices() else "0" + self.sequence_parallel_size: int = sequence_parallel_size + + self.record_memory = record_memory + if record_memory: + torch.cuda.memory._record_memory_history() + configure_ray_worker_logging() + + def get_node_local_rank(self): + return self._local_rank + + def init_worker_process_group(self): + if not torch.distributed.is_initialized(): + # Default torch dist pg init timeout is 10 minutes (600 seconds) + torch.distributed.init_process_group( + backend="nccl", timeout=timedelta(seconds=SKYRL_WORKER_NCCL_TIMEOUT_IN_S) + ) + + # setup device mesh + # TODO: Support TP / PP for additional backends + # NOTE (sumanthrh): Device mesh and mesh rank are rank specific attributes. For the current way the strategy is defined, it is only meant to interact with worker state; not hold worker state. Thus, this should live outside the strategy object. + # This device mesh can be common across all the strategies we use + dp_size = self._world_size // self.sequence_parallel_size + device_mesh = torch.distributed.device_mesh.init_device_mesh( + "cuda", mesh_shape=(dp_size, self.sequence_parallel_size), mesh_dim_names=("dp", "sp") + ) + self.device_mesh = device_mesh + self.mesh_rank = MeshRank( + dp=self.device_mesh.get_local_rank(mesh_dim="dp"), + sp=self.device_mesh.get_local_rank(mesh_dim="sp"), + tp=0, + pp=0, + world_size=self._world_size, + dp_size=self.device_mesh.size(0), + pp_size=1, + ) + + def _seq_parallel_monkey_patch(self, model: PreTrainedModel, use_parent_class: bool = False): + # NOTE (sumanthrh): This sets a global variable that is used during the forward pass for sequence parallelism + # This works because each worker is it's own process and thus different worker types are isolated + # TODO (sumanthrh): We should re-visit this and see if we should adopt a context-manager pattern for sequence parallelism + if self.sequence_parallel_size > 1: + set_ulysses_sequence_parallel_group(self.device_mesh["sp"].get_group()) + apply_monkey_patch( + model=model, ulysses_sp_size=self.sequence_parallel_size, use_parent_class=use_parent_class + ) + + def get_mesh_rank(self): + return self.mesh_rank + + def get_gpu_id(self): + return ray.get_gpu_ids()[0] + + @staticmethod + def _get_current_node_ip(): + address = ray._private.services.get_node_ip_address() + # strip ipv6 address + return address.strip("[]") + + def get_ray_node_id(self): + return ray.get_runtime_context().get_node_id() + + @staticmethod + def _get_free_port(): + with socket.socket() as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + + def get_master_addr_port(self): + return self._master_addr, self._master_port + + # TODO(tgriggs): For numa affinity, pass in the Worker._local_rank for the second arg here. Distinguish 'rank' and 'local_rank' differ here. + def _set_numa_affinity(self, rank): + def local_rank_to_real_gpu_id(local_rank): + cuda_visible_devices = [ + int(x) for x in os.environ.get("CUDA_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7").split(",") + ] + return cuda_visible_devices[local_rank] + + rank = local_rank_to_real_gpu_id(rank) + + global _SET_AFFINITY + if _SET_AFFINITY: + return + + from ctypes.util import find_library + + class bitmask_t(Structure): + _fields_ = [ + ("size", c_ulong), + ("maskp", POINTER(c_ulong)), + ] + + try: + LIBNUMA = CDLL(find_library("numa")) + except Exception as e: + logger.error(f"Skipping NUMA affinity setup because libnuma is not installed: {e}") + _SET_AFFINITY = True + return + + LIBNUMA.numa_parse_nodestring.argtypes = [c_char_p] + LIBNUMA.numa_parse_nodestring.restype = POINTER(bitmask_t) + LIBNUMA.numa_run_on_node_mask.argtypes = [POINTER(bitmask_t)] + LIBNUMA.numa_run_on_node_mask.restype = c_int + LIBNUMA.numa_set_membind.argtypes = [POINTER(bitmask_t)] + LIBNUMA.numa_set_membind.restype = c_void_p + LIBNUMA.numa_num_configured_nodes.argtypes = [] + LIBNUMA.numa_num_configured_nodes.restype = c_int + + def numa_bind(nid: int): + bitmask = LIBNUMA.numa_parse_nodestring(bytes(str(nid), "ascii")) + LIBNUMA.numa_run_on_node_mask(bitmask) + LIBNUMA.numa_set_membind(bitmask) + + numa_nodes = LIBNUMA.numa_num_configured_nodes() + if numa_nodes <= 0: + numa_nodes = 1 + num_gpu_pre_numa_node = max(1, 8 // numa_nodes) + target_nid = min(numa_nodes - 1, self._local_rank // num_gpu_pre_numa_node) + numa_bind(target_nid) + _SET_AFFINITY = True + + +class Worker(DistributedTorchRayActor): + def __init__(self, cfg: DictConfig, *args, **kwargs): + from skyrl_train.weight_sync import get_transfer_strategy_cls + + super().__init__(*args, **kwargs) + self.cfg = cfg + self._transfer_strategy_cls = get_transfer_strategy_cls(self.cfg) + + def init_model(self, *args, **kwargs): + """Initialize worker state (model, and optimizer if applicable) on worker.""" + raise NotImplementedError() + + def empty_cache(self) -> None: + """Empty GPU memory cache on Worker's CUDA device""" + torch.cuda.empty_cache() + + def offload_to_cpu(self, pin_memory=True, non_blocking=True): + """Offload all worker state to CPU. + + After this function runs, only temporary reserved memory and torch's pre-loaded cuda kernels (~ GB) will remain + + Args: + pin_memory: Whether to use pinned/ paged-locked memory on CPU + non_blocking: Whether the operation is non-blocking + """ + raise NotImplementedError() + + def backload_to_gpu(self, non_blocking=True): + """Backload worker state to GPU + + Args: + non_blocking: Whether the operation is non-blocking + """ + raise NotImplementedError() + + def get_cuda_memory(self) -> Dict[str, Any]: + """Get CUDA memory usage on worker's CUDA device.""" + torch.cuda.synchronize() + free, total = torch.cuda.mem_get_info() + return { + "allocated": torch.cuda.memory_allocated(), + "reserved": torch.cuda.memory_reserved(), + "free": free, + "total": total, + } + + def save_memory_snapshot(self, tag: str = ""): + """Save a snapshot of memory usage on the Worker's CUDA device. + + No-ops if record_memory is False. + + Args: + tag: Label for the snapshot (e.g., "forward_backward", "optim_step") + + .. note:: + This function should be called on all the ranks in the worker group simultaneously. + """ + if not self.record_memory: + return + + # Track snapshot count for unique filenames + if not hasattr(self, "_snapshot_count"): + self._snapshot_count = 0 + self._snapshot_count += 1 + + rank = torch.distributed.get_rank() + save_path = os.path.join(self.cfg.trainer.ckpt_path, "memory_snapshots") + if self._local_rank == 0 and not io.exists(save_path): + io.makedirs(save_path, exist_ok=True) + torch.distributed.barrier() + + tag_str = f"_{tag}" if tag else "" + file_name = f"rank_{rank}{tag_str}_{self._snapshot_count}.pickle" + record_memory_path = os.path.join(save_path, file_name) + if io.exists(record_memory_path): + # seeing issues if we don't remove the file first + io.remove(record_memory_path) + torch.cuda.memory._dump_snapshot(record_memory_path) + + async def init_weight_sync_state(self, inference_engine_client: InferenceEngineClient): + """Initialize state for weight syncing with Inference Engine Client + + Creates init info and sender, then sends init info to inference engines + so they can create receivers. + + .. note:: + This function should be called on all the ranks in the worker group simultaneously. + """ + + assert inference_engine_client is not None + + # Create init info on all ranks (it's deterministic from cfg) + init_info = self._transfer_strategy_cls.create_init_info(self.cfg) + + # Create sender on all ranks + # Strategy implementations may have different logic for different ranks + tasks = [ + asyncio.to_thread( + self._transfer_strategy_cls.create_sender, + init_info=init_info, + inference_client=inference_engine_client, + ), + ] + + # Only rank 0 initializes receivers on inference engines + # NOTE: For broadcast strategy, sender and receiver init must run concurrently + # because both need to join the same process group to avoid deadlock + if torch.distributed.get_rank() == 0: + tasks.append(inference_engine_client.init_weight_update_communicator(init_info)) + + results = await asyncio.gather(*tasks) + self._weight_transfer_sender = results[0] # sender is always first task + + # # Register signal handlers for termination only on rank 0 + # NOTE (sumanthrh): This doesn't work yet, and is thus commented out. + # The better way is to just have this specified in __del__, but there is + # no guarattee that __del__ will be called in general. Ray also doesn't + # explictly call __del__ when the actor shuts down. + # It's commented out so that we can fix this in the future. + # atexit.register(self._handle_termination) + + torch.distributed.barrier() + + def forward( + self, + data: TrainingInputBatch, + ) -> TrainingOutputBatch: + """Run forward pass on the input batch in inference mode. + + This is a wrapper around `_forward_micro_batch` that runs in micro batches of `cfg.trainer.micro_forward_batch_size_per_gpu`. + """ + # run in micro batches of cfg.trainer.micro_forward_batch_size_per_gpu + # TODO (sumanthrh): this can be in the policy/critic impl if the micro batch size can be specific to policy, critic, etc. + micro_batches = data.chunk(self.cfg.trainer.micro_forward_batch_size_per_gpu) + + outputs = [] + for micro_batch in micro_batches: + outputs.append(self._forward_micro_batch(micro_batch)) + output = TrainingOutputBatch.cat(outputs) + if output.device is not None and output.device != torch.device("cpu"): + output = output.to("cpu") + return output + + def _forward_micro_batch(self, micro_batch: TrainingInputBatch) -> TrainingOutputBatch: + raise NotImplementedError() + + +# adapted from OpenReasonerZero: https://github.com/Open-Reasoner-Zero/Open-Reasoner-Zero/blob/main/orz/ppo/actors.py +class PPORayActorGroup: + """ + A group of ray actors + Functions start with 'async' should return list of object refs + + Args: + cfg: config object for workers + num_nodes (int): Number of nodes for this actor group. + num_gpus_per_node (int): Number of gpus for this actor group. + ray_actor_type (Type[Worker]): PPO model type that this actor group serve on. + pg (PlacementGroup, optional): Placement group to schedule actor on. + If none, create new placement group automatically. Defaults to None. + num_gpus_per_actor (float, optional): Number of gpus allocated for each actor. + If < 1.0, multiple models can share same gpu. Defaults to 1. + """ + + def __init__( + self, + cfg, + num_nodes, + num_gpus_per_node, + ray_actor_type: Type[Worker], + pg: Optional[PlacementGroup] = None, + num_gpus_per_actor: float = 1.0, + resources: Optional[Dict[str, float]] = None, + num_resources_per_node: Optional[int] = None, + colocate_all: bool = False, + sequence_parallel_size: int = 1, + record_memory: bool = False, + ) -> None: + self.cfg = cfg + self._num_nodes = num_nodes + self._num_gpus_per_node = num_gpus_per_node + self.ray_actor_type = ray_actor_type + + # custom resources, see https://docs.ray.io/en/latest/ray-core/scheduling/resources.html + self._resources = resources + self._num_resources_per_node = num_resources_per_node + + self.colocate_all = colocate_all + self.sequence_parallel_size = sequence_parallel_size + self.record_memory = record_memory + self._initiate_actors(pg, num_gpus_per_actor) + + def _initiate_actors(self, pg: Optional[PlacementGroup], num_gpus_per_actor: float): + """Initialize Ray actors in the worker group. + + Args: + pg: The placement group for the worker group + num_gpus_per_actor: The number of gpus to allocate per actor. + """ + world_size = self._num_nodes * self._num_gpus_per_node + if self.colocate_all: + assert ( + pg is not None + ), "if colocate_all is True, the shared placement group must be provided to PPORayActorGroup" + pg_data = placement_group_table(pg) + assert ( + len(pg_data["bundles"]) == world_size + ), "if colocate_all is True, the number of bundles in the shared placement group must match the world size" + + reordered_bundle_indices = [] + if pg is not None: + pg_data = placement_group_table(pg) + should_reorder_bundles = len(pg_data["bundles"]) == world_size + if should_reorder_bundles: + reordered_bundle_indices = get_reordered_bundle_indices(pg) + + if self._num_gpus_per_node > 1 and pg is None: + bundles = [{"GPU": self._num_gpus_per_node, "CPU": self._num_gpus_per_node} for _ in range(self._num_nodes)] + if self._resources: + resources_name = list(self._resources.keys())[0] + for i in range(len(bundles)): + bundles[i][resources_name] = self._num_resources_per_node + + pg = placement_group(bundles, strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=SKYRL_RAY_PG_TIMEOUT_IN_S) + if pg: + master_actor = self.ray_actor_type.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, + placement_group_bundle_index=reordered_bundle_indices[0] if reordered_bundle_indices else 0, + ), + ).remote( + cfg=self.cfg, + world_size=world_size, + rank=0, + local_rank=0, + master_addr=None, + master_port=None, + sequence_parallel_size=self.sequence_parallel_size, + record_memory=self.record_memory, + ) + else: + master_actor = self.ray_actor_type.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + ).remote( + cfg=self.cfg, + world_size=world_size, + rank=0, + local_rank=0, + master_addr=None, + master_port=None, + sequence_parallel_size=self.sequence_parallel_size, + record_memory=self.record_memory, + ) + self._actor_handlers = [master_actor] + # Create worker actors + if world_size > 1: + master_addr, master_port = ray.get(master_actor.get_master_addr_port.remote()) + for rank in range(1, world_size): + local_rank = rank % self._num_gpus_per_node + + if pg: + worker_actor = self.ray_actor_type.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, + placement_group_bundle_index=( + reordered_bundle_indices[rank] + if reordered_bundle_indices + else rank // self._num_gpus_per_node + ), + ), + ).remote( + cfg=self.cfg, + world_size=world_size, + rank=rank, + local_rank=local_rank, + master_addr=master_addr, + master_port=master_port, + sequence_parallel_size=self.sequence_parallel_size, + record_memory=self.record_memory, + ) + else: + worker_actor = self.ray_actor_type.options( + num_cpus=num_gpus_per_actor, + num_gpus=num_gpus_per_actor, + resources=self._resources, + ).remote( + cfg=self.cfg, + world_size=world_size, + rank=rank, + local_rank=local_rank, + master_addr=master_addr, + master_port=master_port, + sequence_parallel_size=self.sequence_parallel_size, + record_memory=self.record_memory, + ) + self._actor_handlers.append(worker_actor) + + # Initialize process group + logger.info("Initializing process group for RayActorGroup") + ray.get([actor.init_worker_process_group.remote() for actor in self._actor_handlers]) + logger.info("Initialized process group for RayActorGroup") + self.actor_infos = [ActorInfo(actor, ray.get(actor.get_mesh_rank.remote())) for actor in self._actor_handlers] + logger.info(f"Mesh Ranks: {[actor_info.rank for actor_info in self.actor_infos]}") + + def async_init_model( + self, + *args, + **kwargs, + ) -> List[ObjectRef]: + """Asynchronously initialize worker state (model, and optimizer if applicable) from model path + on all the workers. + + Returns: + A list of ray object refs. + """ + return [actor.init_model.remote(*args, **kwargs) for actor in self._actor_handlers] + + def offload_to_cpu(self, nonblocking=False, offload_optimizer=True, offload_model=True): + """Offload all worker state to CPU. + + Args: + nonblocking: Whether this operation is synchronous or asynchronous. + If `nonblocking=True`, then the function returns a list of object refs. + """ + refs = [ + actor.offload_to_cpu.remote(offload_optimizer=offload_optimizer, offload_model=offload_model) + for actor in self._actor_handlers + ] + if nonblocking: + return refs + return ray.get(refs) + + def backload_to_gpu(self, nonblocking=False, backload_optimizer=True, backload_model=True): + """Backload worker state to GPU + + Args: + nonblocking: Whether this operation is synchronous or asynchronous. + If `nonblocking=True`, then the function returns a list of ObjectRefs. + """ + refs = [ + actor.backload_to_gpu.remote(backload_optimizer=backload_optimizer, backload_model=backload_model) + for actor in self._actor_handlers + ] + if nonblocking: + return refs + return ray.get(refs) + + def run_method(self, dispatch_type: str, method_name: str, *args, **kwargs) -> Optional[TrainingOutputBatch]: + """Run a method on all actors using specified dispatch type synchronously. + + The method should either return `None` or a `TrainingOutputBatch` object. + + Args: + dispatch_type: Type of dispatch to use ("mesh" or "pass_through") + method_name: Name of the method to call on actors + *args: Positional arguments to pass to the method + **kwargs: Keyword arguments to pass to the method + + Returns: + Collect results from all the actors. + """ + dispatch_class: Dispatch = DispatchRegistry.get(dispatch_type) + # validate the dispatch args to be sent to `.dispatch` + args, kwargs = dispatch_class.validate_dispatch_args(*args, **kwargs) + + # Dispatch the method call + object_refs = dispatch_class.dispatch(self.actor_infos, method_name, *args, **kwargs) + # Collect results from all the actors + ret = dispatch_class.sync_collect(self.actor_infos, object_refs) + return ret + + def async_run_ray_method(self, dispatch_type: str, method_name: str, *args, **kwargs) -> List[ObjectRef]: + """Run a method on all actors using specified dispatch type asynchronously. + + Args: + dispatch_type: Type of dispatch to use ("mesh" or "pass_through") + method_name: Name of the method to call on actors + *args: Positional arguments to pass to the method + **kwargs: Keyword arguments to pass to the method + + Returns: + List of object references + """ + dispatch_class: Dispatch = DispatchRegistry.get(dispatch_type) + # validate the dispatch args to be sent to `.dispatch` + args, kwargs = dispatch_class.validate_dispatch_args(*args, **kwargs) + + # Dispatch the method call + object_refs = dispatch_class.dispatch(self.actor_infos, method_name, *args, **kwargs) + return object_refs + + async def async_run_method( + self, dispatch_type: str, method_name: str, *args, **kwargs + ) -> Optional[TrainingOutputBatch]: + """Run a method on all actors using specified dispatch type in an asyncio-compatible way. + + Args: + dispatch_type: Type of dispatch to use ("mesh" or "pass_through") + method_name: Name of the method to call on actors + *args: Positional arguments to pass to the method + **kwargs: Keyword arguments to pass to the method + + Returns: + TrainingOutputBatch: concatenated results from all actors + """ + dispatch_class: Dispatch = DispatchRegistry.get(dispatch_type) + # validate the dispatch args to be sent to `.dispatch` + args, kwargs = dispatch_class.validate_dispatch_args(*args, **kwargs) + + # Dispatch the method call + object_refs = dispatch_class.dispatch(self.actor_infos, method_name, *args, **kwargs) + return await dispatch_class.async_collect(self.actor_infos, object_refs) + + +class PolicyWorkerBase(Worker): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.model: nn.Module = None + self.scheduler: LRScheduler = None + self.optimizer: Optimizer = None + self.strategy: DistributedStrategy = None + self.record_memory: bool = False + self.mesh_rank: MeshRank = None + self.policy_loss_fn: Callable = PolicyLossRegistry.get(self.cfg.trainer.algorithm.policy_loss_type) + + def _normalize_mini_batch_size(self): + """ + Initialize micro batch tracking for gradient accumulation. + + The worker no longer needs to know mini batch size - it processes whatever + batch it receives, breaking it into micro batches. Gradient scaling happens + at optim_step time based on how many micro batches were accumulated. + + TODO: Rename to _init_gradient_accumulation_state once Megatron no longer + requires mini-batch normalization in its override. The name is kept for + backwards compatibility with Megatron which still does actual normalization. + """ + if not hasattr(self, "mesh_rank") or self.mesh_rank is None: + raise RuntimeError("mesh_rank must be initialized before calling _normalize_mini_batch_size()") + + # Track micro batches for gradient scaling at optim_step + self._micro_batches_accumulated = 0 + + # Gradient statistics for cross-step metrics (SNR, megabytes edited, update diversity) + self._grad_stats = { + "mean": {}, # Dict[param_name, Tensor] - running mean per param + "var_numerator": {}, # Dict[param_name, Tensor] - running M2 for variance. + "count": 0, # int - number of steps accumulated + } + return + + def _accumulate_gradient_stats(self): + """Accumulate gradient statistics for cross-step metrics (SNR, megabytes edited, update diversity).""" + self._grad_stats["count"] += 1 + n = self._grad_stats["count"] + + for name, param in self.model.named_parameters(): + if param.grad is None: + continue + # Use local shard instead of full_tensor to save memory + grad = param.grad.detach() + if hasattr(grad, "_local_tensor"): + grad = grad._local_tensor + # Move to CPU to save GPU memory + grad = grad.cpu() + + # Initialize on first step + if name not in self._grad_stats["mean"]: + self._grad_stats["mean"][name] = torch.zeros_like(grad) + self._grad_stats["var_numerator"][name] = torch.zeros_like( + grad + ) # stores M2 (sum of squared deviations) + + # Welford's online update for running mean and variance + # (works on DTensors - element-wise ops are supported) + delta = grad - self._grad_stats["mean"][name] + self._grad_stats["mean"][name] += delta / n + delta2 = grad - self._grad_stats["mean"][name] + self._grad_stats["var_numerator"][name] += delta * delta2 # M2 accumulator + + return + + def compute_gradient_metrics(self) -> Dict[str, float | int | list[float]]: + """Compute all 4 gradient-based metrics.""" + grad_metrics = { + "grad:snr": self._compute_snr(), + "grad:megabytes_edited": self._compute_megabytes_edited(), + "grad:update_diversity": self._compute_update_diversity(), + "grad:effective_rank": self._compute_effective_rank(), + } + return grad_metrics + + def _compute_snr(self) -> float: + """ + Compute gradient Signal-to-Noise Ratio across all parameters. + + SNR = |E[∇θ]| / std(∇θ) measures consistency of gradient direction. + High SNR = gradients point consistently in same direction across steps. + Low SNR = noisy/inconsistent gradient directions. + """ + if self._grad_stats["count"] <= 1: + return 0.0 + + snr_values = [] + + for name in self._grad_stats["mean"]: + mean = self._grad_stats["mean"][name] + m2 = self._grad_stats["var_numerator"][name] # This stores M2, not variance + + # Compute variance from M2 (use Bessel's correction for unbiased estimate) + # When count <= 1, variance is undefined, so return 0 + if self._grad_stats["count"] <= 1: + variance = torch.zeros_like(m2) + else: + variance = m2 / (self._grad_stats["count"] - 1) + std = torch.sqrt(variance + 1e-8) # Add epsilon to avoid division by zero + + # Compute per-element SNR and average + snr = torch.abs(mean) / std + snr_values.append(snr.mean().item()) + + if len(snr_values) == 0: + return 0.0 + + return sum(snr_values) / len(snr_values) + + def _compute_megabytes_edited(self) -> float: + """ + Estimate megabytes of information edited based on significant gradient updates. + + A gradient is "significant" if |grad| > 5% of |param|. + Megabytes edited = floor(3.6 * number of significant gradients) / (8 * 1024 * 1024). + """ + if self._grad_stats["count"] == 0: + return 0 + + significant_count = 0 + + for name, param in self.model.named_parameters(): + if name not in self._grad_stats["mean"]: + continue + + mean_grad = self._grad_stats["mean"][name] + # Use local shard instead of full_tensor to save memory + param_data = param.data + if hasattr(param_data, "_local_tensor"): + param_data = param_data._local_tensor + param_magnitude = torch.abs(param_data.cpu()) + + # Count gradients where |mean_grad| > 5% of |param| + # Both mean_grad and param.data are DTensors with same sharding + threshold = 0.05 * param_magnitude + significant = (torch.abs(mean_grad) > threshold).sum().item() + significant_count += significant + + # 3.6 bits per significant gradient update + # source: http://arxiv.org/abs/2505.24832 + return (3.6 * significant_count) / (8 * 1024 * 1024) # conversion bits -> megabytes. + + def _compute_update_diversity(self) -> list[float]: + """ + Compute update diversity as aggregated gradient histogram distribution. + + Aggregates all mean gradients into a single histogram and returns the + normalized probability distribution. This can be used with cosine + similarity to measure gradient pattern differences across environments. + """ + if self._grad_stats["count"] == 0: + return [] + + # Collect all mean gradients into a single tensor (already local shards on CPU) + all_grads = [] + for name in self._grad_stats["mean"]: + g = self._grad_stats["mean"][name] + all_grads.append(g.flatten()) + + if len(all_grads) == 0: + return [] + all_grads = torch.cat(all_grads) + hist = torch.histc(all_grads, bins=10) + hist_sum = hist.sum() + if hist_sum > 0: + hist = hist / hist_sum # Normalize to probability distribution + return hist.tolist() + else: + return [0.0] * 10 # All zeros if no gradients + + def _compute_effective_rank(self, k: int = 64) -> float: + """ + Compute mean normalized effective rank across all 2D gradient matrices. + + Uses randomized SVD (svd_lowrank) for efficiency with k=64 components. + + Effective rank measures the intrinsic dimensionality of the gradient signal. + For each layer's gradient matrix G: + 1. Compute randomized SVD to get top-k singular values + 2. Normalize singular values: p_i = σ_i / Σσ_j + 3. Compute entropy: H = -Σ p_i log(p_i) + 4. Effective rank: exp(H) + 5. Normalize by k to get value in [0, 1] + + Returns the mean normalized effective rank across all 2D layers. + Higher values indicate gradients span more directions (less low-rank). + """ + if self._grad_stats["count"] == 0: + return 0.0 + + ranks = [] + for name in self._grad_stats["mean"]: + mean_grad = self._grad_stats["mean"][name] + + # Only compute for 2D matrices (weight layers) + if mean_grad.dim() != 2: + continue + + # mean_grad is already local shard on CPU, ready for SVD + # Compute top-k singular values using randomized SVD (faster) + try: + q = min(k, min(mean_grad.shape)) + _, s, _ = torch.svd_lowrank(mean_grad.float(), q=q) + except Exception: + continue + + # Normalize to probability distribution + s_sum = s.sum() + if s_sum == 0: + continue + p = s / s_sum + + # Compute entropy (with epsilon for numerical stability) + entropy = -(p * torch.log(p + 1e-10)).sum() + + # Effective rank = exp(entropy), normalized by k (number of components) + eff_rank = torch.exp(entropy).item() + normalized_rank = eff_rank / q + + ranks.append(normalized_rank) + + if len(ranks) == 0: + return 0.0 + + return sum(ranks) / len(ranks) + + def reset_gradient_stats(self): + """Reset gradient statistics for next accumulation period.""" + self._grad_stats = { + "mean": {}, + "var_numerator": {}, + "count": 0, + } + + def forward_backward(self, data: TrainingInputBatch) -> Dict[str, float]: + """ + Perform forward and backward passes for a batch, handling micro-batching internally. + + The batch is split into micro batches based on micro_train_batch_size_per_gpu. + Gradients accumulate across micro batches. Gradient scaling happens at optim_step. + + Args: + data: TrainingInputBatch (already DP-sharded by WorkerDispatch/MeshDispatch) + + Returns: + Aggregated metrics dict across all micro batches + """ + micro_batch_size = self.cfg.trainer.micro_train_batch_size_per_gpu + all_metrics = defaultdict(list) + + for micro_batch in BatchIterator(data, micro_batch_size, drop_last=False): + metrics = self._forward_backward_micro(micro_batch) + self._micro_batches_accumulated += 1 + for k, v in metrics.items(): + all_metrics[k].append(v) + + return reduce_metrics(dict(all_metrics), ignore_keys=["grad:update_diversity"]) + + def _forward_backward_micro(self, experience: Experience) -> Dict[str, float]: + """ + Perform forward and backward pass for one micro batch. + + Loss is not scaled here - gradient scaling happens at optim_step time. + + Args: + experience: Experience object for one micro batch + + Returns: + All-reduced metrics dict for this micro batch + """ + self.model.train() + + experience.to_device(torch.cuda.current_device()) + + sequences = experience.sequences + old_action_log_probs = experience.action_log_probs + base_action_log_probs = ( + experience.base_action_log_probs if experience.base_action_log_probs is not None else None + ) + advantages = experience.advantages + num_actions = experience.num_actions + attention_mask = experience.attention_mask + loss_mask = experience.loss_mask + rollout_action_logprobs = experience.rollout_logprobs + + # TODO (sumanthrh): don't think this does anything for fsdp rn because autocast happens internally + with torch.autocast(dtype=torch.bfloat16, device_type="cuda"): + # actor loss + action_log_probs, output = self.model( + sequences, + num_actions, + attention_mask=attention_mask, + temperature=self.cfg.generator.sampling_params.temperature, + return_output=True, + compute_entropy=True, + entropy_requires_grad=self.cfg.trainer.algorithm.use_entropy_loss, + ) + # loss function + # TODO: recompute advantages + policy_loss, clip_ratio = self.policy_loss_fn( + action_log_probs, + old_action_log_probs, + advantages, + config=self.cfg.trainer.algorithm, + loss_mask=loss_mask, + rollout_logprobs=rollout_action_logprobs, + ) + + # entropy loss + with torch.set_grad_enabled(self.cfg.trainer.algorithm.use_entropy_loss): + # batch_size, seqlen + entropy_BS = output["entropy"] + entropy_BS = entropy_BS[:, -num_actions - 1 : -1] + entropy = masked_mean(entropy_BS, loss_mask) + + if self.cfg.trainer.algorithm.use_entropy_loss: + entropy_loss_term = entropy * self.cfg.trainer.algorithm.entropy_loss_coef + else: + entropy_loss_term = torch.tensor(0.0) + + # kl loss + if self.cfg.trainer.algorithm.use_kl_loss: + kl_loss = compute_approx_kl( + action_log_probs, + base_action_log_probs, + loss_mask=loss_mask, + kl_estimator_type=self.cfg.trainer.algorithm.kl_estimator_type, + ) + kl_loss = masked_mean(kl_loss, loss_mask, dim=-1).mean() + else: + kl_loss = torch.tensor(0.0) + kl_loss_term = kl_loss * self.cfg.trainer.algorithm.kl_loss_coef + + loss = policy_loss + kl_loss_term - entropy_loss_term + self.strategy.backward(loss, self.model, self.optimizer) + + # Accumulate gradient statistics for cross-step metrics + if self.cfg.trainer.policy.track_extra_gradient_metrics: + self._accumulate_gradient_stats() + + status = { + "final_loss": loss.item(), + "policy_loss": policy_loss.item(), + "ppo_clip_ratio": clip_ratio, + "policy_entropy": entropy.item(), + "response_length": num_actions, + "policy_lr": self.scheduler.get_last_lr()[0], + } + if self.cfg.trainer.algorithm.use_kl_loss: + status["policy_kl"] = kl_loss.item() + + # All-reduce metrics across DP workers + status = self.strategy.all_reduce(status) + + return status + + def optim_step(self) -> float: + """ + Scale gradients by 1/micro_batches_accumulated, perform optimizer step, and reset counter. + + Returns: + The gradient norm (before scaling, after clipping) + """ + # Scale accumulated gradients by 1/N to get correct average + if self._micro_batches_accumulated > 0: + scale = 1.0 / self._micro_batches_accumulated + for param in self.model.parameters(): + if param.grad is not None: + param.grad.mul_(scale) + + # Perform optimizer step (includes gradient clipping) + grad_norm = self.strategy.optimizer_step(self.optimizer, self.model, self.scheduler, name="actor") + + # Reset counter for next accumulation cycle + self._micro_batches_accumulated = 0 + + if grad_norm is not None: + grad_norm = grad_norm.detach().cpu().item() + return grad_norm + + def all_reduce_metrics(self, status: Dict[str, int | float | list[float]]) -> Dict[str, int | float | list[float]]: + """ + All-reduce metrics across data parallel workers. + """ + return self.strategy.all_reduce(status) + + def get_lr(self) -> float: + """ + Get current learning rate from scheduler. + """ + return self.scheduler.get_last_lr()[0] + + def barrier(self) -> None: + """ + Synchronization barrier across all workers. + """ + torch.distributed.barrier() + + def save_checkpoint(self, ckpt_dir: Path, tokenizer=None): + self.strategy.save_checkpoint( + model=self.model, + optimizer=self.optimizer, + scheduler=self.scheduler, + ckpt_dir=ckpt_dir, + node_local_rank=self.get_node_local_rank(), + tokenizer=tokenizer, + ) + + def load_checkpoint( + self, ckpt_dir: Path, load_optimizer_states: bool = True, load_lr_scheduler_states: bool = True + ): + _, states = self.strategy.load_checkpoint( + model=self.model, + optimizer=self.optimizer if load_optimizer_states else None, + scheduler=self.scheduler if load_lr_scheduler_states else None, + ckpt_dir=ckpt_dir, + load_optimizer_states=load_optimizer_states, + load_lr_scheduler_states=load_lr_scheduler_states, + ) + return states + + def save_hf_model(self, export_dir: str, tokenizer): + # Save model in HuggingFace safetensors format + self.strategy.save_hf_model( + self.model, + export_dir, + tokenizer=tokenizer, + ) + + def _forward_micro_batch(self, micro_batch: TrainingInputBatch) -> TrainingOutputBatch: + device = torch.cuda.current_device() + micro_batch.to(device) + self.model.eval() + sequences = micro_batch["sequences"] + response_length = micro_batch.metadata["response_length"] + attention_mask = micro_batch["attention_mask"] + + with torch.no_grad(), torch.autocast(dtype=torch.bfloat16, device_type="cuda"): + policy_logprob = self.model( + sequences, + response_length, + attention_mask, + return_output=False, + temperature=self.cfg.generator.sampling_params.temperature, + ) + policy_logprob = policy_logprob.to("cpu") + output = TrainingOutputBatch( + {"output": policy_logprob}, + ) + output.metadata = micro_batch.metadata + return output + + def process_sequences(self, sequences, input_len, eos_token_id, pad_token_id): + return self.model.process_sequences(sequences, input_len, eos_token_id, pad_token_id) + + +class CriticWorkerBase(Worker): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.model: nn.Module = None + self.scheduler: LRScheduler = None + self.optimizer: Optimizer = None + self.strategy: DistributedStrategy = None + self.record_memory: bool = False + self.mesh_rank: MeshRank = None + self.critic_loss_fn: Callable = ppo_critic_loss + + def _normalize_mini_batch_size(self): + """ + Initialize micro batch tracking for gradient accumulation. + + The worker no longer needs to know mini batch size - it processes whatever + batch it receives, breaking it into micro batches. Gradient scaling happens + at optim_step time based on how many micro batches were accumulated. + + TODO: Rename to _init_gradient_accumulation_state once Megatron no longer + requires mini-batch normalization in its override. The name is kept for + backwards compatibility with Megatron which still does actual normalization. + """ + if not hasattr(self, "mesh_rank") or self.mesh_rank is None: + raise RuntimeError("mesh_rank must be initialized before calling _normalize_mini_batch_size()") + + # Track micro batches for gradient scaling at optim_step + self._micro_batches_accumulated = 0 + + def forward_backward(self, data: TrainingInputBatch) -> Dict[str, float]: + """ + Perform forward and backward passes for a batch, handling micro-batching internally. + + The batch is split into micro batches based on micro_train_batch_size_per_gpu. + Gradients accumulate across micro batches. Gradient scaling happens at optim_step. + + Args: + data: TrainingInputBatch (already DP-sharded by WorkerDispatch/MeshDispatch) + + Returns: + Aggregated metrics dict across all micro batches + """ + micro_batch_size = self.cfg.trainer.micro_train_batch_size_per_gpu + all_metrics = defaultdict(list) + + for micro_batch in BatchIterator(data, micro_batch_size, drop_last=False): + metrics = self._forward_backward_micro(micro_batch) + self._micro_batches_accumulated += 1 + for k, v in metrics.items(): + all_metrics[k].append(v) + + return reduce_metrics(dict(all_metrics), ignore_keys=["grad:update_diversity"]) + + def _forward_backward_micro(self, experience: Experience) -> Dict[str, float]: + """ + Perform forward and backward pass for one micro batch. + + Loss is NOT scaled here - gradient scaling happens at optim_step time. + + Args: + experience: Experience object for one micro batch + + Returns: + All-reduced metrics dict for this micro batch + """ + self.model.train() + + experience.to_device(torch.cuda.current_device()) + + sequences = experience.sequences + old_values = experience.values + returns = experience.returns + num_actions = experience.num_actions + attention_mask = experience.attention_mask + loss_mask = experience.loss_mask + + with torch.autocast(dtype=torch.bfloat16, device_type="cuda"): + # critic loss + values, _ = self.model( + sequences, + num_actions=num_actions, + attention_mask=attention_mask, + return_output=True, + ) + # loss function + loss, clipfrac = self.critic_loss_fn( + values, + old_values, + returns, + config=self.cfg.trainer.algorithm, + loss_mask=loss_mask, + ) + # NO loss scaling here - gradient scaling happens at optim_step + self.strategy.backward(loss, self.model, self.optimizer) + + status = { + "critic_loss": loss.item(), + "values_mean": masked_mean(values, loss_mask).item(), + "values_clipfrac": clipfrac, + "critic_lr": self.scheduler.get_last_lr()[0], + } + + # All-reduce metrics across DP workers + status = self.strategy.all_reduce(status) + + return status + + def optim_step(self) -> float: + """ + Scale gradients by 1/micro_batches_accumulated, perform optimizer step, and reset counter. + + Returns: + The gradient norm (before scaling, after clipping) + """ + # Scale accumulated gradients by 1/N to get correct average + if self._micro_batches_accumulated > 0: + scale = 1.0 / self._micro_batches_accumulated + for param in self.model.parameters(): + if param.grad is not None: + param.grad.mul_(scale) + + # Perform optimizer step (includes gradient clipping) + grad_norm = self.strategy.optimizer_step(self.optimizer, self.model, self.scheduler, name="critic") + + # Reset counter for next accumulation cycle + self._micro_batches_accumulated = 0 + + if grad_norm is not None: + grad_norm = grad_norm.detach().cpu().item() + return grad_norm + + def all_reduce_metrics(self, status: Dict[str, int | float | list[float]]) -> Dict[str, int | float | list[float]]: + """ + All-reduce metrics across data parallel workers. + """ + return self.strategy.all_reduce(status) + + def get_lr(self) -> float: + """ + Get current learning rate from scheduler. + """ + return self.scheduler.get_last_lr()[0] + + def barrier(self) -> None: + """ + Synchronization barrier across all workers. + """ + torch.distributed.barrier() + + def _forward_micro_batch( + self, + micro_batch: TrainingInputBatch, + ) -> TrainingOutputBatch: + """Generates critic values.""" + device = torch.cuda.current_device() + micro_batch.to(device) + sequences = micro_batch["sequences"] + response_length = micro_batch.metadata["response_length"] + attention_mask = micro_batch["attention_mask"] + self.model.eval() + with torch.no_grad(), torch.autocast(dtype=torch.bfloat16, device_type="cuda"): + value = self.model( + sequences, + response_length, + attention_mask, + ) + self.model.train() # reset model state + value = value.to("cpu") + output = TrainingOutputBatch( + {"output": value}, + ) + output.metadata = micro_batch.metadata + return output + + def save_hf_model(self, export_dir: str, tokenizer): + # Save model in HuggingFace safetensors format + self.strategy.save_hf_model( + self.model, + export_dir, + tokenizer=tokenizer, + ) + + def save_checkpoint(self, ckpt_dir: str, tokenizer=None): + self.strategy.save_checkpoint( + model=self.model, + optimizer=self.optimizer, + scheduler=self.scheduler, + ckpt_dir=ckpt_dir, + node_local_rank=self.get_node_local_rank(), + tokenizer=tokenizer, + ) + + def load_checkpoint(self, ckpt_dir=None, load_optimizer_states=True, load_lr_scheduler_states=True): + _, states = self.strategy.load_checkpoint( + model=self.model, + optimizer=self.optimizer if load_optimizer_states else None, + scheduler=self.scheduler if load_lr_scheduler_states else None, + ckpt_dir=ckpt_dir, + load_optimizer_states=load_optimizer_states, + load_lr_scheduler_states=load_lr_scheduler_states, + ) + return states + + +class RefWorkerBase(Worker): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.model: nn.Module = None + + def _forward_micro_batch(self, micro_batch: TrainingInputBatch) -> TrainingOutputBatch: + device = torch.cuda.current_device() + micro_batch.to(device) + sequences = micro_batch["sequences"] + response_length = micro_batch.metadata["response_length"] + attention_mask = micro_batch["attention_mask"] + with torch.no_grad(), torch.autocast(dtype=torch.bfloat16, device_type="cuda"): + log_probs = self.model(sequences, response_length, attention_mask, return_output=False) + log_probs = log_probs.to("cpu") + output = TrainingOutputBatch( + {"output": log_probs}, + ) + output.metadata = micro_batch.metadata + return output diff --git a/skyrl-train/skyrl_train/workers/worker_dispatch.py b/skyrl-train/skyrl_train/workers/worker_dispatch.py new file mode 100644 index 0000000000..d9b322fe7b --- /dev/null +++ b/skyrl-train/skyrl_train/workers/worker_dispatch.py @@ -0,0 +1,302 @@ +""" +WorkerDispatch: Manages all actor groups with automatic offload/onload. + +Automatically handles GPU placement: +- Tracks which model is currently on GPU +- If colocation is enabled, offloads other models when one is requested + +The trainer interacts with the worker dispatch if all models are always on GPU. +""" + +from dataclasses import dataclass +from typing import Dict, List, Optional + +import ray +from omegaconf import DictConfig + +from skyrl_train.distributed.dispatch import concatenate_outputs_after_mesh_dispatch +from skyrl_train.training_batch import TrainingInputBatch, TrainingOutputBatch +from skyrl_train.workers.worker import PPORayActorGroup + + +@dataclass +class GPUState: + """Tracks what's on GPU for a model.""" + + model_on_gpu: bool = False + optimizer_on_gpu: bool = False + + +class WorkerDispatch: + """ + Unified dispatch layer that manages all actor groups (policy, critic, ref). + + Handles automatic offload/onload when colocate_all=True. + """ + + def __init__( + self, + cfg: DictConfig, + policy_actor_group: PPORayActorGroup, + critic_actor_group: Optional[PPORayActorGroup] = None, + ref_actor_group: Optional[PPORayActorGroup] = None, + ): + self.cfg = cfg + self.colocate_all = cfg.trainer.placement.colocate_all + self.colocate_policy_ref = cfg.trainer.placement.colocate_policy_ref + + # Actor groups by name. + # TODO: Remove these role-specific identifiers. We will move to using model IDs and add support for generic models beyond these. + self._actor_groups: Dict[str, PPORayActorGroup] = {"policy": policy_actor_group} + if critic_actor_group is not None: + self._actor_groups["critic"] = critic_actor_group + if ref_actor_group is not None: + self._actor_groups["ref"] = ref_actor_group + + # GPU state tracking (only matters when colocated) + self._gpu_state: Dict[str, GPUState] = {name: GPUState() for name in self._actor_groups.keys()} + + def get_lcm_dp_size(self) -> int: + """Get LCM of all models' dp_size.""" + import math + + dp_size = self._actor_groups["policy"].actor_infos[0].rank.dp_size + if "critic" in self._actor_groups: + dp_size = math.lcm(dp_size, self._actor_groups["critic"].actor_infos[0].rank.dp_size) + if "ref" in self._actor_groups: + dp_size = math.lcm(dp_size, self._actor_groups["ref"].actor_infos[0].rank.dp_size) + return dp_size + + def _should_manage_offload(self, model: str) -> bool: + """Check if we need to manage offload for this model.""" + if self.colocate_all: + return True + if self.colocate_policy_ref and model in ("policy", "ref"): + return True + return False + + def _get_colocation_group(self, model: str) -> List[str]: + """Get which models share GPU with the given model.""" + if self.colocate_all: + return list(self._actor_groups.keys()) + elif self.colocate_policy_ref and model in ("policy", "ref"): + return [m for m in ["policy", "ref"] if m in self._actor_groups] + return [model] + + def _ensure_on_gpu(self, model: str, need_optimizer: bool = True, need_model: bool = True) -> None: + """Ensure model is on GPU, offloading others in same colocation group if needed.""" + if not self._should_manage_offload(model): + return + + if model not in self._actor_groups: + return + + group = self._get_colocation_group(model) + + # Offload others in the same colocation group + for other in group: + if other != model and other in self._actor_groups: + state = self._gpu_state[other] + if state.model_on_gpu or state.optimizer_on_gpu: + self._actor_groups[other].offload_to_cpu() + self._gpu_state[other] = GPUState() + + # Backload requested model + state = self._gpu_state[model] + needs_backload = (need_model and not state.model_on_gpu) or (need_optimizer and not state.optimizer_on_gpu) + + if needs_backload: + self._actor_groups[model].backload_to_gpu( + backload_optimizer=need_optimizer, + backload_model=need_model, + ) + if need_model: + self._gpu_state[model].model_on_gpu = True + if need_optimizer: + self._gpu_state[model].optimizer_on_gpu = True + + def _offload(self, model: str, offload_optimizer: bool = True, offload_model: bool = True) -> None: + """Offload model to CPU.""" + if not self._should_manage_offload(model): + return + + if model not in self._actor_groups: + return + + self._actor_groups[model].offload_to_cpu( + offload_optimizer=offload_optimizer, + offload_model=offload_model, + ) + + if offload_model: + self._gpu_state[model].model_on_gpu = False + if offload_optimizer: + self._gpu_state[model].optimizer_on_gpu = False + + def mark_all_offloaded(self) -> None: + """Mark all models as offloaded (call after build_models when colocate_all).""" + for model in self._actor_groups: + self._gpu_state[model] = GPUState() + + def forward(self, model: str, data: TrainingInputBatch) -> TrainingOutputBatch: + """Run inference forward pass. Only loads model (not optimizer).""" + self._ensure_on_gpu(model, need_optimizer=False, need_model=True) + + refs = self._actor_groups[model].async_run_ray_method("mesh", "forward", data=data) + results = ray.get(refs) + + output = concatenate_outputs_after_mesh_dispatch(self._actor_groups[model].actor_infos, results) + return output + + def forward_backward(self, model: str, data: TrainingInputBatch) -> Dict[str, float]: + """Run forward/backward pass. Needs model + optimizer.""" + self._ensure_on_gpu(model, need_optimizer=True, need_model=True) + + refs = self._actor_groups[model].async_run_ray_method("mesh", "forward_backward", data) + statuses = ray.get(refs) + + self._save_memory_snapshot(model, "forward_backward") + return statuses[0] + + def optim_step(self, model: str) -> Optional[float]: + """Run optimizer step. Model should already be on GPU from forward_backward.""" + refs = self._actor_groups[model].async_run_ray_method("pass_through", "optim_step") + grad_norms = ray.get(refs) + + self._save_memory_snapshot(model, "optim_step") + return grad_norms[0] + + def compute_gradient_metrics(self, model: str) -> Dict[str, float]: + """Compute gradient metrics from the worker.""" + refs = self._actor_groups[model].async_run_ray_method("pass_through", "compute_gradient_metrics") + results = ray.get(refs) + return results[0] + + def reset_gradient_stats(self, model: str) -> None: + """Reset gradient statistics accumulation state in the worker.""" + refs = self._actor_groups[model].async_run_ray_method("pass_through", "reset_gradient_stats") + ray.get(refs) + + # TODO(tgriggs): Remove this when Megatron supports forward_backward and optim_step. + def ppo_train(self, model: str, data: TrainingInputBatch) -> Dict[str, float]: + """Run full PPO training loop (for Megatron).""" + self._ensure_on_gpu(model, need_optimizer=True, need_model=True) + + refs = self._actor_groups[model].async_run_ray_method("mesh", "ppo_train", data) + statuses = ray.get(refs) + + return statuses[0].metadata["train_status"] + + def _save_memory_snapshot(self, model: str, tag: str) -> None: + """Save memory snapshot on workers.""" + ray.get( + self._actor_groups[model].async_run_ray_method("pass_through", "save_memory_snapshot", tag=f"{model}_{tag}") + ) + + def save_checkpoint(self, model: str, ckpt_dir: str, tokenizer=None) -> None: + """Save checkpoint for model.""" + self._ensure_on_gpu(model, need_optimizer=True, need_model=True) + + ray.get( + self._actor_groups[model].async_run_ray_method( + "pass_through", "save_checkpoint", ckpt_dir=ckpt_dir, tokenizer=tokenizer + ) + ) + + def load_checkpoint( + self, + model: str, + ckpt_dir: str, + load_optimizer_states: bool = True, + load_lr_scheduler_states: bool = True, + ) -> None: + """Load checkpoint for model.""" + self._ensure_on_gpu(model, need_optimizer=load_optimizer_states, need_model=True) + + ray.get( + self._actor_groups[model].async_run_ray_method( + "pass_through", + "load_checkpoint", + ckpt_dir=ckpt_dir, + load_optimizer_states=load_optimizer_states, + load_lr_scheduler_states=load_lr_scheduler_states, + ) + ) + + def save_hf_model(self, model: str, export_dir: str, tokenizer) -> None: + """Save model in HuggingFace format.""" + self._ensure_on_gpu(model, need_optimizer=False, need_model=True) + + ray.get(self._actor_groups[model].async_run_ray_method("pass_through", "save_hf_model", export_dir, tokenizer)) + + def init_model(self, model: str, model_path: str, num_training_steps: Optional[int] = None) -> None: + """Initialize model from path. Offloads others in colocation group first.""" + # Offload others in colocation group before init + if self._should_manage_offload(model): + group = self._get_colocation_group(model) + for other in group: + if other != model and other in self._actor_groups: + state = self._gpu_state[other] + if state.model_on_gpu or state.optimizer_on_gpu: + self._actor_groups[other].offload_to_cpu() + self._gpu_state[other] = GPUState() + + kwargs = {"model_path": model_path} + if num_training_steps is not None: + kwargs["num_training_steps"] = num_training_steps + + ray.get(self._actor_groups[model].async_init_model(**kwargs)) + + # After init, model is on GPU + self._gpu_state[model].model_on_gpu = True + self._gpu_state[model].optimizer_on_gpu = model != "ref" # ref has no optimizer + + def init_weight_sync_state(self, inference_engine_client) -> None: + """Initialize weight sync state for policy model.""" + ray.get( + self._actor_groups["policy"].async_run_ray_method( + "pass_through", "init_weight_sync_state", inference_engine_client + ) + ) + + def broadcast_to_inference_engines(self, inference_engine_client) -> None: + """Broadcast policy weights to inference engines.""" + ray.get( + self._actor_groups["policy"].async_run_ray_method( + "pass_through", "broadcast_to_inference_engines", inference_engine_client + ) + ) + + def prepare_for_weight_sync(self) -> None: + """Prepare for weight sync: ensure policy model is on GPU, offload optimizer.""" + if not self.colocate_all: + return + # Ensure policy model is on GPU (will offload others in colocation group) + self._ensure_on_gpu("policy", need_optimizer=False, need_model=True) + # Offload optimizer if it's on GPU + if self._gpu_state["policy"].optimizer_on_gpu: + self._offload("policy", offload_optimizer=True, offload_model=False) + + def finish_weight_sync(self) -> None: + """Finish weight sync: offload model.""" + if not self.colocate_all: + return + self._offload("policy", offload_optimizer=False, offload_model=True) + + def empty_cache(self, model: Optional[str] = None) -> None: + """Empty GPU cache for model(s).""" + if model is not None: + ray.get(self._actor_groups[model].async_run_ray_method("pass_through", "empty_cache")) + else: + refs = [] + for group in self._actor_groups.values(): + refs.extend(group.async_run_ray_method("pass_through", "empty_cache")) + ray.get(refs) + + def get_node_ids(self) -> List[str]: + """Get unique node IDs from all actor groups.""" + all_node_ids = [] + for group in self._actor_groups.values(): + node_ids = ray.get(group.async_run_ray_method("pass_through", "get_ray_node_id")) + all_node_ids.extend(node_ids) + return list(set(all_node_ids)) diff --git a/skyrl-train/skyrl_train/workers/worker_utils.py b/skyrl-train/skyrl_train/workers/worker_utils.py new file mode 100644 index 0000000000..9eff062e16 --- /dev/null +++ b/skyrl-train/skyrl_train/workers/worker_utils.py @@ -0,0 +1,94 @@ +import math +from skyrl_train.dataset.replay_buffer import Experience +from typing import List, Dict +from skyrl_train.training_batch import TrainingInputBatch + + +def reduce_metrics( + metrics: Dict[str, List[float]], ignore_keys: list[str] = [] +) -> Dict[str, int | float | list[float]]: + """ + Reduce metrics from a list of entries per key. + + For ignore_keys containing histograms (list of lists), sum frequencies element-wise + and normalize to produce a single aggregated histogram. + """ + reduced_metrics: dict[str, int | float | list[float]] = dict() + for k, v in metrics.items(): + if k in ignore_keys: + # Check if this is a list of histograms (list of lists) + if isinstance(v, list) and len(v) > 0 and isinstance(v[0], list): + # Sum histograms element-wise and normalize + num_bins = len(v[0]) + summed = [0.0] * num_bins + for hist in v: + for i, freq in enumerate(hist): + summed[i] += freq + # Normalize to sum to 1 + total = sum(summed) + if total > 0: + reduced_metrics[k] = [x / total for x in summed] + else: + reduced_metrics[k] = summed + else: + reduced_metrics[k] = v + else: + assert len(v) > 0, f"No metrics for key {k}" + assert all(isinstance(x, (int, float)) for x in v), f"Metrics for key {k} are not all numbers" + reduced_metrics[k] = sum(v) / len(v) + return reduced_metrics + + +class BatchIterator: + """A simple iterator to yield micro batches of data from the training batch.""" + + def __init__(self, data: TrainingInputBatch, sample_batch_size: int, drop_last: bool = False): + self.data = data + self.sample_batch_size = sample_batch_size + self.total_batch_size = data.batch_size + self.drop_last = drop_last + assert not drop_last, "drop_last is not supported yet" + num_micro_batches = self.total_batch_size / self.sample_batch_size + self.num_micro_batches = int(num_micro_batches) if drop_last else math.ceil(num_micro_batches) + # TODO: switch to tensordict.map_iter if possible + self._chunks = self.data.chunk(self.sample_batch_size) + self._iter = iter(self._chunks) + + def __len__(self): + return self.num_micro_batches + + def __iter__(self): + return self + + def __next__(self) -> Experience: + try: + batch = next(self._iter) + exp = self.batch_to_experience(batch) + return exp + except StopIteration: + self._iter = iter(self._chunks) + raise StopIteration + + @staticmethod + def batch_to_experience(batch: TrainingInputBatch): + # TODO (sumanthrh): other keys are not permitted right now, can go into info + # TODO: this conversion is hidden right now, might need to be surfaced in worker explicitly. + exp = Experience( + sequences=batch["sequences"], + action_log_probs=batch["action_log_probs"], + base_action_log_probs=batch["base_action_log_probs"], + values=batch["values"], + returns=batch["returns"], + advantages=batch["advantages"], + attention_mask=batch["attention_mask"], + loss_mask=batch["loss_mask"], + action_mask=batch["response_mask"], + num_actions=batch.metadata["response_length"], # int + rollout_logprobs=batch["rollout_logprobs"] if "rollout_logprobs" in batch else None, + # additional info + # can be used to log metrics etc for micro-batches in the worker + info={}, + # propagate metadata as is + metadata=batch.metadata, + ) + return exp diff --git a/skyrl-train/tasks/harbor-grpo-qwen3-8b.yaml b/skyrl-train/tasks/harbor-grpo-qwen3-8b.yaml new file mode 100644 index 0000000000..4323685f97 --- /dev/null +++ b/skyrl-train/tasks/harbor-grpo-qwen3-8b.yaml @@ -0,0 +1,195 @@ +# Harbor GRPO Training via SkyPilot - Qwen3-8B +# Usage: sky launch skyrl-train/tasks/harbor-grpo-qwen3-8b.yaml --env WANDB_API_KEY= --env DAYTONA_API_KEY= +# +# This task trains GRPO on Harbor-managed sandboxed environments (CodeContests, TerminalBench, etc). +# Harbor handles sandbox lifecycle, agent execution, and verification. +# SkyRL handles training via the HarborGenerator interface. +# +# Model: Qwen/Qwen3-8B (8B parameters) +# GPUs: B200:4 (preferred), H200:4 (fallback) +# Agent: Terminus-2 (configurable via harbor_trial_config) + +name: harbor-grpo-qwen3-8b + +resources: + disk_size: 500 + ports: 6479 + any_of: + - accelerators: B200:4 + cloud: lambda + memory: 750+ + - accelerators: B200:4 + cloud: runpod + memory: 750+ + - accelerators: B200:4 + cloud: primeintellect + memory: 750+ + - accelerators: H200-SXM:4 + cloud: lambda + memory: 750+ + - accelerators: H200-SXM:4 + cloud: runpod + memory: 750+ + - accelerators: H200-SXM:4 + cloud: vast + memory: 750+ + - accelerators: H200-SXM:4 + cloud: primeintellect + memory: 750+ + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + WANDB_API_KEY: "" + # Sandbox provider credentials (set at least one) + DAYTONA_API_KEY: "" + MODAL_TOKEN_ID: "" + MODAL_TOKEN_SECRET: "" + # Harbor dataset (HuggingFace dataset name) + TRAIN_DATASET: "open-thoughts/CodeContests" + EVAL_DATASET: "open-thoughts/OpenThoughts-TB-dev" + # Sandbox provider: daytona or modal + SANDBOX_PROVIDER: "daytona" + # Training parameters + RUN_NAME: "harbor-codecontest" + NUM_EPOCHS: "3" + MAX_MODEL_LEN: "32768" + MINI_BATCH_SIZE: "32" + N_SAMPLES_PER_PROMPT: "8" + EVAL_N_SAMPLES_PER_PROMPT: "4" + LR: "1.0e-6" + # Rate limiting for sandbox provider + TRAJECTORIES_PER_SECOND: "5" + MAX_CONCURRENCY: "512" + # Agent config + AGENT_NAME: "terminus-2" + AGENT_MAX_TURNS: "32" + AGENT_TIMEOUT_SEC: "1200" + +setup: | + set -euo pipefail + cd skyrl-train + + echo "Validating environment variables..." + if [ -z "$WANDB_API_KEY" ]; then echo "ERROR: WANDB_API_KEY is required"; exit 1; fi + if [ "$SANDBOX_PROVIDER" = "daytona" ] && [ -z "$DAYTONA_API_KEY" ]; then echo "ERROR: DAYTONA_API_KEY is required for daytona provider"; exit 1; fi + if [ "$SANDBOX_PROVIDER" = "modal" ] && ([ -z "$MODAL_TOKEN_ID" ] || [ -z "$MODAL_TOKEN_SECRET" ]); then echo "ERROR: MODAL_TOKEN_ID and MODAL_TOKEN_SECRET are required for modal provider"; exit 1; fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + # Install harbor separately to avoid datasets version conflict with core SkyRL deps + pip install --no-deps "harbor @ git+https://github.com/laude-institute/harbor@8c040e1bb010201fd3c75bee3dede2407b9f57cd" + + # Prepare datasets from HuggingFace + DATA_DIR="$HOME/data/harbor" + mkdir -p "$DATA_DIR" + echo "Preparing training dataset: $TRAIN_DATASET" + python examples/harbor/prepare_harbor_dataset.py --dataset "$TRAIN_DATASET" + echo "Preparing eval dataset: $EVAL_DATASET" + python examples/harbor/prepare_harbor_dataset.py --dataset "$EVAL_DATASET" + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + export PYTORCH_ALLOC_CONF=expandable_segments:True + + # Login to Weights & Biases + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + # Ray cluster setup + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then break; fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + DATA_DIR="$HOME/data/harbor" + TRAIN_DIR=$(echo "$TRAIN_DATASET" | sed 's|.*/||') + EVAL_DIR=$(echo "$EVAL_DATASET" | sed 's|.*/||') + CKPTS_DIR="$HOME/$RUN_NAME/ckpts" + EXPORTS_DIR="$HOME/$RUN_NAME/exports" + TRIALS_DIR="$HOME/$RUN_NAME/trials_run" + LOG_DIR="/tmp/skyrl-logs/$RUN_NAME" + + CHAT_TEMPLATE_PATH="skyrl_train/utils/templates/qwen3_acc_thinking.jinja2" + + python -m examples.harbor.entrypoints.main_harbor \ + data.train_data="['$DATA_DIR/$TRAIN_DIR']" \ + data.val_data="['$DATA_DIR/$EVAL_DIR']" \ + trainer.policy.model.path=Qwen/Qwen3-8B \ + generator.served_model_name=Qwen3-8B \ + hydra.searchpath="['file://examples/harbor']" \ + +harbor_trial_config=default \ + ++harbor_trial_config.trials_dir=$TRIALS_DIR \ + ++harbor_trial_config.agent.name=$AGENT_NAME \ + ++harbor_trial_config.agent.override_timeout_sec=$AGENT_TIMEOUT_SEC \ + ++harbor_trial_config.agent.kwargs.max_turns=$AGENT_MAX_TURNS \ + ++harbor_trial_config.environment.type=$SANDBOX_PROVIDER \ + trainer.export_path=$EXPORTS_DIR \ + trainer.ckpt_path=$CKPTS_DIR \ + trainer.log_path=$LOG_DIR \ + trainer.algorithm.advantage_estimator=grpo \ + trainer.algorithm.loss_reduction=seq_mean_token_sum_norm \ + trainer.algorithm.grpo_norm_by_std=false \ + trainer.algorithm.use_kl_loss=false \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_nodes=1 \ + trainer.placement.ref_num_nodes=1 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$TOTAL_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + +generator.engine_init_kwargs.chat_template=$CHAT_TEMPLATE_PATH \ + +generator.engine_init_kwargs.max_model_len=$MAX_MODEL_LEN \ + +generator.engine_init_kwargs.enable_log_requests=false \ + trainer.epochs=$NUM_EPOCHS \ + trainer.eval_batch_size=128 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=$MINI_BATCH_SIZE \ + trainer.policy_mini_batch_size=$MINI_BATCH_SIZE \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=5 \ + trainer.hf_save_interval=5 \ + trainer.algorithm.max_seq_len=$MAX_MODEL_LEN \ + trainer.policy.optimizer_config.lr=$LR \ + generator.n_samples_per_prompt=$N_SAMPLES_PER_PROMPT \ + generator.eval_n_samples_per_prompt=$EVAL_N_SAMPLES_PER_PROMPT \ + generator.apply_overlong_filtering=true \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger=wandb \ + trainer.project_name=harbor \ + trainer.run_name=$RUN_NAME \ + trainer.resume_mode=latest \ + generator.backend=vllm \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.enforce_eager=false \ + generator.enable_http_endpoint=true \ + generator.http_endpoint_host=127.0.0.1 \ + generator.http_endpoint_port=8000 \ + +generator.rate_limit.enabled=true \ + +generator.rate_limit.trajectories_per_second=$TRAJECTORIES_PER_SECOND \ + +generator.rate_limit.max_concurrency=$MAX_CONCURRENCY diff --git a/skyrl-train/tasks/openenv-fleet-grpo-glm-4.7-flash.yaml b/skyrl-train/tasks/openenv-fleet-grpo-glm-4.7-flash.yaml new file mode 100644 index 0000000000..ed30454a09 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-glm-4.7-flash.yaml @@ -0,0 +1,350 @@ +# Fleet Task GRPO Training via SkyPilot - GLM-4.7-Flash +# Usage: sky launch tasks/openenv-fleet-grpo-glm-4.7-flash.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# Zhipu AI GLM-4.7-Flash: Efficient MoE model for coding and agents +# - 30B total params, 3B active (MoE with 5/64 experts) +# - Multi-Headed Latent Attention (MLA) +# - GLM-4.7 family has strong tool-use capabilities +# - vLLM day-zero support +# +# Model: zai-org/GLM-4.7-Flash (30B-A3B MoE) +# GPUs: B200:8 (more parallelism for faster training) +# +# v0.1.0: Initial GLM-4.7-Flash config +# v0.1.1: Install transformers from source for glm4_moe_lite architecture support +# v0.1.2: Use vLLM nightly (required for GLM-4.7-Flash MoE support) +# v0.1.3: Add vLLM + transformers to uv run --with (--isolated doesn't see setup packages) +# v0.1.4: Use vLLM nightly - stable vLLM requires transformers<5 but +# glm4_moe_lite architecture needs transformers 5.x from source +# v0.1.5: Install ALL deps in setup, don't use --isolated (fixes flash-attn ABI mismatch) +# --isolated builds flash-attn from source with wrong PyTorch version +# v0.1.6: Install torchvision after vLLM (transformers 5.x needs compatible torchvision) +# v0.1.7: Force reinstall torchvision from PyTorch CUDA 12.8 index (B200 Blackwell) +# v0.1.8: Install torch+torchvision together from nightly/cu128 to ensure they match +# v0.1.9: Uninstall torchvision - text model doesn't need it (transformers#38065) +# v0.1.10: Install vLLM LAST so its PyTorch/C++ ABI is preserved +# v0.1.11: Uninstall torchvision AFTER vLLM (vLLM reinstalls it as dep) +# v0.1.12: Install vLLM FIRST, then transformers from source AFTER +# vLLM has pinned deps that downgrade transformers to 4.57.x +# Installing transformers after vLLM overrides this downgrade +# v0.1.13: Install vLLM from source (git+) instead of nightly wheels +# Nightly wheels have availability issues, source build is more reliable +# v0.1.14: Revert to nightly wheels - source build requires CUDA toolkit (nvcc) +# which RunPod images don't include (only CUDA runtime) +# v0.1.15: Use exact install command from HuggingFace model card +# --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly +# v0.1.16: Enable preserved thinking for multi-turn tool use +# reasoning_parser=glm45, tool_call_parser=glm47 (from HF model card) +# v0.1.17: Install vLLM BEFORE uv sync (from B200 debugging) +# vLLM brings PyTorch 2.10.0, uv sync after won't downgrade it +# v0.1.18: Use uv sync --inexact to prevent removing vLLM +# v0.1.19: Replace uv sync with uv pip install -e . (uv sync downgrades PyTorch) +# v0.1.20: Increase GPU count from 4 to 8 for faster training +# v0.1.21: Fix GPU names per provider (H200-SXM for RunPod/Vast) +# v0.1.22: Remove Nebius (fabric config issues causing stuck launches) +# v0.1.23: Use --no-deps for editable install (uv pip install -e . still downgrades PyTorch) +# v0.1.24: Don't use 'uv run' in run phase (re-syncs lockfile, downgrades PyTorch) +# v0.1.25: Also remove 'uv run' from setup PREPARE_CMD (same issue) +# v0.1.26: Install missing deps (datasets, ray, etc.) since --no-deps skips them +# v0.1.27: Remove 'ray stop' (kills SkyPilot's Ray cluster that manages the job) +# v0.1.28: Add ALL deps from pyproject.toml (--no-deps skips them all) +# v0.1.29: Install skyrl-gym from local path (not PyPI - different package) +# v0.1.30: Use +generator.reasoning_parser (add new key to Hydra config) +# v0.1.31: DEBUG flash-attn build - check nvcc, CUDA, pyproject.toml settings +# v0.1.32: Skip flash-attn entirely, use FLASHINFER attention backend +# - flash-attn has NO prebuilt wheels for PyTorch 2.10.0 (vLLM nightly) +# - Building from source requires nvcc which RunPod doesn't have +# - FLASHINFER is bundled with vLLM and works without external deps +# - Set VLLM_ATTENTION_BACKEND=FLASHINFER in environment +# v0.1.33: Install flash-attn Python utilities (SKIP_CUDA_BUILD=TRUE) +# - SkyRL training code needs flash_attn.bert_padding (pad_input/unpad_input) +# - These are pure Python functions, no CUDA kernels needed +# - vLLM still uses FLASHINFER for inference attention +# v0.1.34: Create mock flash_attn_2_cuda module +# - flash_attn's __init__.py imports flash_attn_2_cuda at runtime +# - Without CUDA build, this module doesn't exist causing ImportError +# - Create mock module so flash_attn can import (we only need bert_padding) +# v0.1.35: Create minimal flash_attn stub package +# - Only provides bert_padding module that SkyRL needs +# - Copies pad_input/unpad_input from flash-attn source (pure Python) +# - Avoids all CUDA import issues - no flash_attn_2_cuda needed +# - Works in all processes (main, Ray workers, etc.) +# v0.1.36: Extend flash_attn stub with rotary embedding module +# - vLLM imports flash_attn.ops.triton.rotary.apply_rotary internally +# - Add pure PyTorch implementation (no Triton kernel needed) +# - Now supports: bert_padding + ops.triton.rotary +# v0.1.37: Add package metadata (.dist-info) to flash_attn stub +# - transformers calls importlib.metadata.version("flash_attn") +# - This requires flash_attn-*.dist-info/METADATA, not __version__ +# - Create proper dist-info directory with METADATA file +# v0.1.38: Use PEP 440 compliant version (2.7.0+stub not 2.7.0.stub) +# - uv/pip reject non-compliant versions like "2.7.0.stub" +# - Use local version identifier format: "2.7.0+stub" +# v0.1.39: Use SDPA attention instead of flash_attention_2 +# - flash_attn stub can't provide flash_attn_func (CUDA kernels) +# - SDPA is PyTorch's built-in efficient attention (no external deps) +# - Set use_flash_attention_2=false in training config +# v0.1.40: Override attn_implementation in model config +# - use_flash_attention_2=false doesn't work - model config.json overrides it +# - Set attn_implementation=sdpa via model_config_kwargs +# - This overrides the HF config before model instantiation + +name: fleet-task-grpo-glm-4.7-flash + +resources: + disk_size: 300 + ports: 6479 + any_of: + # B200 options + - accelerators: B200:8 + cloud: lambda + memory: 750+ + - accelerators: B200:8 + cloud: runpod + memory: 750+ + # H200 options (use H200-SXM for RunPod/Vast) + - accelerators: H200-SXM:8 + cloud: runpod + memory: 750+ + - accelerators: H200-SXM:8 + cloud: vast + memory: 750+ + # GCP spot H200 fallback (preemptible, 64 GPU quota per region) + - accelerators: H200:8 + cloud: gcp + use_spot: true + memory: 750+ + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + # Use FLASHINFER attention backend for vLLM inference (bundled with vLLM) + # flash_attn stub package provides bert_padding (pure Python) for SkyRL training + VLLM_ATTENTION_BACKEND: "FLASHINFER" + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + MODALITY: "tool_use" + MAX_TURNS: 50 + # GLM-4.7-Flash supports 128K context natively + MAX_INPUT_LENGTH: 48000 + MAX_GENERATE_LENGTH: 8192 + NUM_EPOCHS: 20 + RUN_ID: "" + MAX_TASKS: "" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + # Dataset version: "v3" (20,557 tasks) or "v4" (15,193 tasks, recommended) + DATA_VERSION: "v4" + +setup: | + set -euo pipefail + cd skyrl-train + + echo "Validating environment variables..." + if [ -z "$FLEET_API_KEY" ]; then + echo "ERROR: FLEET_API_KEY is required" + exit 1 + fi + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download" + exit 1 + fi + if [ "$MODALITY" != "tool_use" ] && [ "$MODALITY" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: $MODALITY" + exit 1 + fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + + # GLM-4.7-Flash requires specific install order: + # 1. vLLM nightly FIRST - brings PyTorch 2.10.0 (vLLM compiled against this version) + # 2. flash_attn stub package - minimal Python-only impl of bert_padding (pad_input/unpad_input) + # 3. transformers from source - for glm4_moe_lite architecture (overrides vLLM's 4.x pin) + # 4. skyrl-gym from local - PyPI has different package with same name + # 5. Other deps via pip (NOT uv sync - it enforces lockfile and downgrades PyTorch) + # + # Why not uv sync? The lockfile pins torch==2.9.0 but vLLM nightly needs torch==2.10.0 + # Why --no-deps for main package? Prevents installing vllm/transformers from PyPI + # Why flash_attn stub? Real flash-attn has no prebuilt wheels for torch 2.10, RunPod lacks nvcc + # We only need bert_padding (pure Python), vLLM uses FLASHINFER for attention + + # Step 1: Install vLLM nightly (HuggingFace model card command) + # https://huggingface.co/zai-org/GLM-4.7-Flash + uv pip install vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly + + # Step 2: Create flash_attn stub package (pure Python, no CUDA build) + # - bert_padding: pad_input/unpad_input (used by SkyRL training) + # - ops.triton.rotary: apply_rotary (used by vLLM internally for rotary embeddings) + # Pure PyTorch implementations - avoids all CUDA import issues + # Script is in integrations/fleet/scripts/create_flash_attn_stub.py + python3 integrations/fleet/scripts/create_flash_attn_stub.py + + # Step 3: Install transformers from source for glm4_moe_lite architecture + uv pip install git+https://github.com/huggingface/transformers.git + + # Step 4: Install skyrl-gym from LOCAL path (NOT PyPI - different package!) + uv pip install -e ../skyrl-gym + + # Step 5: Install main package without deps (deps installed separately to avoid version conflicts) + uv pip install -e . --no-deps + + # Step 6: Install remaining core deps from pyproject.toml (--no-deps skipped these) + # einops needed for flash_attn stub (bert_padding uses it) + uv pip install loguru tqdm ninja tensorboard func_timeout hydra-core==1.3.2 accelerate torchdata omegaconf ray==2.51.1 peft debugpy==1.8.0 hf_transfer datasets==4.0.0 tensordict jaxtyping polars s3fs fastapi uvicorn pybind11 setuptools pyarrow einops + + # Step 7: Install runtime deps + uv pip install "litellm>=1.75.5" boto3 wandb awscli + uv pip install "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" fleet-python logfire "mcp>=1.0.0" + + # Cleanup: Uninstall torchvision (vLLM installs it but text models don't need it) + uv pip uninstall torchvision || true + + # Diagnostics: print versions and test imports + echo "=== DIAGNOSTIC: Package versions ===" + pip show torch | grep -E "^(Name|Version)" + pip show transformers | grep -E "^(Name|Version)" + pip show vllm | grep -E "^(Name|Version)" + python3 -c "import flash_attn; print(f'flash_attn stub: {flash_attn.__version__}')" + pip show skyrl-gym | grep -E "^(Name|Version|Location)" + echo "=== DIAGNOSTIC: Test imports ===" + python3 -c "import torch; print(f'torch: {torch.__version__}')" + python3 -c "import transformers; print(f'transformers: {transformers.__version__}')" + python3 -c "import vllm; print(f'vllm: {vllm.__version__}')" + python3 -c "from flash_attn.bert_padding import pad_input, unpad_input; print('flash_attn.bert_padding: OK')" + python3 -c "from flash_attn.ops.triton.rotary import apply_rotary; print('flash_attn.ops.triton.rotary: OK')" + python3 -c "import skyrl_gym; print(f'skyrl_gym loaded from: {skyrl_gym.__file__}')" + echo "=== DIAGNOSTIC: Test vLLM CUDA module ===" + python3 -c "import vllm._C; print('vllm._C imported successfully')" + echo "=== DIAGNOSTIC: FLASHINFER backend ===" + echo "VLLM_ATTENTION_BACKEND=$VLLM_ATTENTION_BACKEND" + echo "=== DIAGNOSTIC: Complete ===" + + mkdir -p $HOME/data/fleet + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" + echo "Downloading dataset from $S3_PATH..." + aws s3 cp "$S3_PATH" "$TASKS_FILE" + TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") + echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + + DATA_DIR="$HOME/data/fleet/${MODALITY}" + PREPARE_CMD="python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" + [ -n "$ENV_KEYS" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" + [ -n "$DIFFICULTY" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" + eval "$PREPARE_CMD" + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + export PYTORCH_ALLOC_CONF=expandable_segments:True + + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + DATA_DIR="$HOME/data/fleet/${MODALITY}" + + # Don't use 'uv run' - it re-syncs lockfile and downgrades PyTorch + python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + # Diagnostics: verify environment at run time + echo "=== RUN DIAGNOSTIC: Package versions ===" + python3 -c "import torch; print(f'torch: {torch.__version__}')" + python3 -c "import transformers; print(f'transformers: {transformers.__version__}')" + python3 -c "import vllm; print(f'vllm: {vllm.__version__}')" + python3 -c "import vllm._C; print('vllm._C imported successfully')" + + # Don't run 'ray stop' - it kills SkyPilot's Ray cluster that manages this job + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + # Diagnostics: test vLLM import in Ray worker + echo "=== RUN DIAGNOSTIC: Testing vLLM import in Ray worker ===" + python3 -c "import ray; print(ray.get(ray.remote(lambda: __import__('vllm').__version__).remote()))" + echo "=== RUN DIAGNOSTIC: Ray worker vLLM import successful ===" + + # Run GRPO training with GLM-4.7-Flash + # MoE model: 30B total, 3B active - efficient training + # All deps installed in setup - use python directly (no --isolated) + python -m integrations.fleet.entrypoints.main_fleet \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file="$TASKS_FILE" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="zai-org/GLM-4.7-Flash" \ + +trainer.policy.model.model_config_kwargs.attn_implementation=sdpa \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$TOTAL_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=24 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=24 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=24 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_ckpts_to_keep=2 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=4 \ + generator.eval_n_samples_per_prompt=3 \ + generator.gpu_memory_utilization=0.7 \ + +generator.reasoning_parser=glm45 \ + +generator.tool_call_parser=glm47 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_${MODALITY}_glm47flash_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_${MODALITY}_glm47flash" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo-qwen3-32b.yaml b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-32b.yaml new file mode 100644 index 0000000000..c98856f709 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-32b.yaml @@ -0,0 +1,209 @@ +# Fleet Task GRPO Training via SkyPilot - Qwen3-32B +# Usage: sky launch tasks/openenv-fleet-grpo-qwen3-32b.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# This task trains GRPO on Fleet-hosted environments using OpenEnv's FleetTaskEnv. +# Larger model (32B) for improved reasoning on complex tool-use tasks. +# +# Model: Qwen/Qwen3-32B (32B parameters) +# GPUs: B200:8 (192GB HBM3e per GPU = 1.5TB total) +# +# v0.4.0: Initial 32B config with google-maps excluded from dataset +# v0.4.1: B200:8 to fix OOM during backward pass +# v0.4.2: disk_size 300→750, max_ckpts_to_keep=1 to fix disk full during checkpoint save +# v0.4.3: Fix workdir ref to main (feat/vl-multimodal-support had broken prepare_dataset.py) +# v0.4.4: S3 checkpoint resume support (keep_local=True, download from S3 via RESUME_RUN_NAME) + +name: fleet-task-grpo-qwen3-32b + +resources: + disk_size: 750 # 32B needs: model (~64GB) + optimizer (~192GB) + checkpoint (~256GB) + headroom + ports: 6479 + any_of: + # B200:8 required for 32B model (OOM on B200:4 during backward pass) + - accelerators: B200:8 + cloud: lambda + memory: 1500+ + - accelerators: B200:8 + cloud: runpod + memory: 1500+ + - accelerators: B200:8 + cloud: vast + memory: 1500+ + - accelerators: B200:8 + cloud: primeintellect + memory: 1500+ + - accelerators: B200:8 + cloud: nebius + memory: 1500+ + # GCP spot H200 fallback (preemptible, 64 GPU quota per region) + - accelerators: H200:8 + cloud: gcp + use_spot: true + memory: 1500+ + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + MODALITY: "tool_use" + MAX_TURNS: 50 + # Context: Qwen3-32B has 128K context, use YaRN to extend + MAX_INPUT_LENGTH: 64000 + MAX_GENERATE_LENGTH: 8192 + NUM_EPOCHS: 20 + RUN_ID: "" + MAX_TASKS: "" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + RESUME_RUN_NAME: "" + # Dataset version: "v3" (20,557 tasks) or "v4" (15,193 tasks, recommended) + DATA_VERSION: "v4" + +setup: | + set -euo pipefail + cd skyrl-train + + echo "Validating environment variables..." + if [ -z "$FLEET_API_KEY" ]; then + echo "ERROR: FLEET_API_KEY is required" + exit 1 + fi + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download" + exit 1 + fi + if [ "$MODALITY" != "tool_use" ] && [ "$MODALITY" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: $MODALITY" + exit 1 + fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + uv pip install wandb boto3 awscli + + uv pip install "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" fleet-python logfire "mcp>=1.0.0" + + mkdir -p $HOME/data/fleet + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" + echo "Downloading dataset from $S3_PATH..." + aws s3 cp "$S3_PATH" "$TASKS_FILE" + TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") + echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + + DATA_DIR="$HOME/data/fleet/${MODALITY}" + PREPARE_CMD="uv run python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" + [ -n "$ENV_KEYS" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" + [ -n "$DIFFICULTY" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" + eval "$PREPARE_CMD" + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + export PYTORCH_ALLOC_CONF=expandable_segments:True + + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + DATA_DIR="$HOME/data/fleet/${MODALITY}" + + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + # Run GRPO training with Qwen3-32B + # Using 8 GPUs: 4 inference engines (TP=2 each) + FSDP training across all 8 + uv run --isolated --extra "$INFERENCE_BACKEND" \ + --with "litellm>=1.75.5" \ + --with "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" \ + --with "fleet-python" \ + --with "logfire" \ + --with "mcp>=1.0.0" \ + --with "boto3" \ + -m integrations.fleet.entrypoints.main_fleet \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file="$TASKS_FILE" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-32B" \ + ++trainer.rope_scaling.rope_type=yarn \ + ++trainer.rope_scaling.factor=2.0 \ + ++trainer.rope_scaling.original_max_position_embeddings=40960 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=4 \ + generator.inference_engine_tensor_parallel_size=2 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=16 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=16 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=16 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_ckpts_to_keep=1 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=5.0e-7 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=8 \ + generator.eval_n_samples_per_prompt=3 \ + generator.gpu_memory_utilization=0.65 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_${MODALITY}_32b_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_${MODALITY}_32b" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b-8gpu.yaml b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b-8gpu.yaml new file mode 100644 index 0000000000..364f2ba276 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b-8gpu.yaml @@ -0,0 +1,225 @@ +# Fleet Task GRPO Training via SkyPilot - Qwen3-8B (8 GPU variant) +# Usage: sky launch tasks/openenv-fleet-grpo-qwen3-8b-8gpu.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# This task trains GRPO on Fleet-hosted environments using OpenEnv's FleetTaskEnv. +# It loads tasks from the Fleet API and runs training with verifier-based rewards. +# Checkpoints are automatically uploaded to S3 to prevent disk exhaustion. +# +# Model: Qwen/Qwen3-8B (8B parameters, instruct-tuned) +# GPUs: B200:8 (preferred), H200:8 (fallback - 141GB HBM3e per GPU) +# +# Batch sizes doubled from 4-GPU variant to utilize 8 GPUs efficiently + +name: fleet-task-grpo-qwen3-8b-8gpu + +resources: + disk_size: 200 # Needs space for: model (~16GB), optimizer (~32GB), Ray temp (~50GB), vLLM cache, checkpoints + ports: 6479 + any_of: + # B200:8 across all providers + - accelerators: B200:8 + cloud: lambda + - accelerators: B200:8 + cloud: runpod + - accelerators: B200:8 + cloud: vast + - accelerators: B200:8 + cloud: primeintellect + - accelerators: B200:8 + cloud: nebius + # Fallback to H200:8 (141GB HBM3e per GPU) + - accelerators: H200:8 + cloud: lambda + - accelerators: H200:8 + cloud: runpod + - accelerators: H200:8 + cloud: vast + - accelerators: H200:8 + cloud: primeintellect + - accelerators: H200:8 + cloud: nebius + # GCP spot H200 fallback (preemptible, 64 GPU quota per region) + - accelerators: H200:8 + cloud: gcp + use_spot: true + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + # These are passed via --env from GHA workflow + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + # Task selection: env_keys to filter (comma-separated, e.g., "github,booking,reddit") + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + # Dataset version: "v3" (20,557 tasks) or "v4" (15,193 tasks, recommended) + DATA_VERSION: "v4" + # Modality: "tool_use" or "computer_use" + MODALITY: "tool_use" + # Training parameters + MAX_TURNS: 50 + # Context budget: Using YaRN (factor=2.5) to extend Qwen3-8B from 32K to 82K context + # Some v5 envs have large tool schemas that exceed 48K, so 64K allows headroom + MAX_INPUT_LENGTH: 64000 + MAX_GENERATE_LENGTH: 8192 # Increased from 4096 to allow thinking + tool calls + NUM_EPOCHS: 20 + RUN_ID: "" + # Optional: limit number of tasks for testing + MAX_TASKS: "" + # AWS credentials (required for S3 dataset download, optional for checkpoint upload) + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + # S3 buckets + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + +setup: | + set -euo pipefail + cd skyrl-train + + # Validate required environment variables + echo "Validating environment variables..." + if [ -z "$FLEET_API_KEY" ]; then + echo "ERROR: FLEET_API_KEY is required" + exit 1 + fi + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download" + exit 1 + fi + if [ "$MODALITY" != "tool_use" ] && [ "$MODALITY" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: $MODALITY" + exit 1 + fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + uv pip install wandb boto3 awscli + + # Install OpenEnv for Fleet environment access (from branch with FleetTaskEnv) + uv pip install "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" fleet-python logfire "mcp>=1.0.0" + + # Download dataset from S3 + mkdir -p $HOME/data/fleet + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" + echo "Downloading dataset from $S3_PATH..." + aws s3 cp "$S3_PATH" "$TASKS_FILE" + TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") + echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + + # Prepare dataset (with optional env/difficulty filters) + DATA_DIR="$HOME/data/fleet/${MODALITY}" + PREPARE_CMD="uv run python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" + [ -n "$ENV_KEYS" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" + [ -n "$DIFFICULTY" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" + eval "$PREPARE_CMD" + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + # Help PyTorch memory allocator handle fragmentation + export PYTORCH_ALLOC_CONF=expandable_segments:True + + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + DATA_DIR="$HOME/data/fleet/${MODALITY}" + + # Login to Weights & Biases + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + # Ray cluster setup - limit object store to prevent disk exhaustion + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 # 10GB limit for object store + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + + # Wait for Ray + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + # Run GRPO training with Fleet task integration + # Batch sizes doubled from 4-GPU variant (32->64, 2->4) + uv run --isolated --extra "$INFERENCE_BACKEND" \ + --with "litellm>=1.75.5" \ + --with "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" \ + --with "fleet-python" \ + --with "logfire" \ + --with "mcp>=1.0.0" \ + --with "boto3" \ + -m integrations.fleet.entrypoints.main_fleet \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file="$TASKS_FILE" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-8B" \ + ++trainer.rope_scaling.rope_type=yarn \ + ++trainer.rope_scaling.factor=2.5 \ + ++trainer.rope_scaling.original_max_position_embeddings=32768 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$TOTAL_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=64 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=64 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=64 \ + trainer.micro_forward_batch_size_per_gpu=2 \ + trainer.micro_train_batch_size_per_gpu=2 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=4 \ + generator.eval_n_samples_per_prompt=3 \ + generator.gpu_memory_utilization=0.7 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_${MODALITY}_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_${MODALITY}" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b-step-wise.yaml b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b-step-wise.yaml new file mode 100644 index 0000000000..4e1d78a723 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b-step-wise.yaml @@ -0,0 +1,223 @@ +# Fleet Task GRPO Training via SkyPilot - Qwen3-8B (Step-Wise) +# Usage: sky launch tasks/openenv-fleet-grpo-qwen3-8b-step-wise.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# Step-wise training variant: processes each turn as a separate episode. +# Enables context management tools (manage_context, check_context). +# ~4x slower than token-in-token-out but required for context management. +# +# Model: Qwen/Qwen3-8B (8B parameters, instruct-tuned) +# GPUs: B200:4 (preferred), H200:4 (fallback - 141GB HBM3e per GPU) +# +# v0.3.6: Initial step-wise config with micro_batch=1 to fix CUDA OOM +# v0.3.7: Increased disk_size 200→500GB to fix OutOfDiskError + +name: fleet-task-grpo-qwen3-8b-step-wise + +resources: + disk_size: 500 + ports: 6479 + any_of: + # Prefer B200:4 (min 750GB RAM) + - accelerators: B200:4 + cloud: lambda + memory: 750+ + - accelerators: B200:4 + cloud: runpod + memory: 750+ + - accelerators: B200:4 + cloud: vast + memory: 750+ + - accelerators: B200:4 + cloud: primeintellect + memory: 750+ + - accelerators: B200:4 + cloud: nebius + memory: 750+ + # Fallback to H200:4 (141GB HBM3e per GPU) + - accelerators: H200:4 + cloud: lambda + memory: 750+ + - accelerators: H200:4 + cloud: runpod + memory: 750+ + - accelerators: H200:4 + cloud: vast + memory: 750+ + - accelerators: H200:4 + cloud: primeintellect + memory: 750+ + - accelerators: H200:4 + cloud: nebius + memory: 750+ + # GCP spot H200 fallback (preemptible, 64 GPU quota per region) + - accelerators: H200:4 + cloud: gcp + use_spot: true + memory: 750+ + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + # Dataset version: "v3" (20,557 tasks) or "v4" (15,193 tasks, recommended) + DATA_VERSION: "v4" + # Modality: "tool_use" or "computer_use" + MODALITY: "tool_use" + MAX_TURNS: 50 + MAX_INPUT_LENGTH: 64000 + MAX_GENERATE_LENGTH: 8192 + NUM_EPOCHS: 20 + RUN_ID: "" + MAX_TASKS: "" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + +setup: | + set -euo pipefail + cd skyrl-train + + echo "Validating environment variables..." + if [ -z "$FLEET_API_KEY" ]; then + echo "ERROR: FLEET_API_KEY is required" + exit 1 + fi + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download" + exit 1 + fi + if [ "$MODALITY" != "tool_use" ] && [ "$MODALITY" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: $MODALITY" + exit 1 + fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + uv pip install wandb boto3 awscli + + uv pip install "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" fleet-python logfire "mcp>=1.0.0" + + mkdir -p $HOME/data/fleet + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" + echo "Downloading dataset from $S3_PATH..." + aws s3 cp "$S3_PATH" "$TASKS_FILE" + TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") + echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + + DATA_DIR="$HOME/data/fleet/${MODALITY}" + PREPARE_CMD="uv run python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" + [ -n "$ENV_KEYS" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" + [ -n "$DIFFICULTY" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" + eval "$PREPARE_CMD" + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + export PYTORCH_ALLOC_CONF=expandable_segments:True + + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + DATA_DIR="$HOME/data/fleet/${MODALITY}" + + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + # Step-wise training with context management tools + uv run --isolated --extra "$INFERENCE_BACKEND" \ + --with "litellm>=1.75.5" \ + --with "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" \ + --with "fleet-python" \ + --with "logfire" \ + --with "mcp>=1.0.0" \ + --with "boto3" \ + -m integrations.fleet.entrypoints.main_fleet \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file="$TASKS_FILE" \ + environment.skyrl_gym.fleet_task.enable_context_tools=true \ + generator.step_wise_trajectories=true \ + generator.inject_context_status=true \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-8B" \ + ++trainer.rope_scaling.rope_type=yarn \ + ++trainer.rope_scaling.factor=2.5 \ + ++trainer.rope_scaling.original_max_position_embeddings=32768 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$TOTAL_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=24 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=24 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=24 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=4 \ + generator.eval_n_samples_per_prompt=3 \ + generator.gpu_memory_utilization=0.7 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_${MODALITY}_step_wise_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_${MODALITY}_step_wise" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b.yaml b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b.yaml new file mode 100644 index 0000000000..445dc56514 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-8b.yaml @@ -0,0 +1,234 @@ +# Fleet Task GRPO Training via SkyPilot - Qwen3-8B +# Usage: sky launch tasks/openenv-fleet-grpo-qwen3-8b.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# This task trains GRPO on Fleet-hosted environments using OpenEnv's FleetTaskEnv. +# It loads tasks from the Fleet API and runs training with verifier-based rewards. +# Checkpoints are automatically uploaded to S3 to prevent disk exhaustion. +# +# Model: Qwen/Qwen3-8B (8B parameters, instruct-tuned) +# GPUs: B200:4 (preferred), H200:4 (fallback - 141GB HBM3e per GPU) +# +# v0.3.5: Reduced train_batch_size (32→24) to fix OOM during convert_to_training_input +# v0.3.4: Enable use_hybrid_env_sampling to ensure every env is sampled every batch +# v0.3.3: Reduced eval_interval (40→20) for more frequent eval checkpoints +# v0.3.2: Increased batch size (4→32) and GPUs (2→4) for hybrid env sampling +# v0.3 OOM fix: gpu_memory_utilization reduced from 0.8 to 0.7 +# v0.3 has 3x more data including google-maps (4.7K tasks) with longer tool schemas +# This leaves more headroom for training activations before vLLM wake_up + +name: fleet-task-grpo-qwen3-8b + +resources: + disk_size: 500 # Needs space for: model (~16GB), optimizer (~32GB), Ray temp (~50GB), vLLM cache, checkpoints, uv worker envs + ports: 6479 + any_of: + # Prefer B200:4 (min 750GB RAM to avoid OOM during convert_to_training_input) + - accelerators: B200:4 + cloud: lambda + memory: 750+ + - accelerators: B200:4 + cloud: runpod + memory: 750+ + - accelerators: B200:4 + cloud: primeintellect + memory: 750+ + # Fallback to H200-SXM:4 (141GB HBM3e per GPU) + # SkyPilot requires "H200-SXM" not "H200" for 4-GPU configs + - accelerators: H200-SXM:4 + cloud: lambda + memory: 750+ + - accelerators: H200-SXM:4 + cloud: runpod + memory: 750+ + - accelerators: H200-SXM:4 + cloud: vast + memory: 750+ + - accelerators: H200-SXM:4 + cloud: primeintellect + memory: 750+ + # GCP spot H200 fallback (preemptible, 64 GPU quota per region) + - accelerators: H200:4 + cloud: gcp + use_spot: true + memory: 750+ + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + # These are passed via --env from GHA workflow + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + # Dataset version: "v3" (20,557 tasks) or "v4" (15,193 tasks, recommended) + DATA_VERSION: "v4" + # Task selection: env_keys to filter (comma-separated, e.g., "github,booking,reddit") + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + # Modality: "tool_use" or "computer_use" + MODALITY: "tool_use" + # Training parameters + MAX_TURNS: 50 + # Context budget: Using YaRN (factor=2.5) to extend Qwen3-8B from 32K to 82K context + # Some v5 envs have large tool schemas that exceed 48K, so 64K allows headroom + MAX_INPUT_LENGTH: 64000 + MAX_GENERATE_LENGTH: 8192 # Increased from 4096 to allow thinking + tool calls + NUM_EPOCHS: 20 + RUN_ID: "" + # Optional: limit number of tasks for testing + MAX_TASKS: "" + # AWS credentials (required for S3 dataset download, optional for checkpoint upload) + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + # S3 buckets + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + +setup: | + set -euo pipefail + cd skyrl-train + + # Validate required environment variables + echo "Validating environment variables..." + if [ -z "$FLEET_API_KEY" ]; then + echo "ERROR: FLEET_API_KEY is required" + exit 1 + fi + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download" + exit 1 + fi + if [ "$MODALITY" != "tool_use" ] && [ "$MODALITY" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: $MODALITY" + exit 1 + fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + uv pip install wandb boto3 awscli + + # Install OpenEnv for Fleet environment access (from branch with FleetTaskEnv) + uv pip install "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" fleet-python logfire "mcp>=1.0.0" + + # Download dataset from S3 + mkdir -p $HOME/data/fleet + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" + echo "Downloading dataset from $S3_PATH..." + aws s3 cp "$S3_PATH" "$TASKS_FILE" + TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") + echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + + # Prepare dataset (with optional env/difficulty filters) + DATA_DIR="$HOME/data/fleet/${MODALITY}" + PREPARE_CMD="uv run python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" + [ -n "$ENV_KEYS" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" + [ -n "$DIFFICULTY" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" + eval "$PREPARE_CMD" + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + # Help PyTorch memory allocator handle fragmentation + export PYTORCH_ALLOC_CONF=expandable_segments:True + + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + DATA_DIR="$HOME/data/fleet/${MODALITY}" + + # Login to Weights & Biases + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + # Ray cluster setup - limit object store to prevent disk exhaustion + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 # 10GB limit for object store + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + + # Wait for Ray + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + # Run GRPO training with Fleet task integration + uv run --isolated --extra "$INFERENCE_BACKEND" \ + --with "litellm>=1.75.5" \ + --with "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" \ + --with "fleet-python" \ + --with "logfire" \ + --with "mcp>=1.0.0" \ + --with "boto3" \ + -m integrations.fleet.entrypoints.main_fleet \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file="$TASKS_FILE" \ + environment.skyrl_gym.fleet_task.ttl_seconds=900 \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-8B" \ + ++trainer.rope_scaling.rope_type=yarn \ + ++trainer.rope_scaling.factor=2.5 \ + ++trainer.rope_scaling.original_max_position_embeddings=32768 \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$TOTAL_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=24 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=24 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=24 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=4 \ + generator.eval_n_samples_per_prompt=3 \ + generator.gpu_memory_utilization=0.7 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_${MODALITY}_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_${MODALITY}" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo-qwen3-vl-8b.yaml b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-vl-8b.yaml new file mode 100644 index 0000000000..b45795fbf0 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-qwen3-vl-8b.yaml @@ -0,0 +1,201 @@ +# Fleet Task GRPO Training via SkyPilot - Qwen3-VL-8B (Vision-Language) +# Usage: sky launch tasks/openenv-fleet-grpo-qwen3-vl-8b.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# VL (Vision-Language) training for computer_use environments with screenshots. +# +# Model: Qwen/Qwen3-VL-8B-Instruct (8B parameters, vision-language) +# GPUs: B200:4 (preferred), H200:4 (fallback - 141GB HBM3e per GPU) +# +# v0.1: Initial VL model support with step-wise training + +name: fleet-task-grpo-qwen3-vl-8b + +resources: + disk_size: 750 + ports: 6479 + any_of: + # Prefer B200:4 (min 750GB RAM) + - accelerators: B200:4 + cloud: lambda + memory: 750+ + - accelerators: B200:4 + cloud: runpod + memory: 750+ + # Fallback to H200-SXM:4 (correct naming for RunPod/Nebius) + - accelerators: H200-SXM:4 + cloud: runpod + memory: 750+ + - accelerators: H200-SXM:4 + cloud: nebius + memory: 750+ + - accelerators: H200-SXM:4 + cloud: lambda + memory: 750+ + # GCP spot fallback (preemptible, 64 GPU quota per region) + - accelerators: B200:8 + cloud: gcp + use_spot: true + - accelerators: H200:8 + cloud: gcp + use_spot: true + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: feat/vl-multimodal-support + +envs: + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + MODALITY: "computer_use" + MAX_TURNS: 20 + MAX_INPUT_LENGTH: 32000 + MAX_GENERATE_LENGTH: 4096 + NUM_EPOCHS: 10 + RUN_ID: "" + MAX_TASKS: "" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + # Dataset version: "v3" (20,557 tasks) or "v4" (15,193 tasks, recommended) + DATA_VERSION: "v4" + +setup: | + set -euo pipefail + cd skyrl-train + + echo "Validating environment variables..." + if [ -z "$FLEET_API_KEY" ]; then + echo "ERROR: FLEET_API_KEY is required" + exit 1 + fi + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download" + exit 1 + fi + if [ "$MODALITY" != "tool_use" ] && [ "$MODALITY" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: $MODALITY" + exit 1 + fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + uv pip install wandb boto3 awscli + + uv pip install "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" fleet-python logfire "mcp>=1.0.0" + + mkdir -p $HOME/data/fleet + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" + echo "Downloading dataset from $S3_PATH..." + aws s3 cp "$S3_PATH" "$TASKS_FILE" + TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") + echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + + DATA_DIR="$HOME/data/fleet/${MODALITY}" + PREPARE_CMD="uv run python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" + [ -n "$ENV_KEYS" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" + [ -n "$DIFFICULTY" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" + eval "$PREPARE_CMD" + +run: | + set -euo pipefail + ulimit -n 65536 + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + export PYTORCH_ALLOC_CONF=expandable_segments:True + + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + DATA_DIR="$HOME/data/fleet/${MODALITY}" + + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + # VL step-wise training (images accumulate across turns) + uv run --isolated --extra "$INFERENCE_BACKEND" \ + --with "litellm>=1.75.5" \ + --with "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" \ + --with "fleet-python" \ + --with "logfire" \ + --with "mcp>=1.0.0" \ + --with "boto3" \ + -m integrations.fleet.entrypoints.main_fleet \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file="$TASKS_FILE" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-VL-8B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$TOTAL_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=24 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=24 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=24 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=4 \ + generator.eval_n_samples_per_prompt=3 \ + generator.gpu_memory_utilization=0.7 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-browser-use-grpo" \ + trainer.run_name="fleet_vl_${MODALITY}_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_vl_${MODALITY}" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo-qwen3_5-35b.yaml b/skyrl-train/tasks/openenv-fleet-grpo-qwen3_5-35b.yaml new file mode 100644 index 0000000000..baa8e8b067 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-qwen3_5-35b.yaml @@ -0,0 +1,134 @@ +# Fleet Task GRPO Training via SkyPilot - Qwen3.5-35B-A3B (MoE, Multi-Node) +# Usage: sky launch tasks/openenv-fleet-grpo-qwen3_5-35b.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# MoE: 35B total, 3B active (256 experts, 9 active/token). GatedDeltaNet architecture. +# 262K native context. All 35B params in memory (~70GB fp16), optimizer ~140GB, gradients ~70GB. +# +# Multi-node (2-node default, 16 GPUs total): 8x B200/H200 per node +# NOTE: Requires vLLM >= 0.17.0 for native Qwen3.5/GDN support + FlashAttention 4 + +name: fleet-task-grpo-qwen3-5-35b + +resources: + disk_size: 750 + memory: 1500+ + ports: 6479 + any_of: + # GKE with GPUDirect-RDMA (fleet-rdma cluster, 3200 Gbps inter-node, spot) + - accelerators: H200:8 + cloud: kubernetes + network_tier: best + use_spot: true + - accelerators: B200:8 + cloud: kubernetes + network_tier: best + use_spot: true + - accelerators: H200-SXM:8 + cloud: runpod + - accelerators: B200:8 + cloud: runpod + - accelerators: B200:8 + cloud: lambda + - accelerators: H200-SXM:8 + cloud: vast + - accelerators: B200:8 + cloud: nebius + - accelerators: H200-SXM:8 + cloud: nebius + - accelerators: B200:8 + cloud: gcp + use_spot: true + image_id: projects/deeplearning-platform-release/global/images/common-cu128-ubuntu-2204-nvidia-570-v20260305 + - accelerators: H200:8 + cloud: gcp + use_spot: true + image_id: projects/deeplearning-platform-release/global/images/common-cu128-ubuntu-2204-nvidia-570-v20260305 + +num_nodes: 2 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + DATA_VERSION: "v55" + ENV_KEYS: "" + DIFFICULTY: "" + MODALITY: "tool_use" + MAX_TURNS: 50 + MAX_INPUT_LENGTH: 96000 + MAX_GENERATE_LENGTH: 4096 + NUM_EPOCHS: 20 + RUN_ID: "" + MAX_TASKS: "" + RESUME_RUN_NAME: "" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + +setup: | + bash skyrl-train/scripts/fleet-common-setup.sh \ + --openenv-branch deniz/fleet_client \ + --extra-setup skyrl-train/scripts/fleet-qwen35-extra-setup.sh + +run: | + bash skyrl-train/scripts/fleet-common-run.sh \ + --use-python-direct --cuda-env "$HOME/.cuda_env" \ + --set-ulimit --no-pytorch-alloc-conf \ + --nccl-heartbeat 1800 -- \ + environment.skyrl_gym.fleet_task.ttl_seconds=900 \ + environment.skyrl_gym.fleet_task.partial_reward=true \ + trainer.algorithm.advantage_estimator=grpo \ + trainer.policy.model.path="Qwen/Qwen3.5-35B-A3B" \ + trainer.flash_attn=true \ + trainer.loss_chunk_size=4096 \ + trainer.use_sample_packing=false \ + +generator.chat_template_kwargs='{enable_thinking:true}' \ + generator.inference_engine_tensor_parallel_size=2 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=8 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=16 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=16 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_ckpts_to_keep=1 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + 'generator.sampling_params.stop=[""]' \ + 'generator.eval_sampling_params.stop=[""]' \ + trainer.policy.optimizer_config.lr=5.0e-7 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=8 \ + generator.eval_n_samples_per_prompt=3 \ + generator.enforce_eager=false \ + generator.gpu_memory_utilization=0.65 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_qwen35_35b_${MODALITY}_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_qwen35_35b_${MODALITY}" \ + trainer.export_path="$HOME/exports" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo-qwen3_5-9b.yaml b/skyrl-train/tasks/openenv-fleet-grpo-qwen3_5-9b.yaml new file mode 100644 index 0000000000..2ac4ce86ae --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo-qwen3_5-9b.yaml @@ -0,0 +1,147 @@ +# Fleet Task GRPO Training via SkyPilot - Qwen3.5-9B (Text-Only / Tool Use) +# Usage: sky launch tasks/openenv-fleet-grpo-qwen3_5-9b.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# Text-only (tool_use) training — replaces Qwen3-8B as the default. +# +# Model: Qwen/Qwen3.5-9B (9B parameters, natively multimodal, early fusion + GatedDeltaNet) +# GPUs: 8x B200/H200 (RunPod, Lambda, Vast, Nebius, GCP) +# +# NOTE: Requires vLLM >= 0.17.0 for native Qwen3.5/GDN support + FlashAttention 4 +# +# v0.1: Initial Qwen3.5-9B tool_use config adapted from CU version +# v0.2: Refactored to use shared scripts (fleet-common-setup/run.sh) + +name: fleet-task-grpo-qwen3-5-9b + +resources: + disk_size: 750 + ports: 6479 + any_of: + # GKE (fleet-rdma cluster, GPUDirect-RDMA for multi-node, spot) + - accelerators: H200:8 + cloud: kubernetes + use_spot: true + - accelerators: B200:8 + cloud: kubernetes + use_spot: true + # 8x GPUs (matching VL branch config) + - accelerators: H200-SXM:8 + cloud: runpod + - accelerators: B200:8 + cloud: runpod + - accelerators: B200:8 + cloud: lambda + - accelerators: H200-SXM:8 + cloud: vast + - accelerators: B200:8 + cloud: nebius + - accelerators: H200-SXM:8 + cloud: nebius + # GCP (spot first, on-demand fallback) + - accelerators: B200:8 + cloud: gcp + use_spot: true + image_id: projects/deeplearning-platform-release/global/images/common-cu128-ubuntu-2204-nvidia-570-v20260305 + - accelerators: H200:8 + cloud: gcp + use_spot: true + image_id: projects/deeplearning-platform-release/global/images/common-cu128-ubuntu-2204-nvidia-570-v20260305 + - accelerators: B200:8 + cloud: gcp + image_id: projects/deeplearning-platform-release/global/images/common-cu128-ubuntu-2204-nvidia-570-v20260305 + - accelerators: H200:8 + cloud: gcp + image_id: projects/deeplearning-platform-release/global/images/common-cu128-ubuntu-2204-nvidia-570-v20260305 + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + # Dataset version: "v54" (v53 with 12 cross-contaminated verifiers patched) + DATA_VERSION: "v54" + # Task selection: env_keys to filter (comma-separated, e.g., "github,booking,reddit") + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + # Modality: text-only tool use + MODALITY: "tool_use" + # Training parameters + MAX_TURNS: 50 + MAX_INPUT_LENGTH: 64000 + MAX_GENERATE_LENGTH: 4096 + NUM_EPOCHS: 20 + RUN_ID: "" + MAX_TASKS: "" + # AWS credentials (required for S3 dataset download, optional for checkpoint upload) + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + # S3 buckets + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + +setup: | + bash skyrl-train/scripts/fleet-common-setup.sh \ + --openenv-branch deniz/fleet_client \ + --extra-setup skyrl-train/scripts/fleet-qwen35-extra-setup.sh \ + --skip-uv-isolated + +run: | + bash skyrl-train/scripts/fleet-common-run.sh \ + --use-python-direct --cuda-env "$HOME/.cuda_env" \ + --set-ulimit --no-pytorch-alloc-conf -- \ + environment.skyrl_gym.fleet_task.ttl_seconds=900 \ + trainer.algorithm.advantage_estimator=grpo \ + trainer.policy.model.path="Qwen/Qwen3.5-9B" \ + trainer.flash_attn=true \ + trainer.loss_chunk_size=4096 \ + trainer.use_sample_packing=false \ + +generator.chat_template_kwargs='{enable_thinking:true}' \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=12 \ + trainer.eval_before_train=true \ + trainer.eval_interval=20 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=24 \ + trainer.use_hybrid_env_sampling=true \ + trainer.min_samples_per_env=1 \ + trainer.policy_mini_batch_size=24 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.sampling_params.temperature=0.9 \ + generator.sampling_params.top_p=0.95 \ + 'generator.sampling_params.stop=[""]' \ + 'generator.eval_sampling_params.stop=[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=4 \ + generator.eval_n_samples_per_prompt=3 \ + generator.enforce_eager=false \ + generator.gpu_memory_utilization=0.85 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_qwen35_${MODALITY}_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_qwen35_${MODALITY}" \ + trainer.export_path="$HOME/exports" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/openenv-fleet-grpo.yaml b/skyrl-train/tasks/openenv-fleet-grpo.yaml new file mode 100644 index 0000000000..e57b7c5b63 --- /dev/null +++ b/skyrl-train/tasks/openenv-fleet-grpo.yaml @@ -0,0 +1,227 @@ +# Fleet Task GRPO Training via SkyPilot +# Usage: sky launch tasks/openenv-fleet-grpo.yaml --env FLEET_API_KEY= --env WANDB_API_KEY= +# +# This task trains GRPO on Fleet-hosted environments using OpenEnv's FleetTaskEnv. +# It loads tasks from the Fleet API and runs training with verifier-based rewards. +# Checkpoints are automatically uploaded to S3 to prevent disk exhaustion. + +name: fleet-task-grpo + +resources: + disk_size: 200 # Needs space for: model (~3GB), optimizer (~6GB), Ray temp (~50GB), vLLM cache, checkpoints + ports: 6479 + any_of: + # Prefer B200 (H100 has unreliable Triton/CUDA environments) + - accelerators: B200:1 + cloud: lambda + - accelerators: B200:1 + cloud: runpod + - accelerators: B200:1 + cloud: vast + - accelerators: B200:1 + cloud: primeintellect + - accelerators: B200:1 + cloud: nebius + # Fallback to H100 + - accelerators: H100:1 + cloud: lambda + - accelerators: H100:1 + cloud: runpod + - accelerators: H100:1 + cloud: vast + - accelerators: H100:1 + cloud: primeintellect + - accelerators: H100:1 + cloud: nebius + # GCP spot H200 fallback (preemptible, 64 GPU quota per region) + - accelerators: H200:1 + cloud: gcp + use_spot: true + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: main + +envs: + # These are passed via --env from GHA workflow + WANDB_API_KEY: "" + FLEET_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + # Dataset version: "v3" or "v4" (v4 has 15,193 tasks, v3 has 20,557 tasks) + DATA_VERSION: "v4" + # Task selection: env_keys to filter (comma-separated, e.g., "github,booking,reddit") + ENV_KEYS: "" + # Difficulty filter: 1=easy, 2=medium, 3=hard (comma-separated, e.g., "1,2" for easy+medium) + DIFFICULTY: "" + # Modality: "tool_use" or "computer_use" + MODALITY: "tool_use" + # Training parameters + MAX_TURNS: 50 + NUM_EPOCHS: 20 + RUN_ID: "" + # Optional: limit number of tasks for testing + MAX_TASKS: "" + # AWS credentials (required for S3 dataset download, optional for checkpoint upload) + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_REGION: "us-east-1" + # S3 buckets + S3_DATASET_BUCKET: "fleet-internal-datasets" + S3_CHECKPOINT_BUCKET: "skyrl-checkpoints" + S3_TRAJECTORY_BUCKET: "skyrl-trajectories" + +setup: | + set -euo pipefail + + # Fix Ray binary permissions (RunPod issue where binaries lose execute permissions) + # Must run BEFORE any Ray commands, including SkyPilot's internal cluster verification + echo "Checking for skypilot-runtime Ray binaries..." + RAY_BIN_DIR="/root/skypilot-runtime/lib/python3.10/site-packages/ray/core/src/ray" + if [ -d "$RAY_BIN_DIR" ]; then + echo "Found Ray binaries at $RAY_BIN_DIR" + echo "Current permissions:" + ls -la "$RAY_BIN_DIR/gcs/" 2>/dev/null || echo " (gcs dir not found)" + echo "Fixing permissions..." + find /root/skypilot-runtime -name "gcs_server" -exec chmod +x {} \; 2>/dev/null || true + find /root/skypilot-runtime -name "raylet" -exec chmod +x {} \; 2>/dev/null || true + find /root/skypilot-runtime -name "plasma_store_server" -exec chmod +x {} \; 2>/dev/null || true + chmod -R +x "$RAY_BIN_DIR" 2>/dev/null || true + echo "Permissions after fix:" + ls -la "$RAY_BIN_DIR/gcs/" 2>/dev/null || echo " (gcs dir not found)" + else + echo "Ray binaries not found at expected path, checking alternatives..." + find /root -name "gcs_server" 2>/dev/null | head -5 || echo " (not found)" + fi + + cd skyrl-train + + # Validate required environment variables + echo "Validating environment variables..." + if [ -z "$FLEET_API_KEY" ]; then + echo "ERROR: FLEET_API_KEY is required" + exit 1 + fi + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 dataset download" + exit 1 + fi + if [ "$MODALITY" != "tool_use" ] && [ "$MODALITY" != "computer_use" ]; then + echo "ERROR: MODALITY must be 'tool_use' or 'computer_use', got: $MODALITY" + exit 1 + fi + echo "Environment validation passed" + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm + uv pip install wandb boto3 awscli + + # Install OpenEnv for Fleet environment access (from branch with FleetTaskEnv) + uv pip install "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" fleet-python logfire "mcp>=1.0.0" + + # Download dataset from S3 + mkdir -p $HOME/data/fleet + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + S3_PATH="s3://${S3_DATASET_BUCKET}/${DATA_VERSION}/openenv/all_${MODALITY}.json" + echo "Downloading dataset from $S3_PATH..." + aws s3 cp "$S3_PATH" "$TASKS_FILE" + TASK_COUNT=$(python3 -c "import json; print(len(json.load(open('$TASKS_FILE'))['tasks']))") + echo "Downloaded $TASK_COUNT tasks for modality: $MODALITY" + + # Prepare dataset (with optional env/difficulty filters) + DATA_DIR="$HOME/data/fleet/${MODALITY}" + PREPARE_CMD="uv run python -m integrations.fleet.prepare_dataset --tasks-json $TASKS_FILE --output-dir $DATA_DIR --modality $MODALITY" + [ -n "$ENV_KEYS" ] && PREPARE_CMD="$PREPARE_CMD --env-filter $ENV_KEYS" + [ -n "$DIFFICULTY" ] && PREPARE_CMD="$PREPARE_CMD --difficulty-filter $DIFFICULTY" + eval "$PREPARE_CMD" + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + + TASKS_FILE="$HOME/data/fleet/tasks_${MODALITY}.json" + DATA_DIR="$HOME/data/fleet/${MODALITY}" + + # Login to Weights & Biases + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + # Ray cluster setup - limit object store to prevent disk exhaustion + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 # 10GB limit for object store + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + + # Wait for Ray + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + + # Run GRPO training with Fleet task integration + uv run --isolated --extra "$INFERENCE_BACKEND" \ + --with "litellm>=1.75.5" \ + --with "git+https://github.com/fleet-ai/OpenEnv.git@deniz/fleet-logfire" \ + --with "fleet-python" \ + --with "logfire" \ + --with "mcp>=1.0.0" \ + --with "boto3" \ + -m integrations.fleet.entrypoints.main_fleet \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + environment.env_class=fleet_task \ + environment.skyrl_gym.fleet_task.tasks_file="$TASKS_FILE" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen2.5-1.5B-Instruct" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$TOTAL_GPUS \ + generator.inference_engine_tensor_parallel_size=1 \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=4 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=4 \ + trainer.policy_mini_batch_size=4 \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=2048 \ + generator.sampling_params.max_generate_length=2048 \ + generator.sampling_params.temperature=0.6 \ + generator.sampling_params.top_p=0.95 \ + generator.sampling_params.stop='[""]' \ + generator.eval_sampling_params.stop='[""]' \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=true \ + generator.max_turns=$MAX_TURNS \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=true \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=false \ + generator.use_conversation_multi_turn=true \ + generator.n_samples_per_prompt=4 \ + generator.eval_n_samples_per_prompt=3 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="fleet-task-grpo" \ + trainer.run_name="fleet_${MODALITY}_${RUN_ID:-$(head -c 4 /dev/urandom | xxd -p)}" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$HOME/ckpts/fleet_${MODALITY}" \ + trainer.dump_data_batch=true diff --git a/skyrl-train/tasks/swe-tasks-grpo-qwen3-8b.yaml b/skyrl-train/tasks/swe-tasks-grpo-qwen3-8b.yaml new file mode 100644 index 0000000000..bf9bf4e5e2 --- /dev/null +++ b/skyrl-train/tasks/swe-tasks-grpo-qwen3-8b.yaml @@ -0,0 +1,157 @@ +# SWE Task GRPO Training via SkyPilot - Qwen3-8B +# Usage: sky launch tasks/swe-tasks-grpo-qwen3-8b.yaml --env WANDB_API_KEY= +# +# This task trains GRPO on swe-task-generator environments using Docker containers. +# Docker images are pulled from Docker Hub (erranli/swe-task-*). +# The agent learns to fix bugs by interacting with the codebase in a container. +# +# Model: Qwen/Qwen3-8B (8B parameters) +# GPUs: Lambda only (bare metal with Docker pre-installed) + +name: swe-tasks-grpo-qwen3-8b + +resources: + disk_size: 200 + ports: 6479 + cloud: lambda + any_of: + - accelerators: B200:2 + - accelerators: H100:2 + +num_nodes: 1 + +workdir: + url: https://github.com/fleet-ai/SkyRL.git + ref: swe-task-generator-integration + +envs: + WANDB_API_KEY: "" + LOGGER: "wandb" + INFERENCE_BACKEND: "vllm" + MAX_TURNS: 20 + MAX_INPUT_LENGTH: 30720 + MAX_GENERATE_LENGTH: 4096 + NUM_EPOCHS: 20 + # Path to swe-task-generator repo (will be cloned in setup) + SWE_TASK_GENERATOR_REPO: "https://github.com/fleet-ai/swe-task-generator.git" + +setup: | + set -euo pipefail + + # Ensure current user can access Docker daemon + if ! docker ps >/dev/null 2>&1; then + sudo usermod -aG docker $USER + sudo chmod 666 /var/run/docker.sock + fi + docker ps # Verify Docker access + + cd skyrl-train + + uv venv --python 3.12 --seed + source .venv/bin/activate + uv sync --extra vllm --extra miniswe + + # Clone swe-task-generator to get task instances + if [ ! -d "$HOME/swe-task-generator" ]; then + git clone $SWE_TASK_GENERATOR_REPO $HOME/swe-task-generator + fi + + # Prepare dataset from task instances + uv pip install pyarrow datasets + python integrations/swe_tasks/prepare_dataset.py \ + --tasks-dir $HOME/swe-task-generator/tasks \ + --output-dir $HOME/data/swe_tasks + + # Pre-pull Docker images for faster training startup + echo "Pre-pulling Docker images..." + docker pull erranli/swe-task-marshmallow-code-marshmallow-2894:latest || true + docker pull erranli/swe-task-marshmallow-code-marshmallow-2892:latest || true + +run: | + set -euo pipefail + cd skyrl-train + source .venv/bin/activate + + TMP_DIR="$HOME/skyrl-tmp" + mkdir -p "$TMP_DIR" + export TMPDIR="$TMP_DIR" + export PYTORCH_ALLOC_CONF=expandable_segments:True + export MSWEA_COST_TRACKING=ignore_errors # Qwen3-8B isn't in litellm's pricing DB + + DATA_DIR="$HOME/data/swe_tasks" + CKPT_PATH="$HOME/ckpts/swe_tasks" + SWE_TASKS_TRAJ_DIR="$HOME/swe_tasks_trajs" + + # Login to Weights & Biases + uv run -- python3 -c "import wandb; wandb.login(relogin=True, key='$WANDB_API_KEY')" + + # Ray cluster setup + export RAY_RUNTIME_ENV_HOOK=ray._private.runtime_env.uv_runtime_env_hook.hook + export RAY_object_store_memory=10000000000 + if ! ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + ray start --head --disable-usage-stats --port 6479 --object-store-memory=10000000000 + fi + + # Wait for Ray + for i in $(seq 1 24); do + if ray status --address 127.0.0.1:6479 >/dev/null 2>&1; then + break + fi + sleep 5 + done + + TOTAL_GPUS=$SKYPILOT_NUM_GPUS_PER_NODE + echo "Running with $TOTAL_GPUS GPU(s)" + + # When colocating, num_inference_engines * TP must equal TOTAL_GPUS + # Use TP=1 and engines=TOTAL_GPUS for maximum throughput + NUM_INFERENCE_ENGINES=$TOTAL_GPUS + TP_SIZE=1 + + # Run GRPO training + uv run --isolated --extra vllm --extra miniswe \ + -m integrations.swe_tasks.entrypoints.main_swe_tasks \ + data.train_data="['${DATA_DIR}/train.parquet']" \ + data.val_data="['${DATA_DIR}/validation.parquet']" \ + trainer.algorithm.advantage_estimator="grpo" \ + trainer.policy.model.path="Qwen/Qwen3-8B" \ + trainer.placement.colocate_all=true \ + trainer.strategy=fsdp2 \ + trainer.placement.policy_num_gpus_per_node=$TOTAL_GPUS \ + trainer.placement.ref_num_gpus_per_node=$TOTAL_GPUS \ + generator.num_inference_engines=$NUM_INFERENCE_ENGINES \ + generator.inference_engine_tensor_parallel_size=$TP_SIZE \ + trainer.epochs=${NUM_EPOCHS} \ + trainer.eval_batch_size=2 \ + trainer.eval_before_train=true \ + trainer.eval_interval=5 \ + trainer.update_epochs_per_batch=1 \ + trainer.train_batch_size=2 \ + trainer.policy_mini_batch_size=2 \ + trainer.micro_forward_batch_size_per_gpu=1 \ + trainer.micro_train_batch_size_per_gpu=1 \ + trainer.dump_data_batch=true \ + trainer.ckpt_interval=10 \ + trainer.max_prompt_length=4096 \ + generator.sampling_params.max_generate_length=$MAX_GENERATE_LENGTH \ + generator.max_input_length=$MAX_INPUT_LENGTH \ + generator.max_turns=$MAX_TURNS \ + trainer.policy.optimizer_config.lr=1.0e-6 \ + trainer.algorithm.use_kl_loss=false \ + generator.backend=$INFERENCE_BACKEND \ + generator.run_engines_locally=True \ + generator.enable_http_endpoint=True \ + generator.http_endpoint_host='127.0.0.1' \ + generator.http_endpoint_port=8001 \ + generator.weight_sync_backend=nccl \ + generator.async_engine=true \ + generator.batched=true \ + generator.n_samples_per_prompt=4 \ + generator.gpu_memory_utilization=0.8 \ + trainer.logger="$LOGGER" \ + trainer.project_name="swe_tasks" \ + trainer.run_name="swe_tasks_8B_$(head -c 4 /dev/urandom | xxd -p)" \ + trainer.resume_mode=latest \ + trainer.ckpt_path="$CKPT_PATH" \ + +generator.swe_tasks_config_path="integrations/swe_tasks/swe_tasks.yaml" \ + +generator.swe_tasks_traj_dir=$SWE_TASKS_TRAJ_DIR diff --git a/skyrl-train/tests/__init__.py b/skyrl-train/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/tests/cpu/__init__.py b/skyrl-train/tests/cpu/__init__.py new file mode 100644 index 0000000000..92ccfcd831 --- /dev/null +++ b/skyrl-train/tests/cpu/__init__.py @@ -0,0 +1 @@ +# CPU Tests diff --git a/skyrl-train/tests/cpu/algorithms/test_losses.py b/skyrl-train/tests/cpu/algorithms/test_losses.py new file mode 100644 index 0000000000..f5904b595f --- /dev/null +++ b/skyrl-train/tests/cpu/algorithms/test_losses.py @@ -0,0 +1,623 @@ +""" +Tests for policy loss functions. + +uv run --isolated --extra dev -- pytest tests/cpu/algorithms/test_losses.py +""" + +import pytest +import torch +from omegaconf import DictConfig + +from skyrl_train.utils.ppo_utils import PolicyLossRegistry, masked_mean + + +# Adapted a good test from NeMO-RL +def test_policy_loss_dual_clip(): + """Tests dual clipping in PolicyLoss function.""" + + device = "cpu" + + # Create test data with a mix of advantages: positive, slightly negative, strongly negative + advantages = torch.tensor([[1.0, -1.0, -4.0]], device=device) + + # Set up logprobs to test different probability ratios + old_log_probs = torch.tensor([[-1.0, -1.0, -3.0]], device=device) + log_probs = torch.tensor([[-1.69315, -1.0, -0.69741]], device=device) # approx log(0.5)-1, log(1)-1, log(10)-3 + + # Create config for dual clipping + config = DictConfig( + { + "eps_clip_low": 0.2, + "eps_clip_high": 0.2, + "clip_ratio_c": 3.0, + "policy_loss_type": "dual_clip", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + # Create loss function with dual clipping + loss_fn = PolicyLossRegistry.get("dual_clip") + + # Calculate expected values + ratio = torch.exp(log_probs - old_log_probs) # approx [0.5, 1.0, 10.0] + assert torch.allclose(ratio, torch.tensor([[0.5, 1.0, 10.0]], device=device), rtol=1e-3) + + # Standard PPO clipping + loss1 = -ratio * advantages # [0.5, -1.0, -40.0] + loss2 = -ratio.clamp(1 - 0.2, 1 + 0.2) * advantages # [0.8, -1.0, -4.8] + max_loss = torch.maximum(loss1, loss2) # [0.5, -1.0, -40.0] + + # Dual clipping + loss3 = -advantages * 3.0 # [-3.0, 3.0, 12.0] + min_loss = torch.min(loss3, max_loss) # [-3.0, 1.0, 12.0] + + # For negative advantages, use dual clipped loss + final_loss = torch.where(advantages < 0, min_loss, max_loss) # [-0.5, 1.0, 12.0] + assert torch.allclose(final_loss, torch.tensor([[-0.5, 1.0, 12.0]], device=device), rtol=1e-3) + expected_loss = final_loss.mean() # -(-12.5/3) = 4.1667 + + # Calculate actual loss + actual_loss, _ = loss_fn(log_probs=log_probs, old_log_probs=old_log_probs, advantages=advantages, config=config) + + # Verify results + torch.testing.assert_close(actual_loss, expected_loss, rtol=1e-3, atol=1e-8) + # close to hand calculated value + assert actual_loss.item() == pytest.approx(4.1667, abs=1e-4) + + +def test_policy_loss_cispo(): + """Tests CISPO in PolicyLoss function.""" + + device = "cpu" + + # Create test data with a mix of advantages: positive, slightly negative, strongly negative + advantages = torch.tensor([[1.0, -1.0, -4.0]], device=device) + + # Set up logprobs to test different probability ratios + old_log_probs = torch.tensor([[-1.0, -1.0, -3.0]], device=device) + log_probs = torch.tensor([[-1.69315, -1.0, -0.69741]], device=device) # approx log(0.5)-1, log(1)-1, log(10)-3 + + # Create config for cispo + config = DictConfig( + { + "cispo": {"cispo_eps_clip_low": 0.2, "cispo_eps_clip_high": 0.2}, + "policy_loss_type": "cispo", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + # Create loss function with cispo + loss_fn = PolicyLossRegistry.get("cispo") + + # Calculate expected values + ratio = torch.exp(log_probs - old_log_probs) # approx [0.5, 1.0, 10.0] + assert torch.allclose(ratio, torch.tensor([[0.5, 1.0, 10.0]], device=device), rtol=1e-3) + + # Hand-calculation for expected loss: + # ratio = [0.5, 1.0, 10.0] + # clamped_ratio = ratio.clamp(0.8, 1.2) = [0.8, 1.0, 1.2] + # advantages = [1.0, -1.0, -4.0] + # log_probs = [-1.69315, -1.0, -0.69741] + # loss_per_token = -advantages * clamped_ratio * log_probs + # loss_per_token[0] = -(1.0 * 0.8 * -1.69315) = 1.35452 + # loss_per_token[1] = -(-1.0 * 1.0 * -1.0) = -1.0 + # loss_per_token[2] = -(-4.0 * 1.2 * -0.69741) = -3.347568 + # mean(loss) = (1.35452 - 1.0 - 3.347568) / 3 = -0.99768266666 + loss = -ratio.clamp(1 - 0.2, 1 + 0.2) * advantages * log_probs + expected_loss = loss.mean() + + # Calculate actual loss + actual_loss, _ = loss_fn( + log_probs=log_probs, + old_log_probs=old_log_probs, + advantages=advantages, + config=config, + ) + + # Verify results + torch.testing.assert_close(actual_loss, expected_loss, rtol=1e-3, atol=1e-8) + # close to hand calculated value + assert actual_loss.item() == pytest.approx(-0.99768266666, abs=1e-4) + + +def test_policy_loss_reduction_modes(): + """Tests different loss_reduction modes in PolicyLoss function. + + Note: token_mean and sequence_mean give the same result when all sequences + have the same length and no mask is applied, but differ when masking creates + different effective sequence lengths. + """ + + device = "cpu" + + clip_eps_low = 0.2 + clip_eps_high = 0.2 + + advantages = torch.tensor( + [ + [2.0, 2.0, 2.0], # sequence 1: consistently higher advantages + [1.0, 1.0, 1.0], # sequence 2: consistently lower advantages + ], + device=device, + ) + + old_log_probs = torch.tensor([[-1.0, -1.0, -1.0], [-1.0, -1.0, -1.0]], device=device) + + log_probs = torch.tensor( + [[-1.5, -0.5, -1.2], [-0.8, -1.3, -0.9]], # ratios ≈ [[0.61, 1.65, 0.83],[1.22, 0.74, 1.11]] + device=device, + ) + + # Create masks to test sequences with different numbers of valid tokens + loss_mask = torch.tensor([[1.0, 1.0, 1.0], [1.0, 0.0, 0.0]], device=device) + + # Create configs for different reduction modes + config_token = DictConfig( + { + "eps_clip_low": clip_eps_low, + "eps_clip_high": clip_eps_high, + "clip_ratio_c": 3.0, + "policy_loss_type": "regular", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + config_seq = DictConfig( + { + "eps_clip_low": clip_eps_low, + "eps_clip_high": clip_eps_high, + "clip_ratio_c": 3.0, + "policy_loss_type": "regular", + "loss_reduction": "sequence_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + # Get loss function + loss_fn = PolicyLossRegistry.get("regular") + + # Test token_mean without mask + loss_token_no_mask, _ = loss_fn(log_probs, old_log_probs, advantages, config_token) + + # Test token_mean with mask + loss_token_with_mask, _ = loss_fn(log_probs, old_log_probs, advantages, config_token, loss_mask) + + # Test sequence_mean without mask + loss_seq_no_mask, _ = loss_fn(log_probs, old_log_probs, advantages, config_seq) + + # Test sequence_mean with mask + loss_seq_with_mask, _ = loss_fn(log_probs, old_log_probs, advantages, config_seq, loss_mask) + + # Manual calculations to verify (using default PolicyLoss parameters) + ratio = torch.exp(log_probs - old_log_probs) + surr1 = ratio * advantages + surr2 = ratio.clamp(1 - clip_eps_low, 1 + clip_eps_high) * advantages # clip_eps_low=0.2, clip_eps_high=0.2 + loss_per_token = -torch.min(surr1, surr2) + + # Expected token_mean without mask: mean of all tokens + expected_token_no_mask = loss_per_token.mean() + + # Expected token_mean with mask: masked mean of all tokens + expected_token_with_mask = (loss_per_token * loss_mask).sum() / (loss_mask.sum() + 1e-8) + + # Expected sequence_mean without mask: mean of sequence means + expected_seq_no_mask = loss_per_token.mean(dim=1).mean() + + # Expected sequence_mean with mask: mean of masked sequence means + seq_means_masked = (loss_per_token * loss_mask).sum(dim=1) / (loss_mask.sum(dim=1) + 1e-8) + expected_seq_with_mask = seq_means_masked.mean() + + # Verify results + torch.testing.assert_close(loss_token_no_mask, expected_token_no_mask, rtol=1e-5, atol=1e-8) + torch.testing.assert_close(loss_token_with_mask, expected_token_with_mask, rtol=1e-5, atol=1e-8) + torch.testing.assert_close(loss_seq_no_mask, expected_seq_no_mask, rtol=1e-5, atol=1e-8) + torch.testing.assert_close(loss_seq_with_mask, expected_seq_with_mask, rtol=1e-5, atol=1e-8) + + # Verify that the two reduction modes give the same results when sequences have equal length and no mask + assert torch.allclose( + loss_token_no_mask, loss_seq_no_mask, rtol=1e-5 + ), "token_mean and sequence_mean should give same results when sequences have equal length and no mask" + # But they should give different results when mask creates different effective sequence lengths + assert not torch.allclose( + loss_token_with_mask, loss_seq_with_mask, rtol=1e-3 + ), "token_mean and sequence_mean with mask should give different results" + + +def test_policy_loss_reduction_edge_cases(): + """Tests edge cases for loss_reduction modes.""" + + device = "cpu" + + # Test with single sequence (should give same result for both modes) + advantages = torch.tensor([[1.0, -1.0, 2.0]], device=device) + old_log_probs = torch.tensor([[-1.0, -1.0, -1.0]], device=device) + log_probs = torch.tensor([[-1.5, -0.5, -1.2]], device=device) + + # Create configs for different reduction modes + config_token = DictConfig( + { + "eps_clip_low": 0.2, + "eps_clip_high": 0.2, + "clip_ratio_c": 3.0, + "policy_loss_type": "regular", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + config_seq = DictConfig( + { + "eps_clip_low": 0.2, + "eps_clip_high": 0.2, + "clip_ratio_c": 3.0, + "policy_loss_type": "regular", + "loss_reduction": "sequence_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + # Get loss function + loss_fn = PolicyLossRegistry.get("regular") + + loss_token, _ = loss_fn(log_probs, old_log_probs, advantages, config_token) + loss_seq, _ = loss_fn(log_probs, old_log_probs, advantages, config_seq) + + # With single sequence, both modes should give same result + torch.testing.assert_close(loss_token, loss_seq, rtol=1e-6, atol=1e-8) + + # Test with completely masked sequence + loss_mask = torch.tensor([[0.0, 0.0, 0.0]], device=device) + loss_token_masked, _ = loss_fn(log_probs, old_log_probs, advantages, config_token, loss_mask) + loss_seq_masked, _ = loss_fn(log_probs, old_log_probs, advantages, config_seq, loss_mask) + + # Should handle zero mask gracefully (due to +1e-8 in denominator) + assert torch.isfinite(loss_token_masked) + assert torch.isfinite(loss_seq_masked) + + +def test_gspo_importance_sampling_levels(): + """Tests GSPO policy loss function with sequence-level importance sampling. + + This test focuses on GSPO's key benefit: stabilizing clipping behavior through sequence-level + importance sampling, which should lead to more consistent training dynamics compared to + token-level importance sampling in standard PPO. + """ + + device = "cpu" + + clip_eps_low = 0.2 + clip_eps_high = 0.2 + + # Create test data with varied sequence lengths and extreme ratios to test clipping stability + # GSPO's benefit is most apparent with sequences of different lengths and high variance + advantages = torch.tensor( + [ + [1.5, 2.0, 1.0, 0.8, 0.5, 0.0, 0.0, 0.0], # long sequence: 5 valid tokens + [3.0, 1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], # short sequence: 2 valid tokens + [0.5, 0.8, 1.2, 2.5, 0.0, 0.0, 0.0, 0.0], # medium sequence: 4 valid tokens + ], + device=device, + ) + + old_log_probs = torch.tensor( + [ + [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], + [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], + [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], + ], + device=device, + ) + + # Create extreme log probability ratios to trigger significant clipping + # This tests GSPO's stability benefits under conditions that would cause unstable clipping + log_probs = torch.tensor( + [ + [0.2, -2.5, -0.3, 0.1, -1.8, -1.0, -1.0, -1.0], # high variance within sequence + [0.8, -0.2, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], # extreme ratios (exp(1.8)≈6.0, exp(0.8)≈2.2) + [-0.5, 0.3, -1.7, 0.4, -1.0, -1.0, -1.0, -1.0], # mixed extreme values + ], + device=device, + ) + + # Create masks for different sequence lengths (key for testing length normalization) + loss_mask = torch.tensor( + [ + [1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], # 5 tokens + [1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], # 2 tokens + [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0], # 4 tokens + ], + device=device, + ) + + # Test standard PPO (token-level importance sampling) + ppo_config = DictConfig( + { + "eps_clip_low": clip_eps_low, + "eps_clip_high": clip_eps_high, + "clip_ratio_c": 3.0, + "policy_loss_type": "regular", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + ppo_loss_fn = PolicyLossRegistry.get("regular") + loss_token, _ = ppo_loss_fn(log_probs, old_log_probs, advantages, ppo_config, loss_mask) + + # Test GSPO (sequence-level importance sampling) + gspo_config = DictConfig( + { + "eps_clip_low": clip_eps_low, + "eps_clip_high": clip_eps_high, + "clip_ratio_c": 3.0, + "policy_loss_type": "gspo", + "loss_reduction": "sequence_mean", # GSPO recommended reduction + "max_seq_len": 4, + "use_tis": False, + } + ) + gspo_loss_fn = PolicyLossRegistry.get("gspo") + loss_sequence, _ = gspo_loss_fn(log_probs, old_log_probs, advantages, gspo_config, loss_mask) + + # Manual calculation for token-level (standard PPO) + log_ratio = log_probs - old_log_probs + ratio_token = log_ratio.exp() + surr1_token = ratio_token * advantages + surr2_token = ratio_token.clamp(1 - clip_eps_low, 1 + clip_eps_high) * advantages + loss_per_token_token = -torch.min(surr1_token, surr2_token) + expected_token = (loss_per_token_token * loss_mask).sum() / (loss_mask.sum() + 1e-8) + + # Calculate token-level clipping ratio + is_clipped_token = (-surr2_token > -surr1_token) & (loss_mask.bool()) + clip_ratio_token = is_clipped_token.float().sum() / loss_mask.sum() + + # Manual calculation for sequence-level (GSPO) + # First compute sequence-level importance weights (key GSPO innovation) + log_importance_weights_seq = masked_mean(log_ratio, loss_mask, dim=-1).unsqueeze(-1) + + # GSPO uses stop gradients: s_i,t(θ) = sg[s_i(θ)] · π_θ(y_i,t|x, y_i, -surr1_sequence) & (loss_mask.bool()) + clip_ratio_sequence = is_clipped_sequence.float().sum() / loss_mask.sum() + + # Verify loss calculations + torch.testing.assert_close(loss_token, expected_token, rtol=1e-5, atol=1e-8) + torch.testing.assert_close(loss_sequence, expected_sequence, rtol=1e-5, atol=1e-8) + + # Core GSPO benefit test: Different clipping behavior + # GSPO should produce different clipping patterns due to sequence-level importance sampling + assert not torch.allclose( + clip_ratio_token, clip_ratio_sequence, rtol=1e-2 + ), f"Clipping ratios should differ: token={clip_ratio_token:.4f} vs sequence={clip_ratio_sequence:.4f}" + + # Test stability: sequence-level should smooth out extreme per-token variations + # Check that sequence-level ratios have lower variance within each sequence + token_ratio_variance = torch.var(ratio_token * loss_mask, dim=-1).mean() + sequence_ratio_variance = torch.var(ratio_sequence * loss_mask, dim=-1).mean() + + # The key insight: GSPO should reduce within-sequence variance by using sequence-averaged ratios + assert sequence_ratio_variance < token_ratio_variance, ( + f"GSPO should reduce ratio variance: sequence={sequence_ratio_variance:.4f} < " + f"token={token_ratio_variance:.4f}" + ) + + # Token-level and sequence-level should give different results due to different importance weighting + assert not torch.allclose( + loss_token, loss_sequence, rtol=1e-3 + ), f"Loss values should differ: token={loss_token:.6f} vs sequence={loss_sequence:.6f}" + + # Test length normalization effect: sequences with different lengths should be handled more uniformly + # This is a key stability benefit of GSPO mentioned in the paper + seq_lengths = loss_mask.sum(dim=-1) # [5, 2, 4] + + # In GSPO, the sequence-level importance weights should be the same across all tokens in a sequence + # This should make the treatment more uniform across different sequence lengths + for seq_idx in range(log_importance_weights_seq.shape[0]): + seq_len = int(seq_lengths[seq_idx]) + if seq_len > 1: + # All importance weights within a sequence should be identical (GSPO property) + seq_weights = log_importance_weights_seq[seq_idx, :seq_len] + assert torch.allclose( + seq_weights, seq_weights[0], rtol=1e-6 + ), f"GSPO should have uniform importance weights within sequence {seq_idx}" + + +def test_clip_cov_policy_loss(): + """Tests Clip-Cov policy loss function with covariance-based correction.""" + + device = "cpu" + torch.manual_seed(42) # For reproducible randomization in clip-cov + + # Create test data + advantages = torch.tensor( + [ + [2.0, -1.0, 1.5, 0.8], + [1.0, 0.5, -2.0, 1.2], + ], + device=device, + ) + + old_log_probs = torch.tensor([[-1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0]], device=device) + + log_probs = torch.tensor([[-0.5, -1.5, -0.8, -1.2], [-1.3, -0.7, -1.8, -0.9]], device=device) + + loss_mask = torch.tensor([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 0.0]], device=device) # Last token masked + + # Create Clip-Cov config + config = DictConfig( + { + "eps_clip_low": 0.2, + "eps_clip_high": 0.2, + "policy_loss_type": "clip_cov", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "clip_cov": {"clip_ratio": 0.5, "clip_cov_lb": -5.0, "clip_cov_ub": 5.0}, # Large ratio for testing + } + ) + + # Get loss function + clip_cov_fn = PolicyLossRegistry.get("clip_cov") + + # Calculate loss + loss, clip_frac = clip_cov_fn(log_probs, old_log_probs, advantages, config, loss_mask) + + # Basic sanity checks + assert torch.isfinite(loss), "Loss should be finite" + assert 0 <= clip_frac <= 1, f"Clip fraction should be between 0 and 1, got {clip_frac}" + + # Compare with regular PPO (should be different due to covariance correction) + regular_config = DictConfig( + { + "eps_clip_low": 0.2, + "eps_clip_high": 0.2, + "policy_loss_type": "regular", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + regular_fn = PolicyLossRegistry.get("regular") + regular_loss, regular_clip_frac = regular_fn(log_probs, old_log_probs, advantages, regular_config, loss_mask) + + # Clip-Cov should give different results due to covariance-based correction + assert not torch.allclose( + loss, regular_loss, rtol=1e-3 + ), f"Clip-Cov and regular PPO should differ: clip_cov={loss:.6f} vs regular={regular_loss:.6f}" + + +def test_kl_cov_policy_loss(): + """Tests KL-Cov policy loss function with covariance-based token selection.""" + + device = "cpu" + torch.manual_seed(42) # For reproducible token selection + + # Create test data + advantages = torch.tensor( + [ + [1.5, -0.5, 2.0, 0.8], + [0.5, 1.0, -1.5, 1.2], + ], + device=device, + ) + + old_log_probs = torch.tensor([[-1.0, -1.0, -1.0, -1.0], [-1.0, -1.0, -1.0, -1.0]], device=device) + + log_probs = torch.tensor([[-0.8, -1.2, -0.6, -1.1], [-1.1, -0.9, -1.4, -0.7]], device=device) + + loss_mask = torch.tensor([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 0.0]], device=device) # Last token masked + + # Create KL-Cov config + config = DictConfig( + { + "policy_loss_type": "kl_cov", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "kl_cov": {"kl_cov_frac": 0.5, "ppo_kl_coef": 1.0}, # Apply KL to 50% of tokens + } + ) + + # Get loss function + kl_cov_fn = PolicyLossRegistry.get("kl_cov") + + # Calculate loss + loss, clip_frac = kl_cov_fn(log_probs, old_log_probs, advantages, config, loss_mask) + + # Basic sanity checks + assert torch.isfinite(loss), "Loss should be finite" + assert clip_frac == 0.0, "KL-Cov should return 0.0 for clipfrac value" + + # Compare with regular PPO (should be different due to KL regularization) + regular_config = DictConfig( + { + "eps_clip_low": 0.2, + "eps_clip_high": 0.2, + "policy_loss_type": "regular", + "loss_reduction": "token_mean", + "max_seq_len": 4, + "use_tis": False, + } + ) + + regular_fn = PolicyLossRegistry.get("regular") + regular_loss, _ = regular_fn(log_probs, old_log_probs, advantages, regular_config, loss_mask) + + # KL-Cov should give different results due to KL regularization on selected tokens + assert not torch.allclose( + loss, regular_loss, rtol=1e-3 + ), f"KL-Cov and regular PPO should differ: kl_cov={loss:.6f} vs regular={regular_loss:.6f}" + + +def test_sapo_policy_loss_basic(): + """Tests SAPO policy loss against a hand-computed expectation.""" + + device = "cpu" + + # Mix of positive and negative advantages so tau_pos / tau_neg both get used + advantages = torch.tensor([[1.0, -1.0, 0.5]], device=device) + + # Simple log-prob configuration to produce non-trivial ratios + old_log_probs = torch.tensor([[-1.0, -1.0, -1.0]], device=device) + # Ratios ≈ [exp(-0.5), exp(0.2), exp(-0.1)] ≈ [0.6065, 1.2214, 0.9048] + log_probs = torch.tensor([[-1.5, -0.8, -1.1]], device=device) + + # SAPO config: uses sequence_mean reduction and distinct tau_pos / tau_neg + config = DictConfig( + { + "policy_loss_type": "sapo", + "loss_reduction": "sequence_mean", + "max_seq_len": 4, + "sapo": {"tau_pos": 1.0, "tau_neg": 2.0}, + } + ) + + loss_fn = PolicyLossRegistry.get("sapo") + + # Actual SAPO loss + actual_loss, actual_clip_ratio = loss_fn( + log_probs=log_probs, + old_log_probs=old_log_probs, + advantages=advantages, + config=config, + ) + + # --- Hand-computed expectation, mirroring sapo_policy_loss implementation --- + + tau_pos = torch.as_tensor(config.sapo.tau_pos, dtype=advantages.dtype, device=advantages.device) + tau_neg = torch.as_tensor(config.sapo.tau_neg, dtype=advantages.dtype, device=advantages.device) + + def gate_function(x, tau): + return torch.sigmoid(tau * (x - 1.0)) * (4.0 / tau) + + log_ratio = log_probs - old_log_probs + log_ratio = torch.clamp(log_ratio, min=-20.0, max=20.0) + ratio = torch.exp(log_ratio) + + taus = torch.where(advantages > 0, tau_pos, tau_neg) + gates = gate_function(ratio, taus) + + loss_per_token = -gates * advantages + # sequence_mean reduction: per-sequence token mean, then batch mean + expected_loss = loss_per_token.mean(dim=-1).mean() + + torch.testing.assert_close(actual_loss, expected_loss, rtol=1e-5, atol=1e-8) + + # SAPO should always report clip_ratio = 0.0 + assert actual_clip_ratio == 0.0 diff --git a/skyrl-train/tests/cpu/conftest.py b/skyrl-train/tests/cpu/conftest.py new file mode 100644 index 0000000000..fd7c45ad64 --- /dev/null +++ b/skyrl-train/tests/cpu/conftest.py @@ -0,0 +1,12 @@ +import pytest +import ray + + +@pytest.fixture(scope="session", autouse=True) +def ray_init(): + """Initialize Ray once for the entire test session.""" + if not ray.is_initialized(): + ray.init() + yield + if ray.is_initialized(): + ray.shutdown() diff --git a/skyrl-train/tests/cpu/dataset/test_dataset.py b/skyrl-train/tests/cpu/dataset/test_dataset.py new file mode 100644 index 0000000000..eb6082a9a2 --- /dev/null +++ b/skyrl-train/tests/cpu/dataset/test_dataset.py @@ -0,0 +1,166 @@ +import pytest +from unittest.mock import MagicMock, patch +from datasets import Dataset +from skyrl_train.dataset import PromptDataset + + +@pytest.fixture +def mock_tokenizer(): + tokenizer = MagicMock() + tokenizer.apply_chat_template.side_effect = lambda x, add_generation_prompt, **kwargs: x + return tokenizer + + +@pytest.fixture +def sample_dataset(): + # 3 samples: one too long, two valid + data = { + "prompt": [ + "short prompt", # length 13 + "a" * 120, # length 120 + "b" * 200, # length 200 (to be filtered out if max len < 200) + ], + "answer": ["a1", "a2", "a3"], + } + return Dataset.from_dict(data) + + +@patch("datasets.load_dataset") +def test_prompt_dataset_filtering(mock_load_dataset, mock_tokenizer, sample_dataset): + mock_load_dataset.return_value = {"train": sample_dataset} + + dataset = PromptDataset( + datasets=["dummy1.parquet"], + tokenizer=mock_tokenizer, + max_prompt_length=150, # should exclude third item + num_workers=1, + prompt_key="prompt", + env_class_key="env_class", + ) + + # Only first two prompts should remain + assert len(dataset) == 2 + messages, env, extra, uid = dataset[0] + assert env is None + assert messages == "short prompt" + assert extra == {"answer": "a1"} + + +def test_collate_fn(): + dataset = PromptDataset.__new__(PromptDataset) # Bypass __init__ + sample_data = [("prompt 1", "env", {"answer": "a1"}, "1"), ("prompt 2", "env", {"answer": "a2"}, "2")] + expected = [ + {"prompt": "prompt 1", "env_class": "env", "env_extras": {"answer": "a1"}, "uid": "1"}, + {"prompt": "prompt 2", "env_class": "env", "env_extras": {"answer": "a2"}, "uid": "2"}, + ] + + output = dataset.collate_fn(sample_data) + assert output == expected + + +@patch("datasets.load_dataset") +def test_prompt_dataset_hf_name_defaults_to_train(mock_load_dataset, mock_tokenizer, sample_dataset): + # When only a dataset name is provided, we default to the 'train' split + mock_load_dataset.return_value = {"train": sample_dataset} + + dataset = PromptDataset( + datasets=["my_hf_dataset"], + tokenizer=mock_tokenizer, + max_prompt_length=150, + num_workers=1, + prompt_key="prompt", + env_class_key="env_class", + ) + + assert len(dataset) == 2 + messages, env, extra, uid = dataset[1] + assert messages == "a" * 120 + assert extra == {"answer": "a2"} + + +@patch("datasets.load_dataset") +def test_prompt_dataset_hf_name_with_split(mock_load_dataset, mock_tokenizer, sample_dataset): + # When a dataset name with split is provided, respect the split + mock_load_dataset.return_value = {"validation": sample_dataset} + + dataset = PromptDataset( + datasets=["my_hf_dataset:validation"], + tokenizer=mock_tokenizer, + max_prompt_length=150, + num_workers=1, + prompt_key="prompt", + env_class_key="env_class", + ) + + assert len(dataset) == 2 + messages, env, extra, uid = dataset[0] + assert messages == "short prompt" + assert extra == {"answer": "a1"} + + +@patch("datasets.load_dataset") +def test_prompt_dataset_hf_missing_train_raises(mock_load_dataset, mock_tokenizer, sample_dataset): + # No split provided and 'train' not available + mock_load_dataset.return_value = {"validation": sample_dataset} + + with pytest.raises(ValueError, match=r"Split `train` not found"): + PromptDataset( + datasets=["my_hf_dataset"], + tokenizer=mock_tokenizer, + max_prompt_length=150, + num_workers=1, + ) + + +@patch("datasets.load_dataset") +def test_prompt_dataset_hf_invalid_split_raises(mock_load_dataset, mock_tokenizer, sample_dataset): + # Split provided but does not exist + mock_load_dataset.return_value = {"train": sample_dataset} + + with pytest.raises(ValueError, match=r"Split `bogus` not found"): + PromptDataset( + datasets=["my_hf_dataset:bogus"], + tokenizer=mock_tokenizer, + max_prompt_length=150, + num_workers=1, + ) + + +def test_prompt_dataset_hf_real_dataset(mock_tokenizer): + # Real integration test with a small public dataset and known split + ds = PromptDataset( + datasets=["HuggingFaceH4/aime_2024:train"], + tokenizer=mock_tokenizer, + max_prompt_length=1024, + num_workers=1, + prompt_key="problem", + env_class_key="env_class", + ) + assert len(ds) > 0 + + +@patch("datasets.load_dataset") +def test_prompt_dataset_uids(mock_load_dataset, sample_dataset, mock_tokenizer): + # When only a dataset name is provided, we default to the 'train' split + mock_load_dataset.return_value = {"train": sample_dataset} + + ds = PromptDataset( + datasets=["my_hf_dataset"], + tokenizer=mock_tokenizer, + max_prompt_length=1024, + num_workers=1, + prompt_key="prompt", + env_class_key="env_class", + ) + rows = [ds[i] for i in range(len(ds))] + uids = [row[3] for row in rows] + + # UID is simply the row index + assert uids[0] == "0" + # UIDs must be unique + assert len(set(uids)) == len(uids) + + rows_again = [ds[i] for i in range(len(ds))] + uids_again = [row[3] for row in rows_again] + # When sampled the second time, UIDs should not change + assert uids_again == uids diff --git a/skyrl-train/tests/cpu/dataset/test_preprocess.py b/skyrl-train/tests/cpu/dataset/test_preprocess.py new file mode 100644 index 0000000000..4143ea9c0a --- /dev/null +++ b/skyrl-train/tests/cpu/dataset/test_preprocess.py @@ -0,0 +1,161 @@ +""" +uv run --isolated --extra dev pytest tests/cpu/dataset/test_preprocess.py +""" + +import pytest +import torch +from omegaconf import OmegaConf +from skyrl_train.dataset.preprocess import ( + convert_prompts_responses_to_batch_tensors, +) + +from unittest.mock import MagicMock + + +@pytest.fixture +def cfg(): + return OmegaConf.create({"trainer": {"max_prompt_length": 10}, "generator": {"max_generate_length": 5}}) + + +# NOTE (sumanthrh): the tests in this file are hardcoded to use the below character-level tokenizer +@pytest.fixture +def tokenizer(): + mock_tokenizer = MagicMock() + mock_tokenizer.pad_token_id = 0 + mock_tokenizer.eos_token_id = 2 + + # encode("abc") -> [97, 98, 99] + def fake_encode(text): + if isinstance(text, list): + return [fake_encode(t) for t in text] + return [ord(c) for c in text] + + mock_tokenizer.encode.side_effect = fake_encode + + # tokenizer("abc") -> {"input_ids": [...], "attention_mask": [...]} + def fake_tokenizer_call(text, **kwargs): + if isinstance(text, list): + dicts = [fake_tokenizer_call(t, **kwargs) for t in text] + return { + "input_ids": [d["input_ids"] for d in dicts], + "attention_mask": [d["attention_mask"] for d in dicts], + } + ids = [ord(c) for c in text] + return { + "input_ids": ids, + "attention_mask": [1] * len(ids), + } + + mock_tokenizer.side_effect = fake_tokenizer_call + + def fake_tokenizer_decode(ids, **kwargs): + return "".join([chr(i) for i in ids]) + + mock_tokenizer.decode.side_effect = fake_tokenizer_decode + + def fake_tokenizer_decode_list(ids, **kwargs): + return [fake_tokenizer_decode(i) for i in ids] + + mock_tokenizer.batch_decode.side_effect = fake_tokenizer_decode_list + + return mock_tokenizer + + +def test_convert_prompts_responses_to_batch_tensors_exact(tokenizer, cfg): + prompts = ["abc", "12345"] + outputs = ["def", "67890"] + prompts = tokenizer(prompts)["input_ids"] + outputs = tokenizer(outputs)["input_ids"] + + loss_masks = [[1, 1, 0], [1, 1, 1, 0, 0]] + rewards = [torch.tensor([0, 1, 0]), torch.tensor([1, 0, 0, 0, 0])] + + sequences, attention_mask, action_mask, ret_rewards, ret_loss_masks, ret_log_probs = ( + convert_prompts_responses_to_batch_tensors( + tokenizer, + prompts, + outputs, + rewards, + loss_masks, + ) + ) + + # loss mask should be the same length as the action mask (padded to the longest input) + assert sequences.shape[0] == len(prompts) + assert action_mask.shape == ret_loss_masks.shape + assert torch.equal(ret_loss_masks[0], torch.tensor([1, 1, 0, 0, 0])) + assert torch.equal(ret_loss_masks[1], torch.tensor([1, 1, 1, 0, 0])) + assert torch.equal(ret_rewards[0], torch.tensor([0, 1, 0, 0, 0])) + assert torch.equal(ret_rewards[1], torch.tensor([1, 0, 0, 0, 0])) + + +def test_convert_prompts_responses_to_batch_tensors_different_lengths(cfg, tokenizer): + # Test with inputs of different lengths + prompts = ["Short", "This is a longer prompt"] + outputs = ["Long response here", "Short"] + prompts = tokenizer(prompts)["input_ids"] + outputs = tokenizer(outputs)["input_ids"] + rewards = [torch.tensor([1.0, 0.5, 0.3]), torch.tensor([0.8])] + loss_masks = [[1, 1, 1], [1]] + + sequences, attention_mask, action_mask, ret_rewards, ret_loss_masks, ret_log_probs = ( + convert_prompts_responses_to_batch_tensors( + tokenizer, + prompts, + outputs, + rewards, + loss_masks, + ) + ) + + max_response_len = max([len(output) for output in outputs]) + + # Check shapes + assert sequences.shape[0] == 2 # batch size + assert attention_mask.shape == sequences.shape + # Tensor.shape can be directly compared with tuples + assert action_mask.shape == (2, max_response_len) + assert ret_rewards.shape == (2, max_response_len) + assert ret_loss_masks.shape == (2, max_response_len) + + # Verify padding is applied correctly + # First input is shorter than second input. the input is left padded + assert sequences[0, 0] == tokenizer.pad_token_id + # second output is shorter than first output. the output is right padded + assert sequences[1, -1] == tokenizer.pad_token_id + + +def test_convert_prompts_responses_to_batch_tensors_empty_input(cfg, tokenizer): + # Test with empty input + prompts = [] + outputs = [] + rewards = [] + loss_masks = [] + + with pytest.raises(AssertionError): + convert_prompts_responses_to_batch_tensors( + tokenizer, + prompts, + outputs, + rewards, + loss_masks, + ) + + +def test_convert_prompts_responses_to_batch_tensors_mismatched_lengths(cfg, tokenizer): + # Test with mismatched input lengths + prompts = ["Hello", "World"] + outputs = ["Response"] + prompts = tokenizer(prompts)["input_ids"] + outputs = tokenizer(outputs)["input_ids"] + rewards = [torch.tensor([1.0])] + loss_masks = [[1]] + + with pytest.raises(AssertionError): + convert_prompts_responses_to_batch_tensors( + tokenizer, + prompts, + outputs, + rewards, + loss_masks, + ) diff --git a/skyrl-train/tests/cpu/distributed/test_dispatch.py b/skyrl-train/tests/cpu/distributed/test_dispatch.py new file mode 100644 index 0000000000..b4d4000c0d --- /dev/null +++ b/skyrl-train/tests/cpu/distributed/test_dispatch.py @@ -0,0 +1,118 @@ +from skyrl_train.training_batch import TrainingInputBatch +from skyrl_train.distributed.dispatch import ( + MeshDispatch, + PassThroughDispatch, + MeshRank, + ActorInfo, + DispatchRegistry, + Dispatch, +) +import ray +import torch +from typing import List, Optional, Union +from ray import ObjectRef +import pytest + + +@ray.remote +class RayActor: + def __init__(self, rank: int, dp_rank: int): + self.rank = rank + self.dp_rank = dp_rank + + def do_work(self, data: TrainingInputBatch): + # intentionally create different outputs for each rank + data["a"] += self.rank + return data + + def dummy(self, a, b): + return + + +class RayActorGroup: + def __init__(self, num_actors: int): + sp_size = 2 + dp_size = num_actors // sp_size + self.actors = [RayActor.remote(i, i % dp_size) for i in range(num_actors)] + self.actor_infos = [ + ActorInfo( + actor, + MeshRank( + dp=i % dp_size, sp=i // dp_size, tp=0, pp=0, world_size=num_actors, dp_size=dp_size, pp_size=1 + ), + ) + for i, actor in enumerate(self.actors) + ] + + def mesh_dispatch_and_collect(self, data: TrainingInputBatch): + object_refs = MeshDispatch.dispatch(self.actor_infos, "do_work", data) + ret = MeshDispatch.sync_collect(self.actor_infos, object_refs) + return ret + + def pass_through_dispatch(self, a, b): + # just pass values as is + object_refs = PassThroughDispatch.dispatch(self.actor_infos, "dummy", a, b) + ret = PassThroughDispatch.sync_collect(self.actor_infos, object_refs) + return ret + + +def test_mesh_dispatch(): + num_actors = 8 + actor_group = RayActorGroup(num_actors) + data = TrainingInputBatch({"a": torch.tensor([1, 2, 3, 4])}) + databatch = actor_group.mesh_dispatch_and_collect(data) + # only dp rank 0, 1, 2, 3, sp 0 will have the contributed to the output. + # In this case, the rank for these are 0, 1, 2, 3. + assert torch.equal(databatch["a"], torch.tensor([1, 3, 5, 7])) + + +def test_pass_through_dispatch(): + num_actors = 8 + actor_group = RayActorGroup(num_actors) + ret = actor_group.pass_through_dispatch(1, 2) + assert ret is None + + +def test_mesh_dispatch_with_mixed(): + num_actors = 8 + actor_group = RayActorGroup(num_actors) + object_refs = MeshDispatch.dispatch( + actor_group.actor_infos, + "do_work", + TrainingInputBatch({"a": torch.tensor([1, 2, 3, 4])}), + ) + object_refs[0] = ray.put(None) + with pytest.raises(AssertionError): + MeshDispatch.sync_collect(actor_group.actor_infos, object_refs) + + +def test_dispatch_registry(): + # add a custom dispatch type + try: + + class CustomDispatch(Dispatch): + @classmethod + def dispatch(cls, actor_infos: List[ActorInfo], method: str, *args, **kwargs) -> List[ObjectRef]: + pass + + @classmethod + def sync_collect( + cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef], nonblocking: bool = False + ) -> Union[List[ObjectRef], TrainingInputBatch]: + pass + + @classmethod + def async_collect( + cls, actor_infos: List[ActorInfo], object_refs: List[ObjectRef] + ) -> Optional[TrainingInputBatch]: + pass + + DispatchRegistry.register("custom", CustomDispatch) + assert DispatchRegistry.get("custom") == CustomDispatch + assert DispatchRegistry.list_registered() == { + "mesh": MeshDispatch, + "pass_through": PassThroughDispatch, + "custom": CustomDispatch, + } + finally: + DispatchRegistry._registry.pop("custom") diff --git a/skyrl-train/tests/cpu/distributed/test_ulysses.py b/skyrl-train/tests/cpu/distributed/test_ulysses.py new file mode 100644 index 0000000000..89344fdb0c --- /dev/null +++ b/skyrl-train/tests/cpu/distributed/test_ulysses.py @@ -0,0 +1,96 @@ +import torch +from unittest.mock import patch +from skyrl_train.distributed.ulysses.monkey_patch import _ulysses_flash_attention_forward + + +def test_basic_forward_no_parallel(): + """Test basic forward pass without sequence parallelism.""" + batch_size = 2 + seq_len = 4 + n_heads = 8 + head_dim = 64 + + # Create input tensors + query_states = torch.randn(batch_size, seq_len, n_heads, head_dim) + key_states = torch.randn(batch_size, seq_len, n_heads, head_dim) + value_states = torch.randn(batch_size, seq_len, n_heads, head_dim) + position_ids = torch.arange(seq_len).unsqueeze(0).repeat(batch_size, 1) + + # Mock the original flash attention forward + with patch("skyrl_train.distributed.ulysses.monkey_patch._flash_attention_forward") as mock_fa: + mock_fa.return_value = torch.randn_like(query_states) + + # Mock sequence parallel size to be 1 + with patch( + "skyrl_train.distributed.ulysses.monkey_patch.get_ulysses_sequence_parallel_world_size", return_value=1 + ): + _ulysses_flash_attention_forward(query_states, key_states, value_states, position_ids=position_ids) + + # Verify the original flash attention was called with correct inputs + mock_fa.assert_called_once() + args, kwargs = mock_fa.call_args + assert torch.equal(args[0], query_states) + assert torch.equal(args[1], key_states) + assert torch.equal(args[2], value_states) + assert torch.equal(kwargs["position_ids"], position_ids) + + +# NOTE (sumanthrh): This test is extremely mocking heavy. +# Extensive mocking is fine here because this unit test is simply checking output metadata i.e tensor shapes +def test_sequence_parallel_forward(): + """Test forward pass with sequence parallelism.""" + batch_size = 2 + seq_len = 8 + n_heads = 8 + head_dim = 64 + sp_size = 2 + + # Create input tensors (already sharded for sequence parallel) + query_states = torch.randn(batch_size, seq_len // sp_size, n_heads, head_dim) + key_states = torch.randn(batch_size, seq_len // sp_size, n_heads // sp_size, head_dim) + value_states = torch.randn(batch_size, seq_len // sp_size, n_heads // sp_size, head_dim) + position_ids = torch.arange(seq_len // sp_size).unsqueeze(0).repeat(batch_size, 1) + + # Mock the original flash attention forward + with patch("skyrl_train.distributed.ulysses.monkey_patch._flash_attention_forward") as mock_fa: + mock_fa.return_value = torch.randn(batch_size, seq_len, n_heads // sp_size, head_dim) + + # Mock sequence parallel size and communication functions + with ( + patch( + "skyrl_train.distributed.ulysses.monkey_patch.get_ulysses_sequence_parallel_world_size", + return_value=sp_size, + ), + patch( + "skyrl_train.distributed.ulysses.monkey_patch.gather_seq_scatter_heads" + ) as mock_gather_seq_scatter_heads, + patch( + "skyrl_train.distributed.ulysses.monkey_patch.gather_heads_scatter_seq" + ) as mock_gather_heads_scatter_seq, + patch("torch.distributed.all_gather") as mock_all_gather, + ): + + # Mock the gather/scatter operations + mock_gather_seq_scatter_heads.side_effect = lambda x, **kwargs: x[:, :, : n_heads // sp_size, :].repeat( + 1, sp_size, 1, 1 + ) + mock_gather_heads_scatter_seq.side_effect = lambda x, **kwargs: x[:, : seq_len // sp_size, :, :].repeat( + 1, 1, sp_size, 1 + ) + # Do nothing - output list remains as is + mock_all_gather.side_effect = lambda output_list, input_tensor, **kwargs: None + + output = _ulysses_flash_attention_forward(query_states, key_states, value_states, position_ids=position_ids) + + # Verify the communication operations were called + assert mock_gather_seq_scatter_heads.call_count == 3 # Called for q, k, v + assert mock_gather_heads_scatter_seq.call_count == 1 # Called for output + assert mock_all_gather.call_count == 1 # Called for position_ids + assert output.shape == (batch_size, seq_len // sp_size, n_heads, head_dim) + # Verify the original flash attention was called + mock_fa.assert_called_once() + args, kwargs = mock_fa.call_args + assert args[0].shape == (batch_size, seq_len, n_heads // sp_size, head_dim) + assert args[1].shape == (batch_size, seq_len, n_heads // sp_size, head_dim) + assert args[2].shape == (batch_size, seq_len, n_heads // sp_size, head_dim) + assert kwargs["position_ids"].shape == (batch_size, seq_len) diff --git a/skyrl-train/tests/cpu/generators/chat_templating_test_constants.py b/skyrl-train/tests/cpu/generators/chat_templating_test_constants.py new file mode 100644 index 0000000000..1b3ff27042 --- /dev/null +++ b/skyrl-train/tests/cpu/generators/chat_templating_test_constants.py @@ -0,0 +1,106 @@ +""" +Expected templated strings used for +skyrl-train/tests/cpu/generators/test_skyrl_gym_generator_chat_templating.py::test_skyrl_gym_generator_chat_templating_exact +""" + +from datetime import date + + +# Produced by expected_str = tokenizer.apply_chat_template(expected_chat_history, tokenize=False) +# where expected_chat_history is: +def get_expected_chat_history(mock_response_text: str): + return [ + {"role": "user", "content": "a"}, + {"role": "assistant", "content": mock_response_text}, + {"role": "user", "content": "1"}, + {"role": "assistant", "content": mock_response_text}, + {"role": "user", "content": "2"}, + {"role": "assistant", "content": mock_response_text}, + ] + + +QWEN2_5_EXPECTED_STR = """<|im_start|>system +You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|> +<|im_start|>user +a<|im_end|> +<|im_start|>assistant +b<|im_end|> +<|im_start|>user +1<|im_end|> +<|im_start|>assistant +b<|im_end|> +<|im_start|>user +2<|im_end|> +<|im_start|>assistant +b<|im_end|> +""" + +LLAMA3_2_EXPECTED_STR = f"""<|begin_of_text|><|start_header_id|>system<|end_header_id|> + +Cutting Knowledge Date: December 2023 +Today Date: {date.today().strftime("%d %b %Y")} + +<|eot_id|><|start_header_id|>user<|end_header_id|> + +a<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +b<|eot_id|><|start_header_id|>user<|end_header_id|> + +1<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +b<|eot_id|><|start_header_id|>user<|end_header_id|> + +2<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +b<|eot_id|>""" + +QWEN3_WITHOUT_THINKING_EXPECTED_STR = """<|im_start|>user +a<|im_end|> +<|im_start|>assistant +b<|im_end|> +<|im_start|>user +1<|im_end|> +<|im_start|>assistant +b<|im_end|> +<|im_start|>user +2<|im_end|> +<|im_start|>assistant + +mock thinking + + +b<|im_end|> +""" + +# Produced by the following code: +# keep_thinking_chat_template = get_custom_chat_template( +# {"source": "name", "name_or_path": "qwen3_with_thinking"} +# ) +# expected_str = tokenizer.apply_chat_template( +# expected_chat_history, tokenize=False, chat_template=keep_thinking_chat_template +# ) +QWEN3_TITO_EXPECTED_STR = """<|im_start|>user +a<|im_end|> +<|im_start|>assistant + +mock thinking + + +b<|im_end|> +<|im_start|>user +1<|im_end|> +<|im_start|>assistant + +mock thinking + + +b<|im_end|> +<|im_start|>user +2<|im_end|> +<|im_start|>assistant + +mock thinking + + +b<|im_end|> +""" diff --git a/skyrl-train/tests/cpu/generators/qwen3_acc_without_thinking.jinja2 b/skyrl-train/tests/cpu/generators/qwen3_acc_without_thinking.jinja2 new file mode 100644 index 0000000000..f66cf2539e --- /dev/null +++ b/skyrl-train/tests/cpu/generators/qwen3_acc_without_thinking.jinja2 @@ -0,0 +1 @@ +{% for message in messages %}{% if (message['role'] != 'assistant') %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% elif (message['role'] == 'assistant')%}{{'<|im_start|>' + message['role'] + '\n'}}{% generation %}{% set full_content = message['content'] %}{% set mycontent = message['content'] %}{% set is_last_message = loop.last and messages[-1]['role'] == 'assistant' %}{% if '' in full_content and not is_last_message %}{% set mycontent = full_content.split('')[-1].lstrip('\n') %}{% endif %}{{mycontent + '<|im_end|>'}}{% endgeneration %}{{'\n'}}{% endif %}{% endfor %} \ No newline at end of file diff --git a/skyrl-train/tests/cpu/generators/test_datatypes.py b/skyrl-train/tests/cpu/generators/test_datatypes.py new file mode 100644 index 0000000000..4d26b47c9a --- /dev/null +++ b/skyrl-train/tests/cpu/generators/test_datatypes.py @@ -0,0 +1,42 @@ +"""Tests for data types used in SkyRLGymGenerator + +uv run --isolated --extra dev -- pytest -s tests/cpu/generators/test_datatypes.py +""" + +import pytest + +from skyrl_train.generators.skyrl_gym_generator import TurnOutput + + +@pytest.mark.parametrize( + "output_ids, observation_ids, output_logprobs, added_eos, expected_loss_mask, expected_logprobs", + [ + # `added_eos` is False - `loss_mask` value is 1.0 for all output tokens. expected logprobs should have value 0.0 for observation tokens + ([1, 2, 3, 4], [100, 101], [0.9, 0.8, 0.7, 0.6], False, [1, 1, 1, 1, 0, 0], [0.9, 0.8, 0.7, 0.6, 0.0, 0.0]), + # `added_eos` is True - `loss_mask` should mask out the last token in `output_ids`. expected logprobs behaviour doesn't change + ( + [1, 2, 3, 4], + [100, 101, 102], + [0.9, 0.8, 0.7, 0.5], + True, + [1, 1, 1, 0, 0, 0, 0], + [0.9, 0.8, 0.7, 0.5, 0.0, 0.0, 0.0], + ), + ], +) +def test_turn_output(output_ids, observation_ids, output_logprobs, added_eos, expected_loss_mask, expected_logprobs): + turn_output = TurnOutput( + output="Dummy", + output_ids=output_ids, + output_logprobs=output_logprobs, + new_obs=[], + obs_ids=observation_ids, + added_eos=added_eos, + reward=1.0, + ) + + # test loss mask + assert turn_output.get_turn_loss_mask() == expected_loss_mask + + # test rollout logprobs + assert turn_output.get_turn_rollout_logprobs() == expected_logprobs diff --git a/skyrl-train/tests/cpu/generators/test_skyrl_gym_generator.py b/skyrl-train/tests/cpu/generators/test_skyrl_gym_generator.py new file mode 100644 index 0000000000..db70dcc1cb --- /dev/null +++ b/skyrl-train/tests/cpu/generators/test_skyrl_gym_generator.py @@ -0,0 +1,1636 @@ +""" +uv run --extra dev --isolated pytest tests/cpu/generators/test_skyrl_gym_generator.py +""" + +import pytest +from typing import List, Dict, Any +from unittest.mock import AsyncMock, MagicMock, patch +import numpy as np + +from skyrl_train.generators.skyrl_gym_generator import SkyRLGymGenerator +from skyrl_train.generators.base import GeneratorInput, GeneratorOutput, ConversationType +from skyrl_train.generators.utils import concatenate_generator_outputs, get_metrics_from_generator_output +from skyrl_gym.envs.base_text_env import BaseTextEnvStepOutput, BaseTextEnv +from skyrl_train.config.utils import get_default_config + + +# Mock constants, where 4 is the eos token id +MOCK_LLM_OUTPUT_IDS = [1, 10, 12, 4] +MOCK_TOKENIZER_ENCODED_IDS = [1, 2, 3, 4] + + +# TODO (erictang000): clean up the mocking for tests in this file +@pytest.fixture +def mock_tokenizer(): + """ + A mock tokenizer that encodes any non-empty string to `MOCK_TOKENIZER_ENCODED_IDS`. + For chat template, if `tokenize=False`, concatenate the content of each message. + If `tokenize=True`, return `MOCK_TOKENIZER_ENCODED_IDS` for each message. + """ + tokenizer = MagicMock() + + def mock_apply_chat_template(x, **kwargs): + if not kwargs.get("tokenize", True): + return "".join([str(i["content"]) for i in x]) + else: + # Check if return_dict is requested + if kwargs.get("return_dict", False): + # Return dictionary format for retokenization path + return { + "input_ids": MOCK_LLM_OUTPUT_IDS.copy(), + "assistant_masks": [1] * len(MOCK_LLM_OUTPUT_IDS), + } + # Non-dict return + if isinstance(x, list) and len(x) > 0 and isinstance(x[0], list): + # Multiple prompts + return [MOCK_TOKENIZER_ENCODED_IDS.copy() for _ in x] + else: + # Single prompt or conversation + return MOCK_TOKENIZER_ENCODED_IDS.copy() + + def mock_encode(x, **kwargs): + if x != "": + return MOCK_TOKENIZER_ENCODED_IDS.copy() + else: + return [] + + tokenizer.apply_chat_template.side_effect = mock_apply_chat_template + tokenizer.decode.side_effect = lambda x: "decoded_output" + tokenizer.encode.side_effect = mock_encode + tokenizer.eos_token_id = 4 + tokenizer.eos_token = "<|end_of_turn|>" + tokenizer.return_value = {"input_ids": MOCK_TOKENIZER_ENCODED_IDS.copy()} # simulate tokenized response + return tokenizer + + +@pytest.fixture +def mock_llm(): + """ + This replaces InferenceEngineClient, where `.generate()` always returns MOCK_LLM_OUTPUT_IDS + for each prompt, with corresponding string output "mocked output". + """ + mock = MagicMock() + + # Mock the new generate method + def mock_generate(input_batch): + num_prompts = len(input_batch["prompts"]) if "prompts" in input_batch else len(input_batch["prompt_token_ids"]) + return { + "responses": ["mocked output"] * num_prompts, + "stop_reasons": ["stop"] * num_prompts, + # say response gets tokenized to 3 tokens + "response_logprobs": [[0.1] * len(MOCK_LLM_OUTPUT_IDS)] * num_prompts, + "response_ids": [MOCK_LLM_OUTPUT_IDS.copy()] * num_prompts, + } + + mock.generate = AsyncMock(side_effect=mock_generate) + return mock + + +@pytest.fixture +def mock_env(): + mock_env_instance = MagicMock() + mock_env_instance.step.side_effect = lambda x: BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "next"}], reward=1.0, done=True, metadata={} + ) + mock_env_instance.close.return_value = None + # MagicMock has all attributes by default, including init_async/step_async/close_async. + # The generator checks hasattr(env, "init_async") to decide whether to await async methods. + # BaseTextEnv does NOT have these, so delete them to match real behavior. + del mock_env_instance.init_async + del mock_env_instance.step_async + del mock_env_instance.close_async + return mock_env_instance + + +@pytest.fixture +def generator_cfg(): + cfg = get_default_config().generator + cfg.sampling_params.max_generate_length = 5 + cfg.sampling_params.logprobs = None + cfg.apply_overlong_filtering = False + cfg.max_input_length = 512 + cfg.batched = True + cfg.max_turns = 1 + cfg.chat_template_kwargs = {} + cfg.chat_template = {"source": "name", "name_or_path": None} + return cfg + + +@pytest.fixture +def mock_env_cfg(): + cfg = MagicMock() + cfg.max_env_workers = 0 + cfg.env_class = "gsm8k" + return cfg + + +def validate_generator_input(input_batch: GeneratorInput) -> bool: + """Validate that input_batch conforms to GeneratorInput TypedDict interface.""" + # Check that input_batch has the required keys + required_keys = {"prompts", "env_extras"} + if not all(key in input_batch for key in required_keys): + return False + + # Validate prompts: List[ConversationType] where ConversationType = List[MessageType] + prompts = input_batch["prompts"] + if not isinstance(prompts, list): + return False + + for conversation in prompts: + if not isinstance(conversation, list): + return False + for message in conversation: + if not isinstance(message, dict): + return False + if not all(isinstance(k, str) and isinstance(v, str) for k, v in message.items()): + return False + + # Validate env_extras: Optional[List[Dict[str, Any]]] + env_extras = input_batch["env_extras"] + if env_extras is not None: + if not isinstance(env_extras, list): + return False + for extra in env_extras: + if not isinstance(extra, dict): + return False + if not all(isinstance(k, str) for k in extra.keys()): + return False + + return True + + +def validate_generator_output(output: GeneratorOutput) -> bool: + """Validate that output conforms to GeneratorOutput TypedDict interface.""" + # Check that output has all required keys + required_keys = { + "prompt_token_ids", + "response_ids", + "rewards", + "loss_masks", + "stop_reasons", + "rollout_metrics", + "rollout_logprobs", + } + if not all(key in output for key in required_keys): + return False + + # Validate prompt_token_ids: List[List[int]] + prompt_token_ids = output["prompt_token_ids"] + if not isinstance(prompt_token_ids, list): + return False + for token_ids in prompt_token_ids: + if not isinstance(token_ids, list): + return False + if not all(isinstance(token, int) for token in token_ids): + return False + + # Validate response_ids: List[List[int]] + response_ids = output["response_ids"] + if not isinstance(response_ids, list): + return False + for token_ids in response_ids: + if not isinstance(token_ids, list): + return False + if not all(isinstance(token, int) for token in token_ids): + return False + + # Validate rewards: List[float] or List[List[float]] + rewards = output["rewards"] + if not isinstance(rewards, list): + return False + is_list_of_float = all(isinstance(r, (int, float)) for r in rewards) + is_list_of_list_float = all(isinstance(r, list) and all(isinstance(x, (int, float)) for x in r) for r in rewards) + if not (is_list_of_float or is_list_of_list_float): + return False + + # Validate loss_masks: List[List[int]] + loss_masks = output["loss_masks"] + if not isinstance(loss_masks, list): + return False + for mask in loss_masks: + if not isinstance(mask, list): + return False + if not all(isinstance(val, int) for val in mask): + return False + + # Validate stop_reasons: Optional[List[str]] + stop_reasons = output["stop_reasons"] + if stop_reasons is not None: + if not isinstance(stop_reasons, list): + return False + if not all(isinstance(reason, str) for reason in stop_reasons): + return False + + # Validate rollout_metrics: Optional[Dict[str, Any]] + rollout_metrics = output["rollout_metrics"] + if rollout_metrics is not None: + if not isinstance(rollout_metrics, dict): + return False + if not all(isinstance(k, str) for k in rollout_metrics.keys()): + return False + + rollout_logprobs = output["rollout_logprobs"] + if rollout_logprobs is not None: + if not isinstance(rollout_logprobs, list): + return False + for sample_logprobs in rollout_logprobs: + if not isinstance(sample_logprobs, list): + return False + if not all(isinstance(val, (int, float)) for val in sample_logprobs): + return False + return True + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +@pytest.mark.parametrize("use_conversation_multi_turn", [True, False]) +@pytest.mark.parametrize("logprobs_setting", [None, 0]) +@pytest.mark.parametrize("mock_llm_output_ids", [[1, 10, 12, 4], [1, 10, 12]]) +async def test_agent_loop_single_turn( + mock_make, + mock_tokenizer, + mock_llm, + mock_env, + generator_cfg, + use_conversation_multi_turn, + logprobs_setting, + mock_llm_output_ids, + mock_env_cfg, +): + """ + This test mocks when we call SkyRLGymGenerator.agent_loop() despite being a single-turn generation. + This is when `batched=False`. Here the environment does nothing. + """ + generator_cfg.use_conversation_multi_turn = use_conversation_multi_turn + generator_cfg.sampling_params.logprobs = logprobs_setting + mock_env.step.side_effect = lambda x: BaseTextEnvStepOutput(observations=[], reward=1.0, done=True, metadata={}) + mock_tokenizer.eos_token_id = 4 # bypass check for eos token id for this test + + mock_make.return_value = mock_env + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + def mock_generate(_): + result = { + "responses": ["4"], + "response_ids": [mock_llm_output_ids.copy()], + "stop_reasons": ["stop"], + } + if logprobs_setting is not None: + result["response_logprobs"] = [[-0.1] * len(mock_llm_output_ids)] + return result + + mock_llm.generate = AsyncMock(side_effect=mock_generate) + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + prompt = [{"role": "user", "content": "What is 2 + 2?"}] + extras = {"answer": "4"} + output = await generator.agent_loop(prompt, mock_env_cfg.env_class, extras, max_tokens=8, max_input_length=512) + + if use_conversation_multi_turn: + expected_response_ids = mock_llm_output_ids + expected_loss_mask = [1] * len(expected_response_ids) + else: + has_eos_in_mock = mock_llm_output_ids and mock_llm_output_ids[-1] == mock_tokenizer.eos_token_id + + expected_response_ids = mock_llm_output_ids.copy() + if has_eos_in_mock: + # Had EOS: removed then re-added, so final IDs same as mock + expected_response_ids = mock_llm_output_ids + else: + # No EOS: just add it + expected_response_ids = mock_llm_output_ids + [mock_tokenizer.eos_token_id] + + expected_loss_mask = [1] * (len(expected_response_ids)) + + if logprobs_setting is not None: + assert output.rollout_logprobs is not None + assert len(output.rollout_logprobs) == len(output.response_ids) + assert all(isinstance(lp, float) for lp in output.rollout_logprobs) + else: + assert output.rollout_logprobs is None + + assert output.response_ids == expected_response_ids + assert output.loss_mask == expected_loss_mask + + if isinstance(output.reward, list): + assert sum(output.reward) == 1.0 + else: + assert output.reward == 1.0 + assert output.stop_reason == "stop" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_generate_batched(mock_make, mock_tokenizer, mock_llm, mock_env, generator_cfg, mock_env_cfg): + mock_make.return_value = mock_env + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + prompts = [[{"role": "user", "content": "What is 3 + 5?"}]] + env_extras = [{"answer": "8"}] + + input_batch: GeneratorInput = { + "prompts": prompts, + "env_extras": env_extras, + "env_classes": [mock_env_cfg.env_class for _ in prompts], # Mock environment class for each prompt + } + + generator_output: GeneratorOutput = await generator.generate(input_batch) + + # uses output from llm directly + assert generator_output["response_ids"][0] == MOCK_LLM_OUTPUT_IDS + + assert generator_output["rewards"][0] == 1.0 + assert generator_output["stop_reasons"][0] == "stop" + assert generator_output["loss_masks"][0] == [1] * len(MOCK_LLM_OUTPUT_IDS) + + +def test_generator_output_concatenation(): + # First ensure that the GeneratorOutput fields are what we expect + expected_fields = [ + "prompt_token_ids", + "response_ids", + "rewards", + "loss_masks", + "stop_reasons", + "rollout_metrics", + "rollout_logprobs", + # optional but present in the signature + "trajectory_ids", + "is_last_step", + ] + assert set(GeneratorOutput.__annotations__.keys()) == set(expected_fields), ( + "GeneratorOutput fields are not what we expect. " + "Please update the test and `concatenate_generator_outputs()` to reflect the new fields." + "It is needed to help Trainer.eval() record the full GeneratorOutput information." + ) + + generator_output_1: GeneratorOutput = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[1, 2], [3, 4]], + "rewards": [1.0, 2.0], + "loss_masks": [[1, 1], [1, 1]], + "stop_reasons": ["stop", "stop"], + "rollout_logprobs": [[0.1, 0.2], [0.3, 0.4]], + } + + generator_output_2: GeneratorOutput = { + "prompt_token_ids": [[5, 6, 7], [8]], + "response_ids": [[5, 6, 7], [8]], + "rewards": [2.0, 3.0], + "loss_masks": [[1, 1, 1], [1]], + "stop_reasons": ["stop", "stop"], + "rollout_logprobs": [[0.5, 0.6, 0.7], [0.8]], + } + + generator_outputs = [generator_output_1, generator_output_2] + concatenated_output = concatenate_generator_outputs(generator_outputs) + + assert concatenated_output["prompt_token_ids"] == [[1, 2], [3, 4], [5, 6, 7], [8]] + assert concatenated_output["response_ids"] == [[1, 2], [3, 4], [5, 6, 7], [8]] + assert concatenated_output["rewards"] == [1.0, 2.0, 2.0, 3.0] + assert concatenated_output["loss_masks"] == [[1, 1], [1, 1], [1, 1, 1], [1]] + assert concatenated_output["stop_reasons"] == ["stop", "stop", "stop", "stop"] + assert concatenated_output["rollout_logprobs"] == [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6, 0.7], [0.8]] + + # Validate rollout metrics + expected_rollout_metrics = { + "generate/min_num_tokens": 1, + "generate/max_num_tokens": 3, + "generate/avg_num_tokens": 2.0, + "generate/std_num_tokens": np.std([2, 2, 3, 1]).item(), + "generate/avg_tokens_non_zero_rewards": 2.0, + "generate/avg_tokens_zero_rewards": 0, + } + assert concatenated_output["rollout_metrics"].keys() == expected_rollout_metrics.keys() + for key, value in expected_rollout_metrics.items(): + np.testing.assert_allclose(concatenated_output["rollout_metrics"][key], value) + + +def test_get_metrics_from_generator_output(): + # Per trajectory rewards, where rewards are List[float] + generator_output: GeneratorOutput = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[1, 2], [3, 4]], + "rewards": [1.0, 2.0], + "loss_masks": [[1, 1], [1, 1]], + "stop_reasons": ["stop", "stop"], + "rollout_logprobs": None, + } + uids = ["a", "b"] + metrics = get_metrics_from_generator_output(generator_output, uids) + assert metrics["avg_score"] == 1.5 + assert metrics["pass_at_n"] == 1.0 + assert metrics["mean_positive_reward"] == 1.5 + + # Per token rewards, where rewards are List[List[float]], so for pass_at_n we use the last + # token's reward to signify the trajectory's reward + generator_output["rewards"] = [[1.0, 0.0], [0.0, 1.0]] + uids = ["a", "b"] + metrics = get_metrics_from_generator_output(generator_output, uids) + assert metrics["avg_score"] == 1.0 + assert metrics["pass_at_n"] == 0.5 + assert metrics["mean_positive_reward"] == 1.0 + + # Mixed rewards with some negative rewards + generator_output["rewards"] = [-1.0, 2.0] + uids = ["a", "b"] + metrics = get_metrics_from_generator_output(generator_output, uids) + assert metrics["avg_score"] == 0.5 + assert metrics["pass_at_n"] == 0.5 + assert metrics["mean_positive_reward"] == 1.0 + + # Mixed per-token rewards with negatives - per-token rewards + # Last tokens: -1.0 and 0.5, neither >= 1.0 so no passes + generator_output["rewards"] = [[1.0, -1.0], [-0.5, 0.5]] + uids = ["a", "b"] + metrics = get_metrics_from_generator_output(generator_output, uids) + assert metrics["avg_score"] == 0.0 + assert metrics["pass_at_n"] == 0.0 + assert metrics["mean_positive_reward"] == 0.75 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("batched", [True, False]) +@patch("skyrl_gym.make") +async def test_generate_interface_compliance( + mock_make, mock_tokenizer, mock_llm, mock_env, generator_cfg, mock_env_cfg, batched +): + """Test that SkyRLGymGenerator.generate() strictly conforms to the TypedDict interface. + + Tests both batched and non-batched modes to ensure interface compliance. + """ + mock_make.return_value = mock_env + # Set the batched mode according to the parameter + generator_cfg.batched = batched + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + # Create test data based on batched mode + if batched: + # For batched mode, test with multiple prompts + prompts: List[ConversationType] = [ + [{"role": "user", "content": "What is 3 + 5?"}], + [{"role": "user", "content": "Solve 10 - 7"}], + ] + env_extras: List[Dict[str, Any]] = [{"answer": "8"}, {"answer": "3"}] + else: + # For non-batched mode, test with single prompt + prompts: List[ConversationType] = [[{"role": "user", "content": "What is 2 * 3?"}]] + env_extras: List[Dict[str, Any]] = [{"answer": "6"}] + env_classes = [mock_env_cfg.env_class for _ in prompts] + + input_batch: GeneratorInput = { + "prompts": prompts, + "env_extras": env_extras, + "env_classes": env_classes, + } + + # Validate input conforms to interface + assert validate_generator_input( + input_batch + ), f"Input does not conform to GeneratorInput interface (batched={batched})" + + # Call generate method + generator_output: GeneratorOutput = await generator.generate(input_batch) + + # Validate output conforms to interface + assert validate_generator_output( + generator_output + ), f"Output does not conform to GeneratorOutput interface (batched={batched})" + + # Additional specific type checks + assert isinstance(generator_output, dict), "Output should be a dictionary" + assert len(generator_output["response_ids"]) == len( + prompts + ), f"Number of responses should match number of prompts (batched={batched})" + assert len(generator_output["rewards"]) == len( + prompts + ), f"Number of rewards should match number of prompts (batched={batched})" + assert len(generator_output["loss_masks"]) == len( + prompts + ), f"Number of loss masks should match number of prompts (batched={batched})" + + # Test with None env_extras to ensure Optional handling works (only test this once) + if batched: + input_batch_with_none: GeneratorInput = { + "prompts": prompts[:1], # Just one prompt + "env_extras": None, + } + + # This should not raise an error even with None env_extras + assert validate_generator_input(input_batch_with_none), "Input with None env_extras should be valid" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("turns_to_exceed", [1, 3]) # Test single-turn and multi-turn scenarios +@patch("skyrl_gym.make") +async def test_length_limit_exceeded_during_conversation( + mock_make, mock_tokenizer, mock_llm, mock_env, generator_cfg, mock_env_cfg, turns_to_exceed +): + """Test that length limit is enforced during multi-turn conversations. + + Tests both single-turn (turns_to_exceed=1) and multi-turn (turns_to_exceed=3) scenarios + to verify length accumulation and limit enforcement. + """ + mock_make.return_value = mock_env + generator_cfg.batched = False # Use agent_loop mode + generator_cfg.max_turns = 5 # Allow multiple turns + generator_cfg.use_conversation_multi_turn = True + generator_cfg.chat_template = {"source": "name", "name_or_path": None} + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + # Configure environment to never set done=True naturally (we want to hit length limit) + def mock_step_never_done(output): + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "next"}], + reward=0.5, + done=False, + metadata={}, + ) + + # We start with initial prompt len 4 due to mock_apply_chat_template + # Each turn, observation is 4 tokens due to mock_encode + mock_env.step.side_effect = mock_step_never_done + max_input_length = 20 # Low limit to trigger length exceeded + + # Mock the new generate method + def mock_generate(input_batch): + num_prompts = len(input_batch["prompts"]) if "prompts" in input_batch else len(input_batch["prompt_token_ids"]) + if turns_to_exceed == 1: + mock_llm_output_ids = [1] * 20 # Enough to exceed limit immediately (4 + 20 + 4 = 28 > 20) + assert ( + len(MOCK_TOKENIZER_ENCODED_IDS) + len(mock_llm_output_ids) + len(MOCK_TOKENIZER_ENCODED_IDS) + > max_input_length + ) + else: + assert turns_to_exceed == 3 + mock_llm_output_ids = [1] * 2 # Enough to exceed limit after 3 turns (4 + (2 + 4) * 3 = 22 > 20) + assert ( + len(MOCK_TOKENIZER_ENCODED_IDS) + + (len(mock_llm_output_ids) + len(MOCK_TOKENIZER_ENCODED_IDS)) * turns_to_exceed + > max_input_length + ) + return { + "responses": ["mocked output"] * num_prompts, + "stop_reasons": ["stop"] * num_prompts, + "response_logprobs": [[0.1] * len(mock_llm_output_ids)] * num_prompts, + "response_ids": [mock_llm_output_ids.copy()] * num_prompts, + } + + mock_llm.generate = AsyncMock(side_effect=mock_generate) + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + prompt = [{"role": "user", "content": "Start conversation"}] + extras = {"test": "value"} + + output = await generator.agent_loop(prompt, "test_env", extras, max_tokens=100, max_input_length=max_input_length) + + # Verify that length limit was hit + assert output.stop_reason == "length", f"Expected stop_reason='length', got '{output.stop_reason}'" + + # Verify environment step was called the expected number of times + expected_calls = turns_to_exceed + assert ( + mock_env.step.call_count == expected_calls + ), f"Expected {expected_calls} environment steps, got {mock_env.step.call_count}" + + # Verify response is still properly formatted + assert isinstance(output.response_ids, list) + assert isinstance(output.loss_mask, list) + assert isinstance(output.reward, float) or isinstance(output.reward, list) + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_multi_turn_response_truncation( + mock_make, mock_tokenizer, mock_llm, mock_env, generator_cfg, mock_env_cfg +): + """ + Tests that in a multi-turn conversation, if the final tokenized response exceeds the + calculated maximum length, it is correctly truncated and the stop reason is set to 'length'. + """ + mock_make.return_value = mock_env + generator_cfg.max_turns = 3 # Ensure multi-turn logic is triggered + generator_cfg.batched = False # Test is for agent_loop + generator_cfg.use_conversation_multi_turn = True + generator_cfg.chat_template = {"source": "name", "name_or_path": None} + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + # Configure environment to run for multiple turns to generate enough tokens for truncation + step_count = 0 + + def mock_step_multi_turn(output): + nonlocal step_count + step_count += 1 + done = step_count >= 10 # Allow many turns to exceed length limit + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "next turn"}], reward=0.5, done=done, metadata={} + ) + + mock_env.step.side_effect = mock_step_multi_turn + + # Define token lengths to control the test + initial_prompt_len = 13 + max_tokens_from_llm = 20 + max_input_len = 50 + + # Expected max response tokens = max_tokens + max_input_length - initial_prompt_length + expected_max_response_tokens = max_tokens_from_llm + max_input_len - initial_prompt_len # 20 + 50 - 13 = 57 + + def mock_apply_chat_template(messages, **kwargs): + if kwargs.get("tokenize", True): + # Return initial prompt tokens + return [1] * initial_prompt_len + else: + # Not used in messages_mode=False + return "".join([msg.get("content", "") for msg in messages]) + + def mock_encode(text, **kwargs): + # This makes observation_ids to always be 13 tokens + return [1] * 13 + + mock_tokenizer.apply_chat_template.side_effect = mock_apply_chat_template + mock_tokenizer.encode.side_effect = mock_encode + + # The intitial prompt is 13 tokens due to mock_apply_chat_template + # Each turn, observation is 13 tokens due to mock_encode and empty system_prompt_ids + # And the LLM response is 4 tokens due to MOCK_LLM_OUTPUT_IDS + # So input_ids are 13, 30, 47, 64. And 64 would cause a break in the loop due to exceeding max_input_len. + # We strip the last observation, which gives us 64 - 13 = 51 tokens, + # then with 51, we get the `input_ids[initial_prompt_length:]`, which makes our final + # response_ids to be 51 - 13 = 38 tokens. So in this case, we are not truncated by expected_max_response_tokens. + expected_final_response_tokens = 38 + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + prompt = [{"role": "user", "content": "Initial prompt"}] + extras = {} + + output = await generator.agent_loop( + prompt, "test_env", extras, max_tokens=max_tokens_from_llm, max_input_length=max_input_len + ) + + # Verify truncation occurred + assert len(output.response_ids) <= expected_max_response_tokens + assert ( + len(output.response_ids) == expected_final_response_tokens + ), f"Expected {expected_final_response_tokens} response tokens, got {len(output.response_ids)}" + assert ( + len(output.loss_mask) == expected_final_response_tokens + ), f"Expected {expected_final_response_tokens} loss mask entries, got {len(output.loss_mask)}" + + # Verify stop reason is "length" due to truncation + assert output.stop_reason == "length", f"Expected stop_reason='length', got '{output.stop_reason}'" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_postprocessed_action_used(mock_make, mock_tokenizer, mock_llm, mock_env, mock_env_cfg, generator_cfg): + """ + Tests that if the environment returns a `postprocessed_action`, it is used + in the chat history instead of the original LLM response. + """ + mock_make.return_value = mock_env + generator_cfg.max_turns = 1 # Single turn + generator_cfg.batched = False + # Override to avoid retokenization path for this test + generator_cfg.chat_template = {"source": "name", "name_or_path": None} + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + postprocessed_response = "This is a clean response." + llm_raw_response = "RAW LLM OUTPUT" + + # Environment step returns a postprocessed version of the LLM response + def mock_step(_): + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "new input"}], + reward=1.0, + done=True, + metadata={}, + postprocessed_action=postprocessed_response, + ) + + mock_env.step.side_effect = mock_step + + # The LLM will output a raw string, which should be overridden + mock_llm.generate.return_value = { + "responses": [llm_raw_response], + "stop_reasons": ["stop"], + } + + def mock_apply_chat_template(messages, **kwargs): + if kwargs.get("tokenize", True): + return [1] * 5 # Initial prompt tokens + else: + return "".join([msg.get("content", "") for msg in messages]) + + def mock_encode(text, **kwargs): + # The key test: postprocessed response should be encoded, not raw LLM output + if postprocessed_response in str(text): + return [42] * 10 # Distinctive tokens for postprocessed response + elif "new input" in str(text): + return [5] * 2 # Observation tokens + else: + return [1] * 3 # Default tokens + + mock_tokenizer.apply_chat_template.side_effect = mock_apply_chat_template + mock_tokenizer.encode.side_effect = mock_encode + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + prompt = [{"role": "user", "content": "Initial input"}] + env_extras = {} + + output = await generator.agent_loop(prompt, "test_env", env_extras, max_tokens=20, max_input_length=50) + + # Check that the postprocessed response tokens (42) are present in response_ids + # This verifies that postprocessed_action was used instead of raw LLM output + assert any( + token == 42 for token in output.response_ids + ), f"Expected postprocessed response tokens (42) in {output.response_ids}" + # Make sure raw LLM tokens (99) are NOT present + assert not any( + token == 99 for token in output.response_ids + ), f"Raw LLM output tokens (99) should not be in {output.response_ids}" + + if isinstance(output.reward, list): + assert sum(output.reward) == 1.0 + else: + assert output.reward == 1.0 + assert output.stop_reason == "stop" + assert len(output.response_ids) == len(output.loss_mask) + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_apply_overlong_filtering_non_batched( + mock_make, mock_tokenizer, mock_llm, mock_env, generator_cfg, mock_env_cfg +): + """ + Test that apply_overlong_filtering correctly zeroes out loss masks for truncated trajectories + in non-batched mode (using agent_loop). + + Tests both truncated and non-truncated responses to verify that: + - Trajectories with responses not ending with eos token have their loss masks zeroed out + - Trajectories with responses ending with eos token keep their original loss masks + """ + mock_make.return_value = mock_env + generator_cfg.apply_overlong_filtering = True # Enable filtering + generator_cfg.batched = False + generator_cfg.max_turns = 1 + generator_cfg.use_conversation_multi_turn = False + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + # Mock out the environment and inference engine generation. + mock_env.step.side_effect = lambda x: BaseTextEnvStepOutput(observations=[], reward=1.0, done=True, metadata={}) + + def mock_apply_chat_template(messages, **kwargs): + if kwargs.get("tokenize", True): + return [1, 2, 3, 4, 5] # 5 tokens for prompt + else: + return "".join([msg.get("content", "") for msg in messages]) + + mock_tokenizer.apply_chat_template.side_effect = mock_apply_chat_template + mock_tokenizer.eos_token_id = 4 # Set EOS token ID + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + # First test: response that doesn't end with eos token (should be filtered) + async def llm_generate_side_effect(input_batch): + + if input_batch.get("sampling_params") is not None: + max_len = input_batch["sampling_params"]["max_generate_length"] + else: + max_len = generator_cfg.sampling_params.max_generate_length + + base_response = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19] # 10 token base + num = len(input_batch["prompts"]) if "prompts" in input_batch else len(input_batch["prompt_token_ids"]) + response_tokens = [base_response[:max_len] for _ in range(num)] + return { + "responses": ["truncated response"] * num, + "stop_reasons": ["length"] * num, + "response_ids": response_tokens, + } + + mock_llm.generate = AsyncMock(side_effect=llm_generate_side_effect) + + input_batch_truncated: GeneratorInput = { + "prompts": [[{"role": "user", "content": "Test prompt"}]], + "env_extras": [{"test": "value"}], + "env_classes": [mock_env_cfg.env_class], + } + + output_truncated = await generator.generate(input_batch_truncated) + + # Verify truncated response has zeroed loss mask + assert len(output_truncated["loss_masks"]) == 1 + assert len(output_truncated["loss_masks"][0]) == 5 + assert output_truncated["loss_masks"][0] == [ + 0, + 0, + 0, + 0, + 0, + ], "Loss mask should be all zeros for response not ending with eos token" + + # Note: The long response gets truncated by max_response_tokens, so it doesn't end with eos token + # Second test: response that ends with eos token (should not be filtered) + # Reset the environment init to ensure clean state + mock_env.init.return_value = ([{"role": "user", "content": "Fresh input"}], {}) + mock_llm.generate = AsyncMock( + return_value={ + "responses": ["normal response"], + "stop_reasons": ["stop"], + "response_ids": [[20, 21, 4]], # 3 tokens, ends with eos token 4 + } + ) + + input_batch_normal: GeneratorInput = { + "prompts": [[{"role": "user", "content": "Another test prompt"}]], + "env_extras": [{"test": "value"}], + "env_classes": [mock_env_cfg.env_class], + } + + output_normal = await generator.generate(input_batch_normal) + + # Verify normal response keeps original loss mask (all 1s) + assert len(output_normal["loss_masks"]) == 1 + assert len(output_normal["loss_masks"][0]) == 3 # 3 response tokens (already includes EOS token) + assert output_normal["loss_masks"][0] == [ + 1, + 1, + 1, + ], "Loss mask should remain as 1s for response ending with eos token" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_apply_overlong_filtering_batched( + mock_make, + mock_tokenizer, + mock_llm, + mock_env, + generator_cfg, + mock_env_cfg, +): + """ + Test that apply_overlong_filtering correctly zeroes out loss masks for truncated trajectories + in batched mode. + + Tests a response that doesn't end with eos token to verify that it gets filtered. + """ + mock_make.return_value = mock_env + generator_cfg.apply_overlong_filtering = True # Enable filtering + generator_cfg.batched = True + generator_cfg.max_turns = 1 + mock_env.init.return_value = ([{"role": "user", "content": "Initial input"}], {}) + + # Mock out environment and inference engine generation. + mock_env.step.side_effect = lambda x: BaseTextEnvStepOutput(observations=[], reward=1.0, done=True, metadata={}) + mock_llm.generate = AsyncMock( + return_value={ + "responses": ["truncated response"], + "stop_reasons": ["length"], + "response_ids": [[10, 11, 12, 13]], + } + ) + + def mock_apply_chat_template(messages, **kwargs): + if kwargs.get("tokenize", True): + return [[1, 2, 3, 4, 5] for _ in messages] # 5 tokens for each prompt + else: + return "".join([msg.get("content", "") for msg in messages]) + + def mock_encode_or_tokenize(text): + return [10, 11, 12, 13] # 4 tokens, doesn't end with eos_token_id=4 + + mock_tokenizer.apply_chat_template.side_effect = mock_apply_chat_template + mock_tokenizer.side_effect = lambda text: {"input_ids": mock_encode_or_tokenize(text)} + mock_tokenizer.eos_token_id = 4 # Set EOS token ID + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] # to make sure observation_ids are encoded correctly + + # Test batched mode with response that doesn't end with eos token + prompts = [[{"role": "user", "content": "Test prompt"}]] + env_extras = [{"test": "value"}] + env_classes = [mock_env_cfg.env_class] + + input_batch: GeneratorInput = { + "prompts": prompts, + "env_extras": env_extras, + "env_classes": env_classes, + } + + generator_output = await generator.generate(input_batch) + + # Verify that the loss mask is zeroed out for the response not ending with eos token + assert len(generator_output["loss_masks"]) == 1 + assert len(generator_output["loss_masks"][0]) == 4 # Should match response length + assert generator_output["loss_masks"][0] == [ + 0, + 0, + 0, + 0, + ], "Loss mask should be all zeros for response not ending with eos token" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_agent_loop_token_level_rewards_multi_turn(mock_make, mock_tokenizer, mock_llm, mock_env_cfg): + """use_conversation_multi_turn=False; verify rewards at assistant turn ends across two steps.""" + # Tokenizer behavior + mock_tokenizer.eos_token_id = 4 + + def apply_chat_template_side_effect(messages, **kwargs): + # initial prompt tokenization + if kwargs.get("tokenize", True): + return [101, 102] + else: + return "".join([m.get("content", "") for m in messages]) + + def encode_side_effect(text, **kwargs): + # one token for each observation + return [77] if text else [] + + mock_tokenizer.apply_chat_template.side_effect = apply_chat_template_side_effect + mock_tokenizer.encode.side_effect = encode_side_effect + + # LLM returns fixed response tokens per step: 3 tokens + eos + async def llm_generate_side_effect(input_batch): + num = ( + len(input_batch["prompt_token_ids"]) if "prompt_token_ids" in input_batch else len(input_batch["prompts"]) + ) # noqa: E501 + return { + "responses": ["aaa"] * num, + "stop_reasons": ["stop"] * num, + "response_logprobs": None, + "response_ids": [[10, 11, 12, mock_tokenizer.eos_token_id] for _ in range(num)], + } + + mock_llm.generate = AsyncMock(side_effect=llm_generate_side_effect) + + # Two-step env with rewards 0.3 then 1.7 + class TwoStepEnv(BaseTextEnv): + def __init__(self): + super().__init__() + self.turns = 0 + + def init(self, prompt): + return prompt, {} + + def step(self, action): + self.turns += 1 + if self.turns == 1: + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "obs1"}], reward=0.3, done=False, metadata={} + ) + else: + return BaseTextEnvStepOutput(observations=[], reward=1.7, done=True, metadata={}) + + mock_make.return_value = TwoStepEnv() + + # Generator config + cfg = get_default_config().generator + cfg.sampling_params.max_generate_length = 50 + cfg.sampling_params.logprobs = None + cfg.apply_overlong_filtering = False + cfg.max_input_length = 512 + cfg.batched = False + cfg.max_turns = 10 + cfg.zero_reward_on_non_stop = False + cfg.use_conversation_multi_turn = False + cfg.chat_template = {"source": "name", "name_or_path": None} + + generator = SkyRLGymGenerator( + generator_cfg=cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + + # Run agent loop + prompt = [{"role": "user", "content": "Q?"}] + extras = {} + out = await generator.agent_loop(prompt, mock_env_cfg.env_class, extras, max_tokens=50, max_input_length=512) + + # Response ids layout: step1 (3 tokens) + obs (1) + step2 (3) + final eos (1) = 8 + assert len(out.response_ids) == 8 + # Indices: 2 (end of step1 assistant), 6 (end of step2 assistant), 7 (manually appended eos token) + # Note that the last reward is placed at the 7 instead of at 6 since we manually move + # it using the flag `appended_eos_token` in skyrl_gym_generator.py + expected_rewards = [0.0, 0.0, 0.3, 0.0, 0.0, 0.0, 0.0, 1.7] + assert isinstance(out.reward, list) + assert out.reward == expected_rewards + assert out.stop_reason == "stop" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_agent_loop_token_level_rewards_multi_turn_conversation_format( + mock_make, mock_tokenizer, mock_llm, mock_env_cfg +): + """use_conversation_multi_turn=True; verify rewards placed at ends of assistant segments before observations.""" + mock_tokenizer.eos_token_id = 4 + + # Tokenizer: initial prompt -> 2 tokens; observation template -> 2 tokens each call + + def apply_chat_template_side_effect(messages, **kwargs): + if kwargs.get("tokenize", True): + # For observations path, generator passes [*base_conversation, *new_obs] with add_generation_prompt=True + return [201, 202] + else: + return "".join([m.get("content", "") for m in messages]) + + mock_tokenizer.apply_chat_template.side_effect = apply_chat_template_side_effect + + # LLM outputs include EOS and are kept in multi-turn path + async def llm_generate_side_effect(input_batch): + num = ( + len(input_batch["prompt_token_ids"]) if "prompt_token_ids" in input_batch else len(input_batch["prompts"]) + ) # noqa: E501 + return { + "responses": ["aaa"] * num, + "stop_reasons": ["stop"] * num, + "response_logprobs": None, + "response_ids": [[10, 11, 12, mock_tokenizer.eos_token_id] for _ in range(num)], + } + + mock_llm.generate = AsyncMock(side_effect=llm_generate_side_effect) + + # Env: two steps with rewards 0.5 then 0.25; first step has an observation, second has none + class MTEnv(BaseTextEnv): + def __init__(self): + super().__init__() + self.turns = 0 + + def init(self, prompt): + return prompt, {} + + def step(self, action): + self.turns += 1 + if self.turns == 1: + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "obs1"}], reward=0.5, done=False, metadata={} + ) + else: + return BaseTextEnvStepOutput(observations=[], reward=0.25, done=True, metadata={}) + + mock_make.return_value = MTEnv() + + # Generator config + cfg = get_default_config().generator + cfg.sampling_params.max_generate_length = 50 + cfg.sampling_params.logprobs = None + cfg.apply_overlong_filtering = False + cfg.max_input_length = 512 + cfg.batched = False + cfg.max_turns = 10 + cfg.zero_reward_on_non_stop = False + cfg.use_conversation_multi_turn = True + cfg.chat_template = {"source": "name", "name_or_path": None} + + mock_env_cfg.env_class = "mt_env" + + generator = SkyRLGymGenerator( + generator_cfg=cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + # Ensure base_conversation_token_ids doesn't shift observation slicing in test + generator.base_conversation_token_ids = [] + + prompt = [{"role": "user", "content": "Q?"}] + extras = {} + out = await generator.agent_loop(prompt, mock_env_cfg.env_class, extras, max_tokens=50, max_input_length=512) + + # Response ids layout: step1 assistant (4 incl. eos) + obs(2) + step2 assistant(4 incl. eos) = 10 + assert len(out.response_ids) == 10 + # Rewards at indices: 3 (end of step1 assistant), 9 (end of step2 assistant) + expected = [0.0] * 10 + expected[3] = 0.5 + expected[9] = 0.25 + assert isinstance(out.reward, list) + assert out.reward == expected + assert out.stop_reason == "stop" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_agent_loop_retokenize_returns_float_reward(mock_make, mock_tokenizer, mock_llm, mock_env_cfg): + """Retokenize mode should return a single float reward (last non-None step reward) because token-level rewards are not yet supported.""" + mock_tokenizer.eos_token_id = 4 + + # Tokenizer: initial prompt ids and final retokenized response with masks + def apply_chat_template_side_effect(messages, **kwargs): + if kwargs.get("return_dict", False): + # Final retokenization output + return {"assistant_masks": [1, 0, 1], "input_ids": [5, 6, 7]} + if kwargs.get("tokenize", True): + return [301, 302] + else: + return "".join([m.get("content", "") for m in messages]) + + mock_tokenizer.apply_chat_template.side_effect = apply_chat_template_side_effect + + # LLM generate in retokenize mode uses prompts; we can return any ids + async def llm_generate_side_effect(input_batch): + num = ( + len(input_batch["prompts"]) if "prompts" in input_batch else len(input_batch["prompt_token_ids"]) + ) # noqa: E501 + return { + "responses": ["bbb"] * num, + "stop_reasons": ["stop"] * num, + "response_logprobs": None, + "response_ids": [[20, 21, 22] for _ in range(num)], + } + + mock_llm.generate = AsyncMock(side_effect=llm_generate_side_effect) + + # Env with rewards: None then 2.5 + class RetokEnv(BaseTextEnv): + def __init__(self): + super().__init__() + self.turns = 0 + + def init(self, prompt): + return prompt, {} + + def step(self, action): + self.turns += 1 + if self.turns == 1: + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "o1"}], reward=None, done=False, metadata={} + ) # noqa: E501 + else: + return BaseTextEnvStepOutput(observations=[], reward=2.5, done=True, metadata={}) + + mock_make.return_value = RetokEnv() + + # Generator config enabling retokenize path + cfg = get_default_config().generator + cfg.sampling_params.max_generate_length = 50 + cfg.sampling_params.logprobs = None + cfg.apply_overlong_filtering = False + cfg.max_input_length = 512 + cfg.batched = False + cfg.max_turns = 10 + cfg.zero_reward_on_non_stop = False + cfg.use_conversation_multi_turn = True + cfg.chat_template = { + "source": "name", + "name_or_path": "qwen3_without_thinking", # TODO: revisit this test once we separate the retokenize config from the custom chat template config + } + + generator = SkyRLGymGenerator( + generator_cfg=cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="Qwen/Qwen3-0.6B", # ensures custom_chat_template is truthy in get_custom_chat_template + ) + # Force retokenize path regardless of model resolution logic if needed + generator.custom_chat_template = "" + + prompt = [{"role": "user", "content": "Q?"}] + extras = {} + out = await generator.agent_loop(prompt, mock_env_cfg.env_class, extras, max_tokens=50, max_input_length=512) + + assert isinstance(out.reward, float) + assert out.reward == 2.5 + assert out.stop_reason == "stop" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_agent_loop_truncation_drops_out_of_range_rewards(mock_make, mock_tokenizer, mock_llm, mock_env_cfg): + """Non-retokenize path: ensure rewards whose indices fall beyond truncated response are ignored.""" + + # Configure tokenizer: initial prompt -> 2 tokens + def apply_chat_template_side_effect(messages, **kwargs): + if kwargs.get("tokenize", True): + return [101, 102] + else: + return "".join([m.get("content", "") for m in messages]) + + mock_tokenizer.apply_chat_template.side_effect = apply_chat_template_side_effect + mock_tokenizer.eos_token_id = 4 + + # LLM returns 4 assistant tokens per turn (no eos here; final EOS appended by generator for non-conv-mt) + async def llm_generate_side_effect(input_batch): + num = len(input_batch["prompt_token_ids"]) if "prompt_token_ids" in input_batch else len(input_batch["prompts"]) + + if input_batch.get("sampling_params") is not None: + max_len = input_batch["sampling_params"]["max_generate_length"] + else: + max_len = cfg.sampling_params.max_generate_length + + base_response = [10, 11, 12, 13] + response_tokens = [base_response[:max_len] for _ in range(num)] + return { + "responses": ["step"] * num, + "stop_reasons": ["stop"] * num, + "response_logprobs": None, + "response_ids": response_tokens, + } + + mock_llm.generate = AsyncMock(side_effect=llm_generate_side_effect) + + # Env with two steps, rewards on both; no observations to keep math simple + class TruncEnv(BaseTextEnv): + def __init__(self): + super().__init__() + self.turns = 0 + self.max_turns = 1 + + def init(self, prompt): + return prompt, {} + + def step(self, action): + self.turns += 1 + if self.turns < self.max_turns: + return BaseTextEnvStepOutput(observations=[], reward=1.0, done=False, metadata={}) + else: + # On the final turn, return the final reward. + return BaseTextEnvStepOutput(observations=[], reward=2.0, done=True, metadata={}) + + def mock_make_func(*args, **kwargs): + return TruncEnv() + + mock_make.side_effect = mock_make_func + + # Generator config: non-retokenize message mode; max_turns=1 so max_response_tokens = max_tokens + cfg = get_default_config().generator + cfg.sampling_params.max_generate_length = 5 # enforce truncation + cfg.sampling_params.logprobs = None + cfg.apply_overlong_filtering = False + cfg.max_input_length = 1000 # prevent earlier length break + cfg.batched = False + cfg.max_turns = 1 + cfg.zero_reward_on_non_stop = False + cfg.use_conversation_multi_turn = False + cfg.chat_template = {"source": "name", "name_or_path": None} + + generator = SkyRLGymGenerator( + generator_cfg=cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + + prompt = [{"role": "user", "content": "Q?"}] + extras = {} + out = await generator.agent_loop(prompt, mock_env_cfg.env_class, extras, max_tokens=5, max_input_length=1000) + + # Untruncated response would be: 4 (step1) + 4 (step2) + 1 (final eos) = 9; we expect truncation to 5 + assert len(out.response_ids) == 5 + assert isinstance(out.reward, list) + assert len(out.reward) == 5 + + # Step1 end index relative should be 4 (0-based) - reward placed at EOS token + # NOTE(Dev): Because we manually append the eos token to the response, the reward is placed at the last token; + # See Charlie's comment in skyrl_gym_generator.py for more details. + + assert out.reward[4] == 2.0 + assert sum(out.reward) == 2.0 + assert out.stop_reason == "stop" + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_step_wise_trajectories_trajectory_ids(mock_make, mock_tokenizer, mock_llm, mock_env_cfg): + """Test step-wise training: validate trajectory_ids field is correctly populated.""" + from skyrl_train.generators.base import TrajectoryID + + mock_tokenizer.eos_token_id = 4 + + def apply_chat_template_side_effect(messages, **kwargs): + if kwargs.get("tokenize", True): + return [201, 202] + else: + return "".join([m.get("content", "") for m in messages]) + + mock_tokenizer.apply_chat_template.side_effect = apply_chat_template_side_effect + + # LLM returns 3 tokens + eos per step + async def llm_generate_side_effect(input_batch): + num = len(input_batch["prompt_token_ids"]) if "prompt_token_ids" in input_batch else len(input_batch["prompts"]) + return { + "responses": ["step"] * num, + "stop_reasons": ["stop"] * num, + "response_logprobs": None, + "response_ids": [[10, 11, 12, mock_tokenizer.eos_token_id] for _ in range(num)], + } + + mock_llm.generate = AsyncMock(side_effect=llm_generate_side_effect) + + # Environment that runs for 2 steps before completing + class MultiStepEnv(BaseTextEnv): + def __init__( + self, + ): + super().__init__() + self.turns = 0 + + def init(self, prompt): + return prompt, {} + + def step(self, action): + self.turns += 1 + if self.turns == 1: + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "obs1"}], reward=0.5, done=False, metadata={} + ) + else: + return BaseTextEnvStepOutput(observations=[], reward=1.0, done=True, metadata={}) + + def mock_make_func(*args, **kwargs): + return MultiStepEnv() + + mock_make.side_effect = mock_make_func + + # Generator config with step_wise_trajectories enabled + cfg = get_default_config().generator + cfg.sampling_params.max_generate_length = 50 + cfg.sampling_params.logprobs = None + cfg.apply_overlong_filtering = False + cfg.max_input_length = 512 + cfg.batched = False + cfg.max_turns = 10 + cfg.zero_reward_on_non_stop = False + cfg.use_conversation_multi_turn = True + cfg.step_wise_trajectories = True + cfg.chat_template = {"source": "name", "name_or_path": None} + + generator = SkyRLGymGenerator( + generator_cfg=cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] + + # Create input with trajectory_ids + prompts = [[{"role": "user", "content": "Q1?"}], [{"role": "user", "content": "Q2?"}]] + env_extras = [{"test": "value1"}, {"test": "value2"}] + trajectory_ids = [ + TrajectoryID(instance_id="uid1", repetition_id=0), + TrajectoryID(instance_id="uid2", repetition_id=0), + ] + + input_batch: GeneratorInput = { + "prompts": prompts, + "env_extras": env_extras, + "env_classes": [mock_env_cfg.env_class for _ in prompts], + "trajectory_ids": trajectory_ids, + } + + generator_output: GeneratorOutput = await generator.generate(input_batch) + + # Validate trajectory_ids field + assert "trajectory_ids" in generator_output, "trajectory_ids should be present in output" + assert generator_output["trajectory_ids"] is not None, "trajectory_ids should not be None" + assert isinstance(generator_output["trajectory_ids"], list), "trajectory_ids should be a list" + + # Each trajectory should produce 2 steps (since env runs for 2 steps) + # So we should have 2 trajectories * 2 steps = 4 total outputs + expected_num_steps = 2 * 2 # 2 trajectories, each with 2 steps + assert len(generator_output["trajectory_ids"]) == expected_num_steps, ( + f"Expected {expected_num_steps} trajectory_ids (2 trajectories * 2 steps), " + f"got {len(generator_output['trajectory_ids'])}" + ) + + # Validate that trajectory_ids match the input trajectory_ids + # For step-wise training, each step should reference the original trajectory + for i, output_traj_id in enumerate(generator_output["trajectory_ids"]): + assert isinstance(output_traj_id, TrajectoryID), f"trajectory_ids[{i}] should be a TrajectoryID instance" + # Each step should correspond to one of the input trajectory_ids + # For trajectory 0, steps 0 and 1 should have instance_id="uid1" + # For trajectory 1, steps 2 and 3 should have instance_id="uid2" + trajectory_idx = i // 2 # Which input trajectory this step belongs to + expected_traj_id = trajectory_ids[trajectory_idx] + assert output_traj_id.instance_id == expected_traj_id.instance_id, ( + f"Step {i} should have instance_id={expected_traj_id.instance_id}, " f"got {output_traj_id.instance_id}" + ) + assert output_traj_id.repetition_id == expected_traj_id.repetition_id, ( + f"Step {i} should have repetition_id={expected_traj_id.repetition_id}, " + f"got {output_traj_id.repetition_id}" + ) + + +@pytest.mark.asyncio +@patch("skyrl_gym.make") +async def test_step_wise_trajectories_basic_output_validation(mock_make, mock_tokenizer, mock_llm, mock_env_cfg): + """Test step-wise training: validate basic output structure and fields.""" + from skyrl_train.generators.base import TrajectoryID + + mock_tokenizer.eos_token_id = 4 + + def apply_chat_template_side_effect(messages, **kwargs): + if kwargs.get("tokenize", True): + return [201, 202] + else: + return "".join([m.get("content", "") for m in messages]) + + mock_tokenizer.apply_chat_template.side_effect = apply_chat_template_side_effect + + # LLM returns 3 tokens + eos per step + async def llm_generate_side_effect(input_batch): + num = len(input_batch["prompt_token_ids"]) if "prompt_token_ids" in input_batch else len(input_batch["prompts"]) + return { + "responses": ["step"] * num, + "stop_reasons": ["stop"] * num, + "response_logprobs": None, + "response_ids": [[10, 11, 12, mock_tokenizer.eos_token_id] for _ in range(num)], + } + + mock_llm.generate = AsyncMock(side_effect=llm_generate_side_effect) + + # Environment that runs for 2 steps before completing + class MultiStepEnv(BaseTextEnv): + def __init__(self): + super().__init__() + self.turns = 0 + + def init(self, prompt): + return prompt, {} + + def step(self, action): + self.turns += 1 + if self.turns == 1: + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": "obs1"}], reward=0.5, done=False, metadata={} + ) + else: + return BaseTextEnvStepOutput(observations=[], reward=1.0, done=True, metadata={}) + + mock_make.return_value = MultiStepEnv() + + # Generator config with step_wise_trajectories enabled + cfg = get_default_config().generator + cfg.sampling_params.max_generate_length = 50 + cfg.sampling_params.logprobs = None + cfg.apply_overlong_filtering = False + cfg.max_input_length = 512 + cfg.batched = False + cfg.max_turns = 10 + cfg.zero_reward_on_non_stop = False + cfg.use_conversation_multi_turn = True + cfg.step_wise_trajectories = True + cfg.chat_template = {"source": "name", "name_or_path": None} + + generator = SkyRLGymGenerator( + generator_cfg=cfg, + skyrl_gym_cfg=mock_env_cfg, + inference_engine_client=mock_llm, + tokenizer=mock_tokenizer, + model_name="test_model", + ) + generator.base_conversation_token_ids = [] + + # Create input with trajectory_ids + prompts = [[{"role": "user", "content": "Q?"}]] + env_extras = [{"test": "value"}] + trajectory_ids = [TrajectoryID(instance_id="uid1", repetition_id=0)] + + input_batch: GeneratorInput = { + "prompts": prompts, + "env_extras": env_extras, + "env_classes": [mock_env_cfg.env_class], + "trajectory_ids": trajectory_ids, + } + + generator_output: GeneratorOutput = await generator.generate(input_batch) + + # Basic output validation: check all required fields are present + required_fields = [ + "prompt_token_ids", + "response_ids", + "rewards", + "loss_masks", + "stop_reasons", + "rollout_metrics", + "rollout_logprobs", + "trajectory_ids", + "is_last_step", + ] + for field in required_fields: + assert field in generator_output, f"Required field '{field}' missing from output" + + # Validate field types and lengths + num_steps = 2 # Environment runs for 2 steps + assert ( + len(generator_output["prompt_token_ids"]) == num_steps + ), f"Expected {num_steps} prompt_token_ids, got {len(generator_output['prompt_token_ids'])}" + assert ( + len(generator_output["response_ids"]) == num_steps + ), f"Expected {num_steps} response_ids, got {len(generator_output['response_ids'])}" + assert ( + len(generator_output["rewards"]) == num_steps + ), f"Expected {num_steps} rewards, got {len(generator_output['rewards'])}" + assert ( + len(generator_output["loss_masks"]) == num_steps + ), f"Expected {num_steps} loss_masks, got {len(generator_output['loss_masks'])}" + assert ( + len(generator_output["stop_reasons"]) == num_steps + ), f"Expected {num_steps} stop_reasons, got {len(generator_output['stop_reasons'])}" + assert ( + len(generator_output["trajectory_ids"]) == num_steps + ), f"Expected {num_steps} trajectory_ids, got {len(generator_output['trajectory_ids'])}" + assert ( + len(generator_output["is_last_step"]) == num_steps + ), f"Expected {num_steps} is_last_step, got {len(generator_output['is_last_step'])}" + + # Validate is_last_step: only the last step should be True + assert generator_output["is_last_step"] == [ + False, + True, + ], f"Expected is_last_step=[False, True], got {generator_output['is_last_step']}" + + # Validate rewards are per-token (List[List[float]]) for step-wise training + for i, reward in enumerate(generator_output["rewards"]): + assert isinstance(reward, list), f"rewards[{i}] should be a list (per-token rewards)" + assert all(isinstance(r, (int, float)) for r in reward), f"rewards[{i}] should contain numeric values" + + # Validate response_ids structure + for i, response_ids in enumerate(generator_output["response_ids"]): + assert isinstance(response_ids, list), f"response_ids[{i}] should be a list" + assert len(response_ids) > 0, f"response_ids[{i}] should not be empty" + assert all(isinstance(token, int) for token in response_ids), f"response_ids[{i}] should contain integers" + + # Validate loss_masks structure + for i, loss_mask in enumerate(generator_output["loss_masks"]): + assert isinstance(loss_mask, list), f"loss_masks[{i}] should be a list" + assert len(loss_mask) == len( + generator_output["response_ids"][i] + ), f"loss_masks[{i}] length should match response_ids[{i}] length" + assert all(isinstance(val, int) for val in loss_mask), f"loss_masks[{i}] should contain integers" + + # Validate stop_reasons + for i, stop_reason in enumerate(generator_output["stop_reasons"]): + assert isinstance(stop_reason, str), f"stop_reasons[{i}] should be a string" diff --git a/skyrl-train/tests/cpu/generators/test_skyrl_gym_generator_chat_templating.py b/skyrl-train/tests/cpu/generators/test_skyrl_gym_generator_chat_templating.py new file mode 100644 index 0000000000..18adc40028 --- /dev/null +++ b/skyrl-train/tests/cpu/generators/test_skyrl_gym_generator_chat_templating.py @@ -0,0 +1,403 @@ +""" +uv run --extra dev --isolated pytest tests/cpu/generators/test_skyrl_gym_generator_chat_templating.py +""" + +import pytest +from typing import Dict, Any +from unittest.mock import AsyncMock, MagicMock +from skyrl_train.generators.skyrl_gym_generator import SkyRLGymGenerator +from skyrl_train.generators.base import GeneratorInput, GeneratorOutput +from omegaconf import OmegaConf + +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from omegaconf import DictConfig +from transformers import AutoTokenizer +from skyrl_gym.envs import register +from skyrl_train.generators.utils import get_custom_chat_template +from skyrl_train.config.utils import get_default_config +from skyrl_train.generators.utils import CUSTOM_CHAT_TEMPLATES +from pathlib import Path +from tests.cpu.generators.chat_templating_test_constants import ( + QWEN2_5_EXPECTED_STR, + LLAMA3_2_EXPECTED_STR, + QWEN3_TITO_EXPECTED_STR, + QWEN3_WITHOUT_THINKING_EXPECTED_STR, + get_expected_chat_history, +) + + +# Setup for formatting tests +class CPUTestEnv(BaseTextEnv): + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + self.max_turns = 3 + + def init(self, prompt): + return prompt, {} + + def step(self, action: str): + self.turns += 1 + done = self.turns >= self.max_turns + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": f"{self.turns}"}] if not done else [], + reward=0, + done=done, + metadata={}, + ) + + +def _register_test_env_if_needed(): + """Register the test env only if it's not already registered.""" + try: + register( + id="cpu_test_env", + entry_point="tests.cpu.generators.test_skyrl_gym_generator_chat_templating:CPUTestEnv", + ) + except Exception: + # Environment already registered, ignore + pass + + +def _build_generator(tokenizer, model_name: str, chat_template_config, extra_overrides: Dict[str, Any] | None = None): + """Helper to create a SkyRLGymGenerator with common config/env settings.""" + # Create default config and apply common generator overrides used across tests + default_cfg = get_default_config() + overrides = { + "sampling_params": {"max_generate_length": 200, "logprobs": None}, + "max_input_length": 200, + "batched": False, + "max_turns": 3, + "zero_reward_on_non_stop": False, + "apply_overlong_filtering": False, + "use_conversation_multi_turn": True, + "chat_template": chat_template_config, + "append_eos_token_after_stop_str_in_multi_turn": True, + } + if extra_overrides: + overrides.update(extra_overrides) + OmegaConf.update(default_cfg, "generator", overrides) + + # Create skryl gym generator + generator_cfg = default_cfg.generator + env_cfg = default_cfg.environment.skyrl_gym + env_cfg.max_env_workers = 0 + return SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=env_cfg, + inference_engine_client=None, # to be replaced per-test + tokenizer=tokenizer, + model_name=model_name, + ) + + +def _default_prompt_and_extras(): + """Standard single-trajectory prompt and extras used throughout tests.""" + prompt = [[{"role": "user", "content": "a"}]] + extras = [{"answer": "4"}] + return prompt, extras + + +def _make_input_batch(prompt, extras): + return {"prompts": prompt, "env_extras": extras, "env_classes": ["cpu_test_env"]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_name,tokenization_codepath,expected_str", + [ + ("Qwen/Qwen2.5-0.5B-Instruct", "tito", QWEN2_5_EXPECTED_STR), + ("unsloth/Llama-3.2-1B-Instruct", "tito", LLAMA3_2_EXPECTED_STR), + # Qwen3: test all three tokenization paths + ("Qwen/Qwen3-0.6B", "tito", QWEN3_TITO_EXPECTED_STR), + ("Qwen/Qwen3-0.6B", "custom_chat_template_from_path", QWEN3_WITHOUT_THINKING_EXPECTED_STR), + ("Qwen/Qwen3-0.6B", "custom_chat_template_builtin", QWEN3_WITHOUT_THINKING_EXPECTED_STR), + ], + ids=[ + "qwen2_5-tito", + "llama3_2-tito", + "qwen3-tito", + "qwen3-custom_chat_template_from_path", + "qwen3-custom_chat_template_builtin", + ], +) +async def test_skyrl_gym_generator_chat_templating_exact(model_name, tokenization_codepath, expected_str): + """ + Tests the behavior of chat templating for various models in multi-turn conversation. + + `tokenization_codepath` being `tito` means token-in-token-out, which is codepath 1 described in + `skyrl_gym_generator.rst`. For Qwen3, we also test `generator.chat_template` being defined. + + We hardcode the expected string in the constants file, so it is easier to check. But we also double + check that those expected strings are correct by applying the chat template on the expected chat history. + """ + # 1. Preparations to mock the generation. + _register_test_env_if_needed() # Register only when needed + tokenizer = AutoTokenizer.from_pretrained(model_name) + mock_llm = MagicMock() + + # Parameterize mock response: Qwen3 uses thinking tokens, others use simple 'b' + mock_response_text = "b" + if "Qwen3" in model_name: + mock_response_text = "\nmock thinking\n\n\n" + mock_response_text + + def mock_generate(input_batch): + num_prompts = len(input_batch["prompts"]) if "prompts" in input_batch else len(input_batch["prompt_token_ids"]) + + mock_llm_output_text = mock_response_text + tokenizer.eos_token + + return { + # no tokenizer.eos_token for responses because `skip_special_tokens` is True in sampling params + "responses": [mock_response_text] * num_prompts, + "stop_reasons": ["stop"] * num_prompts, + "response_logprobs": None, + # add_special_tokens needs to be False, otherwise for instance Llama will always + # add a `<|begin_of_text|>` before the assistant response. + "response_ids": [tokenizer.encode(mock_llm_output_text, add_special_tokens=False)] * num_prompts, + } + + mock_llm.generate = AsyncMock(side_effect=mock_generate) + chat_template_config = None + if "Qwen3" in model_name and tokenization_codepath == "custom_chat_template_from_path": + template_path = Path(__file__).parent / "qwen3_acc_without_thinking.jinja2" + chat_template_config = {"source": "file", "name_or_path": str(template_path)} + elif "Qwen3" in model_name and tokenization_codepath == "custom_chat_template_builtin": + chat_template_config = {"source": "name", "name_or_path": "qwen3_without_thinking"} + else: + chat_template_config = {"source": "name", "name_or_path": None} + # Create a mock generator config + generator = _build_generator(tokenizer, model_name, chat_template_config) + generator.inference_engine_client = mock_llm + + prompt, extras = _default_prompt_and_extras() + input_batch: GeneratorInput = _make_input_batch(prompt, extras) + + generator_output: GeneratorOutput = await generator.generate(input_batch) + + # 2. Double check that the hardcoded expected string is correct by recreating them. + expected_chat_history = get_expected_chat_history(mock_response_text) + if "Qwen3" in model_name and tokenization_codepath == "tito": + keep_thinking_chat_template = get_custom_chat_template( + {"source": "name", "name_or_path": "qwen3_with_thinking"} + ) + assert expected_str == tokenizer.apply_chat_template( + expected_chat_history, tokenize=False, chat_template=keep_thinking_chat_template + ) + else: + assert expected_str == tokenizer.apply_chat_template(expected_chat_history, tokenize=False) + + # 3. Check that the full response is exactly string matching with applying the chat template on history + prompt_str = tokenizer.decode(generator_output["prompt_token_ids"][0]) + resp_str = tokenizer.decode(generator_output["response_ids"][0]) + generator_output_str = prompt_str + resp_str + if tokenization_codepath == "tito" and "Qwen" in model_name: + # For Qwen models, there is an `\n` after the eos token. Our generator follows token-in-token-out, + # so it will not generate anything after the eos token, and hence will not have the `\n`. + # e.g. `<|assistant|>\Some content<|im_end|>\n` for expected_str, but + # `<|assistant|>\Some content<|im_end|>` for generator_output_str. + if expected_str.endswith("\n"): + expected_str = expected_str[:-1] + assert generator_output_str == expected_str + + # 4. Check loss mask exact matches + system_prompt = tokenizer.apply_chat_template( + [{"role": "system", "content": ""}] if "Llama" in model_name else [{}], tokenize=True + ) + empty_user = tokenizer.apply_chat_template([{"role": "user", "content": ""}], tokenize=True) + empty_user_with_generation_prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": ""}], add_generation_prompt=True, tokenize=True + ) + # TODO (erictang000): consider hard coding the full loss mask for each model to avoid copying logic in code + generation_prompt_ids = empty_user_with_generation_prompt[len(empty_user) :] # `<|im_start|>assistant\n` + empty_user = empty_user[len(system_prompt) :] # `<|im_start|>user\n<|im_end|>\n` + + # Build expected_loss_masks + if "Qwen3" in model_name: + # last [1, 0] -- 1 is for eos, 0 is for `\n` + # Qwen3 with thinking content + num_tokens_with_thinking = len(tokenizer.encode(mock_response_text)) + num_tokens_without_thinking = 1 + expected_user_loss_mask = [0] * len(empty_user) + [0] + # `<|im_start|>assistant\n\nmock thinking\n\n\nb<|im_end|>\n` + expected_assistant_loss_mask_with_thinking = ( + [0] * len(generation_prompt_ids) + [1] * num_tokens_with_thinking + [1, 0] + ) + # `<|im_start|>assistant\nb<|im_end|>\n` + expected_assistant_loss_mask_without_thinking = ( + [0] * len(generation_prompt_ids) + [1] * num_tokens_without_thinking + [1, 0] + ) + # `\nmock thinking\n\n\nb<|im_end|>\n` + expected_assistant_no_generation_prompt_loss_mask_with_thinking = [1] * num_tokens_with_thinking + [1, 0] + + if tokenization_codepath == "tito": + # For non-custom_chat_template, `resp_str` directly starts with what the model generates + expected_loss_masks = ( + expected_assistant_no_generation_prompt_loss_mask_with_thinking # \nmock thinking\n\n\nb<|im_end|>\n + + ( + expected_user_loss_mask # <|im_start|>user\n1<|im_end|>\n + + expected_assistant_loss_mask_with_thinking # `<|im_start|>assistant\n\nmock thinking\n\n\nb<|im_end|>\n` + ) + * 2 + ) + expected_loss_masks = expected_loss_masks[:-1] # remove the extra 0 for \n + else: + # For chat templating, the first generation prompt IDs are part of `resp_str`, hence has corresponding mask + expected_loss_masks = ( + expected_assistant_loss_mask_without_thinking # `<|im_start|>assistant\nb<|im_end|>\n` + + expected_user_loss_mask # `<|im_start|>user\n1<|im_end|>\n` + ) * 2 + expected_assistant_loss_mask_with_thinking # last `<|im_start|>assistant\n\nmock thinking\n\n\nb<|im_end|>\n` + else: + # `<|im_start|>assistant\nb<|im_end|>\n` + expected_assistant_loss_mask = [0] * len(generation_prompt_ids) + [ + 1, + 1, + ] # 1 for single response token, 1 for eos + expected_assistant_no_generation_prompt_loss_mask = [1, 1] # 1 for single response token, 1 for eos + if "Qwen" in model_name: + expected_assistant_loss_mask += [0] # extra 0 for \n for qwen templates + expected_assistant_no_generation_prompt_loss_mask += [0] + # `<|im_start|>user\n1<|im_end|>\n` + expected_user_loss_mask = [0] * len(empty_user) + [0] # extra 0 for single observation token + + assert tokenization_codepath == "tito" # we only test custom chat template for Qwen3 models + expected_loss_masks = ( + expected_assistant_no_generation_prompt_loss_mask # b<|im_end|>\n + + ( + expected_user_loss_mask # <|im_start|>user\n1 + + expected_assistant_loss_mask # <|im_start|>assistant\nb<|im_end|>\n + ) + * 2 + ) + if "Qwen" in model_name: + expected_loss_masks = expected_loss_masks[:-1] # remove the extra 0 for \n + assert len(expected_loss_masks) == len(generator_output["loss_masks"][0]) + assert generator_output["loss_masks"][0] == expected_loss_masks + + +def test_qwen3_original_vs_without_thinking_chat_template(): + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-4B") + + messages = [ + {"content": "hi", "role": "system"}, + {"content": "hi", "role": "user"}, + {"content": "thinkinghi", "role": "assistant"}, + {"content": "hi", "role": "user"}, + {"content": "thinkinghi", "role": "assistant"}, + {"content": "hi", "role": "user"}, + ] + + # Apply custom chat template + qwen3_without_thinking_str = tokenizer.apply_chat_template( + messages, chat_template=CUSTOM_CHAT_TEMPLATES["qwen3_without_thinking"], tokenize=False + ) + + # Apply custom chat template from file + file_path = Path(__file__).parent / "qwen3_acc_without_thinking.jinja2" + with open(file_path, "r", encoding="utf-8") as f: + template_from_file = f.read() + + qwen3_without_thinking_str_from_file = tokenizer.apply_chat_template( + messages, chat_template=template_from_file, tokenize=False + ) + + # Apply default chat template + default_template_str = tokenizer.apply_chat_template(messages, chat_template=None, tokenize=False) + + # The original_chat_template should match the tokenizer exactly + assert default_template_str == qwen3_without_thinking_str + assert qwen3_without_thinking_str == qwen3_without_thinking_str_from_file + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_name,tokenization_codepath", + [ + ("Qwen/Qwen2.5-0.5B-Instruct", "tito"), + ("unsloth/Llama-3.2-1B-Instruct", "tito"), + ("Qwen/Qwen3-0.6B", "tito"), + ("Qwen/Qwen3-0.6B", "custom_chat_template_builtin"), + ], +) +async def test_append_eos_after_stop_multi_turn(model_name, tokenization_codepath): + """ + Test the behavior of `append_eos_token_after_stop_str_in_multi_turn`, which is applicable + when `sampling_params.stop` is not `null` and `use_conversation_multi_turn` is `true` in + the ``agent_loop()`` function. + It is used in scripts `examples/search/run_search_conversation_format.sh` and + `examples/text_to_sql/run_skyrl_sql_conversation_format.sh`. + `tokenization_codepath` being `tito` means token-in-token-out, which is codepath 1 described in + `skyrl_gym_generator.rst`. For Qwen3, we also test `generator.chat_template` being defined. + """ + _register_test_env_if_needed() + tokenizer = AutoTokenizer.from_pretrained(model_name) + + stop_tag = "" + mock_text = "b" + stop_tag + + async def make_generator(append_flag: bool): + mock_llm = MagicMock() + + # The LLM engine will generate and return the stop tag, but no EOS token ID. + def mock_generate(input_batch): + num_prompts = ( + len(input_batch["prompts"]) if "prompts" in input_batch else len(input_batch["prompt_token_ids"]) + ) + return { + "responses": [mock_text] * num_prompts, + "stop_reasons": ["stop"] * num_prompts, + "response_logprobs": None, + "response_ids": [tokenizer.encode(mock_text, add_special_tokens=False)] * num_prompts, + } + + mock_llm.generate = AsyncMock(side_effect=mock_generate) + chat_template_config = None + if "Qwen3" in model_name and tokenization_codepath == "custom_chat_template_builtin": + chat_template_config = {"source": "name", "name_or_path": "qwen3_without_thinking"} + else: + chat_template_config = {"source": "name", "name_or_path": None} + extra_overrides = { + "sampling_params": {"max_generate_length": 200, "logprobs": None, "stop": [stop_tag]}, + "append_eos_token_after_stop_str_in_multi_turn": append_flag, + } + gen = _build_generator(tokenizer, model_name, chat_template_config, extra_overrides) + gen.inference_engine_client = mock_llm + return gen + + prompt, extras = _default_prompt_and_extras() + sp = {"stop": [stop_tag]} + + # Case 1: append flag = True + generator_true = await make_generator(True) + out_true: GeneratorOutput = await generator_true.generate( + {"prompts": prompt, "env_extras": extras, "env_classes": ["cpu_test_env"], "sampling_params": sp} + ) + + # Case 2: append flag = False + generator_false = await make_generator(False) + out_false: GeneratorOutput = await generator_false.generate( + {"prompts": prompt, "env_extras": extras, "env_classes": ["cpu_test_env"], "sampling_params": sp} + ) + + # Common assertions + assert out_true["stop_reasons"][0] == "stop" + assert out_false["stop_reasons"][0] == "stop" + assert len(out_true["response_ids"][0]) == len(out_true["loss_masks"][0]) + assert len(out_false["response_ids"][0]) == len(out_false["loss_masks"][0]) + + if "Qwen3" in model_name and not tokenization_codepath == "tito": + # Retokenize path is not affected by append_eos_token_after_stop_str_in_multi_turn + # The chat template does things like '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' + # So regardless of append_eos_token_after_stop_str_in_multi_turn, the last tokens are: + # stop_tag, eos_token_id and \n + last_token_ids = tokenizer.encode(stop_tag + tokenizer.eos_token + "\n") + num_last_tokens = len(last_token_ids) + response_ids_true = out_true["response_ids"][0] + response_ids_false = out_false["response_ids"][0] + assert response_ids_true[-num_last_tokens:] == last_token_ids + assert response_ids_false[-num_last_tokens:] == last_token_ids + assert response_ids_true == response_ids_false + else: + # Non-retokenize path: last token is eos only when append flag is True + last_token_id_true = out_true["response_ids"][0][-1] + last_token_id_false = out_false["response_ids"][0][-1] + assert last_token_id_true == tokenizer.eos_token_id + assert last_token_id_false == tokenizer.encode(mock_text, add_special_tokens=False)[-1] diff --git a/skyrl-train/tests/cpu/generators/test_utils.py b/skyrl-train/tests/cpu/generators/test_utils.py new file mode 100644 index 0000000000..ece00edb67 --- /dev/null +++ b/skyrl-train/tests/cpu/generators/test_utils.py @@ -0,0 +1,798 @@ +""" +uv run --extra dev --isolated pytest tests/cpu/generators/test_utils.py +""" + +import pytest +from skyrl_train.generators.utils import ( + apply_overlong_filtering, + encode_messages_subset, + get_response_ids_and_loss_mask_from_messages, + get_generation_prompt_ids, +) +from transformers import AutoTokenizer + + +@pytest.mark.parametrize( + "loss_masks,response_ids,eos_token_id,expected_masks", + [ + # Test case 1: All responses end with eos token - masks should remain unchanged + ( + [[1, 1, 0, 1], [0, 1, 1, 1], [1, 0, 1]], + [[1, 2, 3, 4], [5, 6, 7, 4], [8, 9, 4]], # All end with eos_token_id=4 + 4, + [[1, 1, 0, 1], [0, 1, 1, 1], [1, 0, 1]], + ), + # Test case 2: No responses end with eos token - all masks should be zeroed + ( + [[1, 1, 0, 1], [0, 1, 1, 1], [1, 0, 1]], + [[1, 2, 3, 5], [5, 6, 7, 8], [8, 9, 10]], # None end with eos_token_id=4 + 4, + [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0]], + ), + # Test case 3: Mixed responses - only non-eos ending masks should be zeroed + ( + [[1, 1, 0, 1], [0, 1, 1, 1], [1, 0, 1, 0, 1]], + [[1, 2, 3, 4], [5, 6, 7, 8], [8, 9, 10, 11, 4]], # First and third end with eos_token_id=4 + 4, + [[1, 1, 0, 1], [0, 0, 0, 0], [1, 0, 1, 0, 1]], + ), + # Test case 4: Empty responses should be zeroed + ( + [[1, 1], [1, 0, 1], [0, 1, 1, 1]], + [[], [1, 2, 3], [4, 5, 6, 7]], # Empty, no eos, no eos (eos_token_id=4) + 4, + [[0, 0], [0, 0, 0], [0, 0, 0, 0]], + ), + # Test case 5: Empty lists + ([], [], 4, []), + # Test case 6: Different eos token id + ( + [[1, 1], [1, 0, 1], [0, 1, 1, 1]], + [[1, 2], [3, 4, 99], [5, 6, 7, 99]], # Second and third end with eos_token_id=99 + 99, + [[0, 0], [1, 0, 1], [0, 1, 1, 1]], + ), + ], +) +def test_apply_overlong_filtering(loss_masks, response_ids, eos_token_id, expected_masks): + """ + Test the apply_overlong_filtering function which implements DAPO Overlong Filtering. + + This function should zero-out every token's mask whenever the response does not end + with the eos token id (i.e. truncated), while leaving other masks unchanged. + """ + result = apply_overlong_filtering(loss_masks, response_ids, eos_token_id) + + assert result == expected_masks, f"Expected {expected_masks}, but got {result}" + + # Verify that the original inputs are not modified (immutability check) + assert len(result) == len(loss_masks), "Result should have same length as input" + + # Check that each individual mask is processed correctly + for i, (original_mask, response, expected_mask) in enumerate(zip(loss_masks, response_ids, expected_masks)): + if len(response) == 0 or response[-1] != eos_token_id: + # Should be all zeros with same length as original + assert result[i] == [0] * len(original_mask), f"Mask {i} should be all zeros for truncated response" + else: + # Should be unchanged + assert result[i] == original_mask, f"Mask {i} should be unchanged for response ending with eos token" + + +def test_apply_overlong_filtering_immutability(): + """ + Test that apply_overlong_filtering doesn't modify the original input lists. + """ + original_loss_masks = [[1, 1, 0, 1], [0, 1, 1]] + original_response_ids = [[1, 2, 3, 4], [5, 6, 7]] # First ends with eos=4, second doesn't + eos_token_id = 4 + + # Create copies to compare against later + loss_masks_copy = [mask[:] for mask in original_loss_masks] # Deep copy of lists + response_ids_copy = [response[:] for response in original_response_ids] # Deep copy of lists + + result = apply_overlong_filtering(original_loss_masks, original_response_ids, eos_token_id) + + # Verify original inputs are unchanged + assert original_loss_masks == loss_masks_copy, "Original loss_masks should not be modified" + assert original_response_ids == response_ids_copy, "Original response_ids should not be modified" + + # Verify result is correct + expected = [[1, 1, 0, 1], [0, 0, 0]] # Second mask zeroed due to not ending with eos + assert result == expected, f"Expected {expected}, got {result}" + + +@pytest.mark.parametrize( + "loss_masks,response_ids", + [ + # Test case 1: More loss_masks than response_ids + ([[1, 1], [0, 1]], [[1, 2]]), + # Test case 2: More response_ids than loss_masks + ([[1, 1]], [[1, 2], [3, 4]]), + # Test case 3: Empty loss_masks but non-empty response_ids + ([], [[1, 2]]), + # Test case 4: Non-empty loss_masks but empty response_ids + ([[1, 0]], []), + ], +) +def test_apply_overlong_filtering_length_mismatch_assertion(loss_masks, response_ids): + """ + Test that apply_overlong_filtering raises AssertionError when loss_masks and response_ids + have different lengths. + """ + eos_token_id = 4 + with pytest.raises(AssertionError, match="loss_masks and response_ids must have the same length"): + apply_overlong_filtering(loss_masks, response_ids, eos_token_id) + + +dummy_chat_template = ( + "{%- for message in messages %}" + "{%- if message['role'] == 'user' %}" + "{{ message['content'] }}\n" + "{%- elif message['role'] == 'assistant' %}" + "{{ message['content'] }}\n" + "{%- elif message['role'] == 'system' %}" + "{{ message['content'] }}\n" + "{%- endif %}" + "{%- endfor %}" + "{%- if add_generation_prompt %}" + "" + "{%- endif %}" +) + + +@pytest.fixture +def tokenizer_w_dummy_template(): + tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-2-7b") + tokenizer.chat_template = dummy_chat_template + return tokenizer + + +@pytest.mark.parametrize( + "messages", + [ + # Test case 1: Single assistant message + [{"role": "assistant", "content": "Hello, I can help you."}], + # Test case 2: Single user message + [{"role": "user", "content": "What is the weather today?"}], + # Test case 3: Multiple messages (user-assistant exchange) + [{"role": "user", "content": "What is 2+2?"}, {"role": "assistant", "content": "The answer is 4."}], + # Test case 4: Multiple messages starting with assistant + [ + {"role": "assistant", "content": "I'm here to help."}, + {"role": "user", "content": "Can you explain Python?"}, + {"role": "assistant", "content": "Python is a programming language."}, + ], + ], +) +def test_encode_messages(messages, tokenizer_w_dummy_template): + # For a simple chat template, the fixed base approach is expected to behave the same + # as `apply_chat_template` + expected_token_ids = tokenizer_w_dummy_template.apply_chat_template(messages) + actual_token_ids = encode_messages_subset(messages, tokenizer_w_dummy_template) + assert expected_token_ids == actual_token_ids + + +@pytest.fixture +def qwen_tokenizer(): + return AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct") + + +@pytest.mark.parametrize( + "messages, expected_str", + [ + # Test case 1: Single assistant message + ( + [{"role": "assistant", "content": "Hello, I can help you."}], + "<|im_start|>assistant\nHello, I can help you.<|im_end|>\n", + ), + # Test case 2: Single user message - additional \n because the expectation is that there is a previous assistant turn + ( + [{"role": "user", "content": "What is the weather today?"}], + "<|im_start|>user\nWhat is the weather today?<|im_end|>\n", + ), + # Test case 3: Multiple messages (user-assistant exchange) + ( + [{"role": "user", "content": "What is 2+2?"}, {"role": "assistant", "content": "The answer is 4."}], + # NOTE: Additional \n because the expectation is that there is a previous assistant turn. + # All tokens after EOS in the previous turn get pushed into the next user/tool message. + "<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\nThe answer is 4.<|im_end|>\n", + ), + # Test case 4: Multiple messages starting with assistant + ( + [ + {"role": "assistant", "content": "I'm here to help."}, + {"role": "user", "content": "Can you explain Python?"}, + {"role": "assistant", "content": "Python is a programming language."}, + ], + "<|im_start|>assistant\nI'm here to help.<|im_end|>\n<|im_start|>user\nCan you explain Python?<|im_end|>\n<|im_start|>assistant\nPython is a programming language.<|im_end|>\n", + ), + ], +) +def test_encode_messages_qwen(messages, expected_str, qwen_tokenizer): + expected_token_ids = qwen_tokenizer.encode(expected_str, add_special_tokens=False) + actual_token_ids = encode_messages_subset(messages, qwen_tokenizer) + assert expected_token_ids == actual_token_ids, f"Got actual tokens: {qwen_tokenizer.decode(actual_token_ids)}" + + +@pytest.fixture +def qwen3_tokenizer(): + return AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + + +THINKING_CONTENT = "\nmock thinking\n\n\n" + + +@pytest.mark.parametrize( + "messages, expected_str", + [ + # Test case 1: Single assistant message + ( + [{"role": "assistant", "content": THINKING_CONTENT + "Hello, I can help you."}], + "<|im_start|>assistant\n" + THINKING_CONTENT + "Hello, I can help you.<|im_end|>\n", + ), + # Test case 2: Single user message - additional \n because the expectation is that there is a previous assistant turn + ( + [{"role": "user", "content": "What is the weather today?"}], + "<|im_start|>user\nWhat is the weather today?<|im_end|>\n", + ), + # Test case 3: Multiple messages (user-assistant exchange) + ( + [ + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": THINKING_CONTENT + "The answer is 4."}, + ], + # NOTE: Additional \n because the expectation is that there is a previous assistant turn. + # All tokens after EOS in the previous turn get pushed into the next user/tool message. + "<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n" + + THINKING_CONTENT + + "The answer is 4.<|im_end|>\n", + ), + # Test case 4: Multiple messages starting with assistant + ( + [ + {"role": "assistant", "content": THINKING_CONTENT + "I'm here to help."}, + {"role": "user", "content": "Can you explain Python?"}, + {"role": "assistant", "content": THINKING_CONTENT + "Python is a programming language."}, + ], + "<|im_start|>assistant\nI'm here to help.<|im_end|>\n<|im_start|>user\nCan you explain Python?<|im_end|>\n<|im_start|>assistant\n" + + THINKING_CONTENT + + "Python is a programming language.<|im_end|>\n", + ), + ], +) +def test_encode_messages_qwen3(messages, expected_str, qwen3_tokenizer): + expected_token_ids = qwen3_tokenizer.encode(expected_str, add_special_tokens=False) + actual_token_ids = encode_messages_subset(messages, qwen3_tokenizer) + assert expected_token_ids == actual_token_ids, f"Got actual tokens: {qwen3_tokenizer.decode(actual_token_ids)}" + + +# ============================================================================ +# Tests for get_response_ids_and_loss_mask_from_messages +# ============================================================================ + + +@pytest.fixture +def llama_tokenizer(): + return AutoTokenizer.from_pretrained("unsloth/Llama-3.2-1B-Instruct") + + +class TestGetResponseIdsAndLossMaskFromMessages: + """ + Tests for `get_response_ids_and_loss_mask_from_messages`. + + Key things to verify: + 1. Generation prompt tokens should have loss mask 0 + 2. Assistant-generated tokens (including EOS) should have loss mask 1 + 3. Tokens after EOS (like `\\n` in Qwen models) should have loss mask 0 + 4. User message tokens should all have loss mask 0 + 5. Total length of response_ids and loss_mask should match + """ + + # ------------------------------------------------------------------ + # Test single assistant message + # ------------------------------------------------------------------ + @pytest.mark.parametrize( + "model_name,content", + [ + ("Qwen/Qwen2.5-0.5B-Instruct", "Hello, I can help you."), + ("unsloth/Llama-3.2-1B-Instruct", "Hello, I can help you."), + ("Qwen/Qwen3-0.6B", "Hello, I can help you."), + ("Qwen/Qwen3-0.6B", THINKING_CONTENT + "Hello, I can help you."), + ], + ids=[ + "qwen2_5-simple", + "llama3_2-simple", + "qwen3-simple", + "qwen3-with-thinking", + ], + ) + def test_single_assistant_message(self, model_name, content): + """Test that a single assistant message has correct loss mask.""" + tokenizer = AutoTokenizer.from_pretrained(model_name) + messages = [{"role": "assistant", "content": content}] + + response_ids, loss_mask, rollout_logprobs = get_response_ids_and_loss_mask_from_messages(messages, tokenizer) + + # Verify lengths match + assert len(response_ids) == len(loss_mask) + assert rollout_logprobs is None + + # Verify the response_ids decode to expected string + decoded = tokenizer.decode(response_ids) + assert content in decoded + + # Verify generation prompt tokens have mask 0 + generation_prompt_ids = get_generation_prompt_ids(tokenizer) + assert loss_mask[: len(generation_prompt_ids)] == [0] * len(generation_prompt_ids) + + # Verify EOS token is present and has mask 1 + assert tokenizer.eos_token_id in response_ids + last_eos_idx = len(response_ids) - 1 - response_ids[::-1].index(tokenizer.eos_token_id) + assert loss_mask[last_eos_idx] == 1 + + # Verify tokens after EOS have mask 0 (like \n in Qwen) + if last_eos_idx < len(response_ids) - 1: + assert all(m == 0 for m in loss_mask[last_eos_idx + 1 :]) + + # Verify tokens between generation prompt and EOS have mask 1 + assert all(m == 1 for m in loss_mask[len(generation_prompt_ids) : last_eos_idx + 1]) + + # ------------------------------------------------------------------ + # Test single user message + # ------------------------------------------------------------------ + @pytest.mark.parametrize( + "model_name", + [ + "Qwen/Qwen2.5-0.5B-Instruct", + "unsloth/Llama-3.2-1B-Instruct", + "Qwen/Qwen3-0.6B", + ], + ids=["qwen2_5", "llama3_2", "qwen3"], + ) + def test_single_user_message(self, model_name): + """Test that a single user message has all zeros in loss mask.""" + tokenizer = AutoTokenizer.from_pretrained(model_name) + messages = [{"role": "user", "content": "What is the weather today?"}] + + response_ids, loss_mask, rollout_logprobs = get_response_ids_and_loss_mask_from_messages(messages, tokenizer) + + # Verify lengths match + assert len(response_ids) == len(loss_mask) + assert rollout_logprobs is None + + # All user message tokens should have mask 0 + assert all(m == 0 for m in loss_mask) + + # Verify the content is in the decoded response + decoded = tokenizer.decode(response_ids) + assert "What is the weather today?" in decoded + + # ------------------------------------------------------------------ + # Test multi-turn conversation (user-assistant-user-assistant) + # ------------------------------------------------------------------ + @pytest.mark.parametrize( + "model_name,assistant_content", + [ + ("Qwen/Qwen2.5-0.5B-Instruct", "The answer is 4."), + ("unsloth/Llama-3.2-1B-Instruct", "The answer is 4."), + ("Qwen/Qwen3-0.6B", "The answer is 4."), + ("Qwen/Qwen3-0.6B", THINKING_CONTENT + "The answer is 4."), + ], + ids=["qwen2_5", "llama3_2", "qwen3-simple", "qwen3-with-thinking"], + ) + def test_multi_turn_user_assistant(self, model_name, assistant_content): + """Test multi-turn conversation with user and assistant messages.""" + tokenizer = AutoTokenizer.from_pretrained(model_name) + messages = [ + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": "And what is 3+3?"}, + {"role": "assistant", "content": assistant_content}, + ] + + response_ids, loss_mask, rollout_logprobs = get_response_ids_and_loss_mask_from_messages(messages, tokenizer) + + # Verify lengths match + assert len(response_ids) == len(loss_mask) + assert rollout_logprobs is None + + # Count assistant messages and verify we have the right number of 1s in the mask + generation_prompt_ids = get_generation_prompt_ids(tokenizer) + + # Verify each message's loss mask is correctly assigned + current_pos = 0 + for msg in messages: + msg_token_ids = encode_messages_subset([msg], tokenizer) + msg_loss_mask = loss_mask[current_pos : current_pos + len(msg_token_ids)] + + if msg["role"] == "user": + # User messages should be all zeros + assert all(m == 0 for m in msg_loss_mask), "User message should have all 0s in loss mask" + else: + # Assistant messages: + # - Generation prompt: 0 + # - Generated tokens (including EOS): 1 + # - Tokens after EOS: 0 + assert msg_loss_mask[: len(generation_prompt_ids)] == [0] * len(generation_prompt_ids) + + assert tokenizer.eos_token_id in msg_token_ids, "Assistant message should contain EOS token" + last_eos_idx = len(msg_token_ids) - 1 - msg_token_ids[::-1].index(tokenizer.eos_token_id) + # Tokens from generation prompt end to EOS (inclusive) should be 1 + assert all(m == 1 for m in msg_loss_mask[len(generation_prompt_ids) : last_eos_idx + 1]) + # Tokens after EOS should be 0 + if last_eos_idx < len(msg_token_ids) - 1: + assert all(m == 0 for m in msg_loss_mask[last_eos_idx + 1 :]) + + current_pos += len(msg_token_ids) + + # ------------------------------------------------------------------ + # Test with assistant_logprobs + # ------------------------------------------------------------------ + @pytest.mark.parametrize( + "model_name", + [ + "Qwen/Qwen2.5-0.5B-Instruct", + "unsloth/Llama-3.2-1B-Instruct", + "Qwen/Qwen3-0.6B", + ], + ids=["qwen2_5", "llama3_2", "qwen3"], + ) + def test_with_assistant_logprobs(self, model_name): + """Test that assistant_logprobs are correctly handled.""" + tokenizer = AutoTokenizer.from_pretrained(model_name) + generation_prompt_ids = get_generation_prompt_ids(tokenizer) + + content = "Hello" + messages = [{"role": "assistant", "content": content}] + + # First, get the message encoding to determine the correct logprobs length + msg_token_ids = encode_messages_subset(messages, tokenizer) + + # Calculate the number of generated tokens (excluding generation prompt and tokens after EOS) + assert tokenizer.eos_token_id in msg_token_ids, "Assistant message should contain EOS token" + last_eos_idx = len(msg_token_ids) - 1 - msg_token_ids[::-1].index(tokenizer.eos_token_id) + num_generated_tokens = last_eos_idx + 1 - len(generation_prompt_ids) + + # Create logprobs matching the generated tokens count + mock_logprobs = [-0.5] * num_generated_tokens + assistant_logprobs = [mock_logprobs] + + response_ids, loss_mask, rollout_logprobs = get_response_ids_and_loss_mask_from_messages( + messages, tokenizer, assistant_logprobs=assistant_logprobs + ) + + # Verify lengths match + assert len(response_ids) == len(loss_mask) + assert len(rollout_logprobs) == len(response_ids) + + # Verify logprobs are 0.0 for generation prompt + assert all(lp == 0.0 for lp in rollout_logprobs[: len(generation_prompt_ids)]) + + # Verify logprobs are -0.5 for generated tokens + # We already asserted EOS exists above, reuse last_eos_idx + assert all(lp == -0.5 for lp in rollout_logprobs[len(generation_prompt_ids) : last_eos_idx + 1]) + # Verify logprobs are 0.0 for tokens after EOS + if last_eos_idx < len(msg_token_ids) - 1: + assert all(lp == 0.0 for lp in rollout_logprobs[last_eos_idx + 1 :]) + + # ------------------------------------------------------------------ + # Test with multiple assistant messages and logprobs + # ------------------------------------------------------------------ + @pytest.mark.parametrize( + "model_name", + [ + "Qwen/Qwen2.5-0.5B-Instruct", + "unsloth/Llama-3.2-1B-Instruct", + "Qwen/Qwen3-0.6B", + ], + ids=["qwen2_5", "llama3_2", "qwen3"], + ) + def test_multi_assistant_with_logprobs(self, model_name): + """Test multiple assistant messages with logprobs.""" + tokenizer = AutoTokenizer.from_pretrained(model_name) + generation_prompt_ids = get_generation_prompt_ids(tokenizer) + + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": "How are you?"}, + {"role": "assistant", "content": "Good"}, + ] + + # Calculate the number of generated tokens for each assistant message + def get_num_generated_tokens(content): + msg = [{"role": "assistant", "content": content}] + msg_token_ids = encode_messages_subset(msg, tokenizer) + assert tokenizer.eos_token_id in msg_token_ids, "Assistant message should contain EOS token" + last_eos_idx = len(msg_token_ids) - 1 - msg_token_ids[::-1].index(tokenizer.eos_token_id) + return last_eos_idx + 1 - len(generation_prompt_ids) + + num_tokens_1 = get_num_generated_tokens("Hello") + num_tokens_2 = get_num_generated_tokens("Good") + + assistant_logprobs = [ + [-0.1] * num_tokens_1, # logprobs for first assistant message + [-0.2] * num_tokens_2, # logprobs for second assistant message + ] + + response_ids, loss_mask, rollout_logprobs = get_response_ids_and_loss_mask_from_messages( + messages, tokenizer, assistant_logprobs=assistant_logprobs + ) + + # Verify lengths match + assert len(response_ids) == len(loss_mask) + assert len(rollout_logprobs) == len(response_ids) + + # Verify user messages have 0.0 logprobs + current_pos = 0 + assistant_idx = 0 + for msg in messages: + msg_token_ids = encode_messages_subset([msg], tokenizer) + msg_logprobs = rollout_logprobs[current_pos : current_pos + len(msg_token_ids)] + + if msg["role"] == "user": + assert all(lp == 0.0 for lp in msg_logprobs) + else: + # Assistant message + expected_lp = -0.1 if assistant_idx == 0 else -0.2 + + # Generation prompt should be 0.0 + assert all(lp == 0.0 for lp in msg_logprobs[: len(generation_prompt_ids)]) + + # Generated tokens should have the expected logprob + assert tokenizer.eos_token_id in msg_token_ids, "Assistant message should contain EOS token" + last_eos_idx = len(msg_token_ids) - 1 - msg_token_ids[::-1].index(tokenizer.eos_token_id) + assert all(lp == expected_lp for lp in msg_logprobs[len(generation_prompt_ids) : last_eos_idx + 1]) + # Tokens after EOS should be 0.0 + if last_eos_idx < len(msg_token_ids) - 1: + assert all(lp == 0.0 for lp in msg_logprobs[last_eos_idx + 1 :]) + + assistant_idx += 1 + + current_pos += len(msg_token_ids) + + # ------------------------------------------------------------------ + # Test error cases + # ------------------------------------------------------------------ + def test_empty_messages_raises(self, qwen_tokenizer): + """Test that empty messages list raises AssertionError.""" + with pytest.raises(AssertionError, match="messages list cannot be empty"): + get_response_ids_and_loss_mask_from_messages([], qwen_tokenizer) + + def test_invalid_role_raises(self, qwen_tokenizer): + """Test that invalid message role raises ValueError.""" + messages = [{"role": "system", "content": "You are a helpful assistant."}] + + with pytest.raises(ValueError, match="Expected message role to be 'user' or 'assistant'"): + get_response_ids_and_loss_mask_from_messages(messages, qwen_tokenizer) + + def test_missing_logprobs_raises(self, qwen_tokenizer): + """Test that missing logprobs for assistant message raises ValueError.""" + messages = [ + {"role": "assistant", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + ] + # Only provide logprobs for one assistant message + generation_prompt_ids = get_generation_prompt_ids(qwen_tokenizer) + msg_token_ids = encode_messages_subset([messages[0]], qwen_tokenizer) + assert qwen_tokenizer.eos_token_id in msg_token_ids, "Assistant message should contain EOS token" + last_eos_idx = len(msg_token_ids) - 1 - msg_token_ids[::-1].index(qwen_tokenizer.eos_token_id) + num_tokens = last_eos_idx + 1 - len(generation_prompt_ids) + + assistant_logprobs = [[-0.5] * num_tokens] # Only one logprobs list for two assistant messages + + with pytest.raises(ValueError, match="Missing logprobs for assistant message"): + get_response_ids_and_loss_mask_from_messages(messages, qwen_tokenizer, assistant_logprobs) + + def test_logprobs_count_mismatch_raises(self, qwen_tokenizer): + """Test that mismatched logprobs count raises ValueError.""" + messages = [{"role": "assistant", "content": "Hello"}] + # Provide wrong number of logprobs + assistant_logprobs = [[-0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5]] # Too many + + with pytest.raises(ValueError, match="Logprobs count.*does not match token count"): + get_response_ids_and_loss_mask_from_messages(messages, qwen_tokenizer, assistant_logprobs) + + # ------------------------------------------------------------------ + # Test exact loss mask values for specific models + # ------------------------------------------------------------------ + def test_qwen2_5_exact_loss_mask(self, qwen_tokenizer): + """Test exact loss mask values for Qwen2.5 model.""" + messages = [ + {"role": "assistant", "content": "b"}, + ] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages(messages, qwen_tokenizer) + + # For Qwen2.5: `<|im_start|>assistant\nb<|im_end|>\n` + # - `<|im_start|>assistant\n` is the generation prompt (mask 0) + # - `b<|im_end|>` is the assistant generated content (mask 1) + # - `\n` is after EOS (mask 0) + generation_prompt_ids = get_generation_prompt_ids(qwen_tokenizer) + gen_prompt_len = len(generation_prompt_ids) + + expected_loss_mask = [0] * gen_prompt_len + [1, 1] + [0] # 1 for 'b', 1 for eos, 0 for \n + assert loss_mask == expected_loss_mask, f"Expected {expected_loss_mask}, got {loss_mask}" + + def test_llama_exact_loss_mask(self, llama_tokenizer): + """Test exact loss mask values for Llama model.""" + messages = [ + {"role": "assistant", "content": "b"}, + ] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages(messages, llama_tokenizer) + + # For Llama: `<|start_header_id|>assistant<|end_header_id|>\n\nb<|eot_id|>` + # - `<|start_header_id|>assistant<|end_header_id|>\n\n` is the generation prompt (mask 0) + # - `b<|eot_id|>` is the assistant generated content (mask 1) + # - No tokens after EOS for Llama + generation_prompt_ids = get_generation_prompt_ids(llama_tokenizer) + gen_prompt_len = len(generation_prompt_ids) + + expected_loss_mask = [0] * gen_prompt_len + [1, 1] # 1 for 'b', 1 for eos + assert loss_mask == expected_loss_mask, f"Expected {expected_loss_mask}, got {loss_mask}" + + def test_qwen3_exact_loss_mask_with_thinking(self, qwen3_tokenizer): + """Test exact loss mask values for Qwen3 model with thinking tokens.""" + thinking_content = "\nmock thinking\n\n\nb" + messages = [ + {"role": "assistant", "content": thinking_content}, + ] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages(messages, qwen3_tokenizer) + + generation_prompt_ids = get_generation_prompt_ids(qwen3_tokenizer) + gen_prompt_len = len(generation_prompt_ids) + + # Get the number of tokens in the thinking content (excluding generation prompt and \n after eos) + content_tokens = qwen3_tokenizer.encode(thinking_content, add_special_tokens=False) + num_content_tokens = len(content_tokens) + + # Expected: [0]*gen_prompt_len + [1]*num_content_tokens + [1 for eos] + [0 for \n] + expected_loss_mask = [0] * gen_prompt_len + [1] * num_content_tokens + [1] + [0] + assert loss_mask == expected_loss_mask, f"Expected {expected_loss_mask}, got {loss_mask}" + + # ------------------------------------------------------------------ + # Test multi-turn exact loss masks + # ------------------------------------------------------------------ + def test_qwen2_5_multi_turn_exact_loss_mask(self, qwen_tokenizer): + """Test exact loss mask for multi-turn conversation with Qwen2.5.""" + messages = [ + {"role": "assistant", "content": "b"}, + {"role": "user", "content": "1"}, + {"role": "assistant", "content": "b"}, + ] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages(messages, qwen_tokenizer) + + generation_prompt_ids = get_generation_prompt_ids(qwen_tokenizer) + gen_prompt_len = len(generation_prompt_ids) + + # First assistant message: [0]*gen_prompt + [1, 1] for 'b' and eos + [0] for \n + # User message: all zeros + # Second assistant message: [0]*gen_prompt + [1, 1] for 'b' and eos + [0] for \n + + user_msg_tokens = encode_messages_subset([{"role": "user", "content": "1"}], qwen_tokenizer) + + expected_loss_mask = ( + [0] * gen_prompt_len + + [1, 1, 0] # first assistant + + [0] * len(user_msg_tokens) # user + + [0] * gen_prompt_len + + [1, 1, 0] # second assistant + ) + + assert loss_mask == expected_loss_mask, f"Expected {expected_loss_mask}, got {loss_mask}" + + def test_llama_multi_turn_exact_loss_mask(self, llama_tokenizer): + """Test exact loss mask for multi-turn conversation with Llama.""" + messages = [ + {"role": "assistant", "content": "b"}, + {"role": "user", "content": "1"}, + {"role": "assistant", "content": "b"}, + ] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages(messages, llama_tokenizer) + + generation_prompt_ids = get_generation_prompt_ids(llama_tokenizer) + gen_prompt_len = len(generation_prompt_ids) + + user_msg_tokens = encode_messages_subset([{"role": "user", "content": "1"}], llama_tokenizer) + + expected_loss_mask = ( + [0] * gen_prompt_len + + [1, 1] # first assistant (no \n after eos for Llama) + + [0] * len(user_msg_tokens) # user + + [0] * gen_prompt_len + + [1, 1] # second assistant + ) + + assert loss_mask == expected_loss_mask, f"Expected {expected_loss_mask}, got {loss_mask}" + + def test_qwen3_multi_turn_exact_loss_mask(self, qwen3_tokenizer): + """Test exact loss mask for multi-turn conversation with Qwen3.""" + messages = [ + {"role": "assistant", "content": "b"}, + {"role": "user", "content": "1"}, + {"role": "assistant", "content": "b"}, + ] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages(messages, qwen3_tokenizer) + + # Verify our assumptions about token structure + generation_prompt_ids = get_generation_prompt_ids(qwen3_tokenizer) + assert len(generation_prompt_ids) == 3, "Qwen3 generation prompt should be 3 tokens: <|im_start|>assistant\\n" + + user_msg_tokens = encode_messages_subset([{"role": "user", "content": "1"}], qwen3_tokenizer) + assert len(user_msg_tokens) == 6, "User message '1' should be 6 tokens: <|im_start|>user\\n1<|im_end|>\\n" + + assistant_msg_tokens = encode_messages_subset([{"role": "assistant", "content": "b"}], qwen3_tokenizer) + # For Qwen3 with content "b": <|im_start|>assistant\n\n\n\n\nb<|im_end|>\n + # = 3 (gen prompt) + 6 (content with thinking + eos) + 1 (\n after eos) = 10 tokens + assert ( + len(assistant_msg_tokens) == 10 + ), f"Assistant message 'b' should be 10 tokens, got {len(assistant_msg_tokens)}" + assert assistant_msg_tokens[-2] == qwen3_tokenizer.eos_token_id, "Second to last token should be EOS" + + # Expected loss mask: + # First assistant: [0,0,0] gen_prompt + [1,1,1,1,1,1] content+eos + [0] \n = 10 tokens + # User: [0,0,0,0,0,0] = 6 tokens + # Second assistant: [0,0,0] gen_prompt + [1,1,1,1,1,1] content+eos + [0] \n = 10 tokens + expected_loss_mask = ( + [0, 0, 0] + + [1, 1, 1, 1, 1, 1] + + [0] # first assistant (10 tokens) + + [0, 0, 0, 0, 0, 0] # user (6 tokens) + + [0, 0, 0] + + [1, 1, 1, 1, 1, 1] + + [0] # second assistant (10 tokens) + ) + + assert len(expected_loss_mask) == 26, "Total should be 26 tokens" + assert loss_mask == expected_loss_mask, f"Expected {expected_loss_mask}, got {loss_mask}" + + def test_qwen3_multi_turn_exact_loss_mask_with_thinking(self, qwen3_tokenizer): + """Test exact loss mask for multi-turn conversation with Qwen3 including thinking content.""" + thinking_content = THINKING_CONTENT + "b" # \nmock thinking\n\n\nb + messages = [ + {"role": "assistant", "content": thinking_content}, + {"role": "user", "content": "1"}, + {"role": "assistant", "content": thinking_content}, + ] + + response_ids, loss_mask, _ = get_response_ids_and_loss_mask_from_messages(messages, qwen3_tokenizer) + + # Verify our assumptions about token structure + generation_prompt_ids = get_generation_prompt_ids(qwen3_tokenizer) + assert len(generation_prompt_ids) == 3, "Qwen3 generation prompt should be 3 tokens" + + user_msg_tokens = encode_messages_subset([{"role": "user", "content": "1"}], qwen3_tokenizer) + assert len(user_msg_tokens) == 6, "User message '1' should be 6 tokens" + + assistant_msg_tokens = encode_messages_subset( + [{"role": "assistant", "content": thinking_content}], qwen3_tokenizer + ) + # For Qwen3 with thinking_content "\nmock thinking\n\n\nb": + # <|im_start|>assistant\n\nmock thinking\n\n\nb<|im_end|>\n + # = 3 (gen prompt) + 9 (content with thinking + eos) + 1 (\n after eos) = 13 tokens + assert ( + len(assistant_msg_tokens) == 13 + ), f"Assistant message with thinking should be 13 tokens, got {len(assistant_msg_tokens)}" + assert assistant_msg_tokens[-2] == qwen3_tokenizer.eos_token_id, "Second to last token should be EOS" + + # Expected loss mask: + # First assistant: [0,0,0] gen_prompt + [1]*9 content+eos + [0] \n = 13 tokens + # User: [0]*6 = 6 tokens + # Second assistant: [0,0,0] gen_prompt + [1]*9 content+eos + [0] \n = 13 tokens + expected_loss_mask = ( + [0, 0, 0] + + [1] * 9 + + [0] # first assistant (13 tokens) + + [0] * 6 # user (6 tokens) + + [0, 0, 0] + + [1] * 9 + + [0] # second assistant (13 tokens) + ) + + assert len(expected_loss_mask) == 32, "Total should be 32 tokens" + assert loss_mask == expected_loss_mask, f"Expected {expected_loss_mask}, got {loss_mask}" diff --git a/skyrl-train/tests/cpu/inf_engines/test_inference_engine_client.py b/skyrl-train/tests/cpu/inf_engines/test_inference_engine_client.py new file mode 100644 index 0000000000..6a9853027f --- /dev/null +++ b/skyrl-train/tests/cpu/inf_engines/test_inference_engine_client.py @@ -0,0 +1,905 @@ +""" +Test for `skyrl-train/skyrl_train/inference_engines/inference_engine_client.py` functinoalities +that can be mocked. Also tests for `skyrl-train/skyrl_train/inference_engines/utils.py`. + +Run with: +uv run --isolated --extra dev pytest tests/cpu/inf_engines/test_inference_engine_client.py +""" + +from http import HTTPStatus +from unittest.mock import patch + +from transformers import AutoTokenizer +from skyrl_train.inference_engines.utils import ( + postprocess_completion_request, + route_prompts_to_engines, + hash_with_sha256, +) +from skyrl_train.inference_engines.inference_engine_client_http_endpoint import ( + ErrorResponse, +) +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.base import InferenceEngineInput, InferenceEngineOutput +from omegaconf import OmegaConf +import asyncio +import pytest +import random +from copy import deepcopy + +# ------------------------------------------- +# tests for postprocess_completion_request +# -------------------------------------------- + + +def test_postprocess_single_string_no_session_id(): + prompt = "hello world" + traj, processed = postprocess_completion_request(prompt, None) + assert traj is None + assert isinstance(processed, list) + assert processed == [prompt] + + +def test_postprocess_single_string_scalar_session_id(): + prompt = "hello world" + traj, processed = postprocess_completion_request(prompt, 123) + assert traj == [123] + assert processed == [prompt] + + +def test_postprocess_single_string_list_session_id_singleton(): + prompt = "hello world" + traj, processed = postprocess_completion_request(prompt, ["abc"]) # accepts str ids + assert traj == ["abc"] + assert processed == [prompt] + + +def test_postprocess_single_string_list_session_id_wrong_len(): + prompt = "hello world" + traj, processed = postprocess_completion_request(prompt, [1, 2]) + assert isinstance(traj, ErrorResponse) + assert processed == [prompt] + assert traj.error.code == HTTPStatus.BAD_REQUEST.value + + +def test_postprocess_single_token_ids_no_session_id(): + prompt = [1, 2, 3] + traj, processed = postprocess_completion_request(prompt, None) + assert traj is None + assert processed == [prompt] + + +def test_postprocess_single_token_ids_scalar_session_id(): + prompt = [1, 2, 3] + traj, processed = postprocess_completion_request(prompt, 7) + assert traj == [7] + assert processed == [prompt] + + +def test_postprocess_single_token_ids_list_session_id_singleton(): + prompt = [1, 2, 3] + traj, processed = postprocess_completion_request(prompt, [8]) + assert traj == [8] + assert processed == [prompt] + + +def test_postprocess_single_token_ids_list_session_id_wrong_len(): + prompt = [1, 2, 3] + traj, processed = postprocess_completion_request(prompt, [8, 9]) + assert isinstance(traj, ErrorResponse) + assert processed == [prompt] + assert traj.error.code == HTTPStatus.BAD_REQUEST.value + + +def test_postprocess_batched_token_ids_no_session_id(): + prompt = [[1, 2], [3, 4, 5]] + traj, processed = postprocess_completion_request(prompt, None) + assert traj is None + assert processed is prompt # unchanged shape + + +def test_postprocess_batched_token_ids_with_matching_session_ids(): + prompt = [[1, 2], [3, 4, 5]] + traj, processed = postprocess_completion_request(prompt, ["a", "b"]) # accepts str ids too + assert traj == ["a", "b"] + assert processed is prompt + + +def test_postprocess_batched_token_ids_with_wrong_session_ids_length(): + prompt = [[1, 2], [3, 4, 5]] + traj, processed = postprocess_completion_request(prompt, [1]) + assert isinstance(traj, ErrorResponse) + assert processed is prompt + assert traj.error.code == HTTPStatus.BAD_REQUEST.value + + +def test_postprocess_batched_strings_no_session_id(): + prompt = ["p0", "p1"] + traj, processed = postprocess_completion_request(prompt, None) + assert traj is None + assert processed is prompt + + +def test_postprocess_batched_strings_with_matching_session_ids(): + prompt = ["p0", "p1", "p2"] + traj, processed = postprocess_completion_request(prompt, [10, 11, 12]) + assert traj == [10, 11, 12] + assert processed is prompt + + +def test_postprocess_batched_strings_with_wrong_session_ids_length(): + prompt = ["p0", "p1", "p2"] + traj, processed = postprocess_completion_request(prompt, [10, 11]) + assert isinstance(traj, ErrorResponse) + assert processed is prompt + assert traj.error.code == HTTPStatus.BAD_REQUEST.value + + +def test_postprocess_batched_strings_with_wrong_session_ids_length_2(): + prompt = ["p0", "p1", "p2"] + traj, processed = postprocess_completion_request(prompt, 10) + assert isinstance(traj, ErrorResponse) + assert processed is prompt + assert traj.error.code == HTTPStatus.BAD_REQUEST.value + + +# ------------------------------------------- +# tests for InferenceEngineClient.completion +# -------------------------------------------- + + +@pytest.mark.parametrize("num_prompts", [1, 50, 100]) +@pytest.mark.parametrize("with_session_id", [True, False]) +@pytest.mark.parametrize("num_engines", [1, 3, 4, 8, 16]) +def test_completion_batched_routing_and_order_preservation(num_prompts, with_session_id, num_engines): + """ + In InferenceEngineClient.completion, when the request is batched, we distribute the batch + and route to engines. If session_id is provided, we map to the corresponding engine; if unprovided, + we split it evenly. While the routing is done by `route_prompts_to_engines`, the aggregation is done + by the client. We expect the aggregated results returned to the user in the original order, and + this test checks exactly that. + + Related test: `test_route_prompts_to_engines_xxx` functions test the specific routing logic, + while this will call `route_prompts_to_engines` and check the end-to-end behavior. + """ + + class MockEngine: + async def completion(self, request_payload): + """ + Given input [i, j, k, ...], return output [f"{i}{i}", f"{j}{j}", f"{k}{k}", ...] with + indices 0, 1, 2, 3, ... + """ + body = request_payload["json"] + my_prompts = body["prompt"] + # Return per-sub-batch indices 0..len-1; client is expected to remap to global order + choices = [] + for i, p in enumerate(my_prompts): + choices.append( + { + "index": i, + "text": f"{p}{p}", + "finish_reason": "stop", + } + ) + num_prompt_tokens = sum(len(p) for p in my_prompts) + num_completion_tokens = num_prompt_tokens * 2 # since we doubled the prompts + return { + "id": "cmpl-mock", + "object": "text_completion", + "model": body.get("model", "dummy-model"), + "choices": choices, + "usage": { + "prompt_tokens": num_prompt_tokens, + "total_tokens": num_prompt_tokens + num_completion_tokens, + "completion_tokens": num_completion_tokens, + "prompt_tokens_details": { + "cached_tokens": num_prompt_tokens, + }, + }, + } + + # Create a minimal config to avoid spinning up HTTP endpoint + cfg = OmegaConf.create( + { + "trainer": { + "policy": {"model": {"path": "dummy-model"}}, + }, + "generator": { + "backend": "vllm", + "enable_http_endpoint": False, + "http_endpoint_host": "127.0.0.1", + "http_endpoint_port": 0, + }, + } + ) + + engines = [MockEngine() for _ in range(num_engines)] + tokenizer = object() # not used by completion() + client = InferenceEngineClient(engines=engines, tokenizer=tokenizer, full_config=cfg) + + prompts = [str(i) for i in range(num_prompts)] + if with_session_id: + session_ids = [random.randint(1, 100) for _ in range(num_prompts)] + else: + session_ids = None + request_payload = { + "json": { + "model": "dummy-model", + "prompt": prompts, + "session_id": session_ids, + "max_tokens": 32, + }, + "headers": {"Content-Type": "application/json"}, + } + + resp = asyncio.run(client.completion(request_payload)) + + assert resp.get("object") != "error" + assert "choices" in resp and len(resp["choices"]) == len(prompts) + # Ensure outputs align with inputs and indices are global order 0..n-1 + expected_texts = [f"{i}{i}" for i in range(num_prompts)] + for i, choice in enumerate(resp["choices"]): + assert choice["index"] == i + assert choice["text"] == expected_texts[i] + + # also check usage aggregation here + global_num_prompt_tokens = sum(len(p) for p in prompts) + global_num_completion_tokens = global_num_prompt_tokens * 2 # since we doubled the prompts + assert resp["usage"] == { + "prompt_tokens": global_num_prompt_tokens, + "total_tokens": global_num_prompt_tokens + global_num_completion_tokens, + "completion_tokens": global_num_completion_tokens, + "prompt_tokens_details": { + "cached_tokens": global_num_prompt_tokens, + }, + } + + +# ------------------------------------------- +# tests for InferenceEngineClient.generate +# -------------------------------------------- + + +@pytest.mark.parametrize("num_prompts", [1, 50, 100]) +@pytest.mark.parametrize("with_session_id", [True, False]) +@pytest.mark.parametrize("num_engines", [1, 3, 4, 8, 16]) +def test_generate_batched_routing_and_order_preservation(num_prompts, with_session_id, num_engines): + """ + See the `test_completion_batched_routing_and_order_preservation` test for more details. + Essentially `InferenceEngineClient.generate` does the same routing and aggregation as + `InferenceEngineClient.completion`. + """ + + class MockEngine: + async def generate(self, input_batch): + # input_batch["prompt_token_ids"] is a local sub-batch list of token id lists + prompt_token_ids = input_batch["prompt_token_ids"] + responses = [] + response_ids = [] + stop_reasons = [] + for ids in prompt_token_ids: + # construct a deterministic text and token output based on first id + base = ids[0] + responses.append(f"{base}{base}") + response_ids.append([base, base]) + stop_reasons.append("stop") + return { + "responses": responses, + "response_ids": response_ids, + "stop_reasons": stop_reasons, + } + + # Minimal config, do not spin up HTTP endpoint + cfg = OmegaConf.create( + { + "trainer": { + "policy": {"model": {"path": "dummy-model"}}, + }, + "generator": { + "backend": "vllm", + "enable_http_endpoint": False, + "http_endpoint_host": "127.0.0.1", + "http_endpoint_port": 0, + }, + } + ) + + engines = [MockEngine() for _ in range(num_engines)] + tokenizer = object() # not used when prompt_token_ids are provided + client = InferenceEngineClient(engines=engines, tokenizer=tokenizer, full_config=cfg) + + # Build token id prompts [[0], [1], ..., [n-1]] + prompt_token_ids = [[i] for i in range(num_prompts)] + if with_session_id: + session_ids = [random.randint(1, 100) for _ in range(num_prompts)] + else: + session_ids = None + + input_batch = { + "prompts": None, + "prompt_token_ids": prompt_token_ids, + "sampling_params": None, + "session_ids": session_ids, + } + + out = asyncio.run(client.generate(input_batch)) + + # Validate reconstruction and ordering + assert len(out["responses"]) == num_prompts + assert len(out["response_ids"]) == num_prompts + assert len(out["stop_reasons"]) == num_prompts + expected_texts = [f"{i}{i}" for i in range(num_prompts)] + for i in range(num_prompts): + assert out["responses"][i] == expected_texts[i] + assert out["response_ids"][i] == [i, i] + assert out["stop_reasons"][i] == "stop" + + +# ----------------------------- +# Test for route_prompts_to_engines function that routes prompts to inference engines +# in inference engine client. +# ------------------------------- + + +def test_route_prompts_to_engines_single_prompt_no_trajectory_random_engine(): + # Force deterministic random routing to engine index 1 + with patch("random.randint", return_value=1): + mapping = route_prompts_to_engines(num_prompts=1, num_inference_engines=4, session_ids=None) + assert mapping == {1: [0]} + + +def test_route_prompts_to_engines_batched_even_split_exact_multiple(): + # 4 prompts, 2 engines => [0,1] and [2,3] + num_prompts = 4 + num_engines = 2 + mapping = route_prompts_to_engines(num_prompts=num_prompts, num_inference_engines=num_engines, session_ids=None) + assert mapping == {0: [0, 1], 1: [2, 3]} + + +def test_route_prompts_to_engines_batched_uneven_split(): + # 5 prompts, 2 engines => ceil(5/2)=3 => [0,1,2] and [3,4] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=2, session_ids=None) + assert mapping == {0: [0, 1, 2], 1: [3, 4]} + + # 5 prompts, 3 engines => ceil(5/3)=2 => [0,1] and [2,3] and [4] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=3, session_ids=None) + assert mapping == {0: [0, 1], 1: [2, 3], 2: [4]} + + # 5 prompts, 4 engines => ceil(5/4)=2 => [0,1] and [2,3] and [4] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=4, session_ids=None) + assert mapping == {0: [0, 1], 1: [2, 3], 2: [4]} + + # 129 prompts, 4 engines => ceil(129/4)=33 => [0,1,2,...,32] and [33,34,35,...,65] and [66,67,68,...,99] and [100,101,102,...,128] + mapping = route_prompts_to_engines(num_prompts=129, num_inference_engines=4, session_ids=None) + assert mapping == {0: list(range(33)), 1: list(range(33, 66)), 2: list(range(66, 99)), 3: list(range(99, 129))} + + +def test_route_prompts_to_engines_batched_more_engines_than_prompts(): + # 2 prompts, 4 engines => size=1 => {0:[0], 1:[1]} + mapping = route_prompts_to_engines(num_prompts=2, num_inference_engines=4, session_ids=None) + assert mapping == {0: [0], 1: [1]} + + +def test_route_prompts_to_engines_with_session_ids_grouping_and_partition(): + num_engines = 4 + # Ensure same session IDs route to the same engine index + sids = ["A", "A", "B", "C", "B"] + # hash A ends in 45, B ends in 44, C ends in 69, with % 4 they become 1, 0, 1 + engine_idx = [hash_with_sha256(sid) % num_engines for sid in sids] # what we do in route_prompts_to_engines + assert engine_idx == [1, 1, 0, 1, 0] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=num_engines, session_ids=sids) + + assert mapping == {1: [0, 1, 3], 0: [2, 4]} + + +def test_route_prompts_to_engines_validation_errors(): + # num_prompts must be > 0 + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=0, num_inference_engines=1, session_ids=None) + + # num_inference_engines must be > 0 + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=1, num_inference_engines=0, session_ids=None) + + # session_ids length must match + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=["x"]) # len 1 != 2 + + # session_ids type checking + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=[1, 2.0]) # float invalid + + # No error + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=[1, 2]) + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=None) + route_prompts_to_engines(num_prompts=1, num_inference_engines=1, session_ids=None) + + +# ------------------------------------------- +# tests for InferenceEngineClient.chat_completion retry logic +# -------------------------------------------- + + +def _make_min_cfg(): + return OmegaConf.create( + { + "trainer": { + "policy": {"model": {"path": "dummy-model"}}, + }, + "generator": { + "backend": "vllm", + "enable_http_endpoint": False, + "http_endpoint_host": "127.0.0.1", + "http_endpoint_port": 0, + }, + } + ) + + +@pytest.mark.asyncio +async def test_chat_completion_retry_accumulates_and_sends_continuations(): + """ + First response aborts with tokens; second aborts with 0 tokens (ignored); + third finishes. Assert: + - Continuation requests append accumulated assistant content with correct role + - continue_final_message/add_generation_prompt flags are set + - remaining max_tokens decreases by accumulated completion tokens + - Final response accumulates content, logprobs, token_ids and recomputes usage correctly + - Each retry request is what we expect the engine to receive + """ + + class MockEngine: + def __init__(self): + self.calls = [] # capture full request payloads {"json":..., "headers":...} + # Pre-programmed partial responses + self.responses = [ + # 1) abort with 1 token "A" + { + "id": "cmpl-1", + "object": "chat.completion", + "model": "dummy-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "A"}, + "finish_reason": "abort", + "logprobs": { + "content": [ + { + "token": "token_id:11", + "logprob": -0.1, + "bytes": [84, 111], + "top_logprobs": [{"token": "token_id:11", "logprob": -0.1, "bytes": [116]}], + }, + ] + }, + "token_ids": [11], + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + # 2) abort with 0 tokens (should be ignored for accumulation) + { + "id": "cmpl-2", + "object": "chat.completion", + "model": "dummy-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": ""}, + "finish_reason": "abort", + "logprobs": {"content": []}, + "token_ids": [], + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 0, "total_tokens": 5}, + }, + # 3) finish with 1 token "B" + { + "id": "cmpl-3", + "object": "chat.completion", + "model": "dummy-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "B"}, + "finish_reason": "stop", + "logprobs": { + "content": [ + { + "token": "token_id:12", + "logprob": -0.1, + "bytes": [84, 111], + "top_logprobs": [{"token": "token_id:12", "logprob": -0.1, "bytes": [116]}], + }, + ] + }, + "token_ids": [12], + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + ] + + async def chat_completion(self, request_payload): + self.calls.append(deepcopy(request_payload)) + idx = len(self.calls) - 1 + assert idx < len(self.responses), f"Unexpected extra call {idx}" + return deepcopy(self.responses[idx]) + + engines = [MockEngine()] + cfg = _make_min_cfg() + client = InferenceEngineClient(engines=engines, tokenizer=object(), full_config=cfg) + + original = { + "json": { + "model": "dummy-model", + "messages": [{"role": "user", "content": "Hi"}], + "max_tokens": 8, + # ask for structures that the client can accumulate + "logprobs": True, + "top_logprobs": 1, + "return_tokens_as_token_ids": True, + }, + "headers": {"Content-Type": "application/json"}, + } + + out = await client.chat_completion(original) + + # Verify engine received 3 calls + assert len(engines[0].calls) == 3 + first_call = engines[0].calls[0] + second_call = engines[0].calls[1] + third_call = engines[0].calls[2] + + # First call should be identical to original json (no continuation flags) + assert first_call["json"] == original["json"] + assert first_call["headers"] == original["headers"] + assert first_call["json"].get("continue_final_message") is None + assert first_call["json"].get("add_generation_prompt") is None + assert first_call["json"]["messages"] == [{"role": "user", "content": "Hi"}] + assert first_call["json"]["max_tokens"] == 8 + + # Second/third calls should be continuation requests + for call in (second_call, third_call): + assert call["headers"] == original["headers"] + # Flags + assert call["json"].get("continue_final_message") is True + assert call["json"].get("add_generation_prompt") is False + # Accumulated assistant message appended with content "A" + assert call["json"]["messages"][-1] == {"role": "assistant", "content": "A"} + # Original user message preserved + assert call["json"]["messages"][0] == {"role": "user", "content": "Hi"} + # Remaining max_tokens reduced by 1 (we already generated one token) + assert call["json"].get("max_tokens") == 7 + # Other params preserved + assert call["json"]["model"] == "dummy-model" + assert call["json"]["logprobs"] is True + assert call["json"]["top_logprobs"] == 1 + assert call["json"]["return_tokens_as_token_ids"] is True + + # Final response should accumulate content/logprobs/token_ids and usage + choice = out["choices"][0] + assert choice["finish_reason"] == "stop" + assert choice["message"]["content"] == "AB" + assert len(choice["logprobs"]["content"]) == 2 + assert choice["logprobs"]["content"][0]["token"] == "token_id:11" + assert choice["logprobs"]["content"][1]["token"] == "token_id:12" + assert choice["token_ids"] == [11, 12] + + # usage: prompt_tokens from base (5), completion_tokens summed (2), total 7 + assert out["usage"]["prompt_tokens"] == 5 + assert out["usage"]["completion_tokens"] == 2 + assert out["usage"]["total_tokens"] == 7 + + +@pytest.mark.asyncio +async def test_chat_completion_retry_resends_original_when_no_tokens_generated_yet(): + """ + First response aborts with 0 tokens, so the next request should resend the original + payload unchanged. Second response finishes; client returns it directly. + """ + + class MockEngine: + def __init__(self): + self.calls = [] # capture full payloads + self.responses = [ + # 1) abort with 0 tokens + { + "id": "cmpl-a1", + "object": "chat.completion", + "model": "dummy-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": ""}, + "finish_reason": "abort", + "logprobs": {"content": []}, + "token_ids": [], + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, + }, + # 2) finish with tokens "XYZ" (3 tokens) + { + "id": "cmpl-a2", + "object": "chat.completion", + "model": "dummy-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "XYZ"}, + "finish_reason": "stop", + "logprobs": { + "content": [ + { + "token": "token_id:21", + "logprob": -0.1, + "bytes": [84, 111], + "top_logprobs": [{"token": "token_id:21", "logprob": -0.1, "bytes": [116]}], + }, + { + "token": "token_id:22", + "logprob": -0.1, + "bytes": [84, 111], + "top_logprobs": [{"token": "token_id:22", "logprob": -0.1, "bytes": [116]}], + }, + { + "token": "token_id:23", + "logprob": -0.1, + "bytes": [84, 111], + "top_logprobs": [{"token": "token_id:23", "logprob": -0.1, "bytes": [116]}], + }, + ] + }, + "token_ids": [21, 22, 23], + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13}, + }, + ] + + async def chat_completion(self, request_payload): + self.calls.append(deepcopy(request_payload)) + return deepcopy(self.responses[len(self.calls) - 1]) + + engines = [MockEngine()] + cfg = _make_min_cfg() + client = InferenceEngineClient(engines=engines, tokenizer=object(), full_config=cfg) + + original = { + "json": { + "model": "dummy-model", + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 16, + "logprobs": True, + "top_logprobs": 1, + }, + "headers": {"Content-Type": "application/json"}, + } + + out = await client.chat_completion(original) + + # Two calls should have been made + assert len(engines[0].calls) == 2 + first_call = engines[0].calls[0] + second_call = engines[0].calls[1] + + # After 0-token abort, the next call should resend the original unchanged + assert first_call["json"] == original["json"] + assert second_call["json"] == original["json"] + assert first_call["headers"] == original["headers"] + assert second_call["headers"] == original["headers"] + # No continuation flags should appear + assert first_call["json"].get("continue_final_message") is None + assert second_call["json"].get("continue_final_message") is None + + # Since finish_reason != abort on the second call and base_response was None, + # client should return the second response directly (no accumulation) + assert out == engines[0].responses[1] + + +# ------------------------------------------- +# tests for InferenceEngineClient.generate retry logic +# -------------------------------------------- + + +@pytest.mark.parametrize("max_tokens_key", ["max_tokens", "max_completion_tokens"]) +@pytest.mark.asyncio +async def test_generate_retry_some_gen_no_gen_finish(max_tokens_key): + """ + Test that generate() with retry logic properly accumulates tokens and adjusts subsequent requests. + + First response aborts with tokens [21, 22]; second aborts with 0 tokens (ignored); + third finishes with tokens [23, 24]. Assert: + - Continuation requests append accumulated tokens to prompt_token_ids + - remaining max_tokens decreases by accumulated tokens + - Final response accumulates all tokens and uses last stop_reason + """ + + class MockEngine: + def __init__(self): + self.calls = [] # capture InferenceEngineInput calls + # Pre-programmed responses + self.responses = [ + # 1) abort with 2 tokens + InferenceEngineOutput( + responses=["something"], # will be ignored since we decode the final output + response_ids=[[21, 22]], + stop_reasons=["abort"], + response_logprobs=[[-0.1, -0.2]], + ), + # 2) abort with 0 tokens (should be ignored) + InferenceEngineOutput( + responses=[""], + response_ids=[[]], + stop_reasons=["abort"], + response_logprobs=None, + ), + # 3) finish with 2 tokens + InferenceEngineOutput( + responses=[" something"], # will be ignored since we decode the final output + response_ids=[[23, 24]], + stop_reasons=["stop"], + response_logprobs=[[-0.3, -0.4]], + ), + ] + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + self.calls.append(deepcopy(input_batch)) + idx = len(self.calls) - 1 + assert idx < len(self.responses), f"Unexpected extra call {idx}" + return deepcopy(self.responses[idx]) + + engines = [MockEngine()] + cfg = _make_min_cfg() + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + client = InferenceEngineClient(engines=engines, tokenizer=tokenizer, full_config=cfg) + + # Original request + prompt_token_ids = [[1, 2, 3, 4, 5]] # 5 prompt tokens + sampling_params = {max_tokens_key: 10, "temperature": 0.7} + + input_batch = InferenceEngineInput( + prompt_token_ids=prompt_token_ids, + sampling_params=sampling_params, + ) + + out = await client.generate(input_batch) + + # Verify engine received 3 calls + assert len(engines[0].calls) == 3 + first_call = engines[0].calls[0] + second_call = engines[0].calls[1] + third_call = engines[0].calls[2] + + # First call should have original prompt + assert first_call["prompt_token_ids"] == [[1, 2, 3, 4, 5]] + assert first_call["sampling_params"][max_tokens_key] == 10 + assert first_call["sampling_params"]["temperature"] == 0.7 + + # Second call should have prompt + first response tokens + assert second_call["prompt_token_ids"] == [[1, 2, 3, 4, 5, 21, 22]] + assert second_call["sampling_params"][max_tokens_key] == 8 # 10 - 2 already generated + assert second_call["sampling_params"]["temperature"] == 0.7 + + # Third call should also have prompt + first response tokens (second was ignored) + assert third_call["prompt_token_ids"] == [[1, 2, 3, 4, 5, 21, 22]] + assert third_call["sampling_params"][max_tokens_key] == 8 # 10 - 2 already generated + assert third_call["sampling_params"]["temperature"] == 0.7 + + # Final response should accumulate all tokens + expected_final_response_ids = [21, 22, 23, 24] + expected_final_text_response = tokenizer.decode(expected_final_response_ids, skip_special_tokens=True) + assert out["responses"] == [expected_final_text_response] + assert out["response_ids"] == [expected_final_response_ids] + assert out["stop_reasons"] == ["stop"] + assert out["response_logprobs"] == [[-0.1, -0.2, -0.3, -0.4]] + + +@pytest.mark.asyncio +async def test_generate_retry_direct_return(): + """ + Test that if the first generate() request doesn't abort, it returns directly without retries. + """ + + class MockEngine: + def __init__(self): + self.calls = [] + # Single response that completes immediately + self.response = InferenceEngineOutput( + responses=["something"], + response_ids=[[21, 22, 23, 24]], + stop_reasons=["stop"], + response_logprobs=[[-0.1, -0.2, -0.3, -0.4]], + ) + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + self.calls.append(deepcopy(input_batch)) + return deepcopy(self.response) + + engines = [MockEngine()] + cfg = _make_min_cfg() + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + client = InferenceEngineClient(engines=engines, tokenizer=tokenizer, full_config=cfg) + + prompt_token_ids = [[1, 2, 3, 4, 5]] + sampling_params = {"max_tokens": 10} + + input_batch = InferenceEngineInput( + prompt_token_ids=prompt_token_ids, + sampling_params=sampling_params, + ) + + out = await client.generate(input_batch) + + # Verify only one call was made + assert len(engines[0].calls) == 1 + + # Verify response is returned as-is + expected_final_response_ids = [21, 22, 23, 24] + # note since we completed in one turn, we return the text response of the first turn returned by + # the underlying engine instead re-tokenizing like others + assert out["responses"] == ["something"] + assert out["response_ids"] == [expected_final_response_ids] + assert out["stop_reasons"] == ["stop"] + assert out["response_logprobs"] == [[-0.1, -0.2, -0.3, -0.4]] + + +@pytest.mark.asyncio +async def test_generate_retry_no_gen_finish(): + """ + First response aborts with 0 tokens; next finishes. + The second request should resend the original unchanged and the final output equals the second response. + """ + final_response_ids = [21, 22, 23] + + class MockEngine: + def __init__(self): + self.calls = [] + self.responses = [ + # 1) abort with 0 tokens + InferenceEngineOutput( + responses=[""], + response_ids=[[]], + stop_reasons=["abort"], + response_logprobs=[[]], + ), + # 2) finish directly + InferenceEngineOutput( + responses=["something"], # will be ignored since we decode the final output + response_ids=[final_response_ids], + stop_reasons=["stop"], + response_logprobs=[[-0.1, -0.1, -0.1]], + ), + ] + + async def generate(self, input_batch: InferenceEngineInput) -> InferenceEngineOutput: + self.calls.append(deepcopy(input_batch)) + return deepcopy(self.responses[len(self.calls) - 1]) + + engines = [MockEngine()] + cfg = _make_min_cfg() + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + client = InferenceEngineClient(engines=engines, tokenizer=tokenizer, full_config=cfg) + + original_prompt_ids = [7, 8, 9] + input_batch = InferenceEngineInput( + prompt_token_ids=[original_prompt_ids], + sampling_params={"max_tokens": 16}, + ) + + out = await client.generate(input_batch) + + # Two calls should have been made, identical inputs since first had 0 tokens + assert len(engines[0].calls) == 2 + first_call, second_call = engines[0].calls + assert first_call["prompt_token_ids"] == [original_prompt_ids] + assert second_call["prompt_token_ids"] == [original_prompt_ids] + assert first_call["sampling_params"]["max_tokens"] == 16 + assert second_call["sampling_params"]["max_tokens"] == 16 + + # Since finish_reason != abort on the second call and no accumulation occurred, + # client should return the second response directly (no aggregation) + # Besides, since we completed in one turn, we return the text response of the first turn returned by + # the underlying engine instead re-tokenizing the accumulated tokens + assert out == engines[0].responses[1] diff --git a/skyrl-train/tests/cpu/inf_engines/test_route_prompts_to_engines.py b/skyrl-train/tests/cpu/inf_engines/test_route_prompts_to_engines.py new file mode 100644 index 0000000000..bcfe361488 --- /dev/null +++ b/skyrl-train/tests/cpu/inf_engines/test_route_prompts_to_engines.py @@ -0,0 +1,86 @@ +""" +Test for route_prompts_to_engines function that routes prompts to inference engines in inference engine client. + +Run with: +uv run --isolated --extra dev pytest tests/cpu/inf_engines/test_route_prompts_to_engines.py +""" + +from unittest.mock import patch + +import pytest + +from skyrl_train.inference_engines.utils import route_prompts_to_engines, hash_with_sha256 + + +def test_single_prompt_no_trajectory_random_engine(): + # Force deterministic random routing to engine index 1 + with patch("random.randint", return_value=1): + mapping = route_prompts_to_engines(num_prompts=1, num_inference_engines=4, session_ids=None) + assert mapping == {1: [0]} + + +def test_batched_even_split_exact_multiple(): + # 4 prompts, 2 engines => [0,1] and [2,3] + num_prompts = 4 + num_engines = 2 + mapping = route_prompts_to_engines(num_prompts=num_prompts, num_inference_engines=num_engines, session_ids=None) + assert mapping == {0: [0, 1], 1: [2, 3]} + + +def test_batched_uneven_split(): + # 5 prompts, 2 engines => ceil(5/2)=3 => [0,1,2] and [3,4] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=2, session_ids=None) + assert mapping == {0: [0, 1, 2], 1: [3, 4]} + + # 5 prompts, 3 engines => ceil(5/3)=2 => [0,1] and [2,3] and [4] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=3, session_ids=None) + assert mapping == {0: [0, 1], 1: [2, 3], 2: [4]} + + # 5 prompts, 4 engines => ceil(5/4)=2 => [0,1] and [2,3] and [4] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=4, session_ids=None) + assert mapping == {0: [0, 1], 1: [2, 3], 2: [4]} + + # 129 prompts, 4 engines => ceil(129/4)=33 => [0,1,2,...,32] and [33,34,35,...,65] and [66,67,68,...,99] and [100,101,102,...,128] + mapping = route_prompts_to_engines(num_prompts=129, num_inference_engines=4, session_ids=None) + assert mapping == {0: list(range(33)), 1: list(range(33, 66)), 2: list(range(66, 99)), 3: list(range(99, 129))} + + +def test_batched_more_engines_than_prompts(): + # 2 prompts, 4 engines => size=1 => {0:[0], 1:[1]} + mapping = route_prompts_to_engines(num_prompts=2, num_inference_engines=4, session_ids=None) + assert mapping == {0: [0], 1: [1]} + + +def test_with_session_ids_grouping_and_partition(): + num_engines = 4 + # Ensure same session IDs route to the same engine index + sids = ["A", "A", "B", "C", "B"] + # hash A ends in 45, B ends in 44, C ends in 69, with % 4 they become 1, 0, 1 + engine_idx = [hash_with_sha256(sid) % num_engines for sid in sids] # what we do in route_prompts_to_engines + assert engine_idx == [1, 1, 0, 1, 0] + mapping = route_prompts_to_engines(num_prompts=5, num_inference_engines=num_engines, session_ids=sids) + + assert mapping == {1: [0, 1, 3], 0: [2, 4]} + + +def test_validation_errors(): + # num_prompts must be > 0 + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=0, num_inference_engines=1, session_ids=None) + + # num_inference_engines must be > 0 + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=1, num_inference_engines=0, session_ids=None) + + # session_ids length must match + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=["x"]) # len 1 != 2 + + # session_ids type checking + with pytest.raises(AssertionError): + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=[1, 2.0]) # float invalid + + # No error + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=[1, 2]) + route_prompts_to_engines(num_prompts=2, num_inference_engines=1, session_ids=None) + route_prompts_to_engines(num_prompts=1, num_inference_engines=1, session_ids=None) diff --git a/skyrl-train/tests/cpu/inf_engines/vllm/utils.py b/skyrl-train/tests/cpu/inf_engines/vllm/utils.py new file mode 100644 index 0000000000..3707f02169 --- /dev/null +++ b/skyrl-train/tests/cpu/inf_engines/vllm/utils.py @@ -0,0 +1,24 @@ +from skyrl_train.inference_engines.vllm.utils import pop_openai_kwargs + + +def test_pop_openai_kwargs(): + """ + Test pop_openai_kwargs with both primary and alias. + Ensure OpenAI kwargs are popped, non-OpenAI kwargs are kept. + """ + engine_kwargs = { + "enable_auto_tools": 1, + "tool_parser": "json", + "reasoning_parser": "my_parser", + "other": "keep", + } + openai_kwargs = pop_openai_kwargs(engine_kwargs) + + assert openai_kwargs == {"enable_auto_tools": True, "tool_parser": "json", "reasoning_parser": "my_parser"} + assert engine_kwargs == {"other": "keep"} + + engine_kwargs = {"enable_auto_tool_choice": 0, "tool_call_parser": "proto"} + openai_kwargs = pop_openai_kwargs(engine_kwargs) + + assert openai_kwargs == {"enable_auto_tools": False, "tool_parser": "proto"} + assert engine_kwargs == {} diff --git a/skyrl-train/tests/cpu/models/test_models.py b/skyrl-train/tests/cpu/models/test_models.py new file mode 100644 index 0000000000..4544aa42f3 --- /dev/null +++ b/skyrl-train/tests/cpu/models/test_models.py @@ -0,0 +1,67 @@ +import torch +from flash_attn.bert_padding import pad_input, unpad_input +import pytest + + +@pytest.fixture +def input_ids(): + return torch.tensor( + [[0, 0, 10, 12, 13, 14, 15, 17, 0, 0, 0], [0, 14, 6, 4, 50, 20, 18, 19, 21, 22, 23]], dtype=torch.long + ) + + +@pytest.fixture +def attention_mask(): + return torch.tensor([[0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0], [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=torch.long) + + +@pytest.fixture +def position_ids(attention_mask): + pos_ids = torch.cumsum(attention_mask, dim=1) - 1 + pos_ids.masked_fill_(attention_mask == 0, 1) + return pos_ids + + +def test_flash_attention_sequence_packing(input_ids, attention_mask, position_ids): + input_ids_packed, nnz_indices, _, _, _ = unpad_input(input_ids.unsqueeze(-1), attention_mask=attention_mask) + input_ids_packed = input_ids_packed.transpose(0, 1) + position_ids_packed, _, _, _, _ = unpad_input(position_ids.unsqueeze(-1), attention_mask=attention_mask) + position_ids_packed = position_ids_packed.transpose(0, 1) + + # (1, nnz) + expected_input_ids_packed = torch.tensor( + [[10, 12, 13, 14, 15, 17, 14, 6, 4, 50, 20, 18, 19, 21, 22, 23]], dtype=torch.long + ) + # (1, nnz) + expected_position_ids_packed = torch.tensor([[0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], dtype=torch.long) + # (nnz,) + expected_nnz_indices = torch.tensor([2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], dtype=torch.int64) + + assert torch.equal(expected_input_ids_packed, input_ids_packed) + assert torch.equal(expected_position_ids_packed, position_ids_packed) + assert torch.equal(nnz_indices, expected_nnz_indices) + + +def test_flash_attention_sequence_unpacking(input_ids, attention_mask, position_ids): + # pack + input_ids_packed, nnz_indices, _, _, _ = unpad_input(input_ids.unsqueeze(-1), attention_mask=attention_mask) + input_ids_packed = input_ids_packed.transpose(0, 1) + position_ids_packed, _, _, _, _ = unpad_input(position_ids.unsqueeze(-1), attention_mask=attention_mask) + position_ids_packed = position_ids_packed.transpose(0, 1) + + # unpack + # add padding back - postprocess logits to be compatible with original tensors + batch_size, seqlen = attention_mask.shape + # (nnz, 1) -> (batch_size, seqlen, 1) + unpacked_input_ids = pad_input( + input_ids_packed.transpose(0, 1), indices=nnz_indices, batch=batch_size, seqlen=seqlen + ) + unpacked_position_ids = pad_input( + position_ids_packed.transpose(0, 1), indices=nnz_indices, batch=batch_size, seqlen=seqlen + ) + unpacked_input_ids = unpacked_input_ids.squeeze(-1) + unpacked_position_ids = unpacked_position_ids.squeeze(-1) + + assert torch.equal(unpacked_input_ids, input_ids) + # mask out the attention mask because the padding value used can differ + assert torch.equal(unpacked_position_ids * attention_mask, position_ids * attention_mask) diff --git a/skyrl-train/tests/cpu/test_eval.py b/skyrl-train/tests/cpu/test_eval.py new file mode 100644 index 0000000000..a09f2f8283 --- /dev/null +++ b/skyrl-train/tests/cpu/test_eval.py @@ -0,0 +1,118 @@ +""" +uv run --isolated --extra dev pytest tests/cpu/test_eval.py +""" + +from unittest.mock import MagicMock + +import pytest +from omegaconf import OmegaConf + +from skyrl_train.evaluate import evaluate +from skyrl_train.generators.base import GeneratorInterface, GeneratorOutput +from tests.cpu.util import example_dummy_config + + +@pytest.fixture +def dummy_config(): + return example_dummy_config() + + +class DummyStatefulDataLoader: + def __init__(self, batches): + self._batches = batches + + def __len__(self): + return len(self._batches) + + def __iter__(self): + return iter(self._batches) + + +class DummyGenerator(GeneratorInterface): + def __init__(self, output: GeneratorOutput): + self.output = output + self.seen_inputs = [] + + async def generate(self, input_batch): + self.seen_inputs.append(input_batch) + return self.output + + +@pytest.mark.asyncio +async def test_evaluate_computes_expected_metrics(dummy_config, tmp_path): + cfg = dummy_config + cfg.generator.backend = "vllm" + cfg.generator.eval_sampling_params = OmegaConf.create( + { + "max_generate_length": 20, + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "min_p": 0.0, + "logprobs": None, + "stop": None, + } + ) + cfg.generator.eval_n_samples_per_prompt = 1 + cfg.environment = OmegaConf.create({"env_class": "gsm8k"}) + cfg.trainer.dump_eval_results = False + cfg.trainer.export_path = str(tmp_path) + + prompts_batch = [ + { + "prompt": [{"role": "user", "content": "question-1"}], + "env_class": None, + "env_extras": {"data_source": "dataset/a"}, + "uid": "uid-1", + }, + { + "prompt": [{"role": "user", "content": "question-2"}], + "env_class": "custom_env", + "env_extras": {"data_source": "dataset/b"}, + "uid": "uid-2", + }, + ] + eval_dataloader = DummyStatefulDataLoader([prompts_batch]) + + generator_output: GeneratorOutput = { + "prompt_token_ids": [[101], [102]], + "response_ids": [[201], [202]], + "rewards": [1.0, 0.0], + "loss_masks": [[1], [1]], + "stop_reasons": ["stop", "stop"], + "rollout_logprobs": None, + } + generator = DummyGenerator(generator_output) + + tokenizer = MagicMock() + tokenizer.decode.side_effect = lambda tokens: "decoded" + + metrics = await evaluate( + eval_dataloader=eval_dataloader, + generator=generator, + cfg=cfg, + global_step=5, + tokenizer=tokenizer, + ) + + expected_metrics = { + "eval/dataset_a/pass_at_1": 1.0, + "eval/dataset_a/variance_per_prompt": 0.0, # single sample per uid + "eval/dataset_a/mean_positive_reward": 1.0, + "eval/dataset_b/pass_at_1": 0.0, + "eval/dataset_b/variance_per_prompt": 0.0, # single sample per uid + "eval/dataset_b/mean_positive_reward": 0.0, + "eval/all/pass_at_1": 0.5, + "eval/all/variance_per_prompt": 0.0, # single sample per uid + "eval/all/mean_positive_reward": 1.0, + } + + for key, expected_value in expected_metrics.items(): + assert metrics[key] == pytest.approx(expected_value) + + assert len(generator.seen_inputs) == 1 + seen_batch = generator.seen_inputs[0] + assert seen_batch["prompts"] == [prompt["prompt"] for prompt in prompts_batch] + assert seen_batch["env_classes"] == ["gsm8k", "custom_env"] + assert seen_batch["env_extras"] == [prompt["env_extras"] for prompt in prompts_batch] + assert seen_batch["batch_metadata"].training_phase == "eval" diff --git a/skyrl-train/tests/cpu/test_flash_attn_stub.py b/skyrl-train/tests/cpu/test_flash_attn_stub.py new file mode 100644 index 0000000000..48e051ec85 --- /dev/null +++ b/skyrl-train/tests/cpu/test_flash_attn_stub.py @@ -0,0 +1,264 @@ +""" +Tests for the flash_attn stub package. + +The stub provides bert_padding functions (pad_input, unpad_input) that SkyRL +training code uses. This test verifies the stub implementation is correct. +""" + +import sys +import tempfile +import os + +import pytest +import torch + + +class TestFlashAttnStubCreation: + """Test that the stub creation script works.""" + + def test_create_stub_script_exists(self): + """Verify the stub creation script exists.""" + script_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "integrations", + "fleet", + "scripts", + "create_flash_attn_stub.py", + ) + assert os.path.exists(script_path), f"Script not found at {script_path}" + + def test_create_stub_in_temp_dir(self): + """Test stub creation in a temporary directory.""" + import importlib.util + + # Load the stub creation module + script_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "integrations", + "fleet", + "scripts", + "create_flash_attn_stub.py", + ) + spec = importlib.util.spec_from_file_location("create_stub", script_path) + stub_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(stub_module) + + # Create stub in temp dir by patching site.getsitepackages + with tempfile.TemporaryDirectory() as tmpdir: + import site + + original_getsitepackages = site.getsitepackages + site.getsitepackages = lambda: [tmpdir] + + try: + stub_module.create_stub() + + # Verify files were created + flash_attn_dir = os.path.join(tmpdir, "flash_attn") + assert os.path.isdir(flash_attn_dir) + assert os.path.isfile(os.path.join(flash_attn_dir, "__init__.py")) + assert os.path.isfile(os.path.join(flash_attn_dir, "bert_padding.py")) + + # Add to path and test import + sys.path.insert(0, tmpdir) + try: + # Force reimport + if "flash_attn" in sys.modules: + del sys.modules["flash_attn"] + if "flash_attn.bert_padding" in sys.modules: + del sys.modules["flash_attn.bert_padding"] + + from flash_attn.bert_padding import pad_input, unpad_input + + assert callable(pad_input) + assert callable(unpad_input) + finally: + sys.path.remove(tmpdir) + finally: + site.getsitepackages = original_getsitepackages + + +class TestBertPaddingFunctions: + """Test the bert_padding pad_input and unpad_input functions.""" + + @pytest.fixture + def bert_padding_module(self): + """Load bert_padding from stub or create inline for testing.""" + import importlib.util + + # Load the stub creation module to get bert_padding code + script_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "integrations", + "fleet", + "scripts", + "create_flash_attn_stub.py", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + import site + + original_getsitepackages = site.getsitepackages + site.getsitepackages = lambda: [tmpdir] + + try: + spec = importlib.util.spec_from_file_location("create_stub", script_path) + stub_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(stub_module) + stub_module.create_stub() + + sys.path.insert(0, tmpdir) + try: + if "flash_attn" in sys.modules: + del sys.modules["flash_attn"] + if "flash_attn.bert_padding" in sys.modules: + del sys.modules["flash_attn.bert_padding"] + + from flash_attn import bert_padding + + yield bert_padding + finally: + sys.path.remove(tmpdir) + if "flash_attn" in sys.modules: + del sys.modules["flash_attn"] + if "flash_attn.bert_padding" in sys.modules: + del sys.modules["flash_attn.bert_padding"] + finally: + site.getsitepackages = original_getsitepackages + + def test_unpad_input_basic(self, bert_padding_module): + """Test unpad_input removes padding correctly.""" + batch_size = 2 + seq_len = 4 + hidden_dim = 8 + + # Create hidden states: (batch, seq, hidden) + hidden_states = torch.randn(batch_size, seq_len, hidden_dim) + + # Attention mask: 1 for valid tokens, 0 for padding + # First sequence has 3 valid tokens, second has 2 + attention_mask = torch.tensor([[1, 1, 1, 0], [1, 1, 0, 0]]) + + unpad_input = bert_padding_module.unpad_input + hidden_unpad, indices, cu_seqlens, max_seqlen, seqlens = unpad_input(hidden_states, attention_mask) + + # Check shapes + total_valid = 3 + 2 # 5 valid tokens total + assert hidden_unpad.shape == (total_valid, hidden_dim) + assert indices.shape == (total_valid,) + assert cu_seqlens.shape == (batch_size + 1,) + assert max_seqlen == 3 # max of (3, 2) + + # Check seqlens + assert seqlens.tolist() == [3, 2] + + # Check cu_seqlens (cumulative) + assert cu_seqlens.tolist() == [0, 3, 5] + + def test_pad_input_basic(self, bert_padding_module): + """Test pad_input restores padding correctly.""" + batch_size = 2 + seq_len = 4 + hidden_dim = 8 + + # Create unpadded hidden states (5 valid tokens) + hidden_unpad = torch.randn(5, hidden_dim) + + # Indices showing where each token came from in flattened (batch*seq) space + # Batch 0: positions 0, 1, 2 (seq positions 0, 1, 2) + # Batch 1: positions 4, 5 (seq positions 0, 1 in batch 1) + indices = torch.tensor([0, 1, 2, 4, 5]) + + pad_input = bert_padding_module.pad_input + hidden_padded = pad_input(hidden_unpad, indices, batch_size, seq_len) + + # Check shape + assert hidden_padded.shape == (batch_size, seq_len, hidden_dim) + + # Check values at valid positions + assert torch.allclose(hidden_padded[0, 0], hidden_unpad[0]) + assert torch.allclose(hidden_padded[0, 1], hidden_unpad[1]) + assert torch.allclose(hidden_padded[0, 2], hidden_unpad[2]) + assert torch.allclose(hidden_padded[1, 0], hidden_unpad[3]) + assert torch.allclose(hidden_padded[1, 1], hidden_unpad[4]) + + # Check padding positions are zero + assert torch.allclose(hidden_padded[0, 3], torch.zeros(hidden_dim)) + assert torch.allclose(hidden_padded[1, 2], torch.zeros(hidden_dim)) + assert torch.allclose(hidden_padded[1, 3], torch.zeros(hidden_dim)) + + def test_roundtrip_unpad_pad(self, bert_padding_module): + """Test that unpad -> pad recovers original (at valid positions).""" + batch_size = 3 + seq_len = 5 + hidden_dim = 16 + + # Create hidden states + hidden_states = torch.randn(batch_size, seq_len, hidden_dim) + + # Variable length sequences + attention_mask = torch.tensor( + [[1, 1, 1, 1, 0], [1, 1, 0, 0, 0], [1, 1, 1, 1, 1]] # 4 tokens # 2 tokens + ) # 5 tokens + + unpad_input = bert_padding_module.unpad_input + pad_input = bert_padding_module.pad_input + + # Unpad + hidden_unpad, indices, cu_seqlens, max_seqlen, seqlens = unpad_input(hidden_states, attention_mask) + + # Pad back + hidden_repadded = pad_input(hidden_unpad, indices, batch_size, seq_len) + + # Check valid positions match original + for b in range(batch_size): + for s in range(seq_len): + if attention_mask[b, s] == 1: + assert torch.allclose(hidden_repadded[b, s], hidden_states[b, s]) + else: + # Padding positions should be zero + assert torch.allclose(hidden_repadded[b, s], torch.zeros(hidden_dim)) + + def test_gradient_flow(self, bert_padding_module): + """Test that gradients flow through unpad/pad operations.""" + batch_size = 2 + seq_len = 4 + hidden_dim = 8 + + hidden_states = torch.randn(batch_size, seq_len, hidden_dim, requires_grad=True) + attention_mask = torch.tensor([[1, 1, 1, 0], [1, 1, 0, 0]]) + + unpad_input = bert_padding_module.unpad_input + pad_input = bert_padding_module.pad_input + + # Forward + hidden_unpad, indices, cu_seqlens, max_seqlen, seqlens = unpad_input(hidden_states, attention_mask) + + # Apply some operation + hidden_unpad_transformed = hidden_unpad * 2 + + # Pad back + hidden_repadded = pad_input(hidden_unpad_transformed, indices, batch_size, seq_len) + + # Backward + loss = hidden_repadded.sum() + loss.backward() + + # Check gradients exist + assert hidden_states.grad is not None + assert hidden_states.grad.shape == hidden_states.shape + + # Gradients at valid positions should be 2 (from the *2 operation) + for b in range(batch_size): + for s in range(seq_len): + if attention_mask[b, s] == 1: + assert torch.allclose(hidden_states.grad[b, s], torch.full((hidden_dim,), 2.0)) + else: + # No gradient flow to padding positions + assert torch.allclose(hidden_states.grad[b, s], torch.zeros(hidden_dim)) diff --git a/skyrl-train/tests/cpu/test_fleet_config.py b/skyrl-train/tests/cpu/test_fleet_config.py new file mode 100644 index 0000000000..c2bfd16034 --- /dev/null +++ b/skyrl-train/tests/cpu/test_fleet_config.py @@ -0,0 +1,60 @@ +"""Tests for Fleet task configuration in SkyRL. + +These tests verify that the Hydra config schema includes fleet_task settings. +""" + +import os + +import yaml + + +class TestFleetTaskConfig: + """Tests for fleet_task config in skyrl_gym_config.""" + + def test_fleet_task_config_exists(self): + """Test that fleet_task config exists in default.yaml.""" + config_path = os.path.join( + os.path.dirname(__file__), + "../../skyrl_train/config/skyrl_gym_config/default.yaml", + ) + + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + assert "fleet_task" in config, "fleet_task section missing from config" + + def test_fleet_task_config_has_required_fields(self): + """Test that fleet_task config has all required fields.""" + config_path = os.path.join( + os.path.dirname(__file__), + "../../skyrl_train/config/skyrl_gym_config/default.yaml", + ) + + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + fleet_task = config["fleet_task"] + + # Required fields + assert "tasks_file" in fleet_task, "tasks_file field missing" + assert "api_key" in fleet_task, "api_key field missing" + assert "ttl_seconds" in fleet_task, "ttl_seconds field missing" + assert "enable_context_tools" in fleet_task, "enable_context_tools field missing" + + def test_fleet_task_config_default_values(self): + """Test that fleet_task config has correct default values.""" + config_path = os.path.join( + os.path.dirname(__file__), + "../../skyrl_train/config/skyrl_gym_config/default.yaml", + ) + + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + fleet_task = config["fleet_task"] + + # Default values + assert fleet_task["tasks_file"] is None, "tasks_file should default to None" + assert fleet_task["api_key"] is None, "api_key should default to None" + assert fleet_task["ttl_seconds"] == 900, "ttl_seconds should default to 900" + assert fleet_task["enable_context_tools"] is False, "enable_context_tools should default to False" diff --git a/skyrl-train/tests/cpu/test_fleet_env.py b/skyrl-train/tests/cpu/test_fleet_env.py new file mode 100644 index 0000000000..844358da61 --- /dev/null +++ b/skyrl-train/tests/cpu/test_fleet_env.py @@ -0,0 +1,878 @@ +"""Unit tests for Fleet task environment. + +These tests require openenv and skyrl_gym to be installed. +They are skipped in CI where these dependencies aren't available. +""" + +import importlib.util +import json +import os +from unittest.mock import MagicMock, patch + +import pytest +from omegaconf import DictConfig + + +# Check if dependencies are available using importlib +OPENENV_AVAILABLE = importlib.util.find_spec("envs") is not None +SKYRL_GYM_AVAILABLE = importlib.util.find_spec("skyrl_gym") is not None + +# Skip all tests in this module if dependencies aren't available +pytestmark = pytest.mark.skipif( + not (OPENENV_AVAILABLE and SKYRL_GYM_AVAILABLE), + reason="OpenEnv or skyrl_gym not installed", +) + +# Only import env module if dependencies are available +if OPENENV_AVAILABLE and SKYRL_GYM_AVAILABLE: + from integrations.fleet.env import ( + load_tasks_from_json, + parse_tool_call, + FleetTaskEnv, + clear_caches, + ) +else: + # Provide dummy implementations for type checking + load_tasks_from_json = None + parse_tool_call = None + FleetTaskEnv = None + clear_caches = None + + +class TestLoadTasksFromJson: + """Tests for load_tasks_from_json function.""" + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + def test_load_array_format(self, tmp_path): + """Test loading tasks from array format JSON.""" + tasks_file = tmp_path / "tasks.json" + tasks_data = [ + {"key": "task-1", "prompt": "Do task 1", "env_id": "env-a"}, + {"key": "task-2", "prompt": "Do task 2", "env_id": "env-b"}, + ] + tasks_file.write_text(json.dumps(tasks_data)) + + result = load_tasks_from_json(str(tasks_file)) + + assert "task-1" in result + assert "task-2" in result + assert result["task-1"]["prompt"] == "Do task 1" + assert result["task-2"]["env_id"] == "env-b" + + def test_load_object_format(self, tmp_path): + """Test loading tasks from object format with 'tasks' key.""" + tasks_file = tmp_path / "tasks.json" + tasks_data = { + "tasks": [ + {"task_key": "task-a", "prompt": "Prompt A"}, + {"task_key": "task-b", "prompt": "Prompt B"}, + ] + } + tasks_file.write_text(json.dumps(tasks_data)) + + result = load_tasks_from_json(str(tasks_file)) + + assert "task-a" in result + assert "task-b" in result + + def test_caching(self, tmp_path): + """Test that tasks are cached after first load.""" + tasks_file = tmp_path / "tasks.json" + tasks_data = [{"key": "task-1", "prompt": "Test"}] + tasks_file.write_text(json.dumps(tasks_data)) + + # First load + result1 = load_tasks_from_json(str(tasks_file)) + + # Modify file (but cache should be used) + tasks_file.write_text(json.dumps([{"key": "task-new", "prompt": "New"}])) + + # Second load should return cached data + result2 = load_tasks_from_json(str(tasks_file)) + + assert result1 is result2 + assert "task-1" in result2 + assert "task-new" not in result2 + + def test_file_not_found(self): + """Test error when file doesn't exist.""" + with pytest.raises(FileNotFoundError, match="Tasks file not found"): + load_tasks_from_json("/nonexistent/path/tasks.json") + + def test_invalid_format(self, tmp_path): + """Test error on invalid JSON format.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps({"invalid": "format"})) + + with pytest.raises(ValueError, match="Invalid JSON format"): + load_tasks_from_json(str(tasks_file)) + + def test_empty_tasks(self, tmp_path): + """Test error when tasks array is empty.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([])) + + with pytest.raises(ValueError, match="No tasks found"): + load_tasks_from_json(str(tasks_file)) + + def test_key_priority(self, tmp_path): + """Test that 'key' takes priority over 'task_key'.""" + tasks_file = tmp_path / "tasks.json" + tasks_data = [ + {"key": "primary-key", "task_key": "secondary-key", "prompt": "Test"}, + ] + tasks_file.write_text(json.dumps(tasks_data)) + + result = load_tasks_from_json(str(tasks_file)) + + assert "primary-key" in result + assert "secondary-key" not in result + + def test_path_expansion(self, tmp_path): + """Test that ~ is expanded in paths.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "test", "prompt": "Test"}])) + + # Create symlink in home + with patch("os.path.expanduser") as mock_expand: + mock_expand.return_value = str(tasks_file) + result = load_tasks_from_json("~/tasks.json") + assert "test" in result + + +class TestParseToolCall: + """Tests for parse_tool_call function.""" + + def test_tool_call_tag(self): + """Test parsing format.""" + action = 'I will search. {"name": "search", "arguments": {"query": "test"}}' + result = parse_tool_call(action) + + assert result is not None + assert result["name"] == "search" + assert result["arguments"] == {"query": "test"} + + def test_function_call_tag(self): + """Test parsing format.""" + action = '{"name": "compute", "arguments": {"x": 42}}' + result = parse_tool_call(action) + + assert result is not None + assert result["name"] == "compute" + assert result["arguments"] == {"x": 42} + + def test_tool_key_normalization(self): + """Test that 'tool' key is normalized to 'name'.""" + action = '{"tool": "my_tool", "params": {"a": 1}}' + result = parse_tool_call(action) + + assert result is not None + assert result["name"] == "my_tool" + assert result["arguments"] == {"a": 1} + + def test_multiline_json(self): + """Test parsing multiline JSON in tool call.""" + action = """ +{ + "name": "complex_tool", + "arguments": { + "list": [1, 2, 3], + "nested": {"key": "value"} + } +} +""" + result = parse_tool_call(action) + + assert result is not None + assert result["name"] == "complex_tool" + assert result["arguments"]["list"] == [1, 2, 3] + + def test_no_tool_call(self): + """Test that None is returned when no tool call found.""" + action = "I don't know how to help with that." + result = parse_tool_call(action) + + assert result is None + + def test_invalid_json(self): + """Test handling of invalid JSON in tags.""" + action = '{"name": invalid json}' + result = parse_tool_call(action) + + assert result is None + + def test_empty_arguments(self): + """Test tool call with no arguments.""" + action = '{"name": "get_time", "arguments": {}}' + result = parse_tool_call(action) + + assert result is not None + assert result["name"] == "get_time" + assert result["arguments"] == {} + + def test_missing_name(self): + """Test that missing name returns None.""" + action = '{"arguments": {"x": 1}}' + result = parse_tool_call(action) + + assert result is None + + +class TestFleetTaskEnvInit: + """Tests for FleetTaskEnv initialization.""" + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + def test_missing_task_key(self, tmp_path): + """Test error when task_key is not provided.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "test", "prompt": "Test"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + + with pytest.raises(ValueError, match="task_key must be provided"): + FleetTaskEnv(env_config, extras={}) + + def test_missing_tasks_file(self): + """Test error when tasks_file is not provided.""" + env_config = DictConfig({}) + + with pytest.raises(ValueError, match="tasks_file must be provided"): + FleetTaskEnv(env_config, extras={"task_key": "test"}) + + def test_task_not_found(self, tmp_path): + """Test error when task_key doesn't exist in file.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "other-task", "prompt": "Test"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + + with pytest.raises(ValueError, match="Task 'missing-task' not found"): + FleetTaskEnv(env_config, extras={"task_key": "missing-task"}) + + def test_missing_api_key(self, tmp_path): + """Test error when FLEET_API_KEY is not set.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task", "prompt": "Test"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + + with patch.dict(os.environ, {}, clear=True): + # Remove FLEET_API_KEY if present + os.environ.pop("FLEET_API_KEY", None) + with pytest.raises(ValueError, match="FLEET_API_KEY must be set"): + FleetTaskEnv(env_config, extras={"task_key": "task"}) + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-api-key"}) + def test_successful_init(self, tmp_path): + """Test successful initialization.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "my-task", "prompt": "Do something", "env_id": "test-env"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "my-task"}) + + assert env.task_key == "my-task" + assert env.task_config["prompt"] == "Do something" + assert env.api_key == "test-api-key" + assert env.max_turns == 50 # default + + @patch.dict(os.environ, {"FLEET_API_KEY": "env-key"}) + def test_api_key_from_config(self, tmp_path): + """Test that config api_key takes priority over env var.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task", "prompt": "Test"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file), "api_key": "config-key"}) + env = FleetTaskEnv(env_config, extras={"task_key": "task"}) + + assert env.api_key == "config-key" + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_max_turns_from_extras(self, tmp_path): + """Test that max_turns can be set via extras.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task", "prompt": "Test"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task", "max_turns": 10}) + + assert env.max_turns == 10 + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_task_not_found_shows_available_keys(self, tmp_path): + """Test that error message shows available task keys.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text( + json.dumps( + [ + {"key": "task-1", "prompt": "Test1"}, + {"key": "task-2", "prompt": "Test2"}, + ] + ) + ) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + + with pytest.raises(ValueError) as exc_info: + FleetTaskEnv(env_config, extras={"task_key": "nonexistent"}) + + assert "Available keys" in str(exc_info.value) + + +class TestFleetTaskEnvMetrics: + """Tests for FleetTaskEnv metrics methods.""" + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_get_metrics(self, tmp_path): + """Test get_metrics returns correct structure.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "github"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.turns = 5 + + metrics = env.get_metrics() + + assert metrics["task_key"] == "task-1" + assert metrics["env_key"] == "github" + assert metrics["turns"] == 5 + + def test_aggregate_metrics_empty(self): + """Test aggregate_metrics with empty list.""" + result = FleetTaskEnv.aggregate_metrics([]) + assert result == {} + + def test_aggregate_metrics(self): + """Test aggregate_metrics calculates correctly.""" + metrics = [ + {"task_key": "t1", "env_key": "github", "turns": 10}, + {"task_key": "t2", "env_key": "github", "turns": 20}, + {"task_key": "t3", "env_key": "booking", "turns": 15}, + ] + + result = FleetTaskEnv.aggregate_metrics(metrics) + + assert result["avg_turns"] == 15.0 + assert result["total_episodes"] == 3 + assert result["env_distribution"]["github"] == 2 + assert result["env_distribution"]["booking"] == 1 + + +class TestFleetTaskEnvInitMethod: + """Tests for FleetTaskEnv.init() method.""" + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_init_creates_openenv_task_env(self, mock_openenv_class, tmp_path): + """Test that init() creates the OpenEnv FleetTaskEnv.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Search for flights", "env_id": "booking"}])) + + # Mock OpenEnv's FleetTaskEnv + mock_openenv_env = MagicMock() + + # Mock reset_async as coroutine (OpenEnv now fetches tools in __init__) + async def mock_reset_async(): + return {"prompt": "Search for flights", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + + # Call init + chat_history, metadata = env.init([]) + + # Verify OpenEnv FleetTaskEnv was created + mock_openenv_class.assert_called_once() + + # Verify return values + assert len(chat_history) == 2 # system + user message + assert chat_history[0]["role"] == "system" + assert chat_history[1]["role"] == "user" + assert "Search for flights" in chat_history[1]["content"] + assert metadata["task_key"] == "task-1" + assert metadata["env_key"] == "booking" + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_init_includes_tools_info(self, mock_openenv_class, tmp_path): + """Test that init() includes tool information in prompt.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Do something", "env_id": "test"}])) + + # Mock OpenEnv with tools (tools are fetched in __init__, returned in reset_async) + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return { + "prompt": "Do something", + "tools": [{"name": "search"}, {"name": "click"}], + "step": 0, + } + + mock_openenv_env.reset_async = mock_reset_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + + chat_history, metadata = env.init([]) + + # Verify tools are mentioned in system prompt + assert "search" in chat_history[0]["content"] + assert "click" in chat_history[0]["content"] + assert "" in chat_history[0]["content"] + assert "" in chat_history[0]["content"] + + # Verify tools in metadata + assert len(metadata["tools"]) == 2 + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_init_openenv_creation_fails(self, mock_openenv_class, tmp_path): + """Test error when OpenEnv FleetTaskEnv creation fails. + + Since OpenEnv now creates the Fleet env and fetches tools in __init__, + this error covers both env creation and tool fetching failures. + """ + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + # Mock OpenEnv to raise error (simulates fleet.make or list_tools failure) + mock_openenv_class.side_effect = Exception("Failed to create environment") + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + + with pytest.raises(RuntimeError, match="Failed to create OpenEnv FleetTaskEnv"): + env.init([]) + + +class TestFleetTaskEnvStep: + """Tests for FleetTaskEnv.step() method.""" + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_step_with_tool_call(self, mock_openenv_class, tmp_path): + """Test step() with a valid tool call.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + # Setup mocks + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + + # Mock step_async to return a coroutine + async def mock_step_async(action): + return ({"observation": "Search results: found 5 items"}, 0.0, False, {}) + + mock_openenv_env.step_async = mock_step_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + # Step with tool call + action = '{"name": "search", "arguments": {"q": "test"}}' + result = env.step(action) + + # Verify result + assert result.done is False + assert len(result.observations) == 1 + assert "Search results" in result.observations[0]["content"] + assert result.metadata["tool_call"]["name"] == "search" + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_step_with_done_signal(self, mock_openenv_class, tmp_path): + """Test step() when agent signals done.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + + async def mock_step_async(action): + return ({}, 1.0, True, {}) + + mock_openenv_env.step_async = mock_step_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + # Step with done signal + result = env.step("Task completed! ") + + assert result.done is True + assert result.reward == 1.0 + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_step_max_turns(self, mock_openenv_class, tmp_path): + """Test step() when max turns reached.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1", "max_turns": 2}) + env.init([]) + + # First step + env.step("action 1") + assert env.turns == 1 + + # Second step (max turns) + result = env.step("action 2") + + assert result.done is True + assert result.metadata["done_reason"] == "max_turns" + assert result.observations == [] + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_step_no_tool_call(self, mock_openenv_class, tmp_path): + """Test step() when no tool call is found.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + # Step without tool call + result = env.step("I'm not sure what to do") + + assert "No tool call found" in result.observations[0]["content"] + + +class TestStepErrorDetection: + """Tests for error detection in tool results. + + Regression tests for the "Action executed." bug where tool results + containing {"error": null} were silently dropped. + """ + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_dict_result_with_error_null_not_dropped(self, mock_openenv_class, tmp_path): + """Test that tool results with 'error': null are NOT dropped. + + MCP tools like fostgres return {"results": [...], "error": null}. + The old code checked `"error" in dict` (key existence) which is True + even when the value is null, causing results to be silently dropped + and replaced with "Action executed." + """ + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "fostgres"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [{"name": "execute_postgresql"}], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + + async def mock_step_async(action): + # Simulate fostgres MCP response: results with error: null + return ( + { + "observation": { + "results": [{"table_name": "orders"}], + "columns": ["table_name"], + "row_count": 1, + "error": None, + } + }, + 0.0, + False, + {}, + ) + + mock_openenv_env.step_async = mock_step_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + action = '{"name": "execute_postgresql", "arguments": {"sql": "SELECT table_name FROM information_schema.tables"}}' + result = env.step(action) + + obs_content = result.observations[0]["content"] + assert "Action executed." not in obs_content + assert "Tool result:" in obs_content + assert "orders" in obs_content + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_dict_result_with_actual_error_detected(self, mock_openenv_class, tmp_path): + """Test that tool results with actual error values are still detected.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "fostgres"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [{"name": "execute_postgresql"}], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + + async def mock_step_async(action): + # Simulate fostgres MCP response: SQL error + return ( + {"observation": {"results": [], "columns": [], "row_count": 0, "error": 'column "foo" does not exist'}}, + 0.0, + False, + {}, + ) + + mock_openenv_env.step_async = mock_step_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + action = '{"name": "execute_postgresql", "arguments": {"sql": "SELECT foo FROM bar"}}' + result = env.step(action) + + obs_content = result.observations[0]["content"] + assert "Error:" in obs_content + assert "column" in obs_content + assert env.tool_errors == 1 + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_dict_result_without_error_key(self, mock_openenv_class, tmp_path): + """Test that dicts without an 'error' key work fine.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + + async def mock_step_async(action): + return ({"observation": {"data": [1, 2, 3]}}, 0.0, False, {}) + + mock_openenv_env.step_async = mock_step_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + action = '{"name": "get_data", "arguments": {}}' + result = env.step(action) + + obs_content = result.observations[0]["content"] + assert "Tool result:" in obs_content + assert "Action executed." not in obs_content + + +class TestFleetTaskEnvClose: + """Tests for FleetTaskEnv.close() method.""" + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_close(self, mock_openenv_class, tmp_path): + """Test close() calls openenv_task_env.close().""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + env.close() + + mock_openenv_env.close.assert_called_once() + assert env.openenv_task_env is None + + @patch("integrations.fleet.env.OpenEnvFleetTaskEnv") + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_close_handles_error(self, mock_openenv_class, tmp_path, capsys): + """Test close() handles errors gracefully.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + mock_openenv_env = MagicMock() + + async def mock_reset_async(): + return {"prompt": "Test", "tools": [], "step": 0} + + mock_openenv_env.reset_async = mock_reset_async + mock_openenv_env.close.side_effect = Exception("Connection error") + mock_openenv_class.return_value = mock_openenv_env + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + env.init([]) + + # Should not raise + env.close() + + # Should print warning + captured = capsys.readouterr() + assert "Warning" in captured.out + assert env.openenv_task_env is None + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_close_without_init(self, tmp_path): + """Test close() when init was never called.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + + # Should not raise + env.close() + assert env.openenv_task_env is None + + +class TestEnableContextTools: + """Tests for enable_context_tools configuration.""" + + def setup_method(self): + """Clear cache before each test.""" + clear_caches() + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_enable_context_tools_from_env_config(self, tmp_path): + """Test that enable_context_tools is read from env_config, not extras. + + This is a regression test for the bug where enable_context_tools + was read from extras (per-sample data) instead of env_config (Hydra config). + """ + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + # Set enable_context_tools in env_config (where Hydra puts it) + env_config = DictConfig( + { + "tasks_file": str(tasks_file), + "enable_context_tools": True, + } + ) + + # extras should NOT have enable_context_tools + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + + assert env.enable_context_tools is True + assert env.context_manager is not None + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_enable_context_tools_default_false(self, tmp_path): + """Test that enable_context_tools defaults to False.""" + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1"}) + + assert env.enable_context_tools is False + assert env.context_manager is None + + @patch.dict(os.environ, {"FLEET_API_KEY": "test-key"}) + def test_extras_enable_context_tools_ignored(self, tmp_path): + """Test that enable_context_tools in extras is ignored. + + This ensures the config is read from the right place (env_config). + """ + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "task-1", "prompt": "Test", "env_id": "test"}])) + + # enable_context_tools in extras should be IGNORED + env_config = DictConfig({"tasks_file": str(tasks_file)}) + env = FleetTaskEnv(env_config, extras={"task_key": "task-1", "enable_context_tools": True}) + + # Should be False because env_config doesn't have it + assert env.enable_context_tools is False + + +class TestClearCaches: + """Tests for clear_caches function.""" + + def test_clear_caches(self, tmp_path): + """Test clear_caches resets all global state.""" + # Create some cached state + tasks_file = tmp_path / "tasks.json" + tasks_file.write_text(json.dumps([{"key": "test", "prompt": "Test"}])) + load_tasks_from_json(str(tasks_file)) + + # Clear caches + clear_caches() + + # Loading again should read from file (we'd see different results if file changed) + tasks_file.write_text(json.dumps([{"key": "new-test", "prompt": "New Test"}])) + result = load_tasks_from_json(str(tasks_file)) + assert "new-test" in result + assert "test" not in result diff --git a/skyrl-train/tests/cpu/test_generator_postprocess.py b/skyrl-train/tests/cpu/test_generator_postprocess.py new file mode 100644 index 0000000000..416777ad26 --- /dev/null +++ b/skyrl-train/tests/cpu/test_generator_postprocess.py @@ -0,0 +1,161 @@ +""" +Test for token-level rewards support in RayPPOTrainer.postprocess_generator_output method. + +Run with: +uv run --isolated --extra dev pytest tests/cpu/test_generator_postprocess.py +""" + +from unittest.mock import MagicMock + +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.generators.base import GeneratorOutput +from skyrl_train.config.utils import get_default_config +from omegaconf import OmegaConf + + +class DummyDataset: + def __len__(self): + return 1 + + def __getitem__(self, idx): + return "dummy" + + def collate_fn(self, batch): + return batch + + +def create_config(batch_size): + default_config = get_default_config() + OmegaConf.update( + default_config, + "trainer", + { + "train_batch_size": batch_size, + "eval_batch_size": batch_size, + "resume_mode": "none", + "seed": 42, + "epochs": 1, + }, + ) + OmegaConf.update( + default_config, + "generator", + { + "n_samples_per_prompt": 1, + }, + ) + return default_config + + +def test_response_level_rewards(): + """Test postprocess_generator_output with response-level rewards (List[float]).""" + + # Test length=1 + config = create_config(1) + trainer = RayPPOTrainer( + cfg=config, + tracker=None, + tokenizer=None, + train_dataset=DummyDataset(), + eval_dataset=None, + inference_engine_client=None, + generator=MagicMock(), + ) + + generator_output: GeneratorOutput = { + "prompt_token_ids": [[1, 2]], + "response_ids": [[3, 4, 5]], + "rewards": [1.0], # Response-level reward + "loss_masks": [[1, 1, 1]], + "stop_reasons": ["stop"], + "rollout_metrics": None, + } + + result = trainer.postprocess_generator_output(generator_output, ["uid1"]) + + # Verify conversion to per-token rewards + assert result["rewards"] == [[0.0, 0.0, 1.0]] + + # Test length=2 + config = create_config(2) + trainer = RayPPOTrainer( + cfg=config, + tracker=None, + tokenizer=None, + train_dataset=DummyDataset(), + eval_dataset=None, + inference_engine_client=None, + generator=MagicMock(), + ) + + generator_output: GeneratorOutput = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[5, 6], [7, 8, 9]], + "rewards": [1.0, 0.5], # Response-level rewards + "loss_masks": [[1, 1], [1, 1, 1]], + "stop_reasons": ["stop", "stop"], + "rollout_metrics": None, + } + + result = trainer.postprocess_generator_output(generator_output, ["uid1", "uid2"]) + + # Verify conversion to per-token rewards + assert result["rewards"] == [[0.0, 1.0], [0.0, 0.0, 0.5]] + + +def test_token_level_rewards(): + """Test postprocess_generator_output with token-level rewards (List[List[float]]).""" + + # Test length=1 + config = create_config(1) + trainer = RayPPOTrainer( + cfg=config, + tracker=None, + tokenizer=None, + train_dataset=DummyDataset(), + eval_dataset=None, + inference_engine_client=None, + generator=MagicMock(), + ) + + per_token_rewards = [[0.1, 0.2, 0.3]] + generator_output: GeneratorOutput = { + "prompt_token_ids": [[1, 2]], + "response_ids": [[3, 4, 5]], + "rewards": per_token_rewards, # Token-level rewards + "loss_masks": [[1, 1, 1]], + "stop_reasons": ["stop"], + "rollout_metrics": None, + } + + result = trainer.postprocess_generator_output(generator_output, ["uid1"]) + + # Verify token-level rewards are unchanged + assert result["rewards"] == per_token_rewards + + # Test length=2 + config = create_config(2) + trainer = RayPPOTrainer( + cfg=config, + tracker=None, + tokenizer=None, + train_dataset=DummyDataset(), + eval_dataset=None, + inference_engine_client=None, + generator=MagicMock(), + ) + + per_token_rewards = [[0.1, 0.3], [0.2, 0.1, 0.1]] + generator_output: GeneratorOutput = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[5, 6], [7, 8, 9]], + "rewards": per_token_rewards, # Token-level rewards + "loss_masks": [[1, 1], [1, 1, 1]], + "stop_reasons": ["stop", "stop"], + "rollout_metrics": None, + } + + result = trainer.postprocess_generator_output(generator_output, ["uid1", "uid2"]) + + # Verify token-level rewards are unchanged + assert result["rewards"] == per_token_rewards diff --git a/skyrl-train/tests/cpu/test_s3_checkpoints.py b/skyrl-train/tests/cpu/test_s3_checkpoints.py new file mode 100644 index 0000000000..e6f30ea218 --- /dev/null +++ b/skyrl-train/tests/cpu/test_s3_checkpoints.py @@ -0,0 +1,361 @@ +""" +Tests for S3 checkpoint management. + +uv run --isolated --extra dev pytest tests/cpu/test_s3_checkpoints.py -v +""" + +import os +import sys +import tempfile +from unittest.mock import patch, MagicMock + +from integrations.fleet.s3_checkpoints import ( + cleanup_old_local_checkpoints, + download_checkpoint_from_s3, + wrap_trainer_with_s3_upload, + S3CheckpointUploader, +) + + +# ============================================================================ +# cleanup_old_local_checkpoints +# ============================================================================ + + +def setup_checkpoint_dirs(tmpdir, steps): + """Create fake global_step_N directories.""" + for step in steps: + os.makedirs(os.path.join(tmpdir, f"global_step_{step}")) + + +def test_cleanup_old_local_checkpoints_keeps_n_most_recent(): + with tempfile.TemporaryDirectory() as tmpdir: + setup_checkpoint_dirs(tmpdir, [5, 10, 15, 20]) + + cleanup_old_local_checkpoints(tmpdir, keep_n=2) + + remaining = sorted(os.listdir(tmpdir)) + assert remaining == ["global_step_15", "global_step_20"] + + +def test_cleanup_old_local_checkpoints_keeps_all_when_fewer_than_n(): + with tempfile.TemporaryDirectory() as tmpdir: + setup_checkpoint_dirs(tmpdir, [10, 20]) + + cleanup_old_local_checkpoints(tmpdir, keep_n=5) + + remaining = sorted(os.listdir(tmpdir)) + assert remaining == ["global_step_10", "global_step_20"] + + +def test_cleanup_old_local_checkpoints_noop_on_empty_dir(): + with tempfile.TemporaryDirectory() as tmpdir: + cleanup_old_local_checkpoints(tmpdir, keep_n=2) + assert os.listdir(tmpdir) == [] + + +def test_cleanup_old_local_checkpoints_noop_on_missing_dir(): + # Should not raise + cleanup_old_local_checkpoints("/nonexistent/path/abc123", keep_n=2) + + +def test_cleanup_old_local_checkpoints_ignores_non_checkpoint_dirs(): + with tempfile.TemporaryDirectory() as tmpdir: + setup_checkpoint_dirs(tmpdir, [10, 20, 30]) + os.makedirs(os.path.join(tmpdir, "some_other_dir")) + + cleanup_old_local_checkpoints(tmpdir, keep_n=1) + + remaining = sorted(os.listdir(tmpdir)) + assert "some_other_dir" in remaining + assert "global_step_30" in remaining + assert "global_step_10" not in remaining + assert "global_step_20" not in remaining + + +# ============================================================================ +# download_checkpoint_from_s3 +# ============================================================================ + + +def test_download_skips_when_no_aws_credentials(): + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": ""}, clear=False): + result = download_checkpoint_from_s3( + ckpt_path=tmpdir, + run_name="test_run", + ) + assert result is False + + +def test_download_skips_when_local_checkpoint_exists(): + with tempfile.TemporaryDirectory() as tmpdir: + # Create existing checkpoint + os.makedirs(os.path.join(tmpdir, "global_step_20")) + with open(os.path.join(tmpdir, "latest_ckpt_global_step.txt"), "w") as f: + f.write("20") + + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret"}, clear=False): + result = download_checkpoint_from_s3( + ckpt_path=tmpdir, + run_name="test_run", + ) + assert result is False + + +def test_download_finds_latest_checkpoint(): + """Verify download picks the highest global_step from S3.""" + mock_boto3 = MagicMock() + mock_s3 = MagicMock() + mock_boto3.client.return_value = mock_s3 + + mock_paginator = MagicMock() + mock_s3.get_paginator.return_value = mock_paginator + + def paginate_side_effect(**kwargs): + if "Delimiter" in kwargs: + return [ + { + "CommonPrefixes": [ + {"Prefix": "fleet-task-grpo/Qwen3-32B/test_run/global_step_10/"}, + {"Prefix": "fleet-task-grpo/Qwen3-32B/test_run/global_step_20/"}, + {"Prefix": "fleet-task-grpo/Qwen3-32B/test_run/global_step_5/"}, + ] + } + ] + else: + return [ + { + "Contents": [ + { + "Key": "fleet-task-grpo/Qwen3-32B/test_run/global_step_20/policy/model.safetensors", + "Size": 1000000, + }, + { + "Key": "fleet-task-grpo/Qwen3-32B/test_run/global_step_20/trainer_state.pt", + "Size": 500, + }, + ] + } + ] + + mock_paginator.paginate.side_effect = paginate_side_effect + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(sys.modules, {"boto3": mock_boto3, "botocore": MagicMock(), "botocore.config": MagicMock()}): + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret"}, clear=False): + result = download_checkpoint_from_s3( + ckpt_path=tmpdir, + run_name="test_run", + ) + + assert result is True + + latest_file = os.path.join(tmpdir, "latest_ckpt_global_step.txt") + assert os.path.exists(latest_file) + with open(latest_file) as f: + assert f.read().strip() == "20" + + assert mock_s3.download_file.call_count == 2 + assert os.path.isdir(os.path.join(tmpdir, "global_step_20")) + + +def test_download_returns_false_when_no_checkpoints_in_s3(): + mock_boto3 = MagicMock() + mock_s3 = MagicMock() + mock_boto3.client.return_value = mock_s3 + + mock_paginator = MagicMock() + mock_s3.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [{"CommonPrefixes": []}] + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(sys.modules, {"boto3": mock_boto3, "botocore": MagicMock(), "botocore.config": MagicMock()}): + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret"}, clear=False): + result = download_checkpoint_from_s3( + ckpt_path=tmpdir, + run_name="test_run", + ) + + assert result is False + + +def test_download_uses_correct_s3_prefix(): + """Verify S3 prefix is constructed from project_name/model_name/run_name.""" + mock_boto3 = MagicMock() + mock_s3 = MagicMock() + mock_boto3.client.return_value = mock_s3 + + mock_paginator = MagicMock() + mock_s3.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [{"CommonPrefixes": []}] + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(sys.modules, {"boto3": mock_boto3, "botocore": MagicMock(), "botocore.config": MagicMock()}): + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret"}, clear=False): + download_checkpoint_from_s3( + ckpt_path=tmpdir, + run_name="fleet_tool_use_32b_d7167c1c", + project_name="fleet-task-grpo", + model_name="Qwen3-32B", + ) + + paginate_call = mock_paginator.paginate.call_args + assert paginate_call.kwargs["Prefix"] == "fleet-task-grpo/Qwen3-32B/fleet_tool_use_32b_d7167c1c/" + + +def test_download_uses_custom_bucket_from_env(): + mock_boto3 = MagicMock() + mock_s3 = MagicMock() + mock_boto3.client.return_value = mock_s3 + + mock_paginator = MagicMock() + mock_s3.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [{"CommonPrefixes": []}] + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(sys.modules, {"boto3": mock_boto3, "botocore": MagicMock(), "botocore.config": MagicMock()}): + with patch.dict( + os.environ, + {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret", "S3_CHECKPOINT_BUCKET": "my-bucket"}, + clear=False, + ): + download_checkpoint_from_s3( + ckpt_path=tmpdir, + run_name="test_run", + ) + + paginate_call = mock_paginator.paginate.call_args + assert paginate_call.kwargs["Bucket"] == "my-bucket" + + +# ============================================================================ +# wrap_trainer_with_s3_upload +# ============================================================================ + + +def _make_mock_trainer(ckpt_path, run_name="test_run", project_name="skyrl", model_path="Qwen/Qwen3-32B"): + trainer = MagicMock() + trainer.cfg.trainer.ckpt_path = ckpt_path + trainer.cfg.trainer.run_name = run_name + trainer.cfg.trainer.project_name = project_name + trainer.cfg.trainer.policy.model.path = model_path + trainer.global_step = 10 + trainer.save_checkpoints = MagicMock() + return trainer + + +def test_wrap_trainer_replaces_save_checkpoints(): + with tempfile.TemporaryDirectory() as tmpdir: + trainer = _make_mock_trainer(tmpdir) + original_save = trainer.save_checkpoints + + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": ""}, clear=False): + wrapped = wrap_trainer_with_s3_upload(trainer) + + assert wrapped.save_checkpoints is not original_save + + +def test_wrap_trainer_save_calls_cleanup_before_save(): + """Verify cleanup happens BEFORE save, not after.""" + with tempfile.TemporaryDirectory() as tmpdir: + trainer = _make_mock_trainer(tmpdir) + call_order = [] + + trainer.save_checkpoints = lambda: call_order.append("save") + + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": ""}, clear=False): + with patch( + "integrations.fleet.s3_checkpoints.cleanup_old_local_checkpoints", + side_effect=lambda *a, **kw: call_order.append("cleanup"), + ): + wrapped = wrap_trainer_with_s3_upload(trainer) + wrapped.save_checkpoints() + + assert call_order == ["cleanup", "save"] + + +def test_wrap_trainer_s3_upload_after_save(): + """When AWS creds are set, verify S3 upload is queued after save.""" + with tempfile.TemporaryDirectory() as tmpdir: + trainer = _make_mock_trainer(tmpdir) + trainer.global_step = 42 + + def fake_save(): + os.makedirs(os.path.join(tmpdir, "global_step_42")) + + trainer.save_checkpoints = fake_save + + with patch.dict( + os.environ, + {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret"}, + clear=False, + ): + with patch("integrations.fleet.s3_checkpoints.cleanup_old_local_checkpoints"): + wrapped = wrap_trainer_with_s3_upload(trainer) + wrapped.save_checkpoints() + + assert wrapped._s3_uploader is not None + + +def test_wrap_trainer_no_s3_without_credentials(): + with tempfile.TemporaryDirectory() as tmpdir: + trainer = _make_mock_trainer(tmpdir) + + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": ""}, clear=False): + wrapped = wrap_trainer_with_s3_upload(trainer) + + assert wrapped._s3_uploader is None + + +def test_wrap_trainer_keep_local_controls_cleanup_count(): + """With S3 enabled, keep_n=1; without S3, keep_n=2.""" + with tempfile.TemporaryDirectory() as tmpdir: + trainer = _make_mock_trainer(tmpdir) + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "key", "AWS_SECRET_ACCESS_KEY": "secret"}, clear=False): + with patch("integrations.fleet.s3_checkpoints.cleanup_old_local_checkpoints") as mock_cleanup: + wrapped = wrap_trainer_with_s3_upload(trainer) + wrapped.save_checkpoints() + mock_cleanup.assert_called_once_with(tmpdir, keep_n=1) + + with tempfile.TemporaryDirectory() as tmpdir: + trainer = _make_mock_trainer(tmpdir) + with patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": ""}, clear=False): + with patch("integrations.fleet.s3_checkpoints.cleanup_old_local_checkpoints") as mock_cleanup: + wrapped = wrap_trainer_with_s3_upload(trainer) + wrapped.save_checkpoints() + mock_cleanup.assert_called_once_with(tmpdir, keep_n=2) + + +# ============================================================================ +# S3CheckpointUploader.keep_local +# ============================================================================ + + +def test_uploader_deletes_local_after_upload(): + """After successful S3 upload, local checkpoint dir should be deleted to free disk.""" + mock_boto3 = MagicMock() + mock_s3 = MagicMock() + mock_boto3.client.return_value = mock_s3 + + with tempfile.TemporaryDirectory() as tmpdir: + ckpt_dir = os.path.join(tmpdir, "global_step_10") + os.makedirs(ckpt_dir) + with open(os.path.join(ckpt_dir, "model.pt"), "w") as f: + f.write("fake") + + with patch.dict( + sys.modules, + { + "boto3": mock_boto3, + "botocore": MagicMock(), + "botocore.config": MagicMock(), + "boto3.s3": MagicMock(), + "boto3.s3.transfer": MagicMock(), + }, + ): + uploader = S3CheckpointUploader(bucket="test-bucket", prefix="test/prefix") + result = uploader._upload_sync(ckpt_dir) + + assert result is True + assert not os.path.exists(ckpt_dir), "Checkpoint dir should be deleted after S3 upload" diff --git a/skyrl-train/tests/cpu/test_train_batch.py b/skyrl-train/tests/cpu/test_train_batch.py new file mode 100644 index 0000000000..d3c255d7cc --- /dev/null +++ b/skyrl-train/tests/cpu/test_train_batch.py @@ -0,0 +1,253 @@ +import pytest +import torch +from skyrl_train.training_batch import TensorBatch +import pickle +import ray +import numpy as np + + +def test_train_batch_initialization(): + # Test basic initialization + batch_size = 3 + seq_len = 4 + sequences = torch.randn(batch_size, seq_len) + attention_mask = torch.ones(batch_size, seq_len) + loss_mask = torch.ones(batch_size, seq_len) + response_mask = torch.ones(batch_size, seq_len) + + data = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + "loss_mask": loss_mask, + "response_mask": response_mask, + } + ) + assert isinstance(data, TensorBatch) + assert data.batch_size == batch_size + assert torch.equal(data["sequences"], sequences) + assert torch.equal(data["attention_mask"], attention_mask) + + +def test_train_batch_validation(): + # Test validation of batch sizes + batch_size = 3 + seq_len = 4 + sequences = torch.randn(batch_size, seq_len) + attention_mask = torch.ones(batch_size - 1, seq_len) # Different size + + with pytest.raises(ValueError, match="Batch size mismatch"): + batch = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + } + ) + TensorBatch(batch=batch, metadata={}) + + +def test_train_batch_chunk(): + batch_size = 4 + seq_len = 3 + sequences = torch.randn(batch_size, seq_len) + attention_mask = torch.ones(batch_size, seq_len) + data = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + } + ) + + chunks = data.chunk(2) + assert len(chunks) == 2 + assert chunks[0].batch_size == 2 + assert chunks[1].batch_size == 2 + assert chunks[0]["sequences"].shape == (2, seq_len) + assert chunks[1]["sequences"].shape == (2, seq_len) + + +def test_train_batch_slice(): + batch_size = 4 + seq_len = 3 + sequences = torch.randn(batch_size, seq_len) + attention_mask = torch.ones(batch_size, seq_len) + data = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + } + ) + + sliced = data.slice(1, 3) + assert len(sliced) == 2 + assert sliced["sequences"].shape == (2, seq_len) + assert sliced["attention_mask"].shape == (2, seq_len) + + +def test_train_batch_to_dtype(): + batch_size = 3 + seq_len = 4 + sequences = torch.randn(batch_size, seq_len) + attention_mask = None + data = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + } + ) + + data.to(dtype=torch.float16) + assert data["sequences"].dtype == torch.float16 + assert data["attention_mask"] is None + + +def test_train_batch_select(): + batch_size = 3 + seq_len = 4 + sequences = torch.randn(batch_size, seq_len) + attention_mask = torch.ones(batch_size, seq_len) + loss_mask = torch.ones(batch_size, seq_len) + metadata = {"info": "test", "extra": "data"} + + data = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + "loss_mask": loss_mask, + } + ) + data.metadata = metadata + + selected = data.select(["sequences", "attention_mask"], ["info"]) + assert "sequences" in selected + assert "attention_mask" in selected + assert "loss_mask" not in selected + assert "info" in selected.metadata + assert "extra" not in selected.metadata + + +def test_train_batch_cat(): + batch_size = 3 + seq_len = 4 + sequences1 = torch.randn(batch_size, seq_len) + attention_mask1 = torch.ones(batch_size, seq_len) + data1 = TensorBatch( + { + "sequences": sequences1, + "attention_mask": attention_mask1, + } + ) + sequences2 = torch.randn(batch_size, seq_len) + attention_mask2 = torch.ones(batch_size, seq_len) + data2 = TensorBatch( + { + "sequences": sequences2, + "attention_mask": attention_mask2, + } + ) + + concatenated = data1.cat([data1, data2]) + assert len(concatenated) == 2 * batch_size + assert concatenated["sequences"].shape == (2 * batch_size, seq_len) + assert concatenated["attention_mask"].shape == (2 * batch_size, seq_len) + + +def test_train_batch_pickle(): + # Test pickle serialization + batch_size = 3 + seq_len = 4 + sequences = torch.randn(batch_size, seq_len) + attention_mask = torch.ones(batch_size, seq_len) + data = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + } + ) + metadata = {"info": "test"} + data.metadata = metadata + + # Serialize + pickled = pickle.dumps(data) + + # Deserialize + unpickled = pickle.loads(pickled) + + # Verify all components are preserved + assert len(unpickled) == len(data) + assert all(torch.equal(unpickled[k], data[k]) for k in data.keys()) + assert unpickled.metadata == data.metadata + + +def test_train_batch_setitem(): + batch_size = 3 + seq_len = 4 + sequences = torch.randn(batch_size, seq_len) + attention_mask = torch.ones(batch_size, seq_len) + data = TensorBatch( + { + "sequences": sequences, + "attention_mask": attention_mask, + } + ) + + # Test setting tensor + new_sequences = torch.randn(batch_size, seq_len) + data["sequences"] = new_sequences + assert torch.equal(data["sequences"], new_sequences) + + # Test invalid tensor shape + with pytest.raises(ValueError, match="Batch size mismatch"): + data["sequences"] = torch.randn(batch_size + 1, seq_len) + + # Test invalid types + # 1. string + with pytest.raises(ValueError, match="must be a tensor"): + data["sequences"] = "invalid" + # 2. numpy array + with pytest.raises(ValueError, match="must be a tensor"): + data["sequences"] = np.zeros((batch_size, seq_len)) + + +def test_train_batch_ray_serialization(): + data = TensorBatch( + **{"a": torch.tensor([1.2, 2.4, 3.6, 4.8]), "b": torch.tensor([4, 5, 6, 7])}, + ) + data.metadata = {"hello": "world"} + + def _task(inp: TensorBatch): + assert inp == data + + _inp_ray = ray.put(data) + ray.remote(_task).remote(_inp_ray) + + +def test_train_batch_repeat(): + batch = {"a": torch.tensor([1, 2, 3]), "b": torch.tensor([4, 5, 6])} + data = TensorBatch(**batch) + data.metadata = {"d": 1, "e": "test"} + repeated = data.repeat(2) + assert len(repeated) == 6 + assert torch.equal(repeated["a"], torch.tensor([1, 2, 3, 1, 2, 3])) + assert torch.equal(repeated["b"], torch.tensor([4, 5, 6, 4, 5, 6])) + assert repeated.metadata == {"d": 1, "e": "test"} + + +def test_train_batch_repeat_interleave(): + batch = {"a": torch.tensor([1, 2, 3]), "b": torch.tensor([4, 5, 6])} + data = TensorBatch(**batch) + data.metadata = {"c": "test"} + repeated = data.repeat_interleave(2) + assert len(repeated) == 6 + assert torch.equal(repeated["a"], torch.tensor([1, 1, 2, 2, 3, 3])) + assert torch.equal(repeated["b"], torch.tensor([4, 4, 5, 5, 6, 6])) + assert repeated.metadata == {"c": "test"} + + +def test_train_batch_get_item(): + batch = {"a": torch.tensor([1, 2, 3, 4]), "b": torch.tensor([4, 5, 6, 7])} + data = TensorBatch(**batch) + data.metadata = {"c": "test"} + new_data = data[:2] + assert torch.equal(new_data["a"], torch.tensor([1, 2])) + assert torch.equal(new_data["b"], torch.tensor([4, 5])) diff --git a/skyrl-train/tests/cpu/test_trainer.py b/skyrl-train/tests/cpu/test_trainer.py new file mode 100644 index 0000000000..a704278e67 --- /dev/null +++ b/skyrl-train/tests/cpu/test_trainer.py @@ -0,0 +1,613 @@ +""" +uv run --isolated --extra dev pytest tests/cpu/test_trainer.py +""" + +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +import torch +from jaxtyping import Float, Integer +from pytest import approx +from skyrl_train.config.utils import get_default_config +from skyrl_train.distributed.dispatch import MeshRank +from skyrl_train.trainer import RayPPOTrainer +from skyrl_train.training_batch import TrainingInputBatch +from skyrl_train.utils.utils import validate_batch_sizes +from skyrl_train.workers.worker import CriticWorkerBase, PolicyWorkerBase +from skyrl_train.workers.worker_utils import BatchIterator + +from tests.cpu.util import example_dummy_config + + +@pytest.fixture +def dummy_config(): + return example_dummy_config() + + +class DummyDataset: + def __len__(self): + return 1 + + def __getitem__(self, idx): + return "dummy" + + def collate_fn(self, batch): + return batch + + +@pytest.fixture +def dummy_tokenizer(): + mock_tokenizer = MagicMock() + mock_tokenizer.pad_token_id = 0 + mock_tokenizer.eos_token_id = 2 + + # encode("abc") -> [97, 98, 99] + mock_tokenizer.encode.side_effect = lambda x: [ord(c) for c in x] + + # tokenizer("abc") -> {"input_ids": [...], "attention_mask": [...]} + def fake_tokenizer_call(text, **kwargs): + ids = [ord(c) for c in text] + return { + "input_ids": ids, + "attention_mask": [1] * len(ids), + } + + mock_tokenizer.side_effect = fake_tokenizer_call + + return mock_tokenizer + + +@pytest.fixture +def dummy_generator(): + return MagicMock() + + +def _get_test_data(trainer: RayPPOTrainer): + trainer.critic_model = MagicMock() # pretend we're using a critic + + batch_size = 2 + total_seq_len = 5 + action_len = 3 + + # Create test data + ret_sequences: Float[torch.Tensor, "batch_size total_seq_len"] = torch.randint(0, 1000, (batch_size, total_seq_len)) + ret_attention_masks: Float[torch.Tensor, "batch_size total_seq_len"] = torch.ones((batch_size, total_seq_len)) + ret_loss_masks: Integer[torch.Tensor, "batch_size total_seq_len"] = torch.stack( + [torch.tensor([1, 1, 0, 0, 0], dtype=torch.int32), torch.tensor([1, 1, 1, 0, 0], dtype=torch.int32)], dim=0 + ) + base_log_probs: Float[torch.Tensor, "batch_size total_seq_len"] = torch.log( + torch.tensor([[0.1, 0.2, 0.3, 0.2, 0.2], [0.25, 0.25, 0.25, 0.15, 0.10]]) + ) + action_log_probs: Float[torch.Tensor, "batch_size total_seq_len"] = torch.log( + torch.tensor([[0.1, 0.3, 0.2, 0.2, 0.2], [0.3, 0.3, 0.2, 0.1, 0.1]]) + ) + action_masks: Integer[torch.Tensor, "batch_size total_seq_len"] = torch.stack( + [torch.tensor([1, 1, 1, 0, 0], dtype=torch.int32), torch.tensor([1, 1, 1, 1, 1], dtype=torch.int32)], dim=0 + ) + actual_response_lengths: Float[torch.Tensor, "batch_size"] = action_masks.sum(dim=-1).to(float) + rewards_all: Float[torch.Tensor, "batch_size total_seq_len"] = torch.stack( + [torch.tensor([0.0, 1.0, 0.0, 0.0, 0.0]), torch.tensor([0.0, 0.0, 1.0, 0.0, 0.0])], dim=0 + ) + values: Float[torch.Tensor, "batch_size action_len"] = torch.randn(batch_size, action_len) + uids: np.ndarray[str] = np.array(["0", "0"]) + + # Run method + data = TrainingInputBatch( + { + "sequences": ret_sequences, + "attention_mask": ret_attention_masks, + "loss_mask": ret_loss_masks, + "base_action_log_probs": base_log_probs, + "action_log_probs": action_log_probs, + "response_mask": action_masks, + "rewards": rewards_all, + "values": values, + }, + ) + data.metadata = { + "uids": uids, + "response_length": action_len, + "avg_response_length": actual_response_lengths.mean().item(), + } + data = trainer.apply_reward_kl_penalty(data) + + return data + + +def test_calculate_kl_create_experience_batched(dummy_config): + trainer = RayPPOTrainer( + cfg=dummy_config, + tracker=None, + tokenizer=None, + train_dataset=DummyDataset(), + eval_dataset=DummyDataset(), + inference_engine_client=None, + generator=dummy_generator, + ) + data = _get_test_data(trainer) + # Assertions + metrics = data.metadata["metrics"] + assert metrics["avg_kl_max"] == approx(0.3143, abs=1e-4) + # Note; the raw KL mean is 0.054, but then the masked mean is different. + assert metrics["avg_kl"] == approx(0.1249, abs=1e-4) + + +@patch("skyrl_train.utils.ppo_utils.compute_advantages_and_returns", new_callable=MagicMock) +def test_calc_advantages_and_returns(mock_compute_adv_and_ret, dummy_config): + trainer = RayPPOTrainer( + cfg=dummy_config, + tracker=None, + tokenizer=None, + train_dataset=DummyDataset(), + eval_dataset=DummyDataset(), + inference_engine_client=None, + generator=dummy_generator, + ) + data = _get_test_data(trainer) + + # Mocked return values + mock_advantages = torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5], [0.6, 0.7, 0.8, 0.9, 1.0]]) + mock_returns = torch.tensor([[0.6, 0.7, 0.8, 0.9, 1.0], [1.1, 1.2, 1.3, 1.4, 1.5]]) + + # Set up mocks + mock_compute_adv_and_ret.return_value = (mock_advantages, mock_returns) + + # Run the method + data = trainer.compute_advantages_and_returns(data) + metrics = data.metadata["metrics"] + + # Assertions + assert torch.allclose(data["advantages"], mock_advantages) + assert torch.allclose(data["returns"], mock_returns) + assert isinstance(metrics, dict) + assert "avg_final_rewards" in metrics + assert "avg_response_length" in metrics + assert "avg_advantages_abs" in metrics + assert metrics["avg_advantages"] == approx( + torch.masked_select(mock_advantages, data["response_mask"].bool()).mean().item(), rel=1e-5 + ) + + +def test_normalize_mini_batch_size(): + """Test the _normalize_mini_batch_size method initializes micro batch tracking. + + Workers don't need to know mini batch sizes per GPU. + They receive batches from the trainer and split them into micro batches. + _normalize_mini_batch_size only initializes micro batch tracking for gradient scaling. + + # TODO: Update naming once Megatron is updated to not be aware of mini batch sizes. + """ + + # Create minimal worker instances for testing + class TestPolicyWorker(PolicyWorkerBase): + def init_model(self, *args, **kwargs): + pass + + def offload_to_cpu(self, pin_memory=True, non_blocking=True): + pass + + def backload_to_gpu(self, non_blocking=True): + pass + + def _forward_micro_batch(self, micro_batch): + pass + + class TestCriticWorker(CriticWorkerBase): + def init_model(self, *args, **kwargs): + pass + + def offload_to_cpu(self, pin_memory=True, non_blocking=True): + pass + + def backload_to_gpu(self, non_blocking=True): + pass + + def _forward_micro_batch(self, micro_batch): + pass + + def create_policy_worker_with_config(dp_size): + """Helper to create policy worker with specific config.""" + cfg = get_default_config() + cfg.trainer.algorithm.policy_loss_type = "regular" + + worker = TestPolicyWorker( + cfg=cfg, + world_size=dp_size, + rank=0, + local_rank=0, + master_addr="localhost", + master_port=12345, + sequence_parallel_size=1, + ) + + # Mock mesh_rank + worker.mesh_rank = MeshRank(dp=0, sp=0, tp=0, pp=0, world_size=dp_size, dp_size=dp_size, pp_size=1) + + return worker + + def create_critic_worker_with_config(dp_size): + """Helper to create critic worker with specific config.""" + cfg = get_default_config() + + worker = TestCriticWorker( + cfg=cfg, + world_size=dp_size, + rank=0, + local_rank=0, + master_addr="localhost", + master_port=12345, + sequence_parallel_size=1, + ) + + # Mock mesh_rank + worker.mesh_rank = MeshRank(dp=0, sp=0, tp=0, pp=0, world_size=dp_size, dp_size=dp_size, pp_size=1) + + return worker + + # Test Case 1: PolicyWorker initializes _micro_batches_accumulated + policy_worker = create_policy_worker_with_config(dp_size=4) + policy_worker._normalize_mini_batch_size() + + assert hasattr(policy_worker, "_micro_batches_accumulated") + assert policy_worker._micro_batches_accumulated == 0 + + # Test Case 2: CriticWorker initializes _micro_batches_accumulated + critic_worker = create_critic_worker_with_config(dp_size=4) + critic_worker._normalize_mini_batch_size() + + assert hasattr(critic_worker, "_micro_batches_accumulated") + assert critic_worker._micro_batches_accumulated == 0 + + # Test Case 3: Single GPU (dp_size=1) for PolicyWorker + policy_worker = create_policy_worker_with_config(dp_size=1) + policy_worker._normalize_mini_batch_size() + + assert hasattr(policy_worker, "_micro_batches_accumulated") + assert policy_worker._micro_batches_accumulated == 0 + + # Test Case 4: Error case - mesh_rank not initialized + policy_worker_no_mesh = create_policy_worker_with_config(dp_size=4) + policy_worker_no_mesh.mesh_rank = None + + with pytest.raises(RuntimeError, match="mesh_rank must be initialized"): + policy_worker_no_mesh._normalize_mini_batch_size() + + +def test_validate_batch_sizes(): + """Test the validate_batch_sizes function with various configurations to trigger all error cases.""" + + def create_test_config( + train_batch_size=128, + policy_mini_batch_size=16, + critic_mini_batch_size=8, + micro_train_batch_size_per_gpu=2, + micro_forward_batch_size_per_gpu=4, + n_samples_per_prompt=2, + policy_num_nodes=1, + policy_num_gpus_per_node=4, + critic_num_nodes=1, + critic_num_gpus_per_node=4, + policy_sequence_parallel_size=1, + critic_sequence_parallel_size=1, + critic_model_path=None, + ): + """Helper to create config for validation testing.""" + cfg = get_default_config() + cfg.trainer.train_batch_size = train_batch_size + cfg.trainer.policy_mini_batch_size = policy_mini_batch_size + cfg.trainer.critic_mini_batch_size = critic_mini_batch_size + cfg.trainer.micro_train_batch_size_per_gpu = micro_train_batch_size_per_gpu + cfg.trainer.micro_forward_batch_size_per_gpu = micro_forward_batch_size_per_gpu + cfg.trainer.placement.policy_num_nodes = policy_num_nodes + cfg.trainer.placement.policy_num_gpus_per_node = policy_num_gpus_per_node + cfg.trainer.placement.critic_num_nodes = critic_num_nodes + cfg.trainer.placement.critic_num_gpus_per_node = critic_num_gpus_per_node + cfg.trainer.policy.sequence_parallel_size = policy_sequence_parallel_size + cfg.trainer.critic.model.path = critic_model_path + cfg.trainer.critic.sequence_parallel_size = critic_sequence_parallel_size + cfg.trainer.algorithm.use_kl_loss = False + cfg.trainer.algorithm.use_kl_in_reward = False + cfg.generator.n_samples_per_prompt = n_samples_per_prompt + return cfg + + # Test Case 1: Valid configuration + cfg = create_test_config() + validate_batch_sizes(cfg) # Should not raise any exceptions + + # Test Case 2: Error case - train_batch_size < policy_mini_batch_size + cfg = create_test_config(train_batch_size=8, policy_mini_batch_size=16) + with pytest.raises(AssertionError): + validate_batch_sizes(cfg) + + # Test Case 3: Error case - train_batch_size < critic_mini_batch_size + cfg = create_test_config(train_batch_size=4, critic_mini_batch_size=8) + with pytest.raises(AssertionError): + validate_batch_sizes(cfg) + + # Test Case 4: Error case - policy_mini_batch_size = 0 + cfg = create_test_config(policy_mini_batch_size=0) + with pytest.raises(AssertionError, match="policy_mini_batch_size must be greater than 0"): + validate_batch_sizes(cfg) + + # Test Case 5: Error case - critic_mini_batch_size = 0 + cfg = create_test_config(critic_mini_batch_size=0, critic_model_path="test") + with pytest.raises(AssertionError, match="critic_mini_batch_size must be greater than 0"): + validate_batch_sizes(cfg) + + # Test Case 6: Error case - micro_train_batch_size_per_gpu = 0 + cfg = create_test_config(micro_train_batch_size_per_gpu=0) + with pytest.raises(AssertionError, match="micro_train_batch_size_per_gpu must be greater than 0"): + validate_batch_sizes(cfg) + + # Test Case 7: Error case - micro_forward_batch_size_per_gpu = 0 + cfg = create_test_config(micro_forward_batch_size_per_gpu=0) + with pytest.raises(AssertionError, match="micro_forward_batch_size_per_gpu must be greater than 0"): + validate_batch_sizes(cfg) + + # Test Case 8: Error case - train_batch_size not divisible by (policy_mini_batch_size * policy_dp_size) + cfg = create_test_config(train_batch_size=100, policy_mini_batch_size=16, policy_num_gpus_per_node=4) + # Should fail because train_batch_size is not evenly divisible by policy batch requirements + with pytest.raises(AssertionError, match="train_batch_size .* should be divisible by policy_mini_batch_size"): + validate_batch_sizes(cfg) + + # Test Case 9: Error case - train_batch_size not divisible by (critic_mini_batch_size * critic_dp_size) + cfg = create_test_config( + train_batch_size=100, + policy_mini_batch_size=5, + critic_mini_batch_size=16, + critic_num_gpus_per_node=4, + critic_model_path="test", + ) + # Should fail because train_batch_size is not evenly divisible by critic batch requirements + with pytest.raises(AssertionError, match="train_batch_size .* should be divisible by critic_mini_batch_size"): + validate_batch_sizes(cfg) + + # Test Case 10: Error case - policy_mini_batch_size_per_gpu not divisible by micro_train_batch_size_per_gpu + cfg = create_test_config( + policy_mini_batch_size=8, n_samples_per_prompt=1, policy_num_gpus_per_node=1, micro_train_batch_size_per_gpu=3 + ) + # Should fail because policy mini batch per GPU is not evenly divisible by micro batch size + with pytest.raises( + AssertionError, + match="normalized policy_mini_batch_size_per_gpu .* should be divisible by micro_train_batch_size_per_gpu", + ): + validate_batch_sizes(cfg) + + # Test Case 11: Error case - critic_mini_batch_size_per_gpu not divisible by micro_train_batch_size_per_gpu + cfg = create_test_config( + train_batch_size=144, + policy_mini_batch_size=12, # Policy validation passes + critic_mini_batch_size=8, # Critic micro batch divisibility fails + n_samples_per_prompt=1, + critic_num_gpus_per_node=1, + micro_train_batch_size_per_gpu=3, + critic_model_path="test", + ) + # Should fail because critic mini batch per GPU is not evenly divisible by micro batch size + with pytest.raises( + AssertionError, + match="normalized critic_mini_batch_size_per_gpu .* should be divisible by micro_train_batch_size_per_gpu", + ): + validate_batch_sizes(cfg) + + # Test Case 12: Valid configuration with sequence parallelism + cfg = create_test_config( + policy_sequence_parallel_size=2, + critic_sequence_parallel_size=2, + policy_num_gpus_per_node=8, + critic_num_gpus_per_node=8, + ) + validate_batch_sizes(cfg) # Should not raise any exceptions + + # Test Case 13: Valid configuration - train_batch_size not divisible by (critic_mini_batch_size * critic_dp_size), but critic model path is None + cfg = create_test_config( + train_batch_size=100, + policy_mini_batch_size=5, + critic_mini_batch_size=16, + critic_num_gpus_per_node=4, + critic_model_path=None, + ) + validate_batch_sizes(cfg) + + # Test Case 14: Valid configuration - critic_mini_batch_size is invalid but critic model is not specified + cfg = create_test_config(critic_mini_batch_size=0, critic_model_path=None) + validate_batch_sizes(cfg) + + # Test Case 15: Error case - train_batch_size_per_gpu not divisible by policy_mini_batch_size_per_gpu + cfg = create_test_config( + train_batch_size=10, + policy_mini_batch_size=5, + policy_num_gpus_per_node=2, + micro_train_batch_size_per_gpu=1, + n_samples_per_prompt=1, + ) + with pytest.raises( + AssertionError, match="policy_train_batch_size_per_gpu .* should be divisible by policy_mini_batch_size_per_gpu" + ): + validate_batch_sizes(cfg) + + # Test Case 16: Error case - train_batch_size_per_gpu not divisible by critic_mini_batch_size_per_gpu + cfg = create_test_config( + train_batch_size=10, + policy_mini_batch_size=10, + policy_num_gpus_per_node=1, + critic_mini_batch_size=5, + critic_num_gpus_per_node=2, + micro_train_batch_size_per_gpu=1, + n_samples_per_prompt=1, + critic_model_path="test", + ) + with pytest.raises( + AssertionError, match="critic_train_batch_size_per_gpu .* should be divisible by critic_mini_batch_size_per_gpu" + ): + validate_batch_sizes(cfg) + + +def test_forward_backward_batch_calculations(): + """Test the key batch calculations and control flow in forward_backward methods. + + FSDP workers use the forward_backward + optim_step pattern: + - forward_backward handles micro-batching internally and accumulates gradients + - optim_step scales gradients by 1/num_accumulated and takes optimizer step + """ + + # Create test configuration + cfg = get_default_config() + cfg.trainer.micro_train_batch_size_per_gpu = 2 + cfg.trainer.update_epochs_per_batch = 1 + cfg.trainer.algorithm.policy_loss_type = "regular" + cfg.generator.sampling_params.temperature = 1.0 + + # Create dummy databatch with known size + batch_size = 12 # This will create 6 micro batches with micro_train_batch_size_per_gpu=2 + response_length = 4 # number of actions + dummy_databatch = TrainingInputBatch( + { + "sequences": torch.randint(0, 100, (batch_size, 10)), # dummy token sequences + "attention_mask": torch.ones(batch_size, 10), + "action_log_probs": torch.randn(batch_size, response_length), + "base_action_log_probs": torch.randn(batch_size, response_length), + "values": torch.randn(batch_size, response_length), + "returns": torch.randn(batch_size, response_length), + "advantages": torch.randn(batch_size, response_length), + "loss_mask": torch.ones(batch_size, response_length), + "response_mask": torch.ones(batch_size, response_length), + "rollout_logprobs": None, + }, + ) + dummy_databatch.metadata = {"global_step": 0, "response_length": response_length} + + # Helper function to create worker with minimal setup + def create_test_worker(worker_class): + worker = worker_class( + cfg=cfg, + world_size=1, + rank=0, + local_rank=0, + master_addr="localhost", + master_port=12345, + sequence_parallel_size=1, + ) + + # Mock dependencies + worker.strategy = MagicMock() + worker.strategy.is_rank_0.return_value = False # Disable progress bars + worker.strategy.all_reduce.return_value = {"loss": 0.5, "lr": 1e-4} + + # Always set model for all worker types + worker.model = MagicMock() + + return worker + + # Test PolicyWorkerBase + policy_worker = create_test_worker(PolicyWorkerBase) + + # Initialize _micro_batches_accumulated (normally done in _normalize_mini_batch_size) + policy_worker._micro_batches_accumulated = 0 + + # Mock _forward_backward_micro to track calls + policy_forward_backward_micro_calls = [] + + def mock_policy_forward_backward_micro(experience): + policy_forward_backward_micro_calls.append(experience) + return {"policy_loss": 0.5, "ppo_clip_ratio": 0.1, "policy_entropy": 2.0, "response_length": response_length} + + policy_worker._forward_backward_micro = mock_policy_forward_backward_micro + policy_worker.record_memory = False + + # Calculate expected values + dataloader = BatchIterator( + dummy_databatch, sample_batch_size=cfg.trainer.micro_train_batch_size_per_gpu, drop_last=False + ) + expected_micro_batches = len(dataloader) # Should be 6 + + # Run forward_backward + with (patch("torch.distributed.barrier"),): + result = policy_worker.forward_backward(dummy_databatch) + + # Verify Policy Worker Results + assert ( + len(policy_forward_backward_micro_calls) == expected_micro_batches + ), f"PolicyWorker: Expected {expected_micro_batches} _forward_backward_micro calls, got {len(policy_forward_backward_micro_calls)}" + + # Verify _micro_batches_accumulated is set correctly + assert policy_worker._micro_batches_accumulated == expected_micro_batches + + # Verify result structure + assert isinstance(result, dict) + assert "policy_loss" in result + + # Test CriticWorkerBase with same pattern + critic_worker = create_test_worker(CriticWorkerBase) + + # Initialize _micro_batches_accumulated (normally done in _normalize_mini_batch_size) + critic_worker._micro_batches_accumulated = 0 + + # Mock _forward_backward_micro for critic + critic_forward_backward_micro_calls = [] + + def mock_critic_forward_backward_micro(experience): + critic_forward_backward_micro_calls.append(experience) + return {"critic_loss": 0.3, "values_mean": 1.0} + + critic_worker._forward_backward_micro = mock_critic_forward_backward_micro + + # Run forward_backward for critic + with (patch("torch.distributed.barrier"),): + result = critic_worker.forward_backward(dummy_databatch) + + # Verify Critic Worker Results + assert ( + len(critic_forward_backward_micro_calls) == expected_micro_batches + ), f"CriticWorker: Expected {expected_micro_batches} _forward_backward_micro calls, got {len(critic_forward_backward_micro_calls)}" + + # Verify _micro_batches_accumulated is set correctly + assert critic_worker._micro_batches_accumulated == expected_micro_batches + + # Verify result structure for critic + assert isinstance(result, dict) + assert "critic_loss" in result + + +def test_validate_batch_sizes_lcm_dp_requirement(): + """Ensure train_batch_size is >= lcm(policy_dp, ref_dp) when ref is used; else >= policy_dp.""" + + def create_config(train_batch_size, policy_dp, ref_dp, include_ref=True): + cfg = get_default_config() + cfg.trainer.train_batch_size = train_batch_size + cfg.trainer.policy_mini_batch_size = train_batch_size + cfg.trainer.critic_mini_batch_size = 1 + cfg.trainer.micro_train_batch_size_per_gpu = 1 + cfg.trainer.micro_forward_batch_size_per_gpu = 1 + cfg.trainer.placement.policy_num_nodes = 1 + cfg.trainer.placement.policy_num_gpus_per_node = policy_dp + cfg.trainer.placement.ref_num_nodes = 1 + cfg.trainer.placement.ref_num_gpus_per_node = ref_dp if include_ref else 1 + cfg.trainer.placement.critic_num_nodes = 1 + cfg.trainer.placement.critic_num_gpus_per_node = 1 + cfg.trainer.policy.sequence_parallel_size = 1 + cfg.trainer.ref.sequence_parallel_size = 1 + cfg.trainer.critic.model.path = None + cfg.trainer.critic.sequence_parallel_size = 1 + cfg.trainer.algorithm.use_kl_loss = include_ref + cfg.trainer.algorithm.use_kl_in_reward = False + cfg.trainer.algorithm.policy_loss_type = "regular" + cfg.generator.n_samples_per_prompt = 1 + return cfg + + # Fail: lcm(2, 3) = 6, but train_batch_size = 5 when ref is used + cfg = create_config(train_batch_size=5, policy_dp=2, ref_dp=3, include_ref=True) + with pytest.raises( + AssertionError, + match=r"least common multiple of the data parallel sizes", + ): + validate_batch_sizes(cfg) + + # Pass: train_batch_size equals lcm(2, 3) = 6 when ref is used + cfg = create_config(train_batch_size=6, policy_dp=2, ref_dp=3, include_ref=True) + validate_batch_sizes(cfg) + + # Pass: ref disabled -> requirement reduces to policy_dp. With policy_dp=2, tbs=2 is valid. + cfg = create_config(train_batch_size=2, policy_dp=2, ref_dp=3, include_ref=False) + validate_batch_sizes(cfg) diff --git a/skyrl-train/tests/cpu/test_trainer_utils.py b/skyrl-train/tests/cpu/test_trainer_utils.py new file mode 100644 index 0000000000..0cab245052 --- /dev/null +++ b/skyrl-train/tests/cpu/test_trainer_utils.py @@ -0,0 +1,1370 @@ +""" +uv run --isolated --extra dev pytest tests/cpu/test_trainer_utils.py +""" + +from skyrl_train.utils.trainer_utils import ( + run_on_each_node, + cleanup_old_checkpoints, + validate_consistency_for_latest_checkpoint, + sanitize_data_source, + calculate_per_dataset_metrics, + dump_per_dataset_eval_results, + handle_dynamic_sampling, + handle_replace_sampling, + handle_filter_sampling, + filter_generator_output, + zero_variance_filter, + validate_generator_output, + build_dataloader, + HybridEnvSampler, +) +from skyrl_train.generators.base import GeneratorInput, GeneratorOutput +from typing import Union +import ray +import os +import tempfile +import pytest +import re +import torch + +from unittest.mock import Mock, patch, mock_open +import json +from tests.cpu.util import example_dummy_config + +BasicType = Union[int, float, str, bool, type(None)] + + +@pytest.fixture +def dummy_config(): + return example_dummy_config() + + +def test_run_on_node_local_rank_0(): + def fn(x): + return x + 1 + + all_nodes = [node for node in ray.nodes() if node.get("CPU", 0) > 0] + # repeat the node ids 4 times to test that the function is called only once per node + node_ids = [all_nodes[i]["NodeID"] for i in range(len(all_nodes))] * 4 + ret = run_on_each_node(node_ids, fn, 1) + assert ret == [2] * len(all_nodes) + + +def setup_mock_ckpts(tmpdir, checkpoint_steps): + """ + Sets up dummy checkpoint directories. + """ + # Create dummy checkpoint directories + for step in checkpoint_steps: + os.makedirs(os.path.join(tmpdir, f"global_step_{step}")) + return + + +def test_cleanup_old_checkpoints(): + """ + Verify that _cleanup_old_checkpoints correctly removes old checkpoints + while keeping the most recent ones. + """ + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 2, 10, 11] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + # 2. Execute + cleanup_old_checkpoints(tmpdir, max_checkpoints=2) + + # 3. Verify + remaining_dirs = sorted(os.listdir(tmpdir)) + expected_remaining = ["global_step_10", "global_step_11"] + + assert len(remaining_dirs) == 2, "Incorrect number of checkpoints remaining" + assert remaining_dirs == expected_remaining, "Did not keep the correct (most recent) checkpoints" + + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 2, 10, 11] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + # 2. Execute + cleanup_old_checkpoints(tmpdir, max_checkpoints=0) + + # 3. Verify + remaining_dirs = sorted(os.listdir(tmpdir)) + + assert len(remaining_dirs) == 0, "Cleanup should have removed all checkpoints" + + # Test cleanup with `current_global_step` less than the highest global step in the folder + # This means that the folder contains checkpoints from a previous run. + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 2, 10, 11] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + # 2. Execute + cleanup_old_checkpoints(tmpdir, max_checkpoints=4) + + remaining_dirs = sorted(os.listdir(tmpdir)) + assert len(remaining_dirs) == 4, "Cleanup should not have removed any checkpoints" + + +def test_cleanup_does_not_run_when_not_needed(): + """ + Verify that cleanup does not remove any checkpoints if the total number + is less than or equal to max_checkpoints. + """ + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 2, 3, 4] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + # 2. Execute + cleanup_old_checkpoints(tmpdir, max_checkpoints=5) + + # 3. Verify + remaining_dirs = sorted(os.listdir(tmpdir)) + assert len(remaining_dirs) == 4, "Cleanup should not have removed any checkpoints" + + +def test_cleanup_with_negative_max_checkpoints(): + """ + Verify that cleanup is disabled when max_checkpoints is -1 + """ + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 2, 3, 4, 5] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + # 2. Execute + cleanup_old_checkpoints(tmpdir, max_checkpoints=-1) + + # 3. Verify + remaining_dirs = sorted(os.listdir(tmpdir)) + assert len(remaining_dirs) == 5, "Cleanup should be disabled when max_checkpoints is -1" + + +def test_validate_consistency_for_latest_checkpoint(): + """ + Verify that `validate_consistency_for_latest_checkpoint` correctly validates the checkpoint folder. + """ + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 2, 3, 4, 5] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + latest_ckpt_file = os.path.join(tmpdir, "latest_ckpt_global_step.txt") + with open(latest_ckpt_file, "w") as f: + f.write("5") + + latest_ckpt_path = os.path.join(tmpdir, "global_step_5") + ckpt_iteration = 5 + + # 2. Execute + validate_consistency_for_latest_checkpoint( + tmpdir, ckpt_iteration, latest_ckpt_path, latest_ckpt_file, save_interval=1 + ) + + +def test_validate_consistency_for_latest_checkpoint_with_inconsistent_folder(): + """ + Verify that `validate_consistency_for_latest_checkpoint` correctly validates the checkpoint folder. + """ + # Example 1: `latest_ckpt_global_step.txt` points to a lower global step than the highest global step in the folder + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 2, 3, 4, 5] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + # change the latest checkpoint file to point to a lower global step + latest_ckpt_file = os.path.join(tmpdir, "latest_ckpt_global_step.txt") + with open(latest_ckpt_file, "w") as f: + f.write("3") + + latest_ckpt_path = os.path.join(tmpdir, "global_step_3") + ckpt_iteration = 3 + save_interval = 1 + + # 2. Execute + with pytest.raises(ValueError, match="Inconsistent checkpoint folder"): + validate_consistency_for_latest_checkpoint( + tmpdir, ckpt_iteration, latest_ckpt_path, latest_ckpt_file, save_interval=save_interval + ) + + # Example 2: `latest_ckpt_global_step.txt` points to a lower global step but it's within the save interval + with tempfile.TemporaryDirectory() as tmpdir: + # 1. Setup + checkpoint_steps = [1, 3, 5] + setup_mock_ckpts(tmpdir, checkpoint_steps=checkpoint_steps) + + # change the latest checkpoint file to point to a lower global step + latest_ckpt_file = os.path.join(tmpdir, "latest_ckpt_global_step.txt") + with open(latest_ckpt_file, "w") as f: + f.write("3") + + save_interval = 2 + latest_ckpt_path = os.path.join(tmpdir, "global_step_3") + ckpt_iteration = 3 + + # 2. Execute + validate_consistency_for_latest_checkpoint( + tmpdir, ckpt_iteration, latest_ckpt_path, latest_ckpt_file, save_interval=save_interval + ) + + +def test_sanitize_data_source_none(): + """Test sanitize_data_source with None input.""" + result = sanitize_data_source(None) + assert result == "unknown" + + +def test_sanitize_data_source_slash_replacement(): + """Test sanitize_data_source replaces slashes with underscores.""" + result = sanitize_data_source("dataset/with/slashes") + assert result == "dataset_with_slashes" + + +def test_sanitize_data_source_normal_string(): + """Test sanitize_data_source with normal string.""" + result = sanitize_data_source("normal_dataset") + assert result == "normal_dataset" + + +def test_calculate_per_dataset_metrics_single_source(): + """Test calculate_per_dataset_metrics with single data source.""" + # Create test data - rewards must be >= 1.0 to count as pass + generator_outputs = { + "rewards": [1.0, 1.0, 0.5], + "prompt_token_ids": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "response_ids": [[10, 11], [12, 13], [14, 15]], + } + uids = ["uid1", "uid2", "uid3"] + data_sources = ["dataset1", "dataset1", "dataset1"] + + result = calculate_per_dataset_metrics(generator_outputs, uids, data_sources, 2) + + # Verify results - actual computed values + # Pass@N: uid1=1.0, uid2=1.0 pass; uid3=0.5 does not pass -> 2/3 + # variance_per_prompt: 0.0 (single sample per uid) + # mean_positive_reward: (1.0 + 1.0 + 0.5) / 3 = 0.8333 + assert "eval/dataset1/pass_at_2" in result + assert "eval/dataset1/variance_per_prompt" in result + assert "eval/dataset1/mean_positive_reward" in result + assert result["eval/dataset1/pass_at_2"] == pytest.approx(2.0 / 3.0) + assert result["eval/dataset1/variance_per_prompt"] == 0.0 # single sample per uid + assert result["eval/dataset1/mean_positive_reward"] == pytest.approx(5.0 / 6.0) + + +def test_calculate_per_dataset_metrics_multiple_sources(): + """Test calculate_per_dataset_metrics with multiple data sources including None.""" + # Create test data with mixed sources - rewards >= 1.0 count as pass + generator_outputs = { + "rewards": [1.0, 0.7, 0.5, 1.0], + "prompt_token_ids": [[1, 2], [3, 4], [5, 6], [7, 8]], + "response_ids": [[10, 11], [12, 13], [14, 15], [16, 17]], + } + uids = ["uid1", "uid2", "uid3", "uid4"] + data_sources = ["dataset1", None, "dataset1", None] + + result = calculate_per_dataset_metrics(generator_outputs, uids, data_sources, 2) + + # Verify results for both datasets - actual computed values + # dataset1: indices 0, 2 -> rewards [1.0, 0.5] -> mean_positive = 0.75, pass@n = 1/2 = 0.5 + # unknown (None): indices 1, 3 -> rewards [0.7, 1.0] -> mean_positive = 0.85, pass@n = 1/2 = 0.5 + assert "eval/dataset1/pass_at_2" in result + assert "eval/dataset1/variance_per_prompt" in result + assert "eval/dataset1/mean_positive_reward" in result + assert "eval/unknown/pass_at_2" in result + assert "eval/unknown/variance_per_prompt" in result + assert "eval/unknown/mean_positive_reward" in result + + assert result["eval/dataset1/pass_at_2"] == 0.5 + assert result["eval/dataset1/variance_per_prompt"] == 0.0 # single sample per uid + assert result["eval/dataset1/mean_positive_reward"] == pytest.approx(0.75) + assert result["eval/unknown/pass_at_2"] == 0.5 + assert result["eval/unknown/variance_per_prompt"] == 0.0 # single sample per uid + assert result["eval/unknown/mean_positive_reward"] == pytest.approx(0.85) + + +@patch("builtins.open", new_callable=mock_open) +def test_dump_per_dataset_eval_results_comprehensive(mock_file): + """Test dump_per_dataset_eval_results comprehensive functionality.""" + # Mock dump directory path + mock_dump_dir = Mock() + mock_dump_dir.__truediv__ = Mock(side_effect=lambda x: f"mock_path/{x}") + + # Mock tokenizer + mock_tokenizer = Mock() + mock_tokenizer.decode.side_effect = lambda x: f"decoded_{x}" + + # Create test data + generator_outputs = { + "prompt_token_ids": [[1, 2], [3, 4], [5, 6]], + "response_ids": [[10, 11], [12, 13], [14, 15]], + "rewards": [0.5, 0.7, 0.9], + "stop_reasons": ["stop1", "stop2", "stop3"], + } + data_sources = ["dataset1", None, "dataset1"] + all_envs = ["env1", "env2", "env3"] + env_extras = [{"extra1": "val1"}, {"extra2": "val2"}, {"extra3": "val3"}] + eval_metrics = {"eval/dataset1/avg_score": 0.8, "eval/unknown/avg_score": 0.6} + + # Call the function + dump_per_dataset_eval_results( + mock_dump_dir, mock_tokenizer, generator_outputs, data_sources, all_envs, env_extras, eval_metrics + ) + + # Verify tokenizer was called for decoding + assert mock_tokenizer.decode.call_count == 6 # 3 prompts + 3 responses + + # Verify files were opened (2 per-dataset files + 1 aggregated file) + assert mock_file.call_count == 3 + + # Verify file writes occurred + handle = mock_file.return_value + assert handle.write.call_count > 0 + + # Verify JSON structure by checking some write calls contain expected data + write_calls = [call[0][0] for call in handle.write.call_args_list] + json_writes = [call for call in write_calls if call.strip() and not call.startswith("Dumped")] + + # At least one JSON line should contain our test data + assert len(json_writes) > 0 + + # Parse one of the JSON writes to verify structure + for write_call in json_writes: + try: + data = json.loads(write_call.strip()) + if "input_prompt" in data: + # This is a per-dataset entry + assert "output_response" in data + assert "score" in data + assert "data_source" in data + break + except json.JSONDecodeError: + continue + + +def test_handle_dynamic_sampling_null_strategy(): + """Test that null strategy returns input unchanged.""" + generator_output = { + "prompt_token_ids": [[1, 2, 3], [4, 5, 6]], + "response_ids": [[7, 8], [9, 10]], + "rewards": [[1.0, 2.0], [3.0, 4.0]], + "loss_masks": [[1, 1], [1, 1]], + "stop_reasons": ["stop", "stop"], + "rollout_metrics": None, + "rollout_logprobs": [[0.16, 0.4], [0.2, 0.3]], + } + uids = ["uid1", "uid2"] + sampling_config = {"type": None} + + result_output, result_uids, keep_sampling, state = handle_dynamic_sampling(generator_output, uids, sampling_config) + + assert result_output == generator_output + assert result_uids == uids + assert keep_sampling is False + assert state is None + + +def test_handle_dynamic_sampling_invalid_strategy(): + """Test that invalid strategy raises ValueError.""" + generator_output = { + "prompt_token_ids": [[1, 2, 3]], + "response_ids": [[7, 8]], + "rewards": [[1.0, 2.0]], + "loss_masks": [[1, 1]], + "rollout_logprobs": [[0.16, 0.4]], + } + uids = ["uid1"] + sampling_config = {"type": "invalid_strategy"} + + with pytest.raises(ValueError, match="Invalid dynamic sampling type: invalid_strategy"): + handle_dynamic_sampling(generator_output, uids, sampling_config) + + +def test_handle_replace_sampling_sufficient_good_samples(): + """Test replace sampling when there are sufficient good samples (>0.3).""" + # Create test data with some good UIDs (high variance) and some bad UIDs (zero variance) + generator_output = { + "prompt_token_ids": [[1, 2], [1, 2], [3, 4], [3, 4], [5, 6], [5, 6]], + "response_ids": [[13, 14], [15, 16], [17, 18], [19, 20], [21, 22], [23, 24]], + "rewards": [ + 1.0, + 2.0, + 1.0, + 1.0, + 3.0, + 4.0, + ], # uid1: [1.0, 2.0] (good), uid2: [1.0, 1.0] (bad), uid3: [3.0, 4.0] (good) + "loss_masks": [[1, 1]] * 6, + "stop_reasons": ["length"] * 6, + "rollout_metrics": None, + "rollout_logprobs": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.25], [0.15, 0.25], [0.1, 0.2], [0.3, 0.4]], + } + uids = ["uid1", "uid1", "uid2", "uid2", "uid3", "uid3"] # 2 samples per prompt + sampling_config = {"n_samples_per_prompt": 2, "min_replace_ratio": 0.3} + + result_output, result_uids, keep_sampling = handle_replace_sampling(generator_output, uids, sampling_config) + + # Should not keep sampling + assert keep_sampling is False + + # Output should have same structure but with replacements + assert len(result_output["prompt_token_ids"]) == 6 + assert len(result_output["response_ids"]) == 6 + assert len(result_output["rewards"]) == 6 + assert len(result_output["rollout_logprobs"]) == 6 + assert len(result_uids) == 6 + + # Check that bad uid2 samples were replaced with good samples + uid2_indices = [i for i, uid in enumerate(result_uids) if uid == "uid2"] + # After replacement, uid2 indices should now contain UIDs from good samples + assert len(uid2_indices) == 0 # uid2 should be completely replaced + + +def test_handle_replace_sampling_insufficient_good_samples(): + """Test replace sampling when there are insufficient good samples (<0.3).""" + generator_output = { + "prompt_token_ids": [[1, 2], [1, 2], [3, 4], [3, 4]], + "response_ids": [[9, 10], [11, 12], [13, 14], [15, 16]], + "rewards": [1.0, 1.0, 2.0, 2.0], # uid1: [1.0, 1.0] (bad), uid2: [2.0, 2.0] (bad) + "loss_masks": [[1, 1]] * 4, + "stop_reasons": ["length"] * 4, + "rollout_metrics": None, + "rollout_logprobs": None, + } + uids = ["uid1", "uid1", "uid2", "uid2"] # 2 samples per prompt + sampling_config = {"n_samples_per_prompt": 2, "min_replace_ratio": 0.3} + + result_output, result_uids, keep_sampling = handle_replace_sampling(generator_output, uids, sampling_config) + + # Should keep sampling due to insufficient good samples + assert keep_sampling is True + + # Output should be unchanged + assert result_output == generator_output + assert result_uids == uids + + +def test_handle_replace_sampling_single_sample_per_prompt(): + """Test replace sampling with single sample per prompt (should always be considered good).""" + generator_output = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[5, 6], [7, 8]], + "rewards": [1.0, 1.0], + "loss_masks": [[1, 1]] * 2, + "stop_reasons": ["stop", "stop"], + "rollout_metrics": None, + "rollout_logprobs": [[0.1, 0.2]], + } + uids = ["uid1", "uid2"] + sampling_config = {"n_samples_per_prompt": 1, "min_replace_ratio": 0.3} + + result_output, result_uids, keep_sampling = handle_replace_sampling(generator_output, uids, sampling_config) + + # Should not keep sampling (single samples are always considered good) + assert keep_sampling is False + + # Output should be unchanged since all samples are good + assert result_output == generator_output + assert result_uids == uids + + +def test_handle_replace_sampling_token_level_rewards(): + """Test replace sampling with token-level rewards (should sum to sequence level).""" + generator_output = { + "prompt_token_ids": [[1, 2], [1, 2], [3, 4], [3, 4]], + "response_ids": [[9, 10], [11, 12, 13], [14, 15], [16]], + "rewards": [[1.0, 2.0], [3.0, 4.0, 5.0], [1.0, 1.0], [1.0]], # Token-level rewards + "loss_masks": [[1, 1]] * 4, + "stop_reasons": ["stop"] * 4, + "rollout_metrics": None, + "rollout_logprobs": None, + } + uids = ["uid1", "uid1", "uid2", "uid2"] # uid1: [3.0, 7.0] (good), uid2: [2.0, 2.0] (bad) + sampling_config = {"n_samples_per_prompt": 2, "min_replace_ratio": 0.3} + + result_output, result_uids, keep_sampling = handle_replace_sampling(generator_output, uids, sampling_config) + + # Should not keep sampling (sufficient good samples) + assert keep_sampling is False + + # Check that replacements occurred + assert len(result_output["rewards"]) == 4 + assert len(result_uids) == 4 + + +def test_handle_filter_sampling_sufficient_prompts(): + """Test filter sampling when we get sufficient prompts in one batch.""" + generator_output = { + "prompt_token_ids": [[1, 2], [1, 2], [3, 4], [3, 4]], + "response_ids": [[9, 10], [11, 12], [13, 14], [15, 16]], + "rewards": [1.0, 2.0, 3.0, 3.0], # uid1: [1.0, 2.0] (good), uid2: [3.0, 3.0] (bad) + "loss_masks": [[1, 1]] * 4, + "stop_reasons": ["stop"] * 4, + "rollout_metrics": None, + "rollout_logprobs": None, + } + uids = ["uid1", "uid1", "uid2", "uid2"] + sampling_config = { + "train_batch_size": 1, # Only need 1 prompt + "n_samples_per_prompt": 2, + "max_sample_batches": 20, + } + + result_output, result_uids, keep_sampling, state = handle_filter_sampling( + generator_output, uids, sampling_config, collected_state={"sample_batch_count": 1} + ) + + # Should not keep sampling (sufficient prompts) + assert keep_sampling is False + assert state is None + + # Should only keep the good uid1 samples + assert len(result_output["prompt_token_ids"]) == 2 + assert len(result_uids) == 2 + assert all(uid == "uid1" for uid in result_uids) + + +def test_handle_filter_sampling_insufficient_prompts_continue(): + """Test filter sampling when we need to continue sampling.""" + generator_output = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[5, 6], [7, 8]], + "rewards": [1.0, 2.0], # Only 1 good prompt + "loss_masks": [[1, 1]] * 2, + "stop_reasons": ["stop"] * 2, + "rollout_metrics": None, + "rollout_logprobs": None, + } + uids = ["uid1", "uid1"] + sampling_config = { + "train_batch_size": 2, # Need 2 prompts + "n_samples_per_prompt": 2, + "max_sample_batches": 20, + } + + collected_state = {"sample_batch_count": 1} + + result_output, result_uids, keep_sampling, state = handle_filter_sampling( + generator_output, uids, sampling_config, collected_state=collected_state + ) + + # Should keep sampling (insufficient prompts) + assert keep_sampling is True + assert result_output is generator_output + assert result_uids is uids + assert state is not None + assert state["num_prompts_in_batch"] == 1 + assert state["sample_batch_count"] == 1 + + +def test_handle_filter_sampling_accumulation(): + """Test filter sampling accumulation across multiple batches.""" + # First batch + generator_output1 = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[5, 6], [7, 8]], + "rewards": [1.0, 2.0], # Good prompt + "loss_masks": [[1, 1]] * 2, + "stop_reasons": ["stop"] * 2, + "rollout_metrics": None, + "rollout_logprobs": None, + } + uids1 = ["uid1", "uid1"] + + # Second batch + generator_output2 = { + "prompt_token_ids": [[9, 10], [11, 12]], + "response_ids": [[13, 14], [15, 16]], + "rewards": [3.0, 4.0], # Another good prompt + "loss_masks": [[1, 1]] * 2, + "stop_reasons": ["stop"] * 2, + "rollout_metrics": None, + "rollout_logprobs": None, + } + uids2 = ["uid2", "uid2"] + + sampling_config = { + "train_batch_size": 2, # Need 2 prompts + "n_samples_per_prompt": 2, + "max_sample_batches": 20, + } + + collected_state = {"sample_batch_count": 1} + + # Process first batch + result1_output, result1_uids, keep_sampling1, state1 = handle_filter_sampling( + generator_output1, uids1, sampling_config, collected_state=collected_state + ) + + assert keep_sampling1 is True # Need more prompts + assert state1["num_prompts_in_batch"] == 1 + + # Process second batch + result2_output, result2_uids, keep_sampling2, state2 = handle_filter_sampling( + generator_output2, uids2, sampling_config, collected_state=state1 + ) + + assert keep_sampling2 is False # Now have enough prompts + assert state2 is None + assert len(result2_output["prompt_token_ids"]) == 4 # Both batches combined + assert len(result2_uids) == 4 + + +def test_handle_filter_sampling_single_sample_per_prompt(): + """Test filter sampling with single sample per prompt.""" + generator_output = { + "prompt_token_ids": [[1, 2], [3, 4]], + "response_ids": [[5, 6], [7, 8]], + "rewards": [1.0, 1.0], # Same rewards but single sample per prompt + "loss_masks": [[1, 1]] * 2, + "stop_reasons": ["stop"] * 2, + "rollout_metrics": None, + "rollout_logprobs": None, + } + uids = ["uid1", "uid2"] # Different UIDs, single sample each + sampling_config = { + "train_batch_size": 2, + "n_samples_per_prompt": 1, + "max_sample_batches": 20, + } + + result_output, result_uids, keep_sampling, state = handle_filter_sampling( + generator_output, uids, sampling_config, collected_state={"sample_batch_count": 1} + ) + + # Should not keep sampling (single samples are always kept) + assert keep_sampling is False + assert state is None + assert len(result_output["prompt_token_ids"]) == 2 + assert len(result_uids) == 2 + + +def test_filter_generator_output(): + """Test the filter_generator_output utility function.""" + generator_output = { + "prompt_token_ids": [[1, 2], [3, 4], [5, 6]], + "response_ids": [[7, 8], [9, 10], [11, 12]], + "rewards": [1.0, 2.0, 3.0], + "loss_masks": [[1, 1]] * 3, + "stop_reasons": ["length", "length", "stop"], + "rollout_metrics": {"metric": "value"}, + "rollout_logprobs": [[0.16, 0.4], [0.1, 0.2], [0.3, 0.4]], + } + kept_indices = [0, 2] # Keep first and third samples + + filtered = filter_generator_output(generator_output, kept_indices) + + assert filtered["prompt_token_ids"] == [[1, 2], [5, 6]] + assert filtered["response_ids"] == [[7, 8], [11, 12]] + assert filtered["rewards"] == [1.0, 3.0] + assert filtered["loss_masks"] == [[1, 1]] * 2 + assert filtered["stop_reasons"] == ["length", "stop"] + assert filtered["rollout_metrics"] == {"metric": "value"} + assert filtered["rollout_logprobs"] == [[0.16, 0.4], [0.3, 0.4]] + + +def test_zero_variance_filter_mixed_groups(): + """uid groups with non-zero variance and singletons are kept; zero-variance duplicates are removed.""" + rewards = [1.0, 2.0, 3.0, 3.0, 5.0] + uids = ["uid1", "uid1", "uid2", "uid2", "uid3"] + + kept_indices = zero_variance_filter(rewards, uids) + + # uid1 has variance > 0 -> keep indices 0,1; uid2 has variance 0 with size>1 -> drop 2,3; uid3 singleton -> keep 4 + assert kept_indices == [0, 1, 4] + + +def test_zero_variance_filter_all_zero_variance_duplicates(): + """All duplicate groups with zero variance should be filtered out, yielding no kept indices.""" + rewards = [1.0, 1.0, 2.0, 2.0] + uids = ["a", "a", "b", "b"] + + kept_indices = zero_variance_filter(rewards, uids) + + assert kept_indices == [] + + +def test_zero_variance_filter_singletons_kept(): + """Singleton groups should always be kept regardless of reward values.""" + rewards = [1.0, 1.0, 1.0] + uids = ["x", "y", "z"] + + kept_indices = zero_variance_filter(rewards, uids) + + assert kept_indices == [0, 1, 2] + + +def test_validate_generator_output_valid_case(): + """Test validate_generator_output with valid case.""" + input_batch = GeneratorInput( + prompts=["prompt1", "prompt2", "prompt3"], + env_classes=["env1", "env2", "env3"], + env_extras=[{"extra": 1}, {"extra": 2}, {"extra": 3}], + sampling_params={"temperature": 0.7}, + ) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2, 3, 4], [5, 6], [7, 8, 9]], + response_ids=[[10, 11, 12], [13, 14], [15]], + rewards=[[0.5, 0.6, 0.7], [0.8, 0.9], [1.0]], # Nested list structure + loss_masks=[[1, 1, 0], [1, 1], [0]], + stop_reasons=["stop", "length", "stop"], + rollout_metrics={"metric1": 0.5, "metric2": 0.6}, + rollout_logprobs=None, + ) + + # Should not raise any exceptions + validate_generator_output(len(input_batch["prompts"]), generator_output) + + # per trajectory rewards should work too + generator_output["rewards"] = [0.5, 0.6, 0.7] + validate_generator_output(len(input_batch["prompts"]), generator_output) + + # valid rollout logprobs + generator_output["rollout_logprobs"] = [[0.11, 0.12, 0.13], [0.2, 0.3], [0.4]] + validate_generator_output(len(input_batch["prompts"]), generator_output) + + +def test_validate_generator_output_empty_response_ids(): + """Test validate_generator_output raises RuntimeError when response_ids is empty.""" + input_batch = GeneratorInput(prompts=["prompt1"], env_classes=["env1"], env_extras=None, sampling_params=None) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2, 3]], + response_ids=[], + rewards=[], + loss_masks=[], + stop_reasons=[], + rollout_logprobs=[], # Empty response_ids + ) + + with pytest.raises(RuntimeError, match="No outputs generated"): + validate_generator_output(len(input_batch["prompts"]), generator_output) + + +def test_validate_generator_output_mismatched_prompts_responses(): + """Test validate_generator_output raises AssertionError when prompts and response_ids lengths don't match.""" + input_batch = GeneratorInput( + prompts=["prompt1", "prompt2", "prompt3"], # 3 prompts + env_classes=["env1", "env2", "env3"], + env_extras=None, + sampling_params=None, + ) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2], [3, 4]], + response_ids=[[7, 8], [9, 10]], # Only 2 responses + rewards=[0.5, 0.7], + loss_masks=[[1, 1], [1, 0]], + stop_reasons=["eos", "eos"], + rollout_logprobs=None, + ) + + with pytest.raises(AssertionError, match=re.escape("Mismatch between prompts (3) and responses (2)")): + validate_generator_output(len(input_batch["prompts"]), generator_output) + + +def test_validate_generator_output_all_loss_masked(): + """Test validate_generator_output logs warning when all outputs are loss masked.""" + input_batch = GeneratorInput( + prompts=["prompt1", "prompt2"], env_classes=["env1", "env2"], env_extras=None, sampling_params=None + ) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2, 3], [4, 5, 6]], + response_ids=[[7, 8], [9, 10]], + rewards=[0.5, 0.7], + loss_masks=[[0, 0], [0, 0]], # All zeros - completely loss masked + stop_reasons=["eos", "eos"], + rollout_logprobs=None, + ) + + # Capture log output to verify warning is issued + with patch("skyrl_train.utils.trainer_utils.logger") as mock_logger: + validate_generator_output(len(input_batch["prompts"]), generator_output) + mock_logger.warning.assert_called_once_with( + "All outputs are loss masked, which may lead to NaN loss, please check your generation logic!!" + ) + + +def test_validate_generator_output_mismatched_list_lengths(): + """Test validate_generator_output raises AssertionError when generator output lists have mismatched lengths.""" + input_batch = GeneratorInput( + prompts=["prompt1", "prompt2"], env_classes=["env1", "env2"], env_extras=None, sampling_params=None + ) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2, 3], [4, 5, 6]], + response_ids=[[7, 8], [9, 10]], # Length 2 + rewards=[0.5, 0.7, 0.9], # Length 3 - mismatch! + loss_masks=[[1, 1], [1, 0]], + stop_reasons=["eos", "eos"], + rollout_logprobs=None, + ) + + with pytest.raises(AssertionError, match="Generator output rewards length must be equal to response_ids length"): + validate_generator_output(len(input_batch["prompts"]), generator_output) + + +def test_validate_generator_output_element_length_mismatch(): + """Test validate_generator_output with element length mismatch.""" + input_batch = GeneratorInput( + prompts=["prompt1", "prompt2", "prompt3"], + env_classes=["env1", "env2", "env3"], + env_extras=[{"extra": 1}, {"extra": 2}, {"extra": 3}], + sampling_params={"temperature": 0.7}, + ) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2, 3, 4], [5, 6], [7, 8, 9]], + response_ids=[[10, 11, 12], [13, 14], [15]], + rewards=[[0.5, 0.6, 0.7], [0.8, 0.9], [1.0]], + loss_masks=[[1, 1], [1], [1, 1]], # loss masks are not the same length as response ids + stop_reasons=["stop", "length", "stop"], + rollout_metrics={"metric1": 0.5, "metric2": 0.6}, + rollout_logprobs=None, + ) + + with pytest.raises(AssertionError, match="Response ids and loss masks must have the same length"): + validate_generator_output(len(input_batch["prompts"]), generator_output) + + generator_output["loss_masks"] = [[1, 1, 1], [1, 1], [1]] # add correct loss masks + generator_output["rewards"] = [[0.5, 0.6], [0.8], [1.0, 2.0]] # add incorrect rewards + with pytest.raises(AssertionError, match="Token rewards and response ids must have the same length"): + validate_generator_output(len(input_batch["prompts"]), generator_output) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + response_ids=[[7, 8], [9, 10], [11, 12]], + rewards=[0.5, 0.7, -0.1], + loss_masks=[[1, 1], [1, 0], [1, 1]], + stop_reasons=["eos", "eos", "length"], + rollout_logprobs=[[0.17, 0.2], [0.9], [0.1, 0.2]], # Second entry has length 1 - mismatch ! + ) + + with pytest.raises(AssertionError, match="Response ids and rollout logprobs must have the same length"): + validate_generator_output(len(input_batch["prompts"]), generator_output) + + +def test_build_dataloader_seeding(dummy_config): + """Test that build_dataloader correctly seeds the dataloader for reproducible shuffling.""" + + # Create a dataset with multiple distinct items to test shuffling + class MultiItemDataset: + def __init__(self, size=10): + self.data = [f"item_{i}" for i in range(size)] + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + return self.data[idx] + + def collate_fn(self, batch): + return batch + + dataset = MultiItemDataset(size=20) + + # Test 1: Same seed should produce same shuffling + config1 = dummy_config.copy() + config1.trainer.seed = 42 + config1.trainer.train_batch_size = 5 + + config2 = dummy_config.copy() + config2.trainer.seed = 42 # Same seed + config2.trainer.train_batch_size = 5 + + # Build dataloaders + dataloader1 = build_dataloader(config1, dataset, is_train=True) + dataloader2 = build_dataloader(config2, dataset, is_train=True) + + # Get first batch from each dataloader + first_batch1 = next(iter(dataloader1)) + first_batch2 = next(iter(dataloader2)) + + # With same seed, first batches should be identical + assert ( + first_batch1 == first_batch2 + ), f"Same seed should produce same first batch, got {first_batch1} vs {first_batch2}" + + # Test 2: Different seeds should produce different shuffling + config3 = dummy_config.copy() + config3.trainer.seed = 123 # Different seed + config3.trainer.train_batch_size = 5 + + dataloader3 = build_dataloader(config3, dataset, is_train=True) + first_batch3 = next(iter(dataloader3)) + + # With different seed, first batch should be different + # Note: There's a tiny chance they could be the same by random chance, but very unlikely with 20 items + assert ( + first_batch1 != first_batch3 + ), f"Different seeds should produce different first batches, but both gave {first_batch1}" + + +def test_validate_generator_output_invalid_rewards(): + """Test validate_generator_output raises AssertionError when rewards is neither List[float-like] nor List[List[float-like]].""" + input_batch = GeneratorInput( + prompts=["prompt1", "prompt2"], env_classes=["env1", "env2"], env_extras=None, sampling_params=None + ) + + generator_output = GeneratorOutput( + prompt_token_ids=[[1, 2, 3], [4, 5, 6]], + response_ids=[[7, 8], [9, 10]], + rewards=[[0.5, 0.6], 0.7], + loss_masks=[[1, 1], [1, 0]], + stop_reasons=["eos", "eos"], + rollout_logprobs=None, + ) + + with pytest.raises( + AssertionError, + match=re.escape("rewards must be `List[float]` or `List[List[float]]`"), + ): + validate_generator_output(len(input_batch["prompts"]), generator_output) + + generator_output["rewards"] = [0.5, 0.7] + validate_generator_output(len(input_batch["prompts"]), generator_output) + + generator_output["rewards"] = [[0.5, 0.6], [0.7, 0.8]] + validate_generator_output(len(input_batch["prompts"]), generator_output) + + +# ============================================================================ +# HybridEnvSampler Tests +# ============================================================================ + + +class MockPromptDataset: + """Mock PromptDataset for testing HybridEnvSampler.""" + + def __init__(self, env_distribution: dict): + """ + Create a mock dataset with specified environment distribution. + + Args: + env_distribution: Dict mapping env_class -> number of samples + e.g., {"booking": 10, "github": 5, "reddit": 3} + """ + self.env_class_key = "env_class" + self.dataframe = [] + + for env_class, count in env_distribution.items(): + for i in range(count): + self.dataframe.append( + { + "env_class": env_class, + "prompt": f"prompt_{env_class}_{i}", + } + ) + + def __len__(self): + return len(self.dataframe) + + def __getitem__(self, idx): + row = self.dataframe[idx] + return row["prompt"], row["env_class"], {}, str(idx) + + def collate_fn(self, batch): + return batch + + +def test_hybrid_env_sampler_minimum_samples_per_env(): + """Test that HybridEnvSampler ensures minimum samples from each environment.""" + # Create dataset with uneven distribution + dataset = MockPromptDataset( + { + "booking": 100, + "github": 50, + "reddit": 20, + } + ) + + batch_size = 30 + min_samples_per_env = 3 + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + drop_last=True, + ) + + # Get first batch (sampler yields batches of indices) + first_batch = next(iter(sampler)) + batch_indices = first_batch + + # Count samples per env in the batch + env_counts = {"booking": 0, "github": 0, "reddit": 0} + for idx in batch_indices: + env_class = dataset.dataframe[idx]["env_class"] + env_counts[env_class] += 1 + + # Verify minimum samples from each environment + for env, count in env_counts.items(): + assert ( + count >= min_samples_per_env + ), f"Environment {env} has {count} samples, expected at least {min_samples_per_env}" + + # Verify total batch size + assert len(batch_indices) == batch_size + + +def test_hybrid_env_sampler_reproducibility(): + """Test that HybridEnvSampler produces same results with same seed.""" + dataset = MockPromptDataset( + { + "env_a": 20, + "env_b": 15, + "env_c": 10, + } + ) + + batch_size = 12 + min_samples_per_env = 2 + + # Create two samplers with same seed + sampler1 = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(123), + ) + + sampler2 = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(123), + ) + + # Get first batch from both (sampler yields batches) + batch1 = next(iter(sampler1)) + batch2 = next(iter(sampler2)) + + # Should be identical + assert batch1 == batch2, "Same seed should produce identical batches" + + +def test_hybrid_env_sampler_different_seeds(): + """Test that different seeds produce different results.""" + dataset = MockPromptDataset( + { + "env_a": 30, + "env_b": 25, + "env_c": 20, + } + ) + + batch_size = 15 + min_samples_per_env = 2 + + sampler1 = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + sampler2 = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(999), + ) + + # Get first batch from each (sampler yields batches) + batch1 = next(iter(sampler1)) + batch2 = next(iter(sampler2)) + + # Should be different (with very high probability) + assert batch1 != batch2, "Different seeds should produce different batches" + + +def test_hybrid_env_sampler_warns_when_min_exceeds_batch(): + """Test that sampler warns and adjusts when min_samples_per_env * num_envs > batch_size.""" + dataset = MockPromptDataset( + { + "env_a": 10, + "env_b": 10, + "env_c": 10, + "env_d": 10, + "env_e": 10, + } + ) + + # 5 envs * 5 min = 25 > batch_size of 10 + batch_size = 10 + min_samples_per_env = 5 + + # Should warn and reduce min_samples_per_env + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + # min_samples_per_env should be reduced to fit: max(1, 10 // 5) = 2 + assert sampler.min_samples_per_env == 2 + + +def test_hybrid_env_sampler_length(): + """Test that __len__ returns correct number of batches.""" + dataset = MockPromptDataset( + { + "env_a": 20, + "env_b": 10, # Smallest: limits batches + "env_c": 15, + } + ) + + batch_size = 9 + min_samples_per_env = 2 + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + # env_b has 10 samples, needs 2 per batch -> 5 batches possible + # __len__ returns number of batches (for batch_sampler compatibility) + expected_batches = 10 // min_samples_per_env # 5 + + assert len(sampler) == expected_batches + + +def test_hybrid_env_sampler_proportional_fill(): + """Test that remaining slots are filled proportionally.""" + # Create dataset with very uneven distribution + dataset = MockPromptDataset( + { + "large_env": 900, # 90% + "small_env": 100, # 10% + } + ) + + batch_size = 100 + min_samples_per_env = 1 + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + # Collect multiple batches to get statistical average + # Sampler yields batches, so flatten them + all_batches = list(sampler) + + total_large = 0 + total_small = 0 + + for batch in all_batches: + for idx in batch: + env = dataset.dataframe[idx]["env_class"] + if env == "large_env": + total_large += 1 + else: + total_small += 1 + + # After minimum allocation (1 each), 98 slots filled proportionally + # Expected: ~90% large, ~10% small (with some variance) + total = total_large + total_small + large_ratio = total_large / total + + # Should be roughly 90% large (allowing ±10% tolerance for randomness) + assert 0.80 <= large_ratio <= 0.95, f"Expected ~90% large_env samples, got {large_ratio*100:.1f}%" + + +def test_hybrid_env_sampler_single_env(): + """Test sampler with only one environment.""" + dataset = MockPromptDataset( + { + "only_env": 50, + } + ) + + batch_size = 10 + min_samples_per_env = 1 + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + # Get first batch (sampler yields batches) + batch = next(iter(sampler)) + + # All should be from the only environment + assert len(batch) == batch_size + for idx in batch: + assert dataset.dataframe[idx]["env_class"] == "only_env" + + +def test_hybrid_env_sampler_many_small_envs(): + """Test sampler with many small environments.""" + # 10 environments with 5 samples each + env_dist = {f"env_{i}": 5 for i in range(10)} + dataset = MockPromptDataset(env_dist) + + batch_size = 20 + min_samples_per_env = 1 + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + # Get first batch (sampler yields batches) + batch = next(iter(sampler)) + + # Count envs in batch + env_counts = {} + for idx in batch: + env = dataset.dataframe[idx]["env_class"] + env_counts[env] = env_counts.get(env, 0) + 1 + + # Each env should have at least 1 sample + for env in env_dist.keys(): + assert env in env_counts, f"Environment {env} missing from batch" + assert env_counts[env] >= min_samples_per_env + + +def test_hybrid_env_sampler_iteration_exhaustion(): + """Test that sampler correctly limits iterations based on smallest env.""" + dataset = MockPromptDataset( + { + "abundant": 100, + "scarce": 6, # Only 6 samples + } + ) + + batch_size = 10 + min_samples_per_env = 2 + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + # scarce has 6, needs 2 per batch -> 3 batches max + all_batches = list(sampler) + + expected_num_batches = 3 + assert len(all_batches) == expected_num_batches, f"Expected {expected_num_batches} batches, got {len(all_batches)}" + + # Verify total samples + total_samples = sum(len(batch) for batch in all_batches) + expected_total = 3 * batch_size # 30 + assert total_samples == expected_total, f"Expected {expected_total} samples, got {total_samples}" + + +def test_hybrid_env_sampler_dataloader_integration(): + """ + Test that HybridEnvSampler works correctly with DataLoader using batch_sampler. + + This test catches the bug where using `sampler=` instead of `batch_sampler=` + caused the collate_fn to receive indices instead of dataset items, resulting in: + ValueError: not enough values to unpack (expected 4, got 1) + """ + from torch.utils.data import DataLoader + + dataset = MockPromptDataset( + { + "booking": 20, + "github": 15, + "reddit": 10, + } + ) + + batch_size = 9 + min_samples_per_env = 2 + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=batch_size, + min_samples_per_env=min_samples_per_env, + generator=torch.Generator().manual_seed(42), + ) + + # This is the correct way - using batch_sampler + dataloader = DataLoader( + dataset, + batch_sampler=sampler, + collate_fn=dataset.collate_fn, + ) + + # Get first batch + batch = next(iter(dataloader)) + + # Verify batch structure - should be list of tuples from __getitem__ + assert len(batch) == batch_size, f"Expected batch size {batch_size}, got {len(batch)}" + + # Each item should be a tuple of (prompt, env_class, extras, uid) + for item in batch: + assert isinstance(item, tuple), f"Expected tuple, got {type(item)}" + assert len(item) == 4, f"Expected 4 elements (prompt, env_class, extras, uid), got {len(item)}" + prompt, env_class, extras, uid = item + assert isinstance(prompt, str), f"Expected prompt to be str, got {type(prompt)}" + assert env_class in ["booking", "github", "reddit"], f"Unexpected env_class: {env_class}" + assert isinstance(extras, dict), f"Expected extras to be dict, got {type(extras)}" + assert isinstance(uid, str), f"Expected uid to be str, got {type(uid)}" + + # Verify minimum samples per environment in the batch + env_counts = {} + for item in batch: + env_class = item[1] + env_counts[env_class] = env_counts.get(env_class, 0) + 1 + + for env, count in env_counts.items(): + assert count >= min_samples_per_env, f"Environment {env} has {count} samples, expected >= {min_samples_per_env}" + + +def test_hybrid_env_sampler_wrong_usage_fails(): + """ + Demonstrate that using sampler= instead of batch_sampler= fails. + + This documents the bug we fixed - the sampler yields batches of indices, + not individual indices, so it must be used with batch_sampler=. + """ + from torch.utils.data import DataLoader + + dataset = MockPromptDataset({"env_a": 10, "env_b": 10}) + + sampler = HybridEnvSampler( + dataset=dataset, + batch_size=6, + min_samples_per_env=1, + generator=torch.Generator().manual_seed(42), + ) + + # WRONG: Using sampler= with a batch-yielding sampler and batch_size + # This causes the dataloader to: + # 1. Get a batch [idx1, idx2, ...] from sampler + # 2. Try to use each element as an index to fetch from dataset + # 3. But since sampler yields whole batches, not indices, this fails + wrong_dataloader = DataLoader( + dataset, + batch_size=6, # DataLoader will try to batch the batches + sampler=sampler, + collate_fn=dataset.collate_fn, + ) + + # This will fail because: + # - sampler yields [0, 3, 5, 1, 4, 2] (a batch of indices) + # - DataLoader tries to use [0, 3, 5, 1, 4, 2] as the index for __getitem__ + # - dataset[list] fails with TypeError + with pytest.raises(TypeError): + next(iter(wrong_dataloader)) diff --git a/skyrl-train/tests/cpu/util.py b/skyrl-train/tests/cpu/util.py new file mode 100644 index 0000000000..bc775b9668 --- /dev/null +++ b/skyrl-train/tests/cpu/util.py @@ -0,0 +1,46 @@ +# utility functions used for CPU tests + +from skyrl_train.config.utils import get_default_config +from omegaconf import OmegaConf + + +def example_dummy_config(): + cfg = get_default_config() + # TODO (sumanthrh): Some of these overrides are no longer needed after reading from the config file + trainer_overrides = { + "project_name": "unit-test", + "run_name": "test-run", + "logger": "tensorboard", + "micro_train_batch_size_per_gpu": 2, + "train_batch_size": 2, + "eval_batch_size": 2, + "update_epochs_per_batch": 1, + "epochs": 1, + "max_prompt_length": 20, + "use_sample_packing": False, + "seed": 42, + "resume_mode": "none", + "algorithm": { + "advantage_estimator": "grpo", + "use_kl_estimator_k3": False, + "use_abs_kl": False, + "kl_estimator_type": "k1", + "use_kl_loss": True, + "kl_loss_coef": 0.0, + "loss_reduction": "token_mean", + "grpo_norm_by_std": True, + }, + } + generator_overrides = { + "sampling_params": {"max_generate_length": 20}, + "n_samples_per_prompt": 1, + "batched": False, + "max_turns": 1, + "enable_http_endpoint": False, + "http_endpoint_host": "127.0.0.1", + "http_endpoint_port": 8000, + } + OmegaConf.update(cfg, "trainer", trainer_overrides) + OmegaConf.update(cfg, "generator", generator_overrides) + + return cfg diff --git a/skyrl-train/tests/cpu/utils/test_io.py b/skyrl-train/tests/cpu/utils/test_io.py new file mode 100644 index 0000000000..61f1561db6 --- /dev/null +++ b/skyrl-train/tests/cpu/utils/test_io.py @@ -0,0 +1,398 @@ +""" +Unit tests for cloud storage I/O utilities. +""" + +import os +import tempfile +import pytest +from unittest.mock import patch, Mock +import torch + + +from skyrl_train.utils.io.io import ( + is_cloud_path, + makedirs, + exists, + open_file, + upload_directory, + download_directory, + local_work_dir, + local_read_dir, + list_dir, +) +from skyrl_train.utils.trainer_utils import ( + list_checkpoint_dirs, + cleanup_old_checkpoints, +) + + +class TestCloudPathDetection: + """Test cloud path detection functionality.""" + + def test_is_cloud_path_s3(self): + """Test S3 path detection.""" + assert is_cloud_path("s3://bucket/path/file.pt") + assert is_cloud_path("s3://my-bucket/checkpoints/global_step_1000/model.pt") + + def test_is_cloud_path_gcs(self): + """Test GCS path detection.""" + assert is_cloud_path("gs://bucket/path/file.pt") + assert is_cloud_path("gcs://bucket/path/file.pt") + + def test_is_local_path(self): + """Test local path detection.""" + assert not is_cloud_path("/local/path/file.pt") + assert not is_cloud_path("./relative/path/file.pt") + assert not is_cloud_path("relative/path/file.pt") + assert not is_cloud_path("C:\\Windows\\path\\file.pt") + + +class TestLocalFileOperations: + """Test file operations for local paths.""" + + def test_makedirs_local(self): + """Test directory creation for local paths.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_dir = os.path.join(temp_dir, "test_checkpoints") + + # Should create directory + makedirs(test_dir) + assert os.path.exists(test_dir) + assert os.path.isdir(test_dir) + + # Should not fail with exist_ok=True (default) + makedirs(test_dir, exist_ok=True) + assert os.path.exists(test_dir) + + def test_makedirs_cloud_path(self): + """Test that makedirs does nothing for cloud paths.""" + # Should not raise an error for cloud paths + makedirs("s3://bucket/path", exist_ok=True) + makedirs("gs://bucket/path", exist_ok=True) + makedirs("gcs://bucket/path", exist_ok=True) + + def test_open_file_text_local(self): + """Test text file operations for local paths using open_file.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_file = os.path.join(temp_dir, "test.txt") + test_content = "test checkpoint step: 1000" + + # Write and read text using open_file + with open_file(test_file, "w") as f: + f.write(test_content) + assert os.path.exists(test_file) + + with open_file(test_file, "r") as f: + read_content = f.read() + assert read_content == test_content + + def test_exists_local(self): + """Test file existence check for local paths.""" + with tempfile.TemporaryDirectory() as temp_dir: + existing_file = os.path.join(temp_dir, "existing.txt") + non_existing_file = os.path.join(temp_dir, "non_existing.txt") + + # Create a file + with open(existing_file, "w") as f: + f.write("test") + + assert exists(existing_file) + assert not exists(non_existing_file) + assert exists(temp_dir) + + def test_list_dir_returns_strings_local(self): + """Test that list_dir returns a list of strings for a local directory.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create files and a subdirectory + file_a = os.path.join(temp_dir, "a.txt") + with open(file_a, "w") as f: + f.write("a") + sub_dir = os.path.join(temp_dir, "sub") + os.makedirs(sub_dir) + file_b = os.path.join(sub_dir, "b.txt") + with open(file_b, "w") as f: + f.write("b") + + entries = list_dir(temp_dir) + print(f"\n\n\nentries: {entries}\n\n\n") + + assert isinstance(entries, list) + assert len(entries) >= 2 + assert all(isinstance(p, str) for p in entries) + + +class TestCheckpointUtilities: + """Test checkpoint-specific utilities.""" + + def test_list_checkpoint_dirs_local(self): + """Test listing checkpoint directories for local paths.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create some checkpoint directories + checkpoint_dirs = [ + "global_step_1000", + "global_step_2000", + "global_step_500", + "other_dir", # Should be ignored + ] + + for dirname in checkpoint_dirs: + os.makedirs(os.path.join(temp_dir, dirname)) + + # List checkpoint directories + found_dirs = list_checkpoint_dirs(temp_dir) + + # Should only include global_step_ directories, sorted + expected = ["global_step_1000", "global_step_2000", "global_step_500"] + assert sorted(found_dirs) == sorted(expected) + + def test_list_checkpoint_dirs_nonexistent(self): + """Test listing checkpoint directories for non-existent path.""" + non_existent_path = "/non/existent/path" + found_dirs = list_checkpoint_dirs(non_existent_path) + assert found_dirs == [] + + def test_cleanup_old_checkpoints_local(self): + """Test cleanup of old checkpoints for local paths.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create checkpoint directories + steps = [1000, 1500, 2000, 2500, 3000] + for step in steps: + checkpoint_dir = os.path.join(temp_dir, f"global_step_{step}") + os.makedirs(checkpoint_dir) + # Add a dummy file to make it more realistic + with open(os.path.join(checkpoint_dir, "model.pt"), "w") as f: + f.write("dummy") + + # Keep only 3 most recent + cleanup_old_checkpoints(temp_dir, max_checkpoints=3) + + # Check remaining directories + remaining_dirs = list_checkpoint_dirs(temp_dir) + remaining_steps = [int(d.split("_")[2]) for d in remaining_dirs] + remaining_steps.sort() + + # Should keep the 3 most recent: 2000, 2500, 3000 + assert remaining_steps == [2000, 2500, 3000] + + def test_cleanup_old_checkpoints_no_cleanup_needed(self): + """Test cleanup when no cleanup is needed.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create only 2 checkpoint directories + steps = [1000, 2000] + for step in steps: + checkpoint_dir = os.path.join(temp_dir, f"global_step_{step}") + os.makedirs(checkpoint_dir) + + # Keep 5 - should not remove any + cleanup_old_checkpoints(temp_dir, max_checkpoints=5) + + # All directories should remain + remaining_dirs = list_checkpoint_dirs(temp_dir) + assert len(remaining_dirs) == 2 + + +class TestCloudFileOperationsMocked: + """Test cloud file operations with mocked fsspec.""" + + @patch("skyrl_train.utils.io.io._get_filesystem") + def test_list_checkpoint_dirs_cloud(self, mock_get_filesystem): + """Test list_checkpoint_dirs with cloud storage.""" + mock_fs = Mock() + mock_fs.exists.return_value = True + mock_fs.ls.return_value = [ + "s3://bucket/checkpoints/global_step_1000", + "s3://bucket/checkpoints/global_step_2000", + "s3://bucket/checkpoints/other_dir", + ] + mock_fs.isdir.side_effect = lambda path: "global_step_" in path + mock_get_filesystem.return_value = mock_fs + + cloud_path = "s3://bucket/checkpoints" + + result = list_checkpoint_dirs(cloud_path) + + # Should return sorted checkpoint directories + expected = ["global_step_1000", "global_step_2000"] + assert sorted(result) == sorted(expected) + + @patch("skyrl_train.utils.io.io._get_filesystem") + def test_cleanup_old_checkpoints_cloud(self, mock_get_filesystem): + """Test cleanup_old_checkpoints with cloud storage.""" + mock_fs = Mock() + mock_fs.exists.return_value = True + mock_fs.ls.return_value = [ + "s3://bucket/checkpoints/global_step_1000", + "s3://bucket/checkpoints/global_step_1500", + "s3://bucket/checkpoints/global_step_2000", + "s3://bucket/checkpoints/global_step_2500", + ] + mock_fs.isdir.return_value = True + mock_get_filesystem.return_value = mock_fs + + cloud_path = "s3://bucket/checkpoints" + + cleanup_old_checkpoints(cloud_path, max_checkpoints=2) + + # Should remove the 2 oldest checkpoints + expected_removes = [ + "s3://bucket/checkpoints/global_step_1000", + "s3://bucket/checkpoints/global_step_1500", + ] + + # Check that remove was called for old checkpoints + actual_removes = [call[0][0] for call in mock_fs.rm.call_args_list] + assert sorted(actual_removes) == sorted(expected_removes) + + +class TestCheckpointScenarios: + """Test realistic checkpoint scenarios.""" + + def test_local_checkpoint_save_load_cycle(self): + """Test a complete checkpoint save/load cycle with local storage.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Simulate trainer checkpoint saving + global_step = 1500 + global_step_folder = os.path.join(temp_dir, f"global_step_{global_step}") + trainer_state_path = os.path.join(global_step_folder, "trainer_state.pt") + latest_checkpoint_file = os.path.join(temp_dir, "latest_ckpt_global_step.txt") + + # Save checkpoint + makedirs(global_step_folder) + + # Simulate saving trainer state (mock torch.save for simplicity) + trainer_state = {"global_step": global_step, "config": {"lr": 0.001}} + with patch("torch.save") as mock_save: + with open_file(trainer_state_path, "wb") as f: + torch.save(trainer_state, f) + mock_save.assert_called_once() + + # Save latest checkpoint info + with open_file(latest_checkpoint_file, "w") as f: + f.write(str(global_step)) + + # Verify checkpoint was saved + assert exists(global_step_folder) + # Note: trainer_state_path won't exist since we mocked torch.save + assert exists(latest_checkpoint_file) + + # Verify latest step can be retrieved + with open_file(latest_checkpoint_file, "r") as f: + ckpt_iteration = int(f.read().strip()) + assert ckpt_iteration == global_step + + +class TestContextManagers: + """Test the local_work_dir and local_read_dir context managers.""" + + def test_local_work_dir_local_path(self): + """Test local_work_dir with a local path.""" + with tempfile.TemporaryDirectory() as base_temp_dir: + test_dir = os.path.join(base_temp_dir, "test_output") + + with local_work_dir(test_dir) as work_dir: + # Should return the same path for local paths + assert work_dir == test_dir + # Directory should be created + assert os.path.exists(work_dir) + assert os.path.isdir(work_dir) + + # Write a test file + test_file = os.path.join(work_dir, "test.txt") + with open(test_file, "w") as f: + f.write("test content") + + # File should still exist after context exit + assert os.path.exists(test_file) + with open(test_file, "r") as f: + assert f.read() == "test content" + + @patch("skyrl_train.utils.io.io.upload_directory") + @patch("skyrl_train.utils.io.io.is_cloud_path") + def test_local_work_dir_cloud_path(self, mock_is_cloud_path, mock_upload_directory): + """Test local_work_dir with a cloud path.""" + mock_is_cloud_path.return_value = True + + cloud_path = "s3://bucket/model" + + with local_work_dir(cloud_path) as work_dir: + # Should get a temporary directory for cloud paths + assert work_dir.startswith("/") + assert "tmp" in work_dir.lower() + assert os.path.exists(work_dir) + assert os.path.isdir(work_dir) + + # Write a test file + test_file = os.path.join(work_dir, "model.txt") + with open(test_file, "w") as f: + f.write("model data") + + # Should have called upload_directory to upload to cloud + mock_upload_directory.assert_called_once() + # First argument should be the temp directory, second should be cloud path + call_args = mock_upload_directory.call_args[0] + assert call_args[1] == cloud_path + # First argument should be the temp directory we worked with + assert call_args[0] == work_dir # This will be the temp dir + + def test_local_read_dir_local_path(self): + """Test local_read_dir with a local path.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create a test file + test_file = os.path.join(temp_dir, "test.txt") + with open(test_file, "w") as f: + f.write("test content") + + with local_read_dir(temp_dir) as read_dir: + # Should return the same path for local paths + assert read_dir == temp_dir + + # Should be able to read the file + read_file = os.path.join(read_dir, "test.txt") + assert os.path.exists(read_file) + with open(read_file, "r") as f: + assert f.read() == "test content" + + @patch("skyrl_train.utils.io.io.download_directory") + @patch("skyrl_train.utils.io.io.is_cloud_path") + def test_local_read_dir_cloud_path(self, mock_is_cloud_path, mock_download_directory): + """Test local_read_dir with a cloud path.""" + mock_is_cloud_path.return_value = True + + cloud_path = "s3://bucket/model" + + with local_read_dir(cloud_path) as read_dir: + # Should get a temporary directory for cloud paths + assert read_dir.startswith("/") + assert "tmp" in read_dir.lower() + assert os.path.exists(read_dir) + assert os.path.isdir(read_dir) + + # Should have called download_directory to download from cloud + mock_download_directory.assert_called_once_with(cloud_path, read_dir) + + def test_local_read_dir_nonexistent_local(self): + """Test local_read_dir with a non-existent local path.""" + non_existent_path = "/non/existent/path/12345" + + with pytest.raises(FileNotFoundError, match="Path does not exist"): + with local_read_dir(non_existent_path): + pass + + +class TestUploadDownload: + """Test upload and download directory functions.""" + + def test_upload_directory_validates_cloud_path(self): + """Test that upload_directory validates destination is a cloud path.""" + with pytest.raises(ValueError, match="Destination must be a cloud path"): + upload_directory("/local/src", "/local/dst") + + def test_download_directory_validates_cloud_path(self): + """Test that download_directory validates source is a cloud path.""" + with pytest.raises(ValueError, match="Source must be a cloud path"): + download_directory("/local/src", "/local/dst") + + +if __name__ == "__main__": + # Run tests with pytest + pytest.main([__file__, "-v"]) diff --git a/skyrl-train/tests/cpu/utils/test_logging_utils.py b/skyrl-train/tests/cpu/utils/test_logging_utils.py new file mode 100644 index 0000000000..ab8a56c6e7 --- /dev/null +++ b/skyrl-train/tests/cpu/utils/test_logging_utils.py @@ -0,0 +1,122 @@ +""" +uv run --isolated --extra dev pytest tests/cpu/utils/test_logging_utils.py +""" + +import pytest + +from skyrl_train.utils.logging_utils import ( + BASE_PROMPT_COLOR, + NEGATIVE_RESPONSE_COLOR, + POSITIVE_RESPONSE_COLOR, + _color_block_format_and_kwargs, + log_example, +) + + +class StubLogger: + """Minimal logger stub capturing calls to .opt().info().""" + + def __init__(self) -> None: + self.last_message = None + self.last_args = None + self.last_kwargs = None + + def opt(self, **kwargs): + # In real loguru this returns a new logger; here we just ignore options. + return self + + def info(self, msg, *args, **kwargs): + self.last_message = msg + self.last_args = args + self.last_kwargs = kwargs + + +def test_color_block_format_and_kwargs_single_line(): + fmt, kwargs = _color_block_format_and_kwargs("hello", "red", "p") + + assert fmt == "{p0}" + assert kwargs == {"p0": "hello"} + + +def test_color_block_format_and_kwargs_multi_line(): + text = "line1\nline2" + fmt, kwargs = _color_block_format_and_kwargs(text, "blue", "x") + + assert fmt == "{x0}\n{x1}" + assert kwargs == {"x0": "line1", "x1": "line2"} + + +@pytest.mark.parametrize( + "reward,expected_color", + [ + (None, NEGATIVE_RESPONSE_COLOR), + (0.0, NEGATIVE_RESPONSE_COLOR), + (-1.0, NEGATIVE_RESPONSE_COLOR), + (0.1, POSITIVE_RESPONSE_COLOR), + ([0.1, 0.2], POSITIVE_RESPONSE_COLOR), + ], +) +def test_log_example_uses_expected_colors_and_reward_string(reward, expected_color): + logger = StubLogger() + + prompt = [{"role": "user", "content": "line1\nline2"}] + response = "out1\nout2" + + log_example(logger, prompt=prompt, response=response, reward=reward) + + # Basic structure checks + assert logger.last_message.startswith("Example:\n Input: ") + assert "Output (Total Reward: {reward}):" in logger.last_message + + # Placeholder keys from helper should be present + assert "p0" in logger.last_kwargs + assert "r0" in logger.last_kwargs + + # Prompt lines kept as-is + assert logger.last_kwargs["p0"] == "[{'role': 'user', 'content': 'line1\\nline2'}]" + + # Response lines kept as-is + assert logger.last_kwargs["r0"] == "out1" + assert logger.last_kwargs["r1"] == "out2" + + # Reward formatting + reward_str = logger.last_kwargs["reward"] + if reward is None: + assert reward_str == "N/A" + else: + # log_example normalizes rewards to a single float + if isinstance(reward, list): + expected_val = float(sum(reward)) + else: + expected_val = float(reward) + assert pytest.approx(float(reward_str), rel=1e-6) == expected_val + + # Color tags should appear in the format string with the correct colors + assert f"<{BASE_PROMPT_COLOR}>" in logger.last_message + assert f"" in logger.last_message + assert f"<{expected_color}>" in logger.last_message + assert f"" in logger.last_message + + +def test_log_example_handles_exceptions_gracefully(monkeypatch, capsys): + """Force an exception inside log_example and ensure the fallback path prints.""" + + def broken_color_block(*args, **kwargs): + raise RuntimeError("boom") + + # Patch the helper to raise + monkeypatch.setattr( + "skyrl_train.utils.logging_utils._color_block_format_and_kwargs", + broken_color_block, + ) + + logger = StubLogger() + log_example(logger, prompt=[{"role": "user", "content": "p"}], response="r", reward=None) + + # And the plain-text fallback should be printed to stdout + captured = capsys.readouterr() + assert "Error pretty printing example" in captured.out + assert "Example:" in captured.out + assert "Input: [{'role': 'user', 'content': 'p'}]" in captured.out + assert "Output (Total Reward: N/A):" in captured.out + assert "r" in captured.out diff --git a/skyrl-train/tests/cpu/utils/test_ppo_utils.py b/skyrl-train/tests/cpu/utils/test_ppo_utils.py new file mode 100644 index 0000000000..7c531c77ea --- /dev/null +++ b/skyrl-train/tests/cpu/utils/test_ppo_utils.py @@ -0,0 +1,682 @@ +""" +Run with: +uv run --isolated --extra dev pytest tests/cpu/utils/test_ppo_utils.py +""" + +import torch +import math +import pytest +from skyrl_train.utils.ppo_utils import ( + reduce_loss, + compute_approx_kl, + compute_gae_advantage_return, + compute_grpo_outcome_advantage, + compute_advantages_and_returns, + AdaptiveKLController, + FixedKLController, + AdvantageEstimatorRegistry, + register_advantage_estimator, + PolicyLossRegistry, + register_policy_loss, + compute_reinforce_plus_plus_outcome_advantage, + compute_rloo_outcome_advantage, +) +import numpy as np + + +@pytest.fixture +def dummy_data(): + log_probs = torch.tensor([[0.2, 0.3, 0.5]]) + log_probs_base = torch.tensor([[0.1, 0.2, 0.4]]) + mask = torch.tensor([[1.0, 1.0, 0.0]]) # last value masked out + return log_probs, log_probs_base, mask + + +@pytest.fixture +def advantage_test_data(): + rewards = torch.tensor([[1.0, 2.0, 3.0]]) + values = torch.tensor([[0.5, 1.0, 1.5]]) + response_mask = torch.tensor([[1.0, 1.0, 1.0]]) + index = np.array(["0", "0", "0"]) + return rewards, values, response_mask, index + + +def test_compute_approx_kl(dummy_data): + log_probs, log_probs_base, mask = dummy_data + kl = compute_approx_kl(log_probs, log_probs_base, mask, kl_estimator_type="k1") + + expected_kl = (log_probs - log_probs_base) * mask + assert torch.allclose(kl, expected_kl), "KL approximation should be log-prob diff masked" + + kl_abs = compute_approx_kl(log_probs, log_probs_base, mask, kl_estimator_type="abs") + expected_abs = (log_probs - log_probs_base).abs() * mask + assert torch.allclose(kl_abs, expected_abs), "KL approximation should be abs(log-prob diff) masked" + + kl_k2 = compute_approx_kl(log_probs, log_probs_base, mask, kl_estimator_type="k2") + expected_k2 = 0.5 * (log_probs - log_probs_base).square() * mask + assert torch.allclose(kl_k2, expected_k2, atol=1e-4), "k2 estimator is not correct" + + kl_k3 = compute_approx_kl(log_probs, log_probs_base, mask, kl_estimator_type="k3") + log_ratio = log_probs - log_probs_base + expected_k3 = (torch.exp(-log_ratio) - 1 + log_ratio) * mask + assert torch.allclose(kl_k3, expected_k3, atol=1e-4), "k3 estimator is not correct" + + +def test_compute_reinforce_plus_plus_outcome_advantage_returns_and_masking(): + """REINFORCE++ returns should be discounted sums with reset after EOS; advantages masked.""" + token_level_rewards = torch.tensor([[1.0, 2.0, 3.0]]) + response_mask = torch.tensor([[1.0, 1.0, 0.0]]) # EOS after second token + + adv, ret = compute_reinforce_plus_plus_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + gamma=1.0, + ) + + expected_ret = torch.tensor([[3.0, 2.0, 3.0]]) + + assert ret.shape == token_level_rewards.shape + assert torch.allclose(ret, expected_ret, atol=1e-5) + # advantages are whitened and then masked; masked positions should be zero + assert adv.shape == token_level_rewards.shape + assert torch.allclose(adv * (1 - response_mask), torch.zeros_like(adv)) + + +def test_compute_reinforce_plus_plus_outcome_advantage_gamma(): + """REINFORCE++ returns should reflect gamma discounting.""" + token_level_rewards = torch.tensor([[1.0, 2.0, 3.0]]) + response_mask = torch.ones_like(token_level_rewards) + + adv, ret = compute_reinforce_plus_plus_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + gamma=0.5, + ) + + expected_ret = torch.tensor([[2.75, 3.50, 3.00]]) + + assert ret.shape == token_level_rewards.shape + assert torch.allclose(ret, expected_ret, atol=1e-5) + assert adv.shape == token_level_rewards.shape + + +def test_compute_rloo_outcome_advantage_basic(): + """RLOO should produce leave-one-out centered scores per group, broadcast across tokens.""" + # Three groups: [6.0, 3.0] -> [3.0, -3.0], [9.0, 12.0] -> [-3.0, 3.0] + # [1.0] -> [0.0] (since there's only one response, the advantage is 0) + token_level_rewards = torch.tensor( + [ + [0.0, 0.0, 6.0], # sum = 6.0, group 0 + [0.0, 0.0, 3.0], # sum = 3.0, group 0 + [0.0, 0.0, 9.0], # sum = 9.0, group 1 + [0.0, 0.0, 12.0], # sum = 12.0, group 1 + [0.0, 0.0, 1.0], # sum = 0.0, group 2 + ] + ) + response_mask = torch.ones_like(token_level_rewards) + index = np.array([0, 0, 1, 1, 2]) + + adv, ret = compute_rloo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + ) + + expected = torch.tensor([3.0, -3.0, -3.0, 3.0, 0.0]).unsqueeze(-1) * response_mask + + assert adv.shape == token_level_rewards.shape + assert torch.allclose(adv, ret), "Advantages and returns should be equal with RLOO" + assert torch.allclose(adv, expected, atol=1e-5) + + +def test_compute_grpo_outcome_advantage(advantage_test_data): + rewards, _, response_mask, index = advantage_test_data + + adv, ret = compute_grpo_outcome_advantage( + token_level_rewards=rewards, + response_mask=response_mask, + index=index, + ) + + assert adv.shape == rewards.shape + assert ret.shape == rewards.shape + assert torch.allclose(adv, ret), "Advantages and returns should be equal with GRPO" + + +def test_compute_grpo_outcome_advantage_norm_std_false(): + """Test GRPO advantage computation with grpo_norm_by_std=False.""" + # Two groups: [6.0, 3.0] mean=4.5, [9.0, 12.0] mean=10.5 + token_level_rewards = torch.tensor( + [ + [1.0, 2.0, 3.0], # sum = 6.0, group 0 + [1.0, 1.0, 1.0], # sum = 3.0, group 0 + [3.0, 3.0, 3.0], # sum = 9.0, group 1 + [4.0, 4.0, 4.0], # sum = 12.0, group 1 + ] + ) + response_mask = torch.ones_like(token_level_rewards) + index = np.array([0, 0, 1, 1]) + + adv, ret = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + grpo_norm_by_std=False, + ) + + # Expected: [6.0-4.5, 3.0-4.5, 9.0-10.5, 12.0-10.5] = [1.5, -1.5, -1.5, 1.5] + expected = torch.tensor([1.5, -1.5, -1.5, 1.5]).unsqueeze(-1) * response_mask + + assert adv.shape == token_level_rewards.shape + assert torch.allclose(adv, ret), "Advantages and returns should be equal with GRPO" + assert torch.allclose(adv, expected, atol=1e-5), f"Expected {expected}, got {adv}" + + +def test_compute_gae_advantage_return(advantage_test_data): + rewards, values, response_mask, index = advantage_test_data + + adv, ret = compute_gae_advantage_return( + token_level_rewards=rewards, + values=values, + response_mask=response_mask, + gamma=1.0, + lambd=1.0, # no discounting for simplicity + ) + + expected_ret = torch.tensor([[6.0, 5.0, 3.0]]) + + # The advantages will be whitened, so we just check the shape and that they're not all zeros + assert adv.shape == rewards.shape + assert not torch.allclose(adv, torch.zeros_like(adv)) + assert ret.shape == expected_ret.shape + assert torch.allclose(ret, expected_ret, atol=1e-5) + + +def test_compute_gae_advantage_return_with_masking(advantage_test_data): + rewards, values, _, _ = advantage_test_data + response_mask = torch.tensor([[1.0, 0.0, 1.0]]) # Mask out the second token + + adv, ret = compute_gae_advantage_return( + token_level_rewards=rewards, + values=values, + response_mask=response_mask, + gamma=1.0, + lambd=1.0, # no discounting for simplicity + ) + + # The returns should be reversed cumulative rewards + expected_ret = torch.tensor([[6.0, 5.0, 3.0]]) + expected_adv = torch.tensor([[0.7071, 0.1768, -0.7071]]) + + assert torch.allclose(ret, expected_ret, atol=1e-5) + assert torch.allclose(adv, expected_adv, atol=1e-4) + + +def test_compute_gae_advantage_return_gamma(advantage_test_data): + rewards, values, response_mask, _ = advantage_test_data + + _, ret = compute_gae_advantage_return( + token_level_rewards=rewards, + values=values, + response_mask=response_mask, + gamma=0.5, + lambd=1.0, + ) + + expected_ret = torch.tensor([[2.7500, 3.5000, 3.0000]]) + assert torch.allclose(ret, expected_ret, atol=1e-5) + + +def test_compute_gae_advantage_return_lam(advantage_test_data): + rewards, values, response_mask, _ = advantage_test_data + + _, ret = compute_gae_advantage_return( + token_level_rewards=rewards, + values=values, + response_mask=response_mask, + lambd=0.5, + gamma=1.0, + ) + + expected_ret = torch.tensor([[3.6250, 4.2500, 3.0000]]) + assert torch.allclose(ret, expected_ret, atol=1e-5) + + +def test_reduce_loss(): + """Test the reduce_loss function with different reduction types.""" + # Test data: 2x3 loss tensor with different valid token counts per sequence + loss = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + loss_mask = torch.tensor([[1.0, 1.0, 1.0], [1.0, 0.0, 0.0]]) # seq0 has 3 tokens, seq1 has 1 token + + # Test token_mean: sum all valid losses / count valid tokens + # Valid losses: [1.0, 2.0, 3.0, 4.0], mean = 10.0/4 = 2.5 + result_token = reduce_loss(loss, loss_mask, "token_mean") + expected_token = torch.tensor(2.5) + assert torch.allclose(result_token, expected_token), f"Expected {expected_token}, got {result_token}" + + # Test sequence_mean: mean of per-sequence means + # Seq 0: (1.0 + 2.0 + 3.0) / 3 = 2.0, Seq 1: 4.0 / 1 = 4.0, batch mean = (2.0 + 4.0) / 2 = 3.0 + result_seq = reduce_loss(loss, loss_mask, "sequence_mean") + expected_seq = torch.tensor(3.0) + assert torch.allclose(result_seq, expected_seq), f"Expected {expected_seq}, got {result_seq}" + + # Test seq_mean_token_sum_norm: sum per sequence / max_len, then batch mean + # Seq 0: (1.0 + 2.0 + 3.0) / 4 = 1.5, Seq 1: 4.0 / 4 = 1.0, batch mean = (1.5 + 1.0) / 2 = 1.25 + max_seq_len = 4 + result_max = reduce_loss(loss, loss_mask, "seq_mean_token_sum_norm", max_seq_len) + expected_max = torch.tensor(1.25) + assert torch.allclose(result_max, expected_max), f"Expected {expected_max}, got {result_max}" + + +def test_adaptive_kl_controller_update(): + controller = AdaptiveKLController(init_kl_coef=0.2, target=0.1, horizon=100) + controller.update(current=0.2, n_steps=10) + + # Expected error: (0.2 / 0.1 - 1) = 1 → clipped to 0.2 + # Mult = 1 + 0.2 * 10 / 100 = 1.02 + expected = 0.2 * 1.02 + assert math.isclose(controller.value, expected, rel_tol=1e-5) + + +def test_fixed_kl_controller(): + controller = FixedKLController(kl_coef=0.1) + controller.update(current=1.0, n_steps=10) + assert controller.value == 0.1 # Should remain unchanged + + +def test_base_function_registry_registration_and_retrieval(): + """Test basic registration and retrieval functionality of BaseFunctionRegistry.""" + + def dummy_function(**kwargs): + return torch.zeros_like(kwargs["token_level_rewards"]), torch.zeros_like(kwargs["token_level_rewards"]) + + # Register function + AdvantageEstimatorRegistry.register("test_basic", dummy_function) + + # Test retrieval + retrieved_func = AdvantageEstimatorRegistry.get("test_basic") + assert retrieved_func == dummy_function + + # Test it's in available list + assert "test_basic" in AdvantageEstimatorRegistry.list_available() + + # Clean up + AdvantageEstimatorRegistry.unregister("test_basic") + + +def test_base_function_registry_error_handling(): + """Test error handling in BaseFunctionRegistry.""" + + def dummy_function(**kwargs): + return None, None + + # Test getting non-existent function + with pytest.raises(ValueError, match="Unknown advantage estimator"): + AdvantageEstimatorRegistry.get("non_existent") + + # Test unregistering non-existent function + with pytest.raises(ValueError, match="not registered"): + AdvantageEstimatorRegistry.unregister("non_existent") + + # Test duplicate registration + AdvantageEstimatorRegistry.register("test_dup", dummy_function) + with pytest.raises(ValueError, match="already registered"): + AdvantageEstimatorRegistry.register("test_dup", dummy_function) + + # Clean up + AdvantageEstimatorRegistry.unregister("test_dup") + + +def test_base_registry_unregister(): + """Test unregistration functionality.""" + + def dummy_function(**kwargs): + return torch.zeros_like(kwargs["token_level_rewards"]), torch.zeros_like(kwargs["token_level_rewards"]) + + # Register and verify + AdvantageEstimatorRegistry.register("test_unregister", dummy_function) + assert "test_unregister" in AdvantageEstimatorRegistry.list_available() + + # Unregister and verify + AdvantageEstimatorRegistry.unregister("test_unregister") + assert "test_unregister" not in AdvantageEstimatorRegistry.list_available() + + +def test_advantage_estimator_registry_specific(): + """Test AdvantageEstimatorRegistry-specific functionality.""" + + @register_advantage_estimator("test_decorator") + def decorated_estimator(**kwargs): + return torch.ones_like(kwargs["token_level_rewards"]), torch.ones_like(kwargs["token_level_rewards"]) + + # Test decorator worked + assert "test_decorator" in AdvantageEstimatorRegistry.list_available() + retrieved = AdvantageEstimatorRegistry.get("test_decorator") + assert retrieved == decorated_estimator + + # Test integration with compute_advantages_and_returns + rewards = torch.tensor([[1.0, 2.0, 3.0]]) + response_mask = torch.tensor([[1.0, 1.0, 1.0]]) + index = np.array(["0", "0", "0"]) + + adv, ret = compute_advantages_and_returns( + token_level_rewards=rewards, response_mask=response_mask, index=index, adv_estimator="test_decorator", config={} + ) + + assert torch.allclose(adv, torch.ones_like(rewards)) + assert torch.allclose(ret, torch.ones_like(rewards)) + + # Clean up + AdvantageEstimatorRegistry.unregister("test_decorator") + + +def test_policy_loss_registry_specific(): + """Test PolicyLossRegistry-specific functionality.""" + from omegaconf import DictConfig + + @register_policy_loss("test_policy_decorator") + def decorated_policy_loss(log_probs, old_log_probs, advantages, config, loss_mask=None, rollout_log_probs=None): + return torch.tensor(1.5), 0.3 + + # Test decorator worked + assert "test_policy_decorator" in PolicyLossRegistry.list_available() + retrieved = PolicyLossRegistry.get("test_policy_decorator") + assert retrieved == decorated_policy_loss + + # Test function execution + config = DictConfig({"policy_loss_type": "test_policy_decorator"}) + loss, clip_ratio = retrieved( + log_probs=torch.tensor([[0.1]]), + old_log_probs=torch.tensor([[0.2]]), + advantages=torch.tensor([[1.0]]), + config=config, + ) + assert loss.item() == 1.5 + assert clip_ratio == 0.3 + + # Test error message includes "Policy loss" + with pytest.raises(ValueError, match="Unknown policy loss"): + PolicyLossRegistry.get("non_existent_policy") + + # Clean up + PolicyLossRegistry.unregister("test_policy_decorator") + + +def test_registry_cross_ray_process(): + """Test that registry works with Ray and that functions can be retrieved and called from different processes""" + try: + import ray + from omegaconf import DictConfig + + if not ray.is_initialized(): + ray.init() + + # Create test functions + def test_policy_loss(log_probs, old_log_probs, advantages, config, loss_mask=None): + return torch.tensor(2.0), 0.5 + + def test_policy_loss_2(log_probs, old_log_probs, advantages, config, loss_mask=None): + return torch.tensor(3.0), 0.6 + + def test_advantage_estimator(**kwargs): + rewards = kwargs["token_level_rewards"] + return rewards * 2, rewards * 3 + + # Test basic registration and retrieval + PolicyLossRegistry.register("cross_process_test", test_policy_loss) + AdvantageEstimatorRegistry.register("cross_process_adv_test", test_advantage_estimator) + + # Test Ray integration + @ray.remote + def test_ray_registry_access(): + policy_loss = PolicyLossRegistry.get("cross_process_test") + adv_estimator = AdvantageEstimatorRegistry.get("cross_process_adv_test") + + loss, clip_ratio = policy_loss( + log_probs=torch.tensor([[0.1]]), + old_log_probs=torch.tensor([[0.2]]), + advantages=torch.tensor([[1.0]]), + config=DictConfig({"policy_loss_type": "cross_process_test"}), + ) + + adv, ret = adv_estimator( + token_level_rewards=torch.tensor([[1.0, 2.0]]), + response_mask=torch.tensor([[1.0, 1.0]]), + index=np.array(["0", "0"]), + ) + return loss, clip_ratio, adv, ret + + # Run Ray task + loss, clip_ratio, adv, ret = ray.get(test_ray_registry_access.remote()) + assert loss.item() == 2.0 + assert clip_ratio == 0.5 + assert adv.shape == torch.Size([1, 2]) + assert ret.shape == torch.Size([1, 2]) + + # test that registration works after ray init as well + PolicyLossRegistry.register("cross_process_test_2", test_policy_loss_2) + loss_2, clip_ratio_2 = PolicyLossRegistry.get("cross_process_test_2")( + log_probs=torch.tensor([[0.1]]), + old_log_probs=torch.tensor([[0.2]]), + advantages=torch.tensor([[1.0]]), + config=DictConfig({"policy_loss_type": "cross_process_test_2"}), + ) + assert loss_2.item() == 3.0 + assert clip_ratio_2 == 0.6 + finally: + PolicyLossRegistry.reset() + AdvantageEstimatorRegistry.reset() + + +def test_registry_named_actor_creation(): + """Test that the registry creates named Ray actors and properly serializes functions.""" + try: + import ray + + if not ray.is_initialized(): + ray.init() + + def test_func(**kwargs): + rewards = kwargs["token_level_rewards"] + return rewards * 2, rewards * 3 + + # Register function (should create/use named actor) + AdvantageEstimatorRegistry.register("named_actor_test", test_func) + + # Verify local retrieval works + retrieved = AdvantageEstimatorRegistry.get("named_actor_test") + assert retrieved == test_func + + # Verify named actor exists and contains function + actor = ray.get_actor("advantage_estimator_registry") + assert actor is not None + + available_in_actor = ray.get(actor.list_available.remote()) + assert "named_actor_test" in available_in_actor + + # Verify function serialization/deserialization + serialized_func = ray.get(actor.get.remote("named_actor_test")) + assert serialized_func is not None + + import cloudpickle + + deserialized_func = cloudpickle.loads(serialized_func) + + # Test deserialized function works + test_rewards = torch.tensor([[1.0, 2.0]]) + result = deserialized_func( + token_level_rewards=test_rewards, + response_mask=torch.tensor([[1.0, 1.0]]), + index=np.array(["0", "0"]), + ) + + assert torch.allclose(result[0], test_rewards * 2) + assert torch.allclose(result[1], test_rewards * 3) + + finally: + AdvantageEstimatorRegistry.reset() + + +def test_registry_reset_after_ray_shutdown(): + """ + Test that the registry resets properly after ray is shutdown. + + This mimics when we run multiple unit tests in a row with ray inits and shutdowns. + """ + + def _register_func_and_verify(): + """Register a function and verify it works.""" + + def test_func(**kwargs): + rewards = kwargs["token_level_rewards"] + return rewards * 2, rewards * 3 + + AdvantageEstimatorRegistry.register("named_actor_test", test_func) + retrieved = AdvantageEstimatorRegistry.get("named_actor_test") + assert retrieved == test_func + actor = ray.get_actor("advantage_estimator_registry") + assert actor is not None + + try: + import ray + + # 1. Initialize ray and register function + if not ray.is_initialized(): + ray.init() + _register_func_and_verify() + + # 2. Shutdown ray + ray.shutdown() + + # 3. Initialize ray, reset registry, and register function + ray.init() + AdvantageEstimatorRegistry.reset() + _register_func_and_verify() + + finally: + ray.shutdown() + + +# ── First-turn baseline tests ────────────────────────────────────────────── + + +def test_grpo_first_turn_baseline_with_hints(): + """First-turn baseline: baseline = mean of raw samples only. + + Setup: 1 group with 4 raw (reward=0) + 2 hinted (rewards 0.6 and 0.0). + Raw-only mean = 0.0, so: + - raw advantages = 0 - 0 = 0 + - hinted sample with 0.6 → advantage = 0.6 + - hinted sample with 0.0 → advantage = 0.0 + """ + # 6 samples, 1 token each (outcome reward) + token_level_rewards = torch.tensor([[0.0], [0.0], [0.0], [0.0], [0.6], [0.0]]) + response_mask = torch.ones_like(token_level_rewards) + index = np.array(["p0", "p0", "p0", "p0", "p0", "p0"]) + is_hinted = np.array([False, False, False, False, True, True]) + + adv, ret = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + grpo_norm_by_std=False, + is_hinted=is_hinted, + ) + + expected = torch.tensor([0.0, 0.0, 0.0, 0.0, 0.6, 0.0]).unsqueeze(-1) * response_mask + assert torch.allclose(adv, expected, atol=1e-5), f"Expected {expected}, got {adv}" + assert torch.allclose(adv, ret), "Advantages and returns should be equal" + + +def test_grpo_first_turn_baseline_no_hints_unchanged(): + """When is_hinted is all False, behavior is identical to standard GRPO.""" + token_level_rewards = torch.tensor( + [ + [1.0, 2.0, 3.0], # sum = 6.0, group 0 + [1.0, 1.0, 1.0], # sum = 3.0, group 0 + [3.0, 3.0, 3.0], # sum = 9.0, group 1 + [4.0, 4.0, 4.0], # sum = 12.0, group 1 + ] + ) + response_mask = torch.ones_like(token_level_rewards) + index = np.array([0, 0, 1, 1]) + is_hinted = np.array([False, False, False, False]) + + adv_with, _ = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + grpo_norm_by_std=False, + is_hinted=is_hinted, + ) + + adv_without, _ = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards.clone(), + response_mask=response_mask, + index=index, + grpo_norm_by_std=False, + ) + + assert torch.allclose( + adv_with, adv_without, atol=1e-5 + ), f"All-False is_hinted should match no is_hinted. Got {adv_with} vs {adv_without}" + + +def test_grpo_first_turn_baseline_mixed_groups(): + """Two groups: group A has hints, group B has no hints. + + Group A: 2 raw (rewards 0, 0) + 1 hinted (reward 0.8) + raw mean = 0.0, advantages: [0, 0, 0.8] + Group B: 2 raw (rewards 3, 5), no hints + raw mean = 4.0, advantages: [-1, 1] + """ + token_level_rewards = torch.tensor([[0.0], [0.0], [0.8], [3.0], [5.0]]) + response_mask = torch.ones_like(token_level_rewards) + index = np.array(["a", "a", "a", "b", "b"]) + is_hinted = np.array([False, False, True, False, False]) + + adv, _ = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + grpo_norm_by_std=False, + is_hinted=is_hinted, + ) + + expected = torch.tensor([0.0, 0.0, 0.8, -1.0, 1.0]).unsqueeze(-1) * response_mask + assert torch.allclose(adv, expected, atol=1e-5), f"Expected {expected}, got {adv}" + + +def test_grpo_first_turn_baseline_with_std_norm(): + """First-turn baseline with std normalization from raw samples only. + + Group: 3 raw (rewards 2, 4, 6) + 1 hinted (reward 10). + Raw mean = 4.0, raw std = std([2, 4, 6]) = 2.0. + Advantages (before mask): + raw: (2-4)/2 = -1, (4-4)/2 = 0, (6-4)/2 = 1 + hinted: (10-4)/2 = 3 + """ + token_level_rewards = torch.tensor([[2.0], [4.0], [6.0], [10.0]]) + response_mask = torch.ones_like(token_level_rewards) + index = np.array(["g", "g", "g", "g"]) + is_hinted = np.array([False, False, False, True]) + + adv, _ = compute_grpo_outcome_advantage( + token_level_rewards=token_level_rewards, + response_mask=response_mask, + index=index, + grpo_norm_by_std=True, + is_hinted=is_hinted, + ) + + raw_scores = torch.tensor([2.0, 4.0, 6.0]) + raw_mean = raw_scores.mean() # 4.0 + raw_std = raw_scores.std() # 2.0 + epsilon = 1e-6 + + expected_vals = [] + for r in [2.0, 4.0, 6.0, 10.0]: + expected_vals.append((r - raw_mean.item()) / (raw_std.item() + epsilon)) + expected = torch.tensor(expected_vals).unsqueeze(-1) * response_mask + + assert torch.allclose(adv, expected, atol=1e-4), f"Expected {expected}, got {adv}" diff --git a/skyrl-train/tests/cpu/utils/test_torch_utils.py b/skyrl-train/tests/cpu/utils/test_torch_utils.py new file mode 100644 index 0000000000..6c51fd33e9 --- /dev/null +++ b/skyrl-train/tests/cpu/utils/test_torch_utils.py @@ -0,0 +1,114 @@ +from skyrl_train.utils.torch_utils import chunked_cross_entropy_from_log_probs, chunked_entropy_from_logits +import torch +import pytest +import math + + +def test_chunked_cross_entropy_from_logprobs(): + # Define a small log-probability tensor (batch_size=2, seqlen=3, vocab_size=4) + logits = [ + [ + [1.0, 2.0, 3.0, 4.0], # example 1, token 1 + [1.0, 0.0, 0.0, 0.0], # token 2 + [0.0, 0.0, 0.0, 0.0], + ], # token 3 (uniform) + [ + [0.0, 0.0, 0.0, 0.0], # example 2, token 1 (uniform) + [1.0, 2.0, 3.0, 4.0], # token 2 + [4.0, 3.0, 2.0, 1.0], + ], # token 3 + ] + logits = torch.tensor(logits, dtype=torch.float32) + logprobs_BSV = torch.log_softmax(logits, dim=-1) # shape: (2, 3, 4) + + result_BS = chunked_cross_entropy_from_log_probs(logprobs_BSV) + + # For uniform logprobs (all zeros before softmax), entropy should be log(vocab_size) = log(4) + expected_uniform_entropy = math.log(4.0) # ≈ 1.386 + + assert torch.allclose(result_BS[0, 2], torch.tensor(expected_uniform_entropy), atol=1e-4) + assert torch.allclose(result_BS[1, 0], torch.tensor(expected_uniform_entropy), atol=1e-4) + + +def test_chunked_entropy_from_logits(): + # Define a small log-probability tensor (batch_size=2, seqlen=3, vocab_size=4) + logits = [ + [ + [1.0, 2.0, 3.0, 4.0], # example 1, token 1 + [1.0, 0.0, 0.0, 0.0], # token 2 + [0.0, 0.0, 0.0, 0.0], + ], # token 3 (uniform) + [ + [0.0, 0.0, 0.0, 0.0], # example 2, token 1 (uniform) + [1.0, 2.0, 3.0, 4.0], # token 2 + [4.0, 3.0, 2.0, 1.0], + ], # token 3 + ] + logits = torch.tensor(logits, dtype=torch.float32) + + result_BS = chunked_entropy_from_logits(logits) + + # For uniform logprobs (all zeros before softmax), entropy should be log(vocab_size) = log(4) + expected_uniform_entropy = math.log(4.0) # ≈ 1.386 + + assert torch.allclose(result_BS[0, 2], torch.tensor(expected_uniform_entropy), atol=1e-4) + assert torch.allclose(result_BS[1, 0], torch.tensor(expected_uniform_entropy), atol=1e-4) + + +def test_chunked_entropy_from_logits_with_attention_mask(): + """Test that attention mask correctly zeros out entropy for padded positions.""" + # Define logits (batch_size=2, seqlen=4, vocab_size=4) + logits = torch.randn(2, 4, 4, dtype=torch.float32) + + # Create attention mask: 1 for valid tokens, 0 for padding + # First sequence: all valid + # Second sequence: first 2 valid, last 2 padded + attention_mask = torch.tensor( + [ + [1, 1, 1, 1], # all valid + [1, 1, 0, 0], # last 2 are padding + ], + dtype=torch.float32, + ) + + # Calculate entropy with attention mask + result_with_mask = chunked_entropy_from_logits(logits, attention_mask=attention_mask) + + # Verify that padded positions have zero entropy + assert result_with_mask[1, 2] == 0.0, "Padded position should have zero entropy" + assert result_with_mask[1, 3] == 0.0, "Padded position should have zero entropy" + + # Verify that valid positions have non-zero entropy + assert result_with_mask[0, 0] > 0.0, "Valid position should have non-zero entropy" + assert result_with_mask[1, 0] > 0.0, "Valid position should have non-zero entropy" + + # Calculate entropy without mask for the valid positions + result_without_mask = chunked_entropy_from_logits(logits) + + # Entropy values for valid positions should be the same with or without mask + assert torch.allclose(result_with_mask[0, :], result_without_mask[0, :], atol=1e-6) + assert torch.allclose(result_with_mask[1, :2], result_without_mask[1, :2], atol=1e-6) + + +def test_chunked_entropy_from_logits_attention_mask_shape_validation(): + """Test that attention mask shape validation works correctly.""" + logits = torch.randn(2, 3, 4, dtype=torch.float32) + + # Wrong shape: different batch size + wrong_mask_batch = torch.ones(3, 3, dtype=torch.float32) + + # For wrong batch size + with pytest.raises(ValueError, match="does not match logits shape"): + chunked_entropy_from_logits(logits, attention_mask=wrong_mask_batch) + + # Wrong shape: different sequence length + wrong_mask_seqlen = torch.ones(2, 4, dtype=torch.float32) + + # For wrong sequence length + with pytest.raises(ValueError, match="does not match logits shape"): + chunked_entropy_from_logits(logits, attention_mask=wrong_mask_seqlen) + + # Correct shape should work + correct_mask = torch.ones(2, 3, dtype=torch.float32) + result = chunked_entropy_from_logits(logits, attention_mask=correct_mask) + assert result.shape == (2, 3) diff --git a/skyrl-train/tests/cpu/weight_sync/__init__.py b/skyrl-train/tests/cpu/weight_sync/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/tests/cpu/weight_sync/test_remote_weight_loader.py b/skyrl-train/tests/cpu/weight_sync/test_remote_weight_loader.py new file mode 100644 index 0000000000..b8b7dea87f --- /dev/null +++ b/skyrl-train/tests/cpu/weight_sync/test_remote_weight_loader.py @@ -0,0 +1,209 @@ +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from skyrl_train.inference_engines.remote_inference_engine import RemoteWeightLoader +from skyrl_train.weight_sync import BroadcastInitInfo, BroadcastWeightUpdateRequest + + +class AsyncContextManagerMock: + """Helper to mock async context managers (for `async with ... as ...`).""" + + def __init__(self, return_value): + self.return_value = return_value + + async def __aenter__(self): + return self.return_value + + async def __aexit__(self, *args): + pass + + +def create_mock_session(mock_response): + """Create a mock aiohttp.ClientSession with proper async behavior. + + Handles both: + - `async with session.post(...) as resp:` (context manager form) + - `resp = await session.post(...)` (direct await form) + """ + mock_session = MagicMock() + + # Create a mock that works both as context manager and as awaitable + class MockPostReturn: + def __init__(self, response): + self.response = response + + async def __aenter__(self): + return self.response + + async def __aexit__(self, *args): + pass + + def __await__(self): + async def _await(): + return self.response + + return _await().__await__() + + mock_session.post = MagicMock(return_value=MockPostReturn(mock_response)) + return mock_session + + +class TestRemoteWeightLoader: + """Tests for RemoteWeightLoader class.""" + + @staticmethod + def make_broadcast_init_info(): + """Create a BroadcastInitInfo for testing.""" + return BroadcastInitInfo( + master_addr="127.0.0.1", + master_port=29500, + rank_offset=1, + world_size=2, + group_name="test_group", + backend="nccl", + model_dtype_str="torch.bfloat16", + override_existing_receiver=True, + ) + + @pytest.mark.parametrize( + "url,backend", + [ + ("http://localhost:8000", "vllm"), + ("http://localhost:9000", "sglang"), + ], + ) + def test_init(self, url, backend): + """Test initialization stores URL and backend correctly.""" + loader = RemoteWeightLoader(url=url, engine_backend=backend) + + assert loader._url == url + assert loader._engine_backend == backend + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "backend,expected_endpoint", + [ + ("vllm", "/init_weight_update_communicator"), + ("sglang", "/init_weights_update_group"), + ], + ) + async def test_init_communicator(self, backend, expected_endpoint): + """Test init_communicator calls correct endpoint for each backend.""" + url = "http://localhost:8000" + loader = RemoteWeightLoader(url=url, engine_backend=backend) + init_info = self.make_broadcast_init_info() + + mock_response = MagicMock() + mock_response.json = AsyncMock(return_value={"success": True}) + + with patch("aiohttp.ClientSession") as mock_session_class: + mock_session = create_mock_session(mock_response) + mock_session_class.return_value = AsyncContextManagerMock(mock_session) + + result = await loader.init_communicator(init_info) + + # Verify correct endpoint called + mock_session.post.assert_called_once() + call_args = mock_session.post.call_args + assert call_args[0][0] == f"{url}{expected_endpoint}" + + # Verify payload matches init_info fields + payload = call_args[1]["json"] + + if backend == "vllm": + # vLLM uses asdict(init_info) - all fields with original names + from dataclasses import asdict + + assert payload == asdict(init_info) + else: + # SGLang uses only the fields SGLang expects + assert payload["master_address"] == init_info.master_addr + assert payload["master_port"] == init_info.master_port + assert payload["rank_offset"] == init_info.rank_offset + assert payload["world_size"] == init_info.world_size + assert payload["group_name"] == init_info.group_name + assert payload["backend"] == init_info.backend + assert "model_dtype_str" not in payload + assert "override_existing_receiver" not in payload + + assert result == {"success": True} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "backend,expected_endpoint", + [ + ("vllm", "/update_weights"), + ("sglang", "/update_weights_from_distributed"), + ], + ) + async def test_load_weights(self, backend, expected_endpoint): + """Test load_weights calls correct endpoint for each backend.""" + url = "http://localhost:8000" + loader = RemoteWeightLoader(url=url, engine_backend=backend) + + mock_response = MagicMock() + mock_response.json = AsyncMock(return_value={"success": True}) + + request = BroadcastWeightUpdateRequest( + names=["model.layer.weight"], + dtypes=["bfloat16"], + shapes=[[4096, 4096]], + ) + + with patch("aiohttp.ClientSession") as mock_session_class: + mock_session = create_mock_session(mock_response) + mock_session_class.return_value = AsyncContextManagerMock(mock_session) + + result = await loader.load_weights(request) + + # Verify correct endpoint called + call_args = mock_session.post.call_args + assert call_args[0][0] == f"{url}{expected_endpoint}" + + # Verify payload + payload = call_args[1]["json"] + if backend == "vllm": + # vLLM uses asdict(request) - plural field names + from dataclasses import asdict + + assert payload == asdict(request) + else: + # SGLang uses singular field names + assert payload["name"] == "model.layer.weight" + assert payload["dtype"] == "bfloat16" + assert payload["shape"] == [4096, 4096] + + assert result == {"success": True} + + @pytest.mark.asyncio + async def test_load_weights_invalid_backend(self): + """Test load_weights raises ValueError for unknown backend.""" + loader = RemoteWeightLoader(url="http://localhost:8000", engine_backend="unknown") + + request = BroadcastWeightUpdateRequest( + names=["model.layer.weight"], + dtypes=["bfloat16"], + shapes=[[4096, 4096]], + ) + + with pytest.raises(ValueError, match="Invalid engine backend"): + await loader.load_weights(request) + + @pytest.mark.asyncio + async def test_destroy_group(self): + """Test destroy_group calls correct endpoint.""" + loader = RemoteWeightLoader(url="http://localhost:8000", engine_backend="vllm") + + mock_response = MagicMock() + mock_response.json = AsyncMock(return_value={"success": True}) + + with patch("aiohttp.ClientSession") as mock_session_class: + mock_session = create_mock_session(mock_response) + mock_session_class.return_value = AsyncContextManagerMock(mock_session) + + result = await loader.destroy_group() + + # Verify correct endpoint called + call_args = mock_session.post.call_args + assert call_args[0][0] == "http://localhost:8000/destroy_weights_update_group" + + assert result == {"success": True} diff --git a/skyrl-train/tests/cpu/weight_sync/test_transfer_strategies.py b/skyrl-train/tests/cpu/weight_sync/test_transfer_strategies.py new file mode 100644 index 0000000000..2d29a6af35 --- /dev/null +++ b/skyrl-train/tests/cpu/weight_sync/test_transfer_strategies.py @@ -0,0 +1,216 @@ +import pytest +from skyrl_train.config.utils import get_default_config +from skyrl_train.weight_sync import ( + get_transfer_strategy_cls, + BroadcastTransferStrategy, + CudaIpcTransferStrategy, + BroadcastInitInfo, + CudaIpcInitInfo, + BroadcastWeightUpdateRequest, + CudaIpcWeightUpdateRequest, + LoraLoadRequest, +) + + +class TestGetTransferStrategyCls: + """Tests for get_transfer_strategy_cls function.""" + + def _make_cfg(self, weight_sync_backend: str, colocate_all: bool): + """Create a config object.""" + cfg = get_default_config() + cfg.generator.weight_sync_backend = weight_sync_backend + cfg.trainer.placement.colocate_all = colocate_all + return cfg + + @pytest.mark.parametrize( + "backend,colocate_all,expected_strategy", + [ + ("nccl", True, CudaIpcTransferStrategy), + ("nccl", False, BroadcastTransferStrategy), + ("gloo", True, BroadcastTransferStrategy), + ("gloo", False, BroadcastTransferStrategy), + ], + ) + def test_returns_correct_strategy(self, backend, colocate_all, expected_strategy): + """Should return correct strategy based on backend and colocate_all.""" + cfg = self._make_cfg(weight_sync_backend=backend, colocate_all=colocate_all) + assert get_transfer_strategy_cls(cfg) is expected_strategy + + +class TestCreateInitInfo: + """Tests for create_init_info static methods.""" + + def _make_cfg( + self, + weight_sync_backend: str = "nccl", + model_dtype: str = "torch.bfloat16", + num_inference_engines: int = 1, + tensor_parallel_size: int = 1, + pipeline_parallel_size: int = 1, + data_parallel_size: int = 1, + override_existing_update_group: str = "enable", + ): + """Create a config object for create_init_info.""" + cfg = get_default_config() + cfg.generator.weight_sync_backend = weight_sync_backend + cfg.generator.model_dtype = model_dtype + cfg.generator.num_inference_engines = num_inference_engines + cfg.generator.inference_engine_tensor_parallel_size = tensor_parallel_size + cfg.generator.inference_engine_pipeline_parallel_size = pipeline_parallel_size + cfg.generator.inference_engine_data_parallel_size = data_parallel_size + cfg.generator.override_existing_update_group = override_existing_update_group + return cfg + + def test_cuda_ipc_create_init_info(self): + """CudaIpcTransferStrategy.create_init_info should create CudaIpcInitInfo with model_dtype_str.""" + cfg = self._make_cfg(model_dtype="torch.float32") + init_info = CudaIpcTransferStrategy.create_init_info(cfg) + + assert isinstance(init_info, CudaIpcInitInfo) + assert init_info.model_dtype_str == "torch.float32" + + def test_broadcast_create_init_info(self, monkeypatch): + """BroadcastTransferStrategy.create_init_info should create BroadcastInitInfo with correct fields.""" + # Mock ray to avoid actual network operations + import skyrl_train.weight_sync.broadcast_strategy as broadcast_module + + monkeypatch.setattr(broadcast_module.ray._private.services, "get_node_ip_address", lambda: "192.168.1.1") + + cfg = self._make_cfg( + weight_sync_backend="gloo", + model_dtype="torch.bfloat16", + num_inference_engines=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + data_parallel_size=1, + override_existing_update_group="enable", + ) + init_info = BroadcastTransferStrategy.create_init_info(cfg) + + assert isinstance(init_info, BroadcastInitInfo) + assert init_info.master_addr == "192.168.1.1" + assert isinstance(init_info.master_port, int) + assert init_info.rank_offset == 1 + # world_size = num_engines * tp * pp * dp + 1 = 2 * 2 * 1 * 1 + 1 = 5 + assert init_info.world_size == 5 + assert init_info.group_name == "skyrl" + assert init_info.backend == "gloo" + assert init_info.model_dtype_str == "torch.bfloat16" + assert init_info.override_existing_receiver is True + + def test_broadcast_create_init_info_override_existing_receiver_disabled(self, monkeypatch): + """BroadcastTransferStrategy.create_init_info should set override_existing_receiver=False when config is 'disable'.""" + import skyrl_train.weight_sync.broadcast_strategy as broadcast_module + + monkeypatch.setattr(broadcast_module.ray._private.services, "get_node_ip_address", lambda: "192.168.1.1") + + cfg = self._make_cfg(override_existing_update_group="disable") + init_info = BroadcastTransferStrategy.create_init_info(cfg) + + assert init_info.override_existing_receiver is False + + +class TestBroadcastWeightUpdateRequest: + """Tests for BroadcastWeightUpdateRequest.""" + + def test_len(self): + """__len__ should return number of weights.""" + request = BroadcastWeightUpdateRequest( + names=["layer1.weight", "layer2.weight"], + dtypes=["bfloat16", "bfloat16"], + shapes=[[4096, 4096], [1024]], + ) + assert len(request) == 2 + + def test_mismatched_lengths_raises(self): + """Mismatched lengths should raise ValueError.""" + with pytest.raises(ValueError, match="must have the same length"): + BroadcastWeightUpdateRequest( + names=["layer1.weight", "layer2.weight"], + dtypes=["bfloat16"], + shapes=[[4096, 4096]], + ) + + +class TestCudaIpcWeightUpdateRequest: + """Tests for CudaIpcWeightUpdateRequest.""" + + def test_serialize_roundtrip(self): + """Serialization/deserialization roundtrip preserves data.""" + request = CudaIpcWeightUpdateRequest( + names=["model.layer.weight"], + dtypes=["bfloat16"], + shapes=[[4096, 4096]], + sizes=[4096 * 4096], + ipc_handles={"gpu-uuid": "test_handle"}, + ) + + data = request.serialize() + result = CudaIpcWeightUpdateRequest.deserialize(data) + + assert result.names == request.names + assert result.dtypes == request.dtypes + assert result.shapes == request.shapes + assert result.sizes == request.sizes + assert result.ipc_handles == request.ipc_handles + + def test_serialize_roundtrip_multiple_weights(self): + """Roundtrip with multiple weights.""" + request = CudaIpcWeightUpdateRequest( + names=["layer1.weight", "layer2.weight", "layer3.bias"], + dtypes=["bfloat16", "bfloat16", "bfloat16"], + shapes=[[4096, 4096], [4096, 1024], [1024]], + sizes=[4096 * 4096, 4096 * 1024, 1024], + ipc_handles={"gpu-0": "handle1"}, + ) + + data = request.serialize() + result = CudaIpcWeightUpdateRequest.deserialize(data) + + assert result.names == request.names + assert result.dtypes == request.dtypes + assert result.shapes == request.shapes + assert result.sizes == request.sizes + assert result.ipc_handles == request.ipc_handles + + def test_deserialize_missing_end_marker(self): + """Missing end marker raises ValueError.""" + + invalid_data = b"some_invalid_data" + + with pytest.raises(ValueError, match="End marker not found"): + CudaIpcWeightUpdateRequest.deserialize(invalid_data) + + def test_deserialize_invalid_data(self): + """Invalid base64/pickle data raises ValueError.""" + from skyrl_train.weight_sync.cuda_ipc_strategy import _IPC_REQUEST_END_MARKER + + invalid_data = b"not_valid_base64!!!" + _IPC_REQUEST_END_MARKER + + with pytest.raises(ValueError, match="Failed to deserialize"): + CudaIpcWeightUpdateRequest.deserialize(invalid_data) + + def test_serialize_aligned_to_4_bytes(self): + """Serialized data is 4-byte aligned.""" + request = CudaIpcWeightUpdateRequest( + names=["test"], + dtypes=["bfloat16"], + shapes=[[10]], + sizes=[10], + ipc_handles={}, + ) + data = request.serialize() + + assert len(data) % 4 == 0 + + +class TestLoraLoadRequest: + """Tests for LoraLoadRequest.""" + + def test_lora_path(self): + """lora_path should be stored correctly with empty defaults for base fields.""" + request = LoraLoadRequest(lora_path="/path/to/lora") + assert request.lora_path == "/path/to/lora" + assert request.names == [] + assert request.dtypes == [] + assert request.shapes == [] diff --git a/skyrl-train/tests/cpu/weight_sync/test_weight_chunk.py b/skyrl-train/tests/cpu/weight_sync/test_weight_chunk.py new file mode 100644 index 0000000000..a20da79eb0 --- /dev/null +++ b/skyrl-train/tests/cpu/weight_sync/test_weight_chunk.py @@ -0,0 +1,96 @@ +import pytest +import torch +from skyrl_train.weight_sync import WeightChunk + + +class TestWeightChunk: + """Tests for WeightChunk dataclass.""" + + def test_initialization_basic(self): + """Test basic initialization with valid data.""" + names = ["layer1.weight", "layer1.bias"] + dtypes = ["torch.float32", "torch.float32"] + shapes = [[4, 3], [4]] + tensors = [torch.randn(4, 3), torch.randn(4)] + + chunk = WeightChunk(names=names, dtypes=dtypes, shapes=shapes, tensors=tensors) + + assert chunk.names == names + assert chunk.dtypes == dtypes + assert chunk.shapes == shapes + assert len(chunk.tensors) == 2 + + def test_validation_length_mismatch(self): + """Test that validation catches length mismatches.""" + with pytest.raises(ValueError, match="All lists must have the same length"): + WeightChunk( + names=["layer1.weight", "layer1.bias"], + dtypes=["torch.float32"], # Wrong length + shapes=[[4, 3], [4]], + tensors=[torch.randn(4, 3), torch.randn(4)], + ) + + with pytest.raises(ValueError, match="All lists must have the same length"): + WeightChunk( + names=["layer1.weight"], + dtypes=["torch.float32"], + shapes=[[4, 3], [4]], # Wrong length + tensors=[torch.randn(4, 3)], + ) + + def test_len(self): + """Test __len__ returns number of parameters.""" + chunk = WeightChunk( + names=["layer1.weight", "layer1.bias", "layer2.weight"], + dtypes=["torch.float32"] * 3, + shapes=[[4, 3], [4], [3, 2]], + tensors=[torch.randn(4, 3), torch.randn(4), torch.randn(3, 2)], + ) + + assert len(chunk) == 3 + + def test_total_numel(self): + """Test total_numel cached property.""" + tensors = [ + torch.randn(4, 3), # 12 elements + torch.randn(4), # 4 elements + torch.randn(3, 2), # 6 elements + ] + chunk = WeightChunk( + names=["layer1.weight", "layer1.bias", "layer2.weight"], + dtypes=["torch.float32"] * 3, + shapes=[[4, 3], [4], [3, 2]], + tensors=tensors, + ) + + assert chunk.total_numel == 12 + 4 + 6 + + def test_total_size_bytes(self): + """Test total_size_bytes cached property.""" + tensors = [ + torch.randn(4, 3, dtype=torch.float32), # 12 * 4 = 48 bytes + torch.randn(4, dtype=torch.float32), # 4 * 4 = 16 bytes + ] + chunk = WeightChunk( + names=["layer1.weight", "layer1.bias"], + dtypes=["torch.float32"] * 2, + shapes=[[4, 3], [4]], + tensors=tensors, + ) + + assert chunk.total_size_bytes == 48 + 16 + + def test_total_size_bytes_mixed_dtypes(self): + """Test total_size_bytes with mixed dtypes.""" + tensors = [ + torch.randn(10, dtype=torch.float32), # 10 * 4 = 40 bytes + torch.randn(10, dtype=torch.bfloat16), # 10 * 2 = 20 bytes + ] + chunk = WeightChunk( + names=["layer1.weight", "layer1.bias"], + dtypes=["torch.float32", "torch.bfloat16"], + shapes=[[10], [10]], + tensors=tensors, + ) + + assert chunk.total_size_bytes == 40 + 20 diff --git a/skyrl-train/tests/cpu/weight_sync/test_weight_extractor_utils.py b/skyrl-train/tests/cpu/weight_sync/test_weight_extractor_utils.py new file mode 100644 index 0000000000..d31644b5ec --- /dev/null +++ b/skyrl-train/tests/cpu/weight_sync/test_weight_extractor_utils.py @@ -0,0 +1,228 @@ +import pytest +import torch +from skyrl_train.weight_sync.weight_extractor_utils import yield_module_grouped_chunks +from skyrl_train.weight_sync import WeightChunk + + +class TestModuleGrouping: + """Tests for yield_module_grouped_chunks utility function.""" + + def test_basic_module_grouping(self): + """Test that parameters are grouped by module correctly.""" + params = { + "model.layers.0.self_attn.q_proj.weight": torch.randn(768, 768), + "model.layers.0.self_attn.k_proj.weight": torch.randn(768, 768), + "model.layers.0.self_attn.v_proj.weight": torch.randn(768, 768), + "model.layers.0.mlp.fc1.weight": torch.randn(3072, 768), + "model.layers.0.mlp.fc2.weight": torch.randn(768, 3072), + } + + def gather_tensor(param): + return param + + def get_shape(name, param, tensor): + return list(tensor.shape) + + chunks = list( + yield_module_grouped_chunks( + params=params, + dtype=torch.float32, + gather_tensor_fn=gather_tensor, + get_shape_fn=get_shape, + batch_size_threshold_gb=0.0, + ) + ) + + # Should have 2 modules: self_attn and mlp + assert len(chunks) == 2 + + # Find self_attn and mlp chunks + self_attn_chunks = [c for c in chunks if "self_attn" in c.names[0]] + mlp_chunks = [c for c in chunks if "mlp" in c.names[0]] + + assert len(self_attn_chunks) == 1 + assert len(mlp_chunks) == 1 + + # Check self_attn has 3 params (q, k, v) + self_attn_chunk = self_attn_chunks[0] + assert len(self_attn_chunk) == 3 + assert "q_proj" in self_attn_chunk.names[0] + assert "k_proj" in self_attn_chunk.names[1] + assert "v_proj" in self_attn_chunk.names[2] + + # Check mlp has 2 params (fc1, fc2) + mlp_chunk = mlp_chunks[0] + assert len(mlp_chunk) == 2 + assert "fc1" in mlp_chunk.names[0] + assert "fc2" in mlp_chunk.names[1] + + @pytest.mark.parametrize( + "params_config,threshold_gb,expected_chunks,expected_params_per_chunk,description", + [ + # No batching: each module in separate chunk + ( + {"model.layers.{}.attn.weight": [(i, 100) for i in range(4)]}, + 0.0, + 4, + [1, 1, 1, 1], + "no batching", + ), + # Small threshold (~1000 bytes): batch 2 modules at a time + # Each param is 100 elements * 4 bytes = 400 bytes + # First batch: 400 + 400 = 800 < 1000, Second batch: 400 + 400 = 800 < 1000 + ( + {"model.layers.{}.attn.weight": [(i, 100) for i in range(4)]}, + 0.000001, + 2, + [2, 2], + "small threshold batches 2 modules", + ), + # Large threshold (1 GB): batch all modules together + ( + {"model.layers.{}.attn.weight": [(i, 100) for i in range(4)]}, + 1.0, + 1, + [4], + "large threshold batches all", + ), + # Module boundaries: 3 params (1200 bytes) exceeds threshold (600 bytes) by 2x + ( + {"model.layer.attn.proj{}.weight": [(i, 100) for i in range(3)]}, + 0.0000006, + 1, + [3], + "module boundary: 3 params exceed threshold", + ), + # Module boundaries: 2 params (800 bytes) exceeds threshold (400 bytes) by 2x + ( + {"model.layer.attn.proj{}.weight": [(i, 100) for i in range(2)]}, + 0.0000004, + 1, + [2], + "module boundary: 2 params exceed threshold", + ), + # Module boundaries: 5 params (2000 bytes) exceeds threshold (500 bytes) by 4x + ( + {"model.layer.attn.proj{}.weight": [(i, 100) for i in range(5)]}, + 0.0000005, + 1, + [5], + "module boundary: 5 params exceed threshold", + ), + ], + ) + def test_batching_with_different_thresholds( + self, params_config, threshold_gb, expected_chunks, expected_params_per_chunk, description + ): + """Test batching behavior with various threshold values and module structures.""" + # Build params dict from config + # params_config is {"template": [(idx, size), ...]} + params = {} + for template, indices_and_sizes in params_config.items(): + for idx, size in indices_and_sizes: + param_name = template.format(idx) + params[param_name] = torch.randn(size, dtype=torch.float32) + + chunks = list( + yield_module_grouped_chunks( + params=params, + dtype=torch.float32, + gather_tensor_fn=lambda p: p, + get_shape_fn=lambda n, p, t: list(t.shape), + batch_size_threshold_gb=threshold_gb, + ) + ) + + assert len(chunks) == expected_chunks + for i, expected_params in enumerate(expected_params_per_chunk): + assert len(chunks[i]) == expected_params + + def test_gather_tensor_callback(self): + """Test that gather_tensor_fn callback is called correctly.""" + original = torch.randn(10, 10, dtype=torch.float32) + params = {"model.layer.weight": original} + + def gather_tensor(param): + # Simulate gathering by adding 1.0 + return param + 1.0 + + chunks = list( + yield_module_grouped_chunks( + params=params, + dtype=torch.bfloat16, + gather_tensor_fn=gather_tensor, + get_shape_fn=lambda n, p, t: list(t.shape), + ) + ) + + assert len(chunks) == 1 + tensor = chunks[0].tensors[0] + # Should be bfloat16 (dtype cast in utils) and have +1.0 applied (from gather) + assert tensor.dtype == torch.bfloat16 + expected = (original + 1.0).to(torch.bfloat16) + assert torch.allclose(tensor, expected) + + def test_get_shape_callback(self): + """Test that get_shape_fn callback is called correctly.""" + params = {"model.layer.weight": torch.randn(10, 20)} + + def get_shape(name, param, tensor): + # Return custom shape + return [999, 888] + + chunks = list( + yield_module_grouped_chunks( + params=params, + dtype=torch.float32, + gather_tensor_fn=lambda p: p, + get_shape_fn=get_shape, + ) + ) + + assert len(chunks) == 1 + assert chunks[0].shapes[0] == [999, 888] + + def test_empty_params(self): + """Test with empty params dict.""" + params = {} + + chunks = list( + yield_module_grouped_chunks( + params=params, + dtype=torch.float32, + gather_tensor_fn=lambda p: p, + get_shape_fn=lambda n, p, t: list(t.shape), + ) + ) + + assert len(chunks) == 0 + + def test_chunk_properties(self): + """Test that returned chunks have correct WeightChunk properties.""" + params = { + "model.layer.attn.weight": torch.randn(10, 10), + "model.layer.attn.bias": torch.randn(10), + } + + chunks = list( + yield_module_grouped_chunks( + params=params, + dtype=torch.float32, + gather_tensor_fn=lambda p: p, + get_shape_fn=lambda n, p, t: list(t.shape), + ) + ) + + assert len(chunks) == 1 + chunk = chunks[0] + + # Check all required fields are present + assert isinstance(chunk, WeightChunk) + assert len(chunk.names) == 2 + assert len(chunk.dtypes) == 2 + assert len(chunk.shapes) == 2 + assert len(chunk.tensors) == 2 + + # Check total_numel + expected_numel = 10 * 10 + 10 + assert chunk.total_numel == expected_numel diff --git a/skyrl-train/tests/gpu/__init__.py b/skyrl-train/tests/gpu/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/tests/gpu/conftest.py b/skyrl-train/tests/gpu/conftest.py new file mode 100644 index 0000000000..fd8766fe5e --- /dev/null +++ b/skyrl-train/tests/gpu/conftest.py @@ -0,0 +1,13 @@ +import pytest +import ray +from tests.gpu.utils import ray_init_for_tests + + +@pytest.fixture +def ray_init_fixture(): + if ray.is_initialized(): + ray.shutdown() + ray_init_for_tests() + yield + # call ray shutdown after a test regardless + ray.shutdown() diff --git a/skyrl-train/tests/gpu/gpu_ci/__init__.py b/skyrl-train/tests/gpu/gpu_ci/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-train/tests/gpu/gpu_ci/conftest.py b/skyrl-train/tests/gpu/gpu_ci/conftest.py new file mode 100644 index 0000000000..e6f35b11cb --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/conftest.py @@ -0,0 +1,40 @@ +import pytest +import ray +from loguru import logger +from functools import lru_cache +from skyrl_train.utils.utils import peer_access_supported + + +@lru_cache(5) +def log_once(msg): + logger.info(msg) + return None + + +@pytest.fixture +def ray_init_fixture(): + if ray.is_initialized(): + ray.shutdown() + + # TODO (team): maybe we should use the default config and use prepare_runtime_environment in some way + env_vars = {"VLLM_USE_V1": "1", "VLLM_ENABLE_V1_MULTIPROCESSING": "0", "VLLM_ALLOW_INSECURE_SERIALIZATION": "1"} + + if not peer_access_supported(max_num_gpus_per_node=2): + log_once("Disabling NCCL P2P for CI environment") + env_vars.update( + { + "NCCL_P2P_DISABLE": "1", + "NCCL_SHM_DISABLE": "1", + } + ) + + # needed for megatron tests + env_vars["CUDA_DEVICE_MAX_CONNECTIONS"] = "1" + env_vars["NVTE_FUSED_ATTN"] = "0" + + logger.info(f"Initializing Ray with environment variables: {env_vars}") + ray.init(runtime_env={"env_vars": env_vars}) + + yield + # call ray shutdown after a test regardless + ray.shutdown() diff --git a/skyrl-train/tests/gpu/gpu_ci/distributed/test_fsdp_strategy.py b/skyrl-train/tests/gpu/gpu_ci/distributed/test_fsdp_strategy.py new file mode 100644 index 0000000000..72d87e505d --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/distributed/test_fsdp_strategy.py @@ -0,0 +1,65 @@ +""" +uv run --isolated --extra dev pytest tests/gpu/gpu_ci/distributed/test_fsdp_strategy.py +""" + +from skyrl_train.model_wrapper import HFModelWrapper +import os +import torch.distributed as dist +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.distributed_c10d import init_process_group +from skyrl_train.distributed.fsdp_strategy import FSDPStrategy +from skyrl_train.config.utils import get_default_config +from skyrl_train.utils.trainer_utils import get_rope_scaling_config, get_rope_theta_config +from skyrl_train.utils.utils import get_free_port + +MODEL_NAME = "llamafactory/tiny-random-Llama-3" + + +def test_fsdp1_wrap_policy(): + cfg = get_default_config() + cfg.trainer.policy.model.path = MODEL_NAME + cfg.trainer.strategy = "fsdp" + os.environ["RANK"] = "0" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(get_free_port()) + init_process_group(backend="nccl", rank=0, world_size=1) + strategy = FSDPStrategy( + fsdp_config=cfg.trainer.policy.fsdp_config, + optimizer_config=None, + model_config=cfg.trainer.policy.model, + fsdp_strategy=cfg.trainer.strategy, + seed=cfg.trainer.seed, + micro_train_batch_size_per_gpu=cfg.trainer.micro_train_batch_size_per_gpu, + num_training_steps=None, + ) + strategy.setup_distributed() + + wrapped_model = HFModelWrapper( + MODEL_NAME, + use_flash_attention_2=cfg.trainer.flash_attn, + # NOTE (sumanthrh): Model initialization should always be in fp32 + # during training + bf16=False, + lora_rank=cfg.trainer.policy.model.lora.rank, + lora_alpha=cfg.trainer.policy.model.lora.alpha, + lora_dropout=cfg.trainer.policy.model.lora.dropout, + target_modules=cfg.trainer.policy.model.lora.target_modules, + exclude_modules=cfg.trainer.policy.model.lora.exclude_modules, + sequence_parallel_size=cfg.trainer.policy.sequence_parallel_size, + use_sample_packing=cfg.trainer.use_sample_packing, + use_torch_compile=cfg.trainer.policy.use_torch_compile, + rope_scaling=get_rope_scaling_config(cfg.trainer), + rope_theta=get_rope_theta_config(cfg.trainer), + ) + + model, _, _ = strategy.prepare( + (wrapped_model, None, None), + ) + num_wrapped_modules = 0 + for name, module in model.model.named_modules(): + if isinstance(module, FSDP): + num_wrapped_modules += 1 + # need at least 1 inner layer to be wrapped + assert num_wrapped_modules > 1 + + dist.destroy_process_group() diff --git a/skyrl-train/tests/gpu/gpu_ci/test_engine_generation.py b/skyrl-train/tests/gpu/gpu_ci/test_engine_generation.py new file mode 100644 index 0000000000..d59e0eea8b --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_engine_generation.py @@ -0,0 +1,259 @@ +""" +# Run only vllm tests (requires vllm extra): +uv run --isolated --extra dev --extra vllm pytest tests/gpu/gpu_ci/test_engine_generation.py -m "vllm" + +# Run only sglang tests (requires sglang extra): +uv run --isolated --extra dev --extra sglang pytest tests/gpu/gpu_ci/test_engine_generation.py -m "sglang" +""" + +import pytest +import hydra +from skyrl_train.inference_engines.ray_wrapped_inference_engine import create_ray_wrapped_inference_engines +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +import asyncio +from tests.gpu.utils import are_responses_similar, get_test_prompts, init_remote_inference_servers +from transformers import AutoTokenizer +from omegaconf import DictConfig +from skyrl_train.inference_engines.base import InferenceEngineInput +from skyrl_train.entrypoints.main_base import config_dir + +MODEL = "Qwen/Qwen2.5-0.5B-Instruct" + + +def get_test_actor_config() -> DictConfig: + """Get base config with test-specific overrides.""" + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = MODEL + + cfg.generator.sampling_params.temperature = 0.0 + cfg.generator.sampling_params.top_p = 1 + cfg.generator.sampling_params.top_k = -1 + cfg.generator.sampling_params.max_generate_length = 1024 + cfg.generator.sampling_params.min_p = 0.0 + cfg.generator.sampling_params.logprobs = None + + return cfg + + +def init_ray_inference_engines( + backend: str, tp_size: int, pp_size: int, dp_size: int, config: DictConfig +) -> InferenceEngineClient: + """Initialize ray-wrapped inference engines for the specified backend""" + tokenizer = AutoTokenizer.from_pretrained(MODEL) + engine = create_ray_wrapped_inference_engines( + num_inference_engines=1, + tensor_parallel_size=tp_size, + pipeline_parallel_size=pp_size, + data_parallel_size=dp_size, + model_dtype="bfloat16", + pretrain=MODEL, + seed=42, + vllm_v1_disable_multiproc=True, + enable_prefix_caching=True, + enforce_eager=True, + shared_pg=None, + gpu_memory_utilization=0.8, + inference_engine_enable_sleep=False, + async_engine=True, + max_num_batched_tokens=32768, + max_num_seqs=1024, + tokenizer=tokenizer, + backend=backend, + ) + client = InferenceEngineClient(engine, tokenizer, config) + return client + + +async def run_batch_generation(client, prompts, sampling_params): + engine_input = InferenceEngineInput(prompts=prompts, sampling_params=sampling_params) + engine_output = await client.generate(engine_input) + return engine_output["responses"], engine_output["stop_reasons"] + + +async def run_single_generation(client, prompts, sampling_params): + tasks = [] + for prompt in prompts: + engine_input = InferenceEngineInput(prompts=[prompt], sampling_params=sampling_params) + task = client.generate(engine_input) + tasks.append(task) + + results = await asyncio.gather(*tasks) + + responses = [] + finish_reasons = [] + for result in results: + responses.extend(result["responses"]) + finish_reasons.extend(result["stop_reasons"]) + + return responses, finish_reasons + + +async def run_batch_generation_with_tokens(client, prompt_token_ids, sampling_params): + engine_input = InferenceEngineInput(prompt_token_ids=prompt_token_ids, sampling_params=sampling_params) + engine_output = await client.generate(engine_input) + return engine_output["responses"], engine_output["stop_reasons"] + + +async def run_single_generation_with_tokens(client, prompt_token_ids, sampling_params): + tasks = [] + for tokens in prompt_token_ids: + engine_input = InferenceEngineInput(prompt_token_ids=[tokens], sampling_params=sampling_params) + task = client.generate(engine_input) + tasks.append(task) + + results = await asyncio.gather(*tasks) + + responses = [] + finish_reasons = [] + for result in results: + responses.extend(result["responses"]) + finish_reasons.extend(result["stop_reasons"]) + + return responses, finish_reasons + + +@pytest.mark.parametrize( + "backend,tp_size,pp_size,dp_size", + [ + pytest.param("vllm", 2, 1, 1, marks=pytest.mark.vllm), + pytest.param("vllm", 2, 1, 2, marks=pytest.mark.vllm), + pytest.param("vllm", 2, 2, 1, marks=pytest.mark.vllm), # TP=2, PP=2 + # TODO(Charlie): add TP > 1 tests for sglang when we support it + pytest.param("sglang", 1, 1, 1, marks=pytest.mark.sglang), + ], + ids=["vllm_tp2", "vllm_dp2", "vllm_tp2_pp2", "sglang"], +) +def test_inference_engines_generation(ray_init_fixture, backend: str, tp_size: int, pp_size: int, dp_size: int): + """ + Tests generation with both remote and ray-wrapped engines for the specified backend. + """ + cfg = get_test_actor_config() + cfg.generator.backend = backend + + prompts = get_test_prompts(MODEL) + tokenizer = AutoTokenizer.from_pretrained(MODEL) + + try: + llm_client, remote_server_process = init_remote_inference_servers(tp_size, backend, tokenizer, cfg, MODEL) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + + # Batched generation + remote_batch_responses, batch_finish_reasons = asyncio.run( + run_batch_generation(llm_client, prompts, sampling_params) + ) + assert len(remote_batch_responses) == len( + prompts + ), f"Number of responses should match number of prompts, got {len(remote_batch_responses)} responses but {len(prompts)} prompts" + assert len(batch_finish_reasons) == len( + prompts + ), f"Number of finish reasons should match number of prompts, got {len(batch_finish_reasons)} finish reasons but {len(prompts)} prompts" + + # Single generation (ie, submit individual requests) + remote_single_responses, single_finish_reasons = asyncio.run( + run_single_generation(llm_client, prompts, sampling_params) + ) + assert len(remote_single_responses) == len( + prompts + ), f"Number of responses should match number of prompts, got {len(remote_single_responses)} responses but {len(prompts)} prompts" + assert len(single_finish_reasons) == len( + prompts + ), f"Number of finish reasons should match number of prompts, got {len(single_finish_reasons)} finish reasons but {len(prompts)} prompts" + + # Ensure batched and single generation outputs are (roughly) the same + for i in range(len(prompts)): + if not are_responses_similar(remote_batch_responses[i], remote_single_responses[i], tolerance=0.01): + print( + f"Remote batch and single generation responses are not similar, got batch={remote_batch_responses[i]} and single={remote_single_responses[i]}" + ) + finally: + if "remote_server_process" in locals(): + remote_server_process.terminate() + remote_server_process.wait() + + # Get responses from Ray engine + llm_client = init_ray_inference_engines(backend, tp_size, pp_size, dp_size, cfg) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + + # Batched generation + local_batch_responses, batch_finish_reasons = asyncio.run( + run_batch_generation(llm_client, prompts, sampling_params) + ) + assert len(local_batch_responses) == len( + prompts + ), f"Number of responses should match number of prompts, got {len(local_batch_responses)} responses but {len(prompts)} prompts" + assert len(batch_finish_reasons) == len( + prompts + ), f"Number of finish reasons should match number of prompts, got {len(batch_finish_reasons)} finish reasons but {len(prompts)} prompts" + + # Single generation (ie, submit individual requests) + local_single_responses, single_finish_reasons = asyncio.run( + run_single_generation(llm_client, prompts, sampling_params) + ) + assert len(local_single_responses) == len( + prompts + ), f"Number of responses should match number of prompts, got {len(local_single_responses)} responses but {len(prompts)} prompts" + assert len(single_finish_reasons) == len( + prompts + ), f"Number of finish reasons should match number of prompts, got {len(single_finish_reasons)} finish reasons but {len(prompts)} prompts" + + # Ensure batched and single generation outputs are (roughly) the same + for i in range(len(prompts)): + if not are_responses_similar(local_batch_responses[i], local_single_responses[i], tolerance=0.01): + print( + f"Local batch and single generation responses are not similar, got batch={local_batch_responses[i]} and single={local_single_responses[i]}" + ) + + # Finally, ensure that remote and local outputs are (roughly) the same + for i in range(len(prompts)): + if not are_responses_similar(remote_batch_responses[i], local_batch_responses[i], tolerance=0.01): + print( + f"Remote and local batch generation responses are not similar, got remote={remote_batch_responses[i]} and local={local_batch_responses[i]}" + ) + + +@pytest.mark.parametrize( + "backend,tp_size,dp_size", + [ + pytest.param("vllm", 2, 2, marks=pytest.mark.vllm), + # TODO(Charlie): add TP > 1 tests for sglang when we support it + pytest.param("sglang", 1, 1, marks=pytest.mark.sglang), + ], + ids=["vllm_dp2", "sglang"], +) +def test_token_based_generation(ray_init_fixture, backend: str, tp_size: int, dp_size: int): + """Test generation using prompt_token_ids for the specified backend.""" + + cfg = get_test_actor_config() + cfg.generator.backend = backend + + prompts = get_test_prompts(MODEL, 3) + tokenizer = AutoTokenizer.from_pretrained(MODEL) + prompt_token_ids = tokenizer.apply_chat_template( + prompts, add_generation_prompt=True, tokenize=True, return_dict=True + )["input_ids"] + + llm_client = init_ray_inference_engines(backend, tp_size=tp_size, pp_size=1, dp_size=dp_size, config=cfg) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + + # Test batch generation with tokens + token_batch_responses, _ = asyncio.run( + run_batch_generation_with_tokens(llm_client, prompt_token_ids, sampling_params) + ) + assert len(token_batch_responses) == len(prompts) + + # Test single generation with tokens + token_single_responses, _ = asyncio.run( + run_single_generation_with_tokens(llm_client, prompt_token_ids, sampling_params) + ) + assert len(token_single_responses) == len(prompts) + + # Compare with prompt-based generation + prompt_responses, _ = asyncio.run(run_batch_generation(llm_client, prompts, sampling_params)) + + # Outputs should be similar since we're using the same inputs + for i in range(len(prompts)): + if not are_responses_similar([token_batch_responses[i]], [prompt_responses[i]], tolerance=0.01): + print(f"Token and prompt responses differ: token={token_batch_responses[i]}, prompt={prompt_responses[i]}") diff --git a/skyrl-train/tests/gpu/gpu_ci/test_inference_engine_client_http_endpoint.py b/skyrl-train/tests/gpu/gpu_ci/test_inference_engine_client_http_endpoint.py new file mode 100644 index 0000000000..687ef81696 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_inference_engine_client_http_endpoint.py @@ -0,0 +1,818 @@ +""" +Test the HTTP endpoint with LiteLLM and policy weight sync. + +This uses the same workflow as test_policy_local_engines_e2e.py, but with the HTTP endpoint instead of +the inference client engine. Only requires 1 GPU. + +# Run only vllm tests (requires vllm extra): +uv run --isolated --extra dev --extra vllm pytest tests/gpu/gpu_ci/test_inference_engine_client_http_endpoint.py -m "vllm" +# Run only sglang tests (requires sglang extra): +uv run --isolated --extra dev --extra sglang pytest tests/gpu/gpu_ci/test_inference_engine_client_http_endpoint.py -m "sglang" +""" + +import json +import pytest +import asyncio +from http import HTTPStatus +from typing import Any, Dict, List, Union, Tuple +from pathlib import Path +import ray +import hydra +import threading +import requests +import aiohttp +from omegaconf import DictConfig +from pydantic import BaseModel +import litellm +from litellm import completion as litellm_completion +from litellm import acompletion as litellm_async_completion +from litellm import atext_completion as litellm_async_text_completion + +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.base import ConversationType +from tests.gpu.utils import init_worker_with_type, get_test_prompts +from skyrl_train.entrypoints.main_base import config_dir +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.inference_engines.inference_engine_client_http_endpoint import ( + serve, + wait_for_server_ready, + shutdown_server, +) +from tests.gpu.gpu_ci.test_engine_generation import init_remote_inference_servers +from tests.gpu.utils import init_inference_engines +from concurrent.futures import ThreadPoolExecutor + +from transformers import AutoTokenizer + +MODEL_QWEN2_5 = "Qwen/Qwen2.5-0.5B-Instruct" +MODEL_QWEN3 = "Qwen/Qwen3-0.6B" +TP_SIZE = 1 +SERVER_HOST = "127.0.0.1" + + +# Disable aiohttp transport in litellm to avoid unclosed connector warnings. +# This makes litellm use httpx's default transport instead of aiohttp. +# This is safe for tests since we don't need the performance benefits of aiohttp. +litellm.disable_aiohttp_transport = True + + +def _get_test_sampling_params(backend: str, cfg: DictConfig, endpoint: str) -> Dict[str, Any]: + assert endpoint in ["chat_completions", "completions"] + sampling_params = get_sampling_params_for_backend(backend, cfg.generator.sampling_params) + sampling_params["logprobs"] = True + if endpoint == "chat_completions": + sampling_params["top_logprobs"] = 1 + sampling_params["return_tokens_as_token_ids"] = True + return sampling_params + + +def get_test_actor_config(num_inference_engines: int, model: str) -> DictConfig: + """Get base config with test-specific overrides.""" + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + # Override specific parameters + cfg.trainer.policy.model.path = model + cfg.trainer.critic.model.path = "" + cfg.trainer.placement.policy_num_gpus_per_node = TP_SIZE * num_inference_engines + cfg.generator.async_engine = True + cfg.generator.num_inference_engines = num_inference_engines + cfg.generator.inference_engine_tensor_parallel_size = TP_SIZE + cfg.generator.run_engines_locally = True + + return cfg + + +# ------------------------------------------ +# Helper functions for setting up HTTP server +# ------------------------------------------ + + +def set_up_http_server(client: InferenceEngineClient) -> Tuple[threading.Thread, int]: + def _find_available_port(host: str) -> int: + """Find an available port by binding to port 0.""" + import socket + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind((host, 0)) + return s.getsockname()[1] + + # Find an available port + server_port = _find_available_port(SERVER_HOST) + + # Start server in background thread + def run_server(): + serve(client, host=SERVER_HOST, port=server_port, log_level="warning") + + server_thread = threading.Thread(target=run_server, daemon=True) + server_thread.start() + + # Wait for server to be ready + wait_for_server_ready(host=SERVER_HOST, port=server_port, max_wait_seconds=30) + + return server_thread, server_port + + +# -------------------------------------- +# Helper functions for checking outputs +# -------------------------------------- + + +def _check_chat_completions_outputs(outputs, test_type, num_samples, backend): + # check error + for output in outputs: + assert not ("error" in output or output.get("object", "") == "error"), f"Error in output: {output}" + print_n = 5 + assert len(outputs) == num_samples + print(f"First {print_n} generated responses out of {num_samples} using {test_type}:") + for i, output in enumerate(outputs[:print_n]): + print(f"{i}: {output['choices'][0]['message']['content'][:100]}...") + + # Check response structure + for response_data in outputs: + if test_type == "litellm": + # litellm returns a pydantic object + response_data = response_data.model_dump() + + if test_type != "litellm": + # Cannot check for litellm because it returns it has its own pydantic object + if backend == "vllm": + from vllm.entrypoints.openai.protocol import ChatCompletionResponse + + ChatCompletionResponse.model_validate(response_data) # will raise error if invalid + else: + # TODO(Charlie): add sglang checkings once we support it for http endpoint + raise ValueError(f"Unsupported backend: {backend}") + + for key in ["id", "object", "created", "model", "choices"]: + assert key in response_data + assert response_data[key] is not None + + for i, choice in enumerate(response_data["choices"]): + assert "index" in choice and "message" in choice and "finish_reason" in choice + assert choice["index"] == i and choice["finish_reason"] in ["stop", "length"] + message = choice["message"] + assert "role" in message and "content" in message and message["role"] == "assistant" + + # check token_logprobs + choice = response_data["choices"][i] + assert "logprobs" in choice + assert choice["logprobs"]["content"] is not None + # tokens are token_id: because we request `return_tokens_as_token_ids` from vllm + assert choice["logprobs"]["content"][0]["token"].split(":")[1].isdigit() + + +def _check_completions_outputs(prompts, outputs, test_type, backend): + # check error + for output in outputs: + assert not ("error" in output or output.get("object", "") == "error"), f"Error in output: {output}" + + num_outputs = sum(len(output["choices"]) for output in outputs) + assert num_outputs == len(prompts) + + print_n = 5 + # Content checks + print(f"First {print_n} generated responses out of {num_outputs} using completions:") + # First flatten it output[i][choices] into a single list + choice_list = [output["choices"] for output in outputs] + choice_list = [item for sublist in choice_list for item in sublist] + for i, output in enumerate(choice_list[:print_n]): + data = output.model_dump() if test_type == "litellm" else output + # CompletionResponse uses 'text' field + preview = data.get("text") or str(data)[0:100] + print(f"Prompt {i}: {prompts[i][:300]}...") + print(f"Output {i}: {preview[:100]}...") + + # Formatting checks + for response_data in outputs: + if test_type == "litellm": + response_data = response_data.model_dump() + + if test_type != "litellm": + if backend == "vllm": + from vllm.entrypoints.openai.protocol import CompletionResponse + + CompletionResponse.model_validate(response_data) + else: + raise ValueError(f"Unsupported backend: {backend}") + + for key in ["id", "object", "created", "model", "choices"]: + assert key in response_data + assert response_data[key] is not None + + for i, choice in enumerate(response_data["choices"]): + assert "index" in choice and "text" in choice and "finish_reason" in choice + assert choice["index"] == i and choice["finish_reason"] in ["stop", "length"] + + choice = response_data["choices"][i] + assert "logprobs" in choice and choice["logprobs"] is not None + assert "tokens" in choice["logprobs"] + + +# ------------------------------ +# Tests for HTTP endpoint +# ------------------------------ + + +@pytest.mark.vllm +def test_http_endpoint_completions_routing_and_batching(ray_init_fixture): + """ + Since /completions endpoint supports both single and batched requests, and we support + either using session_id or not, we test all combinations. + + We test with 2 engines of TP=1 to check if routing works. + """ + batched_list = [False, True] + with_traj_list = [False, True] + + try: + # 1. Build engine + cfg = get_test_actor_config(num_inference_engines=2, model=MODEL_QWEN2_5) + cfg.trainer.placement.colocate_all = True + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + sampling_params = _get_test_sampling_params("vllm", cfg, "completions") + client, _ = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL_QWEN2_5, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=1, # since we do not explicitly sync weights + ) + tokenizer = AutoTokenizer.from_pretrained(MODEL_QWEN2_5) + + server_thread, server_port = set_up_http_server(client) + base_url = f"http://{SERVER_HOST}:{server_port}/v1" + + # 2. Build prompts + num_samples = 20 + test_prompts_conv_list: List[ConversationType] = get_test_prompts(MODEL_QWEN2_5, num_samples=num_samples) + text_prompts = [ + tokenizer.apply_chat_template(conv, add_generation_prompt=True, tokenize=False) + for conv in test_prompts_conv_list + ] + + for batched in batched_list: + for with_traj in with_traj_list: + if not batched: + outputs = [] + for i, p in enumerate(text_prompts): + payload = {"model": MODEL_QWEN2_5, "prompt": p, **sampling_params} + if with_traj: + payload["session_id"] = i + outputs.append(requests.post(f"{base_url}/completions", json=payload).json()) + else: + payload = {"model": MODEL_QWEN2_5, "prompt": text_prompts, **sampling_params} + if with_traj: + payload["session_id"] = list(range(len(text_prompts))) + outputs = [requests.post(f"{base_url}/completions", json=payload).json()] + + _check_completions_outputs(text_prompts, outputs, "request_posting", "vllm") + finally: + shutdown_server(host=SERVER_HOST, port=server_port, max_wait_seconds=5) + if server_thread.is_alive(): + server_thread.join(timeout=5) + + +# NOTE(Charlie): we do not test OpenAI client because it throws error when unsupported sampling params +# are passed into OpenAI.chat.completions.create() (e.g. min_tokens, skip_special_tokens, etc.), +# while these sampling params are used in vllm/sglang. Therefore, we instead use LiteLLM. +@pytest.mark.vllm +def test_http_endpoint_openai_api_with_weight_sync(ray_init_fixture): + """ + Test the HTTP endpoint /chat/completions and /completions with policy weight sync. + + For `/completions`, we test by sending each prompt in its own request. For batching, + behavior, we test in `test_http_endpoint_completions_routing_and_batching`. + + Besides we only tests single engine case. + """ + test_types = ["request_posting", "aiohttp_client_session", "litellm"] + endpoints = ["chat_completions", "completions"] + try: + # 1. Set up engine + cfg = get_test_actor_config(num_inference_engines=1, model=MODEL_QWEN2_5) + cfg.trainer.placement.colocate_all = True + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + client, pg = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL_QWEN2_5, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=2, # since we explicitly sync weights + ) + tokenizer = AutoTokenizer.from_pretrained(MODEL_QWEN2_5) + + server_thread, server_port = set_up_http_server(client) + base_url = f"http://{SERVER_HOST}:{server_port}/v1" + + # Weight sync + policy = init_worker_with_type( + "policy", + shared_pg=pg, + colocate_all=cfg.trainer.placement.colocate_all, + num_gpus_per_node=cfg.generator.inference_engine_tensor_parallel_size, + cfg=cfg, + ) + ray.get(policy.async_run_ray_method("pass_through", "init_weight_sync_state", client)) + asyncio.run(client.reset_prefix_cache()) + ray.get(policy.async_run_ray_method("pass_through", "broadcast_to_inference_engines", client)) + + # 2. Do tests + num_samples = 20 + test_prompts_conv_list: List[ConversationType] = get_test_prompts(MODEL_QWEN2_5, num_samples=num_samples) + # For /completions, we test both string and token IDs input + test_prompts_half_str_half_tokens_list: List[Union[str, List[int]]] = [ + tokenizer.apply_chat_template(conv, add_generation_prompt=True, tokenize=False) + for conv in test_prompts_conv_list[: num_samples // 2] + ] + [ + tokenizer.apply_chat_template(conv, add_generation_prompt=True, tokenize=True) + for conv in test_prompts_conv_list[num_samples // 2 :] + ] + + def _generate_outputs(test_type, endpoint): + def payload_builder(session_id, prompt): + if endpoint == "chat_completions": + return { + "model": MODEL_QWEN2_5, + "messages": prompt, + "session_id": session_id, + **sampling_params, + } + else: + return { + "model": MODEL_QWEN2_5, + "prompt": prompt, + "session_id": session_id, + **sampling_params, + } + + if endpoint == "chat_completions": + path = "chat/completions" + prompt_iterable = test_prompts_conv_list + sampling_params = _get_test_sampling_params("vllm", cfg, "chat_completions") + + else: + path = "completions" + prompt_iterable = test_prompts_half_str_half_tokens_list + sampling_params = _get_test_sampling_params("vllm", cfg, "completions") + + if test_type == "request_posting": + + def generate_output(session_id, prompt): + return requests.post( + f"{base_url}/{path}", + json=payload_builder(session_id, prompt), + ).json() + + with ThreadPoolExecutor() as executor: + output_tasks = [ + executor.submit(generate_output, session_id, prompt) + for session_id, prompt in enumerate(prompt_iterable) + ] + outputs = [task.result() for task in output_tasks] + + elif test_type == "aiohttp_client_session": + + async def generate_outputs_async(): + conn = aiohttp.TCPConnector(limit=0, limit_per_host=0) + async with aiohttp.ClientSession( + connector=conn, timeout=aiohttp.ClientTimeout(total=None) + ) as session: + headers = {"Content-Type": "application/json"} + output_tasks = [] + for session_id, prompt in enumerate(prompt_iterable): + payload = payload_builder(session_id, prompt) + output_tasks.append(session.post(f"{base_url}/{path}", json=payload, headers=headers)) + responses = await asyncio.gather(*output_tasks) + return [await response.json() for response in responses] + + outputs = asyncio.run(generate_outputs_async()) + + elif test_type == "litellm": + + async def generate_outputs_async(): + async def generate_output(session_id, prompt): + if endpoint == "chat_completions": + return await litellm_async_completion( + model=f"openai/{MODEL_QWEN2_5}", + messages=prompt, + api_base=base_url, + api_key="DUMMY_KEY", + session_id=session_id, + **sampling_params, + ) + else: + return await litellm_async_text_completion( + model=f"openai/{MODEL_QWEN2_5}", + prompt=[prompt], + api_base=base_url, + api_key="DUMMY_KEY", + session_id=[session_id], + **sampling_params, + ) + + tasks = [generate_output(session_id, prompt) for session_id, prompt in enumerate(prompt_iterable)] + return await asyncio.gather(*tasks) + + outputs = asyncio.run(generate_outputs_async()) + + else: + raise ValueError(f"Invalid test type: {test_type}") + + return outputs + + for test_type in test_types: + for endpoint in endpoints: + print(f"Testing {test_type} with {endpoint}") + outputs = _generate_outputs(test_type, endpoint) + if endpoint == "chat_completions": + _check_chat_completions_outputs(outputs, test_type, num_samples, "vllm") + else: + _check_completions_outputs(test_prompts_half_str_half_tokens_list, outputs, test_type, "vllm") + + finally: + shutdown_server(host=SERVER_HOST, port=server_port, max_wait_seconds=5) + if server_thread.is_alive(): + server_thread.join(timeout=5) + + +@pytest.mark.parametrize( + "backend,tp_size", + [ + pytest.param("vllm", 2, marks=pytest.mark.vllm), + # TODO(Charlie): add TP > 1 tests for sglang when we support it + # TODO(Charlie): sglang remote server not supported for /chat/completion + # yet because we have skip_tokenizer_init=True. Fix by getting tokens + # via return logprobs instead. + # pytest.param("sglang", 1, marks=pytest.mark.sglang), + ], + # ids=["vllm", "sglang"], + ids=["vllm"], +) +def test_http_endpoint_with_remote_servers(ray_init_fixture, backend, tp_size): + """Test sending both /chat/completions and /completions requests to remote servers.""" + endpoints = ["chat_completions", "completions"] + + try: + # 1. Initialize InferenceEngineClient client with remote servers + cfg = get_test_actor_config(num_inference_engines=1, model=MODEL_QWEN2_5) + cfg.generator.backend = backend + tokenizer = AutoTokenizer.from_pretrained(MODEL_QWEN2_5) + + client, remote_server_process = init_remote_inference_servers(tp_size, backend, tokenizer, cfg, MODEL_QWEN2_5) + + # 2. Start HTTP endpoint in background thread using serve function directly + server_thread, server_port = set_up_http_server(client) + base_url = f"http://{SERVER_HOST}:{server_port}/v1" + + # 3. Generate outputs using litellm and check outputs + num_samples = 20 + test_prompts_conv_list: List[ConversationType] = get_test_prompts(MODEL_QWEN2_5, num_samples=num_samples) + test_prompts_half_str_half_tokens_list: List[Union[str, List[int]]] = [ + tokenizer.apply_chat_template(conv, add_generation_prompt=True, tokenize=False) + for conv in test_prompts_conv_list[: num_samples // 2] + ] + [ + tokenizer.apply_chat_template(conv, add_generation_prompt=True, tokenize=True) + for conv in test_prompts_conv_list[num_samples // 2 :] + ] + + def _generate_outputs(endpoint): + if endpoint == "chat_completions": + sampling_params = _get_test_sampling_params(backend, cfg, "chat_completions") + prompt_iterable = test_prompts_conv_list + else: + sampling_params = _get_test_sampling_params(backend, cfg, "completions") + prompt_iterable = test_prompts_half_str_half_tokens_list + + # Default concurrency limit is 100 due to HTTP client pool capacity. + async def generate_outputs_async(): + async def generate_output(session_id, prompt): + if endpoint == "chat_completions": + return await litellm_async_completion( + model=f"openai/{MODEL_QWEN2_5}", + messages=prompt, + api_base=base_url, + api_key="DUMMY_KEY", + session_id=session_id, + **sampling_params, + ) + else: + return await litellm_async_text_completion( + model=f"openai/{MODEL_QWEN2_5}", + prompt=[prompt], + api_base=base_url, + api_key="DUMMY_KEY", + session_id=[session_id], + **sampling_params, + ) + + tasks = [generate_output(session_id, prompt) for session_id, prompt in enumerate(prompt_iterable)] + return await asyncio.gather(*tasks) + + return asyncio.run(generate_outputs_async()) + + for endpoint in endpoints: + outputs = _generate_outputs(endpoint) + if endpoint == "chat_completions": + _check_chat_completions_outputs(outputs, "litellm", num_samples, backend) + else: + _check_completions_outputs(test_prompts_half_str_half_tokens_list, outputs, "litellm", backend) + + # 4. Shutdown server + shutdown_server(host=SERVER_HOST, port=server_port, max_wait_seconds=5) + if server_thread.is_alive(): + server_thread.join(timeout=5) + + finally: + shutdown_server(host=SERVER_HOST, port=server_port, max_wait_seconds=5) + if "remote_server_process" in locals(): + remote_server_process.terminate() + remote_server_process.wait() + + +@pytest.mark.vllm +def test_structured_generation(ray_init_fixture): + try: + cfg = get_test_actor_config(num_inference_engines=1, model=MODEL_QWEN2_5) + cfg.trainer.placement.colocate_all = True # Use colocate for simplicity + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + + client, _ = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL_QWEN2_5, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=1, # since we do not explicitly sync weights + ) + + server_thread, server_port = set_up_http_server(client) + base_url = f"http://{SERVER_HOST}:{server_port}/v1" + + class TestSchema(BaseModel): + name: str + job: str + + prompt = [ + { + "role": "user", + "content": f"Introduce yourself in JSON format briefly, following the schema {TestSchema.model_json_schema()}.", + }, + ] + + output = litellm_completion( + model=f"openai/{MODEL_QWEN2_5}", + api_base=base_url, + api_key="DUMMY_KEY", + messages=prompt, + max_tokens=1024, + response_format={ + "type": "json_schema", + "json_schema": { + "name": "TestSchema", + "schema": TestSchema.model_json_schema(), + "strict": True, + }, + }, + ) + + # assert is valid json + text = output.choices[0].message.content + assert json.loads(text) is not None, f"Output is not valid JSON: {text}" + finally: + shutdown_server(host=SERVER_HOST, port=server_port, max_wait_seconds=5) + if server_thread.is_alive(): + server_thread.join(timeout=5) + + +# TODO(Charlie): sglang has slightly different error response format. We need to handle it. +@pytest.mark.vllm +def test_http_endpoint_error_handling(ray_init_fixture): + """ + Test error handling for various invalid requests. + """ + try: + cfg = get_test_actor_config(num_inference_engines=2, model=MODEL_QWEN2_5) + cfg.trainer.placement.colocate_all = True + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + + client, _ = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL_QWEN2_5, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=1, # since we do not explicitly sync weights + ) + + server_thread, server_port = set_up_http_server(client) + base_url = f"http://{SERVER_HOST}:{server_port}" + + # Test 1: Invalid request - streaming not supported, raised by SkyRL + response = requests.post( + f"{base_url}/v1/chat/completions", + json={"model": MODEL_QWEN2_5, "messages": [{"role": "user", "content": "Hello"}], "stream": True}, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST # 400 + error_data = response.json() + print(f"Error data: {error_data}") + assert "Streaming is not supported" in error_data["error"]["message"] + + # Test 2: OAI can take fields not listed in the protocol + response = requests.post( + f"{base_url}/v1/chat/completions", + json={"model": MODEL_QWEN2_5, "messages": [{"role": "user", "content": "Hello"}], "xxx": "yyy"}, + ) + assert response.status_code == HTTPStatus.OK # 200 + + # Test 3: Invalid request - missing required fields, raised by SkyRL + response = requests.post( + f"{base_url}/v1/chat/completions", + json={ + "model": MODEL_QWEN2_5, + # Missing messages field + }, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST # 400 + error_data = response.json() + print(f"Error data: {error_data}") + assert error_data["error"]["message"] == "The field `messages` is required in your `/chat/completions` request." + + # Test 4: Invalid request - malformed JSON, raised by SkyRL + response = requests.post( + f"{base_url}/v1/chat/completions", data="some invalid json", headers={"Content-Type": "application/json"} + ) + assert response.status_code == HTTPStatus.BAD_REQUEST # 400 + error_data = response.json() + print(f"Error data: {error_data}") + assert "Invalid JSON error" in error_data["error"]["message"] # JSON decode error + + # Test 5: Invalid request - empty messages array, raised by SkyRL + response = requests.post(f"{base_url}/v1/chat/completions", json={"model": MODEL_QWEN2_5, "messages": []}) + assert response.status_code == HTTPStatus.BAD_REQUEST # 400 + error_data = response.json() + print(f"Error data: {error_data}") + assert error_data["error"]["message"] == "The field `messages` in `/chat/completions` cannot be an empty list." + + # Test 6: Wrong model name, raised by SkyRL + response = requests.post( + f"{base_url}/v1/chat/completions", + json={"model": "wrong_model", "messages": [{"role": "user", "content": "Hello"}]}, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST # 400 + error_data = response.json() + print(f"Error data: {error_data}") + assert "Model name mismatch" in error_data["error"]["message"] + + # Test 7: Health check endpoint should work + response = requests.get(f"{base_url}/health") + assert response.status_code == HTTPStatus.OK # 200 + health_data = response.json() + assert health_data["status"] == "healthy" + + # Tests below are for `/completions` endpoint. + # e.g. session id wrong length, etc. + # Additional tests for /v1/completions + # C1: streaming not supported + response = requests.post( + f"{base_url}/v1/completions", + json={"model": MODEL_QWEN2_5, "prompt": "Hello", "stream": True}, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + # C2: wrong model + response = requests.post( + f"{base_url}/v1/completions", + json={"model": "wrong_model", "prompt": "Hello"}, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + # C3: malformed json + response = requests.post( + f"{base_url}/v1/completions", + data="some invalid json", + headers={"Content-Type": "application/json"}, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + # C4: n > 1 + response = requests.post( + f"{base_url}/v1/completions", + json={"model": MODEL_QWEN2_5, "prompt": "Hello", "n": 2}, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + error_data = response.json() + assert "n is not supported in SkyRL for /completions " in error_data["error"]["message"] + + # When batched and session_id wrong length -> 400 from server or client-side error + bad_payload = {"model": MODEL_QWEN2_5, "prompt": ["hi", "hello", "ok"], "session_id": [0, 1]} + r = requests.post(f"{base_url}/v1/completions", json=bad_payload) + assert r.status_code == HTTPStatus.BAD_REQUEST + + finally: + shutdown_server(host=SERVER_HOST, port=server_port, max_wait_seconds=5) + if server_thread.is_alive(): + server_thread.join(timeout=5) + + +@pytest.mark.vllm +@pytest.mark.parametrize("use_custom_template", [False, True]) +def test_http_endpoint_custom_chat_template(ray_init_fixture, use_custom_template): + """ + Test the HTTP endpoint /chat/completions with and without custom chat template. + Check the output correspondingly. + """ + try: + # 1. Set up engine + cfg = get_test_actor_config(num_inference_engines=1, model=MODEL_QWEN3) + cfg.trainer.placement.colocate_all = True # Use colocate for simplicity + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + template_path = "skyrl_train/utils/templates/qwen3_acc_thinking.jinja2" + engine_init_kwargs = {} + if use_custom_template: + # use relative path to workspace root + # __file__ is skyrl-train/tests/gpu/gpu_ci/test_inference_engine_client_http_endpoint.py + repo_root = Path(__file__).parent.parent.parent.parent + engine_init_kwargs["chat_template"] = str(repo_root / template_path) + + client, _ = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL_QWEN3, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=1, # since we do not explicitly sync weights + engine_init_kwargs=engine_init_kwargs, + ) + + server_thread, server_port = set_up_http_server(client) + base_url = f"http://{SERVER_HOST}:{server_port}/v1" + + # 2. Send request + # Test that the custom template will not strip thinking tokens, unlike the default template. + messages = [ + { + "role": "user", + "content": "Hello", + }, + { + "role": "assistant", + "content": "Thinking...Hello", + }, + { + "role": "user", + "content": "Hello", + }, + ] + payload = { + "model": MODEL_QWEN3, + "messages": messages, + "max_tokens": 10, + "return_token_ids": True, + } + + response = requests.post(f"{base_url}/chat/completions", json=payload) + assert response.status_code == HTTPStatus.OK + data = response.json() + + # 3. Check output + assert "choices" in data and len(data["choices"]) > 0 + content = data["choices"][0]["message"]["content"] + assert isinstance(content, str) + + # 4. Check thinking tokens stripped or not + assert "prompt_token_ids" in data, f"prompt_token_ids not found in response. Keys: {data.keys()}" + prompt_token_ids = data["prompt_token_ids"] + tokenizer = AutoTokenizer.from_pretrained(MODEL_QWEN3) + prompt_str = tokenizer.decode(prompt_token_ids) + + if use_custom_template: + # The custom template qwen3_acc_thinking.jinja2 will keep the thinking tokens. + assert "" in prompt_str and "" in prompt_str + else: + # Default template strips thinking tokens + assert "" not in prompt_str and "" not in prompt_str + + finally: + shutdown_server(host=SERVER_HOST, port=server_port, max_wait_seconds=5) + if server_thread.is_alive(): + server_thread.join(timeout=5) diff --git a/skyrl-train/tests/gpu/gpu_ci/test_lora.py b/skyrl-train/tests/gpu/gpu_ci/test_lora.py new file mode 100644 index 0000000000..118a82974c --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_lora.py @@ -0,0 +1,93 @@ +""" +# Run vllm tests (requires vllm extra): +uv run --isolated --extra dev --extra vllm pytest tests/gpu/gpu_ci/test_lora.py +""" + +import pytest +import asyncio +import ray +import hydra +from omegaconf import DictConfig + +from tests.gpu.utils import init_worker_with_type, get_test_prompts, init_inference_engines, run_inference +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.entrypoints.main_base import config_dir + +MODEL = "Qwen/Qwen2.5-0.5B-Instruct" + + +def get_test_actor_config(enable_lora: bool = False) -> DictConfig: + """Get base config with test-specific overrides.""" + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + # Override specific parameters + cfg.trainer.policy.model.path = MODEL + cfg.trainer.critic.model.path = "" + cfg.trainer.placement.policy_num_gpus_per_node = 2 + cfg.generator.async_engine = True + cfg.generator.num_inference_engines = 1 + cfg.generator.run_engines_locally = True + + # LoRA configuration + if enable_lora: + cfg.trainer.policy.model.lora.rank = 32 + cfg.trainer.policy.model.lora.alpha = 32 + cfg.trainer.policy.model.lora.dropout = 0.1 + cfg.trainer.policy.model.lora.target_modules = "all-linear" + + return cfg + + +@pytest.mark.parametrize( + ("colocate_all", "weight_sync_backend", "strategy", "backend", "tp_size"), + [ + pytest.param(False, "nccl", "fsdp", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(True, "nccl", "fsdp", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(False, "nccl", "fsdp2", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(True, "nccl", "fsdp2", "vllm", 2, marks=pytest.mark.vllm), + ], + ids=[ + "no_colocate_nccl_fsdp_vllm", + "colocate_nccl_fsdp_vllm", + "no_colocate_nccl_fsdp2_vllm", + "colocate_nccl_fsdp2_vllm", + ], +) +def test_policy_local_engines_e2e(ray_init_fixture, colocate_all, weight_sync_backend, strategy, backend, tp_size): + """ + Tests initalizing the policy actor group and inference engine, syncing weights, and performing generation. + """ + cfg = get_test_actor_config(enable_lora=True) + cfg.trainer.placement.colocate_all = colocate_all + cfg.generator.weight_sync_backend = weight_sync_backend + cfg.trainer.strategy = strategy + cfg.generator.backend = backend + cfg.generator.inference_engine_tensor_parallel_size = tp_size + + # If colocate is True, this will load the engine, sleep, and wake up the engine + client, pg = init_inference_engines( + model=MODEL, + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend=backend, + sleep_level=1, # since we explicitly sync weights + enable_lora=True, # Enable LoRA for this test + ) + + policy = init_worker_with_type( + "policy", + shared_pg=pg, + colocate_all=cfg.trainer.placement.colocate_all, + num_gpus_per_node=cfg.generator.inference_engine_tensor_parallel_size, + cfg=cfg, + ) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + ray.get(policy.async_run_ray_method("pass_through", "init_weight_sync_state", client)) + asyncio.run(client.reset_prefix_cache()) + ray.get(policy.async_run_ray_method("pass_through", "broadcast_to_inference_engines", client)) + outputs = asyncio.run(run_inference(client, get_test_prompts(MODEL), sampling_params)) + print(f"Example output: {outputs['responses'][0]}, {outputs['stop_reasons'][0]}") diff --git a/skyrl-train/tests/gpu/gpu_ci/test_megatron_worker.py b/skyrl-train/tests/gpu/gpu_ci/test_megatron_worker.py new file mode 100644 index 0000000000..928bd9df45 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_megatron_worker.py @@ -0,0 +1,765 @@ +""" +Run with: +uv run --isolated --extra dev --extra mcore -- pytest tests/gpu/gpu_ci/test_megatron_worker.py +""" + +import ray +import pytest +import hydra +from omegaconf import DictConfig +import torch +import asyncio +from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig +from omegaconf import OmegaConf +from tests.gpu.utils import ( + init_worker_with_type, + ray_init_for_tests, + get_rank_0_memory, + init_inference_engines, + run_inference, + get_test_prompts, + Timer, +) +from skyrl_train.utils.utils import print_mem, validate_cfg +from skyrl_train.entrypoints.main_base import config_dir +from skyrl_train.distributed.dispatch import concatenate_outputs_after_mesh_dispatch +from skyrl_train.utils.torch_utils import logprobs_from_logits +from skyrl_train.training_batch import TrainingInputBatch +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend + + +MODEL_NAME = "Qwen/Qwen3-0.6B" +# TODO (erictang000): we would prefer to use this smaller MoE model for testing, but seeing incorrect logprobs when using EP > 1 +# this might be a model specific mbridge issue - see if this persists when we transition to Megatron-Bridge +# MOE_MODEL_NAME = "Qwen/Qwen1.5-MoE-A2.7B" +MOE_MODEL_NAME = "Qwen/Qwen3-30B-A3B" + + +def get_test_actor_config(model_name=MODEL_NAME) -> DictConfig: + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = model_name + cfg.trainer.micro_forward_batch_size_per_gpu = 2 + cfg.trainer.micro_train_batch_size_per_gpu = 2 + cfg.trainer.use_sample_packing = False + cfg.trainer.logger = "console" + if "moonlight" in model_name: + cfg.trainer.policy.megatron_config.transformer_config_kwargs = OmegaConf.create( + {"num_layers_in_last_pipeline_stage": 13} + ) + if "Qwen3-30B" in model_name or "Qwen1.5-MoE" in model_name: + cfg.trainer.gradient_checkpointing_use_reentrant = True + + validate_cfg(cfg) + + return cfg + + +def get_test_training_batch(batch_size=4) -> TrainingInputBatch: + """ + Returns a test training batch with padded seqs and attention masks + + Gives a batch of 4 sequences with variable amounts of left padding, and variable response lengths/amounts of right padding + Attention masks are 1 for non-padding tokens, 0 for padding tokens + The rest of the fields are filled with dummy data + """ + assert batch_size % 4 == 0, "batch size must be divisible by 4" + num_repeats = batch_size // 4 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True) + + sentences = [ + "<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.", + "<|im_start|>user\nThe selling price of a bicycle that had sold $220 last year was increased by 15", + "What is the new price? Let's think step by step and output the final answer after `####`.<|im_end|>\n", + "<|im_start|>assistant\nTo find the new price of the bicycle after the increase,", + ] * num_repeats + + sequences = [tokenizer.encode(sentence) for sentence in sentences] + attention_masks = [[1] * len(seq) for seq in sequences] + num_actions = 10 + # max seq len 1 longer than the longest sequence so we always have some padding + max_seq_length = max([len(seq) for seq in sequences]) + 7 + + pad_token_id = tokenizer.pad_token_id + pad_before = [4, 0, 1, 6] * num_repeats + pad_after = [max_seq_length - len(seq) - pad_before[i] for i, seq in enumerate(sequences)] + + for i, (pad_before, pad_after) in enumerate(zip(pad_before, pad_after)): + sequences[i] = [pad_token_id] * pad_before + sequences[i] + [pad_token_id] * pad_after + attention_masks[i] = [0] * pad_before + attention_masks[i] + [0] * pad_after + + attention_masks = torch.tensor(attention_masks) + sequences = torch.tensor(sequences) + + data = TrainingInputBatch( + { + "sequences": sequences, + "attention_mask": attention_masks, + "action_log_probs": torch.tensor([[0.1] * num_actions] * batch_size), + "base_action_log_probs": torch.tensor([[0.2] * num_actions] * batch_size), + "rollout_logprobs": torch.tensor([[0.11] * num_actions] * batch_size), + "values": torch.tensor([[0.1] * num_actions] * batch_size), + "returns": torch.tensor([[0.1] * num_actions] * batch_size), + "advantages": torch.tensor([[0.5] * num_actions] * batch_size), + "loss_mask": torch.tensor([[1] * num_actions] * batch_size), + "response_mask": torch.tensor([[1] * num_actions] * batch_size), + } + ) + data.metadata = {"response_length": num_actions} + return data + + +@pytest.mark.parametrize( + ("colocate_all", "inference_tp", "megatron_tp", "megatron_pp", "megatron_ep", "megatron_etp", "lora"), + [(True, 4, 2, 2, 1, None, False), (False, 2, 2, 1, 1, None, False), (True, 4, 2, 2, 1, None, True)], + ids=["colocate_all", "non_colocated", "colocate_all_lora"], +) +@pytest.mark.megatron +def test_megatron_policy_weight_sync( + colocate_all, inference_tp, megatron_tp, megatron_pp, megatron_ep, megatron_etp, lora +): + """ + Test that we can sync weights between policy and inference for megatron then run inference + """ + try: + cfg = get_test_actor_config(model_name=MODEL_NAME) + if lora: + cfg.trainer.policy.model.lora.rank = 16 + cfg.trainer.policy.model.lora.alpha = 16 + cfg.trainer.placement.colocate_all = colocate_all + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "megatron" + cfg.generator.backend = "vllm" + cfg.generator.inference_engine_tensor_parallel_size = inference_tp + + # set tp and pp to 2 to check that gather for weight sync works correctly + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = megatron_tp + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = megatron_pp + cfg.trainer.policy.megatron_config.expert_model_parallel_size = megatron_ep + cfg.trainer.policy.megatron_config.expert_tensor_parallel_size = megatron_etp + + # If colocate is True, this will load the engine, sleep, and wake up the engine + client, pg = init_inference_engines( + model=MODEL_NAME, + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + sleep_level=2, # since we explicitly sync weights + ) + + asyncio.run(client.sleep()) + + policy = init_worker_with_type( + "policy", + shared_pg=pg, + colocate_all=cfg.trainer.placement.colocate_all, + num_gpus_per_node=cfg.generator.inference_engine_tensor_parallel_size, + cfg=cfg, + ) + ray.get(policy.async_run_ray_method("pass_through", "init_weight_sync_state", client)) + asyncio.run(client.wake_up(tags=["weights"])) + # TODO (erictang000): improve this timing + # currently this is ~30 seconds for a 14B MoE model (on 8xL40S) + # or ~20 seconds on 8xH100 + # ~75 seconds on 8xH100 for Qwen3-30B-A3B + with Timer("sync_weights"): + ray.get(policy.async_run_ray_method("pass_through", "broadcast_to_inference_engines", client)) + + policy.offload_to_cpu() + asyncio.run(client.wake_up(tags=["kv_cache"])) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + outputs = asyncio.run(run_inference(client, get_test_prompts(MODEL_NAME), sampling_params)) + + print(f"Example output: {outputs['responses'][0]}, {outputs['stop_reasons'][0]}") + finally: + ray.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_type", "tp", "pp", "cp", "ep", "etp", "gpus_per_node", "use_sample_packing", "lora"), + [ + ("policy", 2, 1, 1, 1, None, 2, False, False), + # ref has same forward pass as policy - just duplicate one test to test setup + ("ref", 2, 1, 1, 1, None, 2, False, False), + ("policy", 2, 2, 1, 1, None, 4, False, False), + ("policy", 2, 2, 1, 1, None, 4, True, False), + ("policy", 2, 2, 1, 1, None, 4, True, True), + ("policy", 1, 1, 2, 1, None, 2, True, False), + ("policy", 2, 1, 2, 1, None, 4, True, False), + ("policy", 4, 1, 1, 4, 1, 4, True, False), + ], + ids=[ + "tp2_pp1_policy", + "tp2_pp1_ref", + "tp2_pp2_policy_unpacked", + "tp2_pp2_policy_seq_packing", + "tp2_pp2_lora", + "cp_2_policy_seq_packing", + "tp_2_cp_2_policy_seq_packing", + "tp4_pp1_cp1_ep4_etp1_policy_seq_packing", + ], +) +@pytest.mark.megatron +async def test_megatron_forward( + ray_init_fixture, worker_type, tp, pp, cp, ep, etp, gpus_per_node, use_sample_packing, lora +): + """ + Test that the Megatron forward pass is numerically equivalent to just running a huggingface model forward. + """ + cfg = get_test_actor_config(model_name=MOE_MODEL_NAME if ep > 1 else MODEL_NAME) + #### Megatron forward pass #### + cfg.trainer.strategy = "megatron" + cfg.trainer.placement.policy_num_gpus_per_node = gpus_per_node + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = tp + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = pp + cfg.trainer.policy.megatron_config.context_parallel_size = cp + cfg.trainer.policy.megatron_config.expert_model_parallel_size = ep + cfg.trainer.policy.megatron_config.expert_tensor_parallel_size = etp + cfg.trainer.use_sample_packing = use_sample_packing + batch = get_test_training_batch(max(4, gpus_per_node)) + + if ep > 1: + transformer_config_kwargs = OmegaConf.to_container( + cfg.trainer.policy.megatron_config.transformer_config_kwargs, resolve=True + ) + transformer_config_kwargs["num_layers"] = 2 + cfg.trainer.policy.megatron_config.transformer_config_kwargs = transformer_config_kwargs + + if lora: + cfg.trainer.policy.model.lora.rank = 16 + cfg.trainer.policy.model.lora.alpha = 16 + + actor_group = init_worker_with_type( + worker_type, + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + action_log_probs_refs = actor_group.async_run_ray_method("mesh", "forward", data=batch) + all_rank_action_log_probs = ray.get(action_log_probs_refs) + action_log_probs_megatron = concatenate_outputs_after_mesh_dispatch( + actor_group.actor_infos, all_rank_action_log_probs + )["output"] + + ray.shutdown() + ray_init_for_tests() + + #### Huggingface forward pass #### + # now run the huggingface model forward + @ray.remote(num_gpus=1) + def run_hf_forward(batch, model_name): + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True, dtype=torch.bfloat16) + if ep > 1: + config.num_hidden_layers = 2 + model = AutoModelForCausalLM.from_pretrained(model_name, config=config, dtype=torch.bfloat16) + model.eval() + model.to("cuda") + sequences_fwd = batch["sequences"] + attention_mask = batch["attention_mask"] + num_actions = batch.metadata["response_length"] + + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + + sequences_rolled = torch.roll(sequences_fwd, shifts=-1, dims=1).to("cuda") + + sequences_fwd, attention_mask, position_ids = ( + sequences_fwd.to("cuda"), + attention_mask.to("cuda"), + position_ids.to("cuda"), + ) + with torch.no_grad(), torch.autocast(dtype=torch.bfloat16, device_type="cuda"): + output = model(sequences_fwd, attention_mask=attention_mask, position_ids=position_ids) + log_probs = logprobs_from_logits(output["logits"], sequences_rolled) + action_log_probs = log_probs[:, -num_actions - 1 : -1].to("cpu").detach() + + return attention_mask.to("cpu").detach(), action_log_probs.to("cpu").detach(), num_actions + + attention_mask, action_log_probs, num_actions = ray.get( + run_hf_forward.remote(batch, MOE_MODEL_NAME if ep > 1 else MODEL_NAME) + ) + + #### Compare results #### + # compare just non-padding tokens + # Create response mask: 1 for valid response tokens, 0 for padding + response_mask = attention_mask[:, -num_actions:].bool() + + # Only compare valid (non-padding) response tokens + action_log_probs_masked = action_log_probs[response_mask] + action_log_probs_megatron_masked = action_log_probs_megatron[response_mask] + + print(f"Comparing {action_log_probs_masked.numel()} valid response tokens") + print(f"HF sample: {action_log_probs_masked[:5]}") + print(f"Megatron sample: {action_log_probs_megatron_masked[:5]}") + + # max diff + max_diff = torch.max(torch.abs(action_log_probs_masked - action_log_probs_megatron_masked)) + print(f"Max diff: {max_diff}") + + # average diff + avg_diff = torch.mean(torch.abs(action_log_probs_masked - action_log_probs_megatron_masked)) + print(f"Avg diff: {avg_diff}") + + if ep == 1: + assert max_diff < 4e-1, f"Max diff {max_diff} is too large" + + if ep == 1: + assert avg_diff < 7e-2, f"Avg diff {avg_diff} is too large" + else: + # allow larger tolerance in diff for the 30B-MoE model due to larger model size + assert avg_diff < 1.6e-1, f"Avg diff {avg_diff} is too large" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tp", "pp", "cp", "ep", "etp", "gpus_per_node"), + [ + (2, 2, 1, 1, None, 4), + (4, 1, 1, 4, 1, 4), + ], + ids=[ + "tp2_pp2_policy", + "tp4_pp1_cp1_ep4_etp1_policy", + ], +) +@pytest.mark.megatron +async def test_megatron_lora_forward(ray_init_fixture, tp, pp, cp, ep, etp, gpus_per_node): + """ + Test that the Megatron + lora forward pass is numerically equivalent to just running a megatron model forward. + """ + cfg = get_test_actor_config(model_name=MOE_MODEL_NAME if ep > 1 else MODEL_NAME) + #### Megatron forward pass #### + cfg.trainer.strategy = "megatron" + cfg.trainer.placement.policy_num_gpus_per_node = gpus_per_node + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = tp + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = pp + cfg.trainer.policy.megatron_config.context_parallel_size = cp + cfg.trainer.policy.megatron_config.expert_model_parallel_size = ep + cfg.trainer.policy.megatron_config.expert_tensor_parallel_size = etp + cfg.trainer.use_sample_packing = True + batch = get_test_training_batch(max(4, gpus_per_node)) + + if ep > 1: + transformer_config_kwargs = OmegaConf.to_container( + cfg.trainer.policy.megatron_config.transformer_config_kwargs, resolve=True + ) + transformer_config_kwargs["num_layers"] = 4 + cfg.trainer.policy.megatron_config.transformer_config_kwargs = transformer_config_kwargs + + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + action_log_probs_refs = actor_group.async_run_ray_method("mesh", "forward", data=batch) + all_rank_action_log_probs = ray.get(action_log_probs_refs) + action_log_probs_full = concatenate_outputs_after_mesh_dispatch(actor_group.actor_infos, all_rank_action_log_probs)[ + "output" + ] + + ray.shutdown() + ray_init_for_tests() + + #### Megatron forward pass #### + cfg.trainer.strategy = "megatron" + cfg.trainer.placement.policy_num_gpus_per_node = gpus_per_node + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = tp + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = pp + cfg.trainer.policy.megatron_config.context_parallel_size = cp + cfg.trainer.policy.megatron_config.expert_model_parallel_size = ep + cfg.trainer.policy.megatron_config.expert_tensor_parallel_size = etp + cfg.trainer.use_sample_packing = True + batch = get_test_training_batch(max(4, gpus_per_node)) + + # set lora this time + cfg.trainer.policy.model.lora.rank = 16 + cfg.trainer.policy.model.lora.alpha = 16 + + if ep > 1: + transformer_config_kwargs = OmegaConf.to_container( + cfg.trainer.policy.megatron_config.transformer_config_kwargs, resolve=True + ) + transformer_config_kwargs["num_layers"] = 4 + cfg.trainer.policy.megatron_config.transformer_config_kwargs = transformer_config_kwargs + + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + action_log_probs_refs = actor_group.async_run_ray_method("mesh", "forward", data=batch) + all_rank_action_log_probs = ray.get(action_log_probs_refs) + action_log_probs_lora = concatenate_outputs_after_mesh_dispatch(actor_group.actor_infos, all_rank_action_log_probs)[ + "output" + ] + + #### Compare results #### + # compare just non-padding tokens + print(f"Comparing {action_log_probs_full.numel()} valid response tokens") + print(f"Full sample: {action_log_probs_full[:5]}") + print(f"Lora sample: {action_log_probs_lora[:5]}") + + # max diff + max_diff = torch.max(torch.abs(action_log_probs_full - action_log_probs_lora)) + print(f"Max diff: {max_diff}") + + assert max_diff == 0, "Numerical difference between LoRA and full fine tuning at init should be 0" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_type", "tp", "pp", "cp", "ep", "etp", "gpus_per_node", "use_sample_packing", "use_entropy_loss", "lora"), + [ + ("policy", 2, 2, 1, 1, 1, 4, True, False, False), + ("policy", 2, 2, 1, 1, 1, 4, True, True, False), + ("policy", 2, 2, 1, 1, 1, 4, True, False, True), + ("policy", 2, 2, 1, 1, 1, 4, False, False, False), + ("policy", 2, 1, 2, 1, 1, 4, True, False, False), + ("policy", 4, 1, 1, 4, 1, 4, True, False, False), + ("policy", 4, 1, 1, 4, 1, 4, True, False, True), + ], + ids=[ + "tp2_pp2_policy_seq_packing", + "tp2_pp2_policy_seq_packing_with_entropy_loss", + "tp2_pp2_policy_lora", + "tp2_pp2_policy_unpacked", + "tp2_cp2_policy_seq_packing", + "tp4_pp1_cp1_ep4_etp1_policy_seq_packing", + "tp4_pp1_cp1_ep4_etp1_policy_seq_packing_lora", + ], +) +@pytest.mark.megatron +async def test_megatron_train( + ray_init_fixture, worker_type, tp, pp, cp, ep, etp, gpus_per_node, use_sample_packing, use_entropy_loss, lora +): + """ + Full test: initialize actor group, send dummy experience to training_step, validate output. + """ + cfg = get_test_actor_config(model_name=MODEL_NAME if ep == 1 else MOE_MODEL_NAME) + batch = get_test_training_batch(batch_size=gpus_per_node) + + cfg.trainer.strategy = "megatron" + cfg.trainer.placement.policy_num_gpus_per_node = gpus_per_node + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = tp + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = pp + cfg.trainer.policy.megatron_config.context_parallel_size = cp + cfg.trainer.policy.megatron_config.expert_model_parallel_size = ep + cfg.trainer.policy.megatron_config.expert_tensor_parallel_size = etp + cfg.trainer.use_sample_packing = use_sample_packing + if use_entropy_loss: + cfg.trainer.algorithm.use_entropy_loss = True + cfg.trainer.algorithm.entropy_loss_coef = 0.01 + if lora: + cfg.trainer.policy.model.lora.rank = 16 + cfg.trainer.policy.model.lora.alpha = 16 + + if ep > 1: + transformer_config_kwargs = OmegaConf.to_container( + cfg.trainer.policy.megatron_config.transformer_config_kwargs, resolve=True + ) + transformer_config_kwargs["num_layers"] = 2 + cfg.trainer.policy.megatron_config.transformer_config_kwargs = transformer_config_kwargs + + # set batch sizes correctly + cfg.trainer.train_batch_size = gpus_per_node + cfg.trainer.policy_mini_batch_size = gpus_per_node + cfg.generator.n_samples_per_prompt = 1 + cfg.trainer.micro_train_batch_size_per_gpu = 1 + + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_nodes=cfg.trainer.placement.policy_num_nodes, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + with Timer(f"megatron training step tp{tp} pp{pp} cp{cp} ep{ep} etp{etp}"): + batch.metadata["global_step"] = 0 + results_megatron = ray.get(actor_group.async_run_ray_method("pass_through", "ppo_train", batch)) + results_megatron = [results_megatron[i].metadata["train_status"] for i in range(len(results_megatron))] + + memory = ray.get(actor_group.async_run_ray_method("pass_through", "get_cuda_memory")) + memory = memory[0] + print_mem("memory after training step", memory) + + for result in results_megatron: + assert isinstance(result, dict), "Result should be a dictionary of training stats" + assert "policy_loss" in result + assert "policy_lr" in result + assert "ppo_clip_ratio" in result + assert "policy_entropy" in result + for k, v in result.items(): + assert isinstance(v, (int, float)), f"{k} should be an int or float" + + ray.shutdown() + ray_init_for_tests() + + cfg.trainer.strategy = "fsdp2" + # NOTE (erictang000): need to set sample packing to false here due to metric calculation differences + # between use_sample_packing true/false for FSDP (no diff for megatron) + # this shouldn't be the case, but tracking here: https://github.com/NovaSky-AI/SkyRL/issues/211 + # + tested that this does not affect convergence + cfg.trainer.use_sample_packing = False + if ep > 1: + cfg.trainer.policy.fsdp_config.cpu_offload = True + + if ep > 1: + model_config_kwargs = OmegaConf.to_container(cfg.trainer.policy.model_config_kwargs, resolve=True) + model_config_kwargs["num_hidden_layers"] = 2 + cfg.trainer.policy.model_config_kwargs = model_config_kwargs + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_nodes=cfg.trainer.placement.policy_num_nodes, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + # FSDP uses forward_backward + optim_step instead of ppo_train + batch.metadata["global_step"] = 0 + results_fsdp = ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", batch)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + # Get learning rate from worker + lr_results = ray.get(actor_group.async_run_ray_method("pass_through", "get_lr")) + for i, result in enumerate(results_fsdp): + result["policy_lr"] = lr_results[i] + + print("megatron results: ", results_megatron[0]) + print("\n\n") + print("fsdp results: ", results_fsdp[0]) + + keys_to_compare = ["policy_loss", "policy_lr", "ppo_clip_ratio", "policy_entropy", "policy_kl", "final_loss"] + for i, result in enumerate(results_fsdp): + for k in keys_to_compare: + if k == "policy_entropy": + # TODO: make entropy calculation only apply to non-padding tokens for all backends + # because the logits for padding tokens are all 0 for the non-sample packing case in megatron + # the entropy calculation is different (fsdp has random logits for padding tokens) + continue + assert isinstance(result[k], (int, float)), f"{k} should be an int or float" + assert abs(result[k] - results_megatron[i][k]) < 1.5e-1, f"diff in {k} is too large!" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_type", "tp", "pp", "gpus_per_node"), + [ + ("policy", 1, 2, 2), + ], +) +@pytest.mark.megatron +async def test_megatron_dp(ray_init_fixture, worker_type, tp, pp, gpus_per_node): + """ + Full test: initialize actor group, send dummy experience to training_step, validate output. + """ + cfg = get_test_actor_config() + batch = get_test_training_batch(16) + + cfg.trainer.strategy = "megatron" + cfg.trainer.placement.policy_num_gpus_per_node = gpus_per_node + + # try tp=2, pp=2 first + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = tp + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = pp + + # set batch sizes correctly + cfg.trainer.train_batch_size = 64 + cfg.trainer.policy_mini_batch_size = len(batch["sequences"]) # self.policy_mini_batch_size_per_gpu = 2 * 1 / 1 = 2 + cfg.generator.n_samples_per_prompt = 1 + cfg.trainer.micro_train_batch_size_per_gpu = 4 + + # set torch profiler config + cfg.trainer.policy.megatron_config.torch_profiler_config.enable = False + cfg.trainer.policy.megatron_config.torch_profiler_config.ranks = [0] + cfg.trainer.policy.megatron_config.torch_profiler_config.save_path = f"/home/ray/megatron_prof/tp{tp}_pp{pp}/" + + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + # call ppo_train with a batch of size 4 per gpu + batch.metadata["global_step"] = 0 + results_megatron = ray.get(actor_group.async_run_ray_method("mesh", "ppo_train", batch)) + results_megatron = [results_megatron[i].metadata["train_status"] for i in range(len(results_megatron))] + + memory = ray.get(actor_group.async_run_ray_method("pass_through", "get_cuda_memory")) + memory = memory[0] + print_mem("memory after training step", memory) + + for result in results_megatron: + assert isinstance(result, dict), "Result should be a dictionary of training stats" + assert "policy_loss" in result + assert "policy_lr" in result + assert "ppo_clip_ratio" in result + assert "policy_entropy" in result + for k, v in result.items(): + assert isinstance(v, (int, float)), f"{k} should be an int or float" + + ray.shutdown() + ray_init_for_tests() + + # check the grad norm for the same thing but with pp=1, tp=1, dp=4 + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = 1 + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = 1 + + # set torch profiler config + cfg.trainer.policy.megatron_config.torch_profiler_config.save_path = "/home/ray/megatron_prof/dp4/" + + # set batch sizes correctly + cfg.trainer.train_batch_size = 64 + cfg.trainer.policy_mini_batch_size = len(batch["sequences"]) # self.policy_mini_batch_size_per_gpu = 8 * 1 / 4 = 2 + cfg.generator.n_samples_per_prompt = 1 + cfg.trainer.micro_train_batch_size_per_gpu = 4 + + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + results_megatron_dp = ray.get(actor_group.async_run_ray_method("mesh", "ppo_train", batch)) + results_megatron_dp = [results_megatron_dp[i].metadata["train_status"] for i in range(len(results_megatron_dp))] + + print("megatron results: ", results_megatron) + print("\n\n") + print("megatron results dp: ", results_megatron_dp) + + keys_to_compare = ["policy_loss", "policy_lr", "ppo_clip_ratio", "policy_entropy", "policy_kl", "raw_grad_norm"] + for i, result in enumerate(results_megatron_dp): + for k in keys_to_compare: + assert isinstance(result[k], (int, float)), f"{k} should be an int or float" + assert abs(result[k] - results_megatron[i][k]) < 1.5e-1, f"diff in {k} is too large!" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_type"), + [ + "policy", + ], +) +@pytest.mark.megatron +async def test_megatron_offload_memory_and_correctness(ray_init_fixture, worker_type): + """ + Test that offloading model memory to cpu lowers memory usage and that correctness + is maintained after backloading and running a training step. + + steps: + 1. Initialize actor group with the specified worker class. + 2. Offload model to CPU and check memory usage. + 3. Backload model to GPU and check memory usage. + 4. Run a training step with dummy experience (with optimizer step) + 5. Offload model to CPU again and check memory usage. + 6. Backload model to GPU and check memory usage. + 7. Run another training step and ensure output consistency. + """ + cfg = get_test_actor_config(MOE_MODEL_NAME) # use MoE model for testing + cfg.trainer.strategy = "megatron" + # 0 learning rate and wd so we can optimizer step to free gradients but still check results are the same + getattr(cfg.trainer, worker_type).optimizer_config.lr = 0 + getattr(cfg.trainer, worker_type).optimizer_config.weight_decay = 0 + + cfg.trainer.placement.policy_num_gpus_per_node = 4 + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = 2 + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = 2 + cfg.trainer.policy.megatron_config.context_parallel_size = 1 + cfg.trainer.policy.megatron_config.expert_model_parallel_size = 2 + cfg.trainer.policy.megatron_config.expert_tensor_parallel_size = 1 + cfg.trainer.policy.megatron_config.optimizer_config_kwargs.use_precision_aware_optimizer = False + transformer_config_kwargs = OmegaConf.to_container( + cfg.trainer.policy.megatron_config.transformer_config_kwargs, resolve=True + ) + transformer_config_kwargs["num_layers"] = 2 + cfg.trainer.policy.megatron_config.transformer_config_kwargs = transformer_config_kwargs + actor_group = init_worker_with_type( + worker_type, + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + get_rank_0_memory(actor_group, "After init") + # offload then backload first (no training step) + actor_group.offload_to_cpu() + + initial_offload_mem = get_rank_0_memory(actor_group, "After initial offload") + + # Backload to GPU + actor_group.backload_to_gpu() + get_rank_0_memory(actor_group, "Before training") + + batch = get_test_training_batch() + results = ray.get(actor_group.async_run_ray_method("pass_through", "ppo_train", batch)) + + after_training = get_rank_0_memory(actor_group, "After training") + + # Offload model to CPU + actor_group.offload_to_cpu(offload_optimizer=True, offload_model=False) + after_offload_optimizer = get_rank_0_memory(actor_group, "After optimizer offload") + + assert ( + after_offload_optimizer < after_training + ), f"Memory after offload optimizer should be less than after training: {after_offload_optimizer} bytes, after training: {after_training} bytes" + + actor_group.offload_to_cpu(offload_optimizer=False, offload_model=True) + after_offload = get_rank_0_memory(actor_group, "After model offload") + + assert ( + after_offload < after_offload_optimizer + ), f"Memory after offload model should be less than after offload optimizer: {after_offload} bytes, after offload optimizer: {after_offload_optimizer} bytes" + + # check that allocated memory is similar to initial offload memory + delta = abs(initial_offload_mem - after_offload) + assert ( + delta < 4e8 # 400MB (should be close to 0 diff) + ), f"Memory after training step + offload is not similar to initial offloaded memory: {delta} bytes. Initial offload mem: {initial_offload_mem}, after offload mem: {after_offload} bytes" + + # also check that allocated memory goes down after offloading + delta_forward = after_training - after_offload + assert delta_forward > 0, f"Memory after offloading should be less than after forward pass: {delta_forward} bytes" + + # Backload model to GPU + actor_group.backload_to_gpu(backload_optimizer=True, backload_model=False) + after_backload_optimizer = get_rank_0_memory(actor_group, "After backload optimizer") + assert ( + after_backload_optimizer > after_offload + ), f"Memory after backload optimizer should be greater than after offload: {after_backload_optimizer} bytes, after offload: {after_offload} bytes" + + actor_group.backload_to_gpu(backload_optimizer=False, backload_model=True) + after_backload = get_rank_0_memory(actor_group, "After backload model") + assert ( + after_backload > after_backload_optimizer + ), f"Memory after backload model should be greater than after backload optimizer: {after_backload} bytes, after backload optimizer: {after_backload_optimizer} bytes" + + get_rank_0_memory(actor_group, "After backload") + + # Run training again and ensure output consistency + results_backload = ray.get(actor_group.async_run_ray_method("pass_through", "ppo_train", batch)) + + for i, result in enumerate(results): + result_backload = results_backload[i] + for k, v in result.items(): + assert k in result_backload + assert v == result_backload[k], f"Results mismatch for {k}: {v} != {result_backload[k]}" diff --git a/skyrl-train/tests/gpu/gpu_ci/test_model_wrapper.py b/skyrl-train/tests/gpu/gpu_ci/test_model_wrapper.py new file mode 100644 index 0000000000..5839e7c6c6 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_model_wrapper.py @@ -0,0 +1,238 @@ +""" +uv run --isolated --extra dev pytest tests/gpu/gpu_ci/test_model_wrapper.py +""" + +from skyrl_train.model_wrapper import HFModelWrapper +import torch +import pytest +from unittest.mock import MagicMock, patch +from transformers import AutoTokenizer +import ray +import torch.distributed as dist +from torch.distributed.distributed_c10d import _get_default_group +from skyrl_train.workers.worker import set_ulysses_sequence_parallel_group, apply_monkey_patch + +MODEL_NAME = "llamafactory/tiny-random-Llama-3" +TOKENIZER_NAME = "unsloth/Meta-Llama-3.1-8B" + + +def get_dummy_inputs(): + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME, trust_remote_code=True) + input_without_padding = [10, 14, 200, 3, 40, tokenizer.eos_token_id] + input_ids = [tokenizer.pad_token_id] * 3 + input_without_padding + [tokenizer.pad_token_id] + attention_mask = [0] * 3 + [1] * len(input_without_padding) + [0] + num_actions = 4 + input_ids = torch.tensor(input_ids).unsqueeze(0) + attention_mask = torch.tensor(attention_mask).unsqueeze(0) + input_without_padding = torch.tensor(input_without_padding).unsqueeze(0) + return input_ids, attention_mask, num_actions, input_without_padding + + +@patch("skyrl_train.model_wrapper.logprobs_from_logits") +def test_hf_model_wrapper_fwd_with_sample_packing(mock_logprobs_from_logits): + model = HFModelWrapper( + pretrain_or_model=MODEL_NAME, + use_flash_attention_2=True, + bf16=False, + sequence_parallel_size=1, + use_sample_packing=True, + ) + model.model.eval() + model.model.to("cuda") + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME, trust_remote_code=True) + input_ids, attention_mask, num_actions, input_without_padding = get_dummy_inputs() + actual_actions = input_ids[:, -num_actions:] + # mock the actual forward + input_ids, attention_mask, input_without_padding = ( + input_ids.to("cuda"), + attention_mask.to("cuda"), + input_without_padding.to("cuda"), + ) + # forward is with packed sequence length + model.model.forward = MagicMock(return_value={"logits": torch.randn(1, 6, 1000).to("cuda")}) + # just roll the inputs over for return value. This means that the model is correctly predicting the next token. + # The last token will be the prediction beyond the eos token, just set to pad token id + mock_return_value = torch.roll(input_without_padding, shifts=-1, dims=-1) + mock_return_value[:, -1] = tokenizer.pad_token_id + mock_logprobs_from_logits.return_value = mock_return_value + + action_log_probs = model(input_ids, num_actions, attention_mask) + expected_log_probs = actual_actions.to("cuda") + assert torch.equal( + action_log_probs, expected_log_probs + ), f"Expected log probs to be {expected_log_probs} but got {action_log_probs}" + + +@patch("skyrl_train.model_wrapper.logprobs_from_logits") +def test_hf_model_wrapper_fwd_without_sample_packing(mock_logprobs_from_logits): + model = HFModelWrapper( + pretrain_or_model=MODEL_NAME, + use_flash_attention_2=True, + bf16=False, + sequence_parallel_size=1, + use_sample_packing=False, + ) + model.model.eval() + model.model.to("cuda") + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME, trust_remote_code=True) + input_ids, attention_mask, num_actions, input_without_padding = get_dummy_inputs() + actual_actions = input_ids[:, -num_actions:] + # mock the actual forward + input_ids, attention_mask, input_without_padding = ( + input_ids.to("cuda"), + attention_mask.to("cuda"), + input_without_padding.to("cuda"), + ) + # forward is with packed sequence length + model.model.forward = MagicMock(return_value={"logits": torch.randn(1, 10, 1000).to("cuda")}) + # just roll the inputs over. This means that the model is correctly predicting the next token. + mock_return_value = torch.roll(input_ids, shifts=-1, dims=-1) + # The last token will be the prediction beyond the eos token, just set to pad token id + mock_return_value[:, -1] = tokenizer.pad_token_id + mock_logprobs_from_logits.return_value = mock_return_value + + action_log_probs = model(input_ids, num_actions, attention_mask) + expected_log_probs = actual_actions.to("cuda") + assert torch.equal( + action_log_probs, expected_log_probs + ), f"Expected log probs to be {expected_log_probs} but got {action_log_probs}" + + +@ray.remote(num_gpus=1) +class ActorTask: + def __init__(self, rank, world_size, sequence_parallel_size, use_sample_packing): + self.rank = rank + self.world_size = world_size + self.sequence_parallel_size = sequence_parallel_size + + # Initialize distributed environment + dist.init_process_group(backend="nccl", init_method="tcp://localhost:23456", world_size=world_size, rank=rank) + + # Create model with sequence parallelism + self.model = HFModelWrapper( + pretrain_or_model=MODEL_NAME, + use_flash_attention_2=True, + bf16=True, + sequence_parallel_size=sequence_parallel_size, + use_sample_packing=use_sample_packing, + ) + group = _get_default_group() + set_ulysses_sequence_parallel_group(group) + apply_monkey_patch(model=self.model.model, ulysses_sp_size=self.sequence_parallel_size, use_parent_class=False) + self.model.model.eval() + self.model.model.to("cuda") + + def forward(self, input_ids, attention_mask, num_actions): + input_ids = input_ids.to("cuda") + attention_mask = attention_mask.to("cuda") + return self.model(input_ids, num_actions, attention_mask, return_output=True, compute_entropy=True) + + def cleanup(self): + dist.destroy_process_group() + + +@pytest.mark.parametrize("use_sample_packing", [True, False]) +def test_actor_model_fwd_with_sequence_parallelism(ray_init_fixture, use_sample_packing): + + # Create input sequence + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME, trust_remote_code=True) + input_ids = [ + [tokenizer.pad_token_id] * 3 + [10, 14, 200, 3, 40, tokenizer.eos_token_id] + [tokenizer.pad_token_id], + [tokenizer.pad_token_id, 12, 13, 220, 1000, 3, 40, tokenizer.eos_token_id] + [tokenizer.pad_token_id] * 2, + ] + attention_mask = [[0] * 3 + [1] * 6 + [0], [0] + [1] * 7 + [0] * 2] + num_actions = 4 + + # Convert to tensors + input_ids = torch.tensor(input_ids) + attention_mask = torch.tensor(attention_mask) + + # First run without sequence parallelism + model_no_sp = HFModelWrapper( + pretrain_or_model=MODEL_NAME, + use_flash_attention_2=True, + bf16=True, + sequence_parallel_size=1, + use_sample_packing=use_sample_packing, + ) + model_no_sp.model.eval() + model_no_sp.model.to("cuda:0") + + # Forward pass without sequence parallelism + with torch.no_grad(): + logprobs_no_sp, output_no_sp = model_no_sp( + input_ids.to("cuda:0"), num_actions, attention_mask.to("cuda:0"), compute_entropy=True, return_output=True + ) + + # Now run with sequence parallelism using Ray + world_size = 2 + sequence_parallel_size = 2 + + # Create Ray tasks + actors = [ + ActorTask.remote(rank, world_size, sequence_parallel_size, use_sample_packing) for rank in range(world_size) + ] + + # Run forward pass with sequence parallelism + results_sp = ray.get([actor.forward.remote(input_ids, attention_mask, num_actions) for actor in actors]) + + # Verify outputs match + # # Since we're using sequence parallelism, each GPU processes half the sequence + # # and the outputs should be gathered and match the non-parallel output + for i, (logprob_sp, output_sp) in enumerate(results_sp): + assert torch.allclose( + logprob_sp, logprobs_no_sp + ), f"Logprobs with sequence parallelism don't match logprobs without sequence parallelism for rank {i}" + assert torch.allclose( + output_no_sp["entropy"], output_sp["entropy"] + ), "Entropy with sequence parallelism doesn't match entropy without sequence parallelism" + + # Cleanup + ray.get([actor.cleanup.remote() for actor in actors]) + + +def test_actor_entropy_consistency_sample_packing(): + """Tests that entropy calculation is consistent with and without sample packing""" + + # Create input sequence + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME, trust_remote_code=True) + input_ids = [ + [tokenizer.pad_token_id] * 3 + [10, 14, 200, 3, 40, tokenizer.eos_token_id] + [tokenizer.pad_token_id], + [tokenizer.pad_token_id, 12, 13, 220, 1000, 3, 40, tokenizer.eos_token_id] + [tokenizer.pad_token_id] * 2, + ] + attention_mask = [[0] * 3 + [1] * 6 + [0], [0] + [1] * 7 + [0] * 2] + num_actions = 4 + + # Convert to tensors + input_ids = torch.tensor(input_ids) + attention_mask = torch.tensor(attention_mask) + + # First run without sample packing + model = HFModelWrapper( + pretrain_or_model=MODEL_NAME, + use_flash_attention_2=True, + bf16=True, + sequence_parallel_size=1, + use_sample_packing=False, + ) + + model.model.eval() + model.model.to("cuda:0") + + with torch.no_grad(): + _, output_no_pack = model( + input_ids.to("cuda:0"), num_actions, attention_mask.to("cuda:0"), compute_entropy=True, return_output=True + ) + + # Switch to using sample packing + model.use_sample_packing = True + + with torch.no_grad(): + _, output_pack = model( + input_ids.to("cuda:0"), num_actions, attention_mask.to("cuda:0"), compute_entropy=True, return_output=True + ) + + # Entropy results should be very close + assert torch.allclose( + output_no_pack["entropy"], output_pack["entropy"] + ), "Entropy with sample packing doesn't match entropy without sample packing" diff --git a/skyrl-train/tests/gpu/gpu_ci/test_pause_and_continue_generation.py b/skyrl-train/tests/gpu/gpu_ci/test_pause_and_continue_generation.py new file mode 100644 index 0000000000..81e6da43f0 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_pause_and_continue_generation.py @@ -0,0 +1,355 @@ +""" +Test pause and continue generation with inference engine client HTTP endpoint. + +uv run --isolated --extra dev --extra vllm pytest tests/gpu/gpu_ci/test_pause_and_continue_generation.py -m "vllm" +""" + +import pytest +import asyncio +from tests.gpu.gpu_ci.test_inference_engine_client_http_endpoint import get_test_actor_config +from tests.gpu.utils import init_inference_engines, get_test_prompts +from skyrl_train.inference_engines.base import ConversationType +from transformers import AutoTokenizer +from typing import List +from skyrl_train.inference_engines.inference_engine_client_http_endpoint import ( + serve, + wait_for_server_ready, + shutdown_server, +) +import threading +import requests + +MODEL = "Qwen/Qwen2.5-0.5B-Instruct" +TP_SIZE = 2 +SERVER_PORT = 8123 +SERVER_HOST = "127.0.0.1" + + +@pytest.mark.vllm +def test_continue_generation_vllm_engine_chat_completion(ray_init_fixture): + """ + We send 6 requests via `/chat/completions` to two engines concurrently with vLLM `max_num_seqs=2` + so that in each engine, 2 run and 1 wait. We ignore eos and let model geneate 2048 tokens. + We pause and then resume generation twice in the middle. We expect each response to + finish with reason `length` and have exactly `max_tokens` completion tokens. + """ + server_thread = None + num_engines = 2 + num_requests = 6 + max_num_seqs = 2 + try: + # 1. Build engine and start server + cfg = get_test_actor_config(num_inference_engines=num_engines, model=MODEL) + cfg.trainer.placement.colocate_all = True + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + sampling_params = { + "max_tokens": 2048, + "stop": None, + "stop_token_ids": None, + "ignore_eos": True, + "stream": False, + "temperature": 0.0, + # Ensure logprobs and token ids are returned for accumulation checks + "logprobs": True, + "top_logprobs": 1, + "return_tokens_as_token_ids": True, + } + client, _ = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=1, + # We test aborting 2 running requests and 1 waiting requests + max_num_seqs=max_num_seqs, + ) + + def run_server(): + serve(client, host=SERVER_HOST, port=SERVER_PORT, log_level="warning") + + server_thread = threading.Thread(target=run_server, daemon=True) + server_thread.start() + wait_for_server_ready(host=SERVER_HOST, port=SERVER_PORT, max_wait_seconds=30) + base_url = f"http://{SERVER_HOST}:{SERVER_PORT}/v1" + + # 2. Prepare input + messages: List[ConversationType] = get_test_prompts(MODEL, num_samples=1)[0] + + # 3. Fire 6 concurrent HTTP requests, then pause/resume mid-flight + results = {} + + def send_request(i: int): + r = requests.post( + f"{base_url}/chat/completions", + json={"model": MODEL, "messages": messages, **sampling_params}, + ) + # Store minimal structured result for assertions + content_type = r.headers.get("Content-Type", "") + resp_json = r.json() if content_type.startswith("application/json") else {} + results[i] = { + "status_code": r.status_code, + "text": r.text, + "response": resp_json, + } + + threads = [threading.Thread(target=send_request, args=(i,), daemon=True) for i in range(num_requests)] + for t in threads: + t.start() + + # Let the requests start and enqueue; with max_num_seqs=2, 2 run and 1 wait + asyncio.run(asyncio.sleep(1)) + + # Pause then resume while requests are in-flight + asyncio.run(client.pause_generation()) + asyncio.run(client.resume_generation()) + # Run for another two seconds, then pause and resume again + asyncio.run(asyncio.sleep(2)) + asyncio.run(client.pause_generation()) + asyncio.run(client.resume_generation()) + + # Wait for all requests to finish + for t in threads: + t.join(timeout=180) + + # Ensure we collected all num_requests results + assert len(results) == num_requests, f"Expected {num_requests} responses, got {len(results)}" + + # 4. Validate each output: finish_reason is length and completion_tokens == max_tokens + for i in range(num_requests): + assert i in results, f"Missing result for index {i}" + cur = results[i] + assert cur.get("status_code") == 200, f"Request {i} failed: {cur.get('status_code')} {cur.get('text')}" + out = cur["response"] + assert "choices" in out and len(out["choices"]) == 1, f"Invalid choices for request {i}: {out}" + assert ( + out["choices"][0].get("finish_reason") == "length" + ), f"Request {i} finish_reason is not 'length': {out['choices'][0].get('finish_reason')}" + + choice = out["choices"][0] + logprobs = choice["logprobs"] + token_count_from_logprobs = len(logprobs["content"]) + print(f"Output first 1500 chars: {choice['message']['content'][:1500]}...") + + # Check completion tokens + assert ( + out["usage"]["completion_tokens"] == sampling_params["max_tokens"] + ), f"Request {i} expected completion_tokens={sampling_params['max_tokens']}, got {out['usage']['completion_tokens']}" + assert ( + token_count_from_logprobs == sampling_params["max_tokens"] + ), f"Request {i} expected {sampling_params['max_tokens']} tokens from logprobs, got {token_count_from_logprobs}" + + # Spot-check structure of each logprob entry: token contains token_id and top_logprobs length matches request + top_logprobs = sampling_params["top_logprobs"] + for entry in logprobs["content"]: + # tokens are token_id: when return_tokens_as_token_ids=True + parts = str(entry["token"]).split(":") + assert ( + len(parts) >= 2 and parts[-1].isdigit() + ), f"Request {i} token field not token_id:int: {entry['token']}" + assert ( + len(entry["top_logprobs"]) == top_logprobs + ), f"Request {i} expected top_logprobs len {top_logprobs}, got {len(entry['top_logprobs'])}" + # Check prompt tokens + prompt_tokens = client.tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=True) + assert ( + len(prompt_tokens) == out["usage"]["prompt_tokens"] + ), f"Request {i} expected {len(prompt_tokens)} tokens from prompt, got {out['usage']['prompt_tokens']}" + # TODO(Charlie): after we bump vllm such that it supports returnining tokens, check `choice["token_ids"]` + # TODO(Charlie): after we add model version to the output, check that as well + finally: + shutdown_server(host=SERVER_HOST, port=SERVER_PORT, max_wait_seconds=5) + if server_thread is not None and server_thread.is_alive(): + server_thread.join(timeout=5) + + +@pytest.mark.vllm +def test_continue_generation_generate_vllm_engine_generation(ray_init_fixture): + """ + Identical to `test_continue_generation_vllm_engine_chat_completion` except we use `generate()` + instead of `/chat/completions`. + + Launch 6 concurrent single-request generate() calls against two engines with vLLM `max_num_seqs=2` + so that in each engine, 2 run and 1 wait. Ignore EOS and request a long generation (2048 tokens). + Pause and then resume generation twice mid-flight. Expect each request to finish with reason `length` + and have exactly `max_tokens` completion tokens (i.e., len(response_ids[0]) == max_tokens and + len(response_logprobs[0]) == max_tokens). + """ + num_engines = 2 + num_requests = 6 + max_num_seqs = 2 + + # 1. Build engines (no HTTP server needed for generate()) + cfg = get_test_actor_config(num_inference_engines=num_engines, model=MODEL) + cfg.trainer.placement.colocate_all = True + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + sampling_params = { + "max_tokens": 2048, + "stop": None, + "stop_token_ids": None, + "ignore_eos": True, + "temperature": 0.0, + # Request token logprobs (vLLM SamplingParams expects an int for how many to return) + "logprobs": 1, + } + client, _ = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=1, + max_num_seqs=max_num_seqs, + ) + + # 2. Prepare a single ConversationType prompt; each generate() call will be single-request + messages: List[ConversationType] = get_test_prompts(MODEL, num_samples=1)[0] + + # 3. Fire 6 concurrent client.generate() single-request calls, then pause/resume mid-flight + async def run_requests_then_pause(): + async def one_req(i: int): + engine_input = { + "prompts": [messages], # single request path + "prompt_token_ids": None, + "sampling_params": dict(sampling_params), + "session_ids": [i], + } + return await client.generate(engine_input) + + tasks = [asyncio.create_task(one_req(i)) for i in range(num_requests)] + # Let requests start and enqueue; with max_num_seqs=2, 2 run and 1 wait per engine + await asyncio.sleep(1) + # Pause then resume while requests are in-flight + await client.pause_generation() + await client.resume_generation() + # Run for another two seconds, then pause and resume again + await asyncio.sleep(2) + await client.pause_generation() + await client.resume_generation() + return await asyncio.gather(*tasks) + + outputs = asyncio.run(run_requests_then_pause()) + + # 4. Validate each output: stop_reason is "length" and tokens/logprobs == max_tokens + assert len(outputs) == num_requests, f"Expected {num_requests} outputs, got {len(outputs)}" + for i, out in enumerate(outputs): + # InferenceEngineOutput shape checks + assert "responses" in out and "response_ids" in out and "stop_reasons" in out + assert len(out["responses"]) == 1 and len(out["response_ids"]) == 1 and len(out["stop_reasons"]) == 1 + assert out["stop_reasons"][0] == "length", f"Request {i} stop_reason is not 'length': {out['stop_reasons'][0]}" + # Check completion tokens via response_ids + token_ids = out["response_ids"][0] + assert ( + len(token_ids) == sampling_params["max_tokens"] + ), f"Request {i} expected {sampling_params['max_tokens']} tokens, got {len(token_ids)}" + # Check response_logprobs length + assert "response_logprobs" in out, f"Request {i} missing response_logprobs" + assert ( + len(out["response_logprobs"][0]) == sampling_params["max_tokens"] + ), f"Request {i} expected {sampling_params['max_tokens']} logprobs, got {len(out['response_logprobs'][0])}" + # Check string output is + assert out["responses"][0] == client.tokenizer.decode(token_ids, skip_special_tokens=True) + # Print a preview to aid debugging + print(f"Output first 1500 chars: {out['responses'][0][:1500]}...") + + +@pytest.mark.vllm +def test_abort_generation_vllm_engine(ray_init_fixture): + """ + We send 4 requests that are really long to `InferenceEngineClient.engines[0].chat_completion` + and then call abort. We set max_num_seqs=2 to test aborting 2 running requests and 2 waiting + requests. We expect 2 requests to be returned with completion_tokens=0 and 2 with non-zero + completion_tokens. We also expect the finish_reason to be "abort" for all requests. + """ + # 1. Build engine + cfg = get_test_actor_config(num_inference_engines=1, model=MODEL) + cfg.trainer.placement.colocate_all = True + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + # We generate 8192 tokens ad ignore eos. + sampling_params = { + "max_tokens": 8192, + "stop": None, + "stop_token_ids": None, + "ignore_eos": True, + "stream": False, + } + client, _ = init_inference_engines( + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend="vllm", + model=MODEL, + num_inference_engines=cfg.generator.num_inference_engines, + sleep_level=1, + # We test aborting 2 running requests and 2 waiting requests + max_num_seqs=2, + ) + + tokenizer = AutoTokenizer.from_pretrained(MODEL) + + for api in ["chat_completion", "completion"]: + + # 2. Build 4 chat prompts that have no early stops + convs: List[ConversationType] = [ + [ + {"role": "system", "content": "You are a token generator that keeps talking endlessly."}, + {"role": "user", "content": "Write a very long rambling response without ending."}, + ] + for _ in range(4) + ] + + # 3. Fire 4 concurrent requests directly to engine[0] + async def run_requests_then_pause(): + async def one_req(i: int): + if api == "chat_completion": + body = { + "model": MODEL, + "messages": convs[i], + **sampling_params, + } + return await client.engines[0].chat_completion({"json": body, "headers": {}}) + else: + # completions: prompt is a string + prompt_str = tokenizer.apply_chat_template(convs[i], add_generation_prompt=True, tokenize=False) + body = { + "model": MODEL, + "prompt": prompt_str, + **sampling_params, + } + return await client.engines[0].completion({"json": body, "headers": {}}) + + tasks = [asyncio.create_task(one_req(i)) for i in range(4)] + # Wait to let it run a bit, then pause generation + await asyncio.sleep(1) + await client.pause_generation() + return await asyncio.gather(*tasks) + + outputs = asyncio.run(run_requests_then_pause()) + + # 5. Validate outputs: each should be a ChatCompletionResponse; finish_reason is either "abort" or "length" + num_completion_tokens_is_zero = 0 + for out in outputs: + assert "choices" in out and len(out["choices"]) == 1 + if out["usage"]["completion_tokens"] == 0: + num_completion_tokens_is_zero += 1 + assert out["choices"][0].get("finish_reason") == "abort" + + # Two requests should have never got to run because we have max_num_seqs=2, and yet they should + # be aborted. + assert ( + num_completion_tokens_is_zero == 2 + ), f"Expected 2 requests with completion_tokens=0, got {num_completion_tokens_is_zero}." + + # Unpause for the next API run + asyncio.run(client.resume_generation()) diff --git a/skyrl-train/tests/gpu/gpu_ci/test_policy_local_engines_e2e.py b/skyrl-train/tests/gpu/gpu_ci/test_policy_local_engines_e2e.py new file mode 100644 index 0000000000..14864dd143 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_policy_local_engines_e2e.py @@ -0,0 +1,105 @@ +""" +# Run only vllm tests (requires vllm extra): +uv run --isolated --extra dev --extra vllm pytest tests/gpu/gpu_ci/test_policy_local_engines_e2e.py -m "vllm" + +# Run only sglang tests (requires sglang extra): +uv run --isolated --extra dev --extra sglang pytest tests/gpu/gpu_ci/test_policy_local_engines_e2e.py -m "sglang" +""" + +import pytest +import asyncio +import ray +import hydra +from omegaconf import DictConfig + +from tests.gpu.utils import init_worker_with_type, get_test_prompts, init_inference_engines, run_inference +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.entrypoints.main_base import config_dir + +MODEL = "Qwen/Qwen2.5-0.5B-Instruct" + + +def get_test_actor_config() -> DictConfig: + """Get base config with test-specific overrides.""" + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + # Override specific parameters + cfg.trainer.policy.model.path = MODEL + cfg.trainer.critic.model.path = "" + cfg.trainer.placement.policy_num_gpus_per_node = 2 + cfg.generator.async_engine = True + cfg.generator.num_inference_engines = 1 + cfg.generator.run_engines_locally = True + + return cfg + + +@pytest.mark.parametrize( + ("colocate_all", "weight_sync_backend", "strategy", "backend", "tp_size"), + [ + pytest.param(False, "nccl", "fsdp", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(True, "nccl", "fsdp", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(False, "gloo", "fsdp", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(True, "gloo", "fsdp", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(False, "nccl", "fsdp2", "vllm", 2, marks=pytest.mark.vllm), + pytest.param(True, "nccl", "fsdp2", "vllm", 2, marks=pytest.mark.vllm), + # TODO(Charlie): add TP > 1 tests for sglang when we support it + pytest.param(False, "nccl", "fsdp2", "sglang", 1, marks=pytest.mark.sglang), + pytest.param(True, "nccl", "fsdp2", "sglang", 1, marks=pytest.mark.sglang), + pytest.param(False, "gloo", "fsdp", "sglang", 1, marks=pytest.mark.sglang), + pytest.param(True, "gloo", "fsdp", "sglang", 1, marks=pytest.mark.sglang), + ], + ids=[ + "no_colocate_nccl_fsdp_vllm", + "colocate_nccl_fsdp_vllm", + "no_colocate_gloo_fsdp_vllm", + "colocate_gloo_fsdp_vllm", + "no_colocate_nccl_fsdp2_vllm", + "colocate_nccl_fsdp2_vllm", + "no_colocate_nccl_fsdp2_sglang", + "colocate_nccl_fsdp2_sglang", + "no_colocate_gloo_fsdp_sglang", + "colocate_gloo_fsdp_sglang", + ], +) +def test_policy_local_engines_e2e(ray_init_fixture, colocate_all, weight_sync_backend, strategy, backend, tp_size): + """ + Tests initalizing the policy actor group and inference engine, syncing weights, and performing generation. + """ + try: + cfg = get_test_actor_config() + cfg.trainer.placement.colocate_all = colocate_all + cfg.generator.weight_sync_backend = weight_sync_backend + cfg.trainer.strategy = strategy + cfg.generator.backend = backend + cfg.generator.inference_engine_tensor_parallel_size = tp_size + + # If colocate is True, this will load the engine, sleep, and wake up the engine + client, pg = init_inference_engines( + model=MODEL, + cfg=cfg, + use_local=True, + async_engine=cfg.generator.async_engine, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + colocate_all=cfg.trainer.placement.colocate_all, + backend=backend, + sleep_level=2, # since we explicitly sync weights + ) + + policy = init_worker_with_type( + "policy", + shared_pg=pg, + colocate_all=cfg.trainer.placement.colocate_all, + num_gpus_per_node=cfg.generator.inference_engine_tensor_parallel_size, + cfg=cfg, + ) + ray.get(policy.async_run_ray_method("pass_through", "init_weight_sync_state", client)) + asyncio.run(client.reset_prefix_cache()) + ray.get(policy.async_run_ray_method("pass_through", "broadcast_to_inference_engines", client)) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + outputs = asyncio.run(run_inference(client, get_test_prompts(MODEL), sampling_params)) + + print(f"Example output: {outputs['responses'][0]}, {outputs['stop_reasons'][0]}") + finally: + ray.shutdown() diff --git a/skyrl-train/tests/gpu/gpu_ci/test_save_load_checkpoint.py b/skyrl-train/tests/gpu/gpu_ci/test_save_load_checkpoint.py new file mode 100644 index 0000000000..7918bb4b04 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_save_load_checkpoint.py @@ -0,0 +1,198 @@ +""" +For FSDP and FSDP2, run: +uv run --isolated --extra dev -- pytest tests/gpu/gpu_ci/test_save_load_checkpoint.py -m "not megatron" + +For Megatron, run: +uv run --isolated --extra dev --extra mcore -- pytest tests/gpu/gpu_ci/test_save_load_checkpoint.py -m "megatron" +""" + +import ray +import pytest +import hydra +import torch +import os +import shutil +import json +from omegaconf import DictConfig +from transformers import AutoTokenizer + +from skyrl_train.utils.utils import print_mem +from tests.gpu.utils import init_worker_with_type, make_dummy_training_batch, get_model_logits_from_actor, validate_cfg +from skyrl_train.entrypoints.main_base import config_dir + +MODEL_NAME = "Qwen/Qwen3-0.6B" +CKPT_PATH = "$HOME/ckpts/test/" +NUM_GPUS = 4 + + +def run_one_training_step( + actor_group, + strategy, + data=None, + megatron_batch=None, +): + """Run forward_backward + optim_step to perform one training step.""" + if strategy == "megatron": + assert megatron_batch is not None, "Megatron requires a TrainingInputBatch for ppo_train" + return ray.get(actor_group.async_run_ray_method("mesh", "ppo_train", megatron_batch)) + else: + assert data is not None, f"{strategy} requires a TrainingInputBatch for forward_backward" + ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", data=data)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + + +def get_test_actor_config(strategy: str) -> DictConfig: + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = MODEL_NAME + cfg.trainer.placement.policy_num_gpus_per_node = NUM_GPUS + cfg.trainer.strategy = strategy + + cfg.trainer.ckpt_path = CKPT_PATH + cfg.trainer.export_path = CKPT_PATH + cfg.trainer.logger = "console" + + validate_cfg(cfg) + + return cfg + + +@pytest.mark.parametrize( + ("strategy", "lora"), + [ + ("fsdp", False), + ("fsdp2", False), + pytest.param("megatron", False, marks=pytest.mark.megatron), + pytest.param("megatron", True, marks=[pytest.mark.megatron, pytest.mark.lora]), + ], +) +def test_save_load_checkpoint(ray_init_fixture, strategy, lora): + """ + Test checkpointing logic by: + 1. Creating model and doing one training step + 2. Saving checkpoint + 3. Doing second training step and recording model logits + 4. Loading checkpoint + 5. Repeating second training step and comparing logits + """ + cfg = get_test_actor_config(strategy) + if lora: + cfg.trainer.policy.model.lora.rank = 32 + cfg.trainer.policy.model.lora.alpha = 32 + + try: + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + + checkpoint_dir = None + # Create dummy training batches for training steps + dp_size = actor_group.actor_infos[0].rank.dp_size + dummy_batch_1 = make_dummy_training_batch(batch_size=dp_size) # First training step + dummy_batch_2 = make_dummy_training_batch(batch_size=dp_size) # Second training step + + # Ensure the second batch is different from the first + dummy_batch_2["sequences"] = torch.randint(100, 200, dummy_batch_2["sequences"].shape, device="cpu") + + # For Megatron, build training batches and reuse the second one pre/post checkpoint resume + if "megatron" in strategy: + from tests.gpu.gpu_ci.test_megatron_worker import get_test_training_batch + + dp_size = actor_group.actor_infos[0].rank.dp_size + train_batch_1 = get_test_training_batch(dp_size if dp_size % NUM_GPUS == 0 else NUM_GPUS) + train_batch_2 = get_test_training_batch(dp_size if dp_size % NUM_GPUS == 0 else NUM_GPUS) + else: + train_batch_1 = None + train_batch_2 = None + + # Step 1: Do initial training step + run_one_training_step( + actor_group, + strategy, + data=dummy_batch_1, + megatron_batch=train_batch_1, + ) + + checkpoint_path = os.path.expandvars(os.path.join(cfg.trainer.ckpt_path, "global_step_1", "policy")) + checkpoint_dir = os.path.expandvars(os.path.join(cfg.trainer.ckpt_path, "global_step_1")) # Store for cleanup + + # Step 2: Save checkpoint + ray.get( + actor_group.async_run_ray_method( + "pass_through", "save_checkpoint", ckpt_dir=checkpoint_path, tokenizer=tokenizer + ) + ) + + # Step 2.1: Make sure that offloading still works after saving checkpoint + memory_after_saving = ray.get(actor_group.async_run_ray_method("pass_through", "get_cuda_memory"))[0] + print_mem("memory after saving checkpoint", memory_after_saving) + + actor_group.offload_to_cpu() + + memory_after_offloading = ray.get(actor_group.async_run_ray_method("pass_through", "get_cuda_memory"))[0] + print_mem("memory after offloading", memory_after_offloading) + + assert ( + memory_after_offloading["allocated"] < memory_after_saving["allocated"] + ), f"Memory after offloading should be less than after saving: {memory_after_offloading} bytes < {memory_after_saving} bytes" + actor_group.backload_to_gpu() + + # check that relevant files are saved + huggingface_dir = os.path.join(checkpoint_path, "huggingface") + expected_files = ["config.json", "generation_config.json", "tokenizer.json"] + for file in expected_files: + assert os.path.exists( + os.path.join(huggingface_dir, file) + ), f"File {file} not found in huggingface directory" + if "fsdp" in strategy: + fsdp_config_path = os.path.join(checkpoint_path, "fsdp_config.json") + with open(fsdp_config_path, "r") as f: + fsdp_config = json.load(f) + assert fsdp_config["fsdp_strategy"] == strategy + assert fsdp_config["world_size"] == NUM_GPUS + + # Step 3: Do second training step and record results + run_one_training_step( + actor_group, + strategy, + data=dummy_batch_2, + megatron_batch=train_batch_2, + ) + + # Create test input for comparing model outputs + dp_size = actor_group.actor_infos[0].rank.dp_size + test_input = torch.randint(0, 1000, (dp_size, 20), device="cpu") # batch_size=dp_size, seq_len=20 + attention_mask = torch.ones_like(test_input) + + # Step 4: Get logits after the second training step (this should be different from after checkpoint load) + logits_after_second_training = get_model_logits_from_actor(actor_group, test_input, attention_mask) + + # Step 5: Load checkpoint via strategy's load_checkpoint method + assert os.path.exists(checkpoint_path), f"Checkpoint directory {checkpoint_path} does not exist" + ray.get(actor_group.async_run_ray_method("pass_through", "load_checkpoint", ckpt_dir=checkpoint_path)) + + # Step 6: Now repeat the exact same second training step + run_one_training_step( + actor_group, + strategy, + data=dummy_batch_2, + megatron_batch=train_batch_2, + ) + + # Get logits after loading checkpoint and repeating second training + logits_after_reload_and_training = get_model_logits_from_actor(actor_group, test_input, attention_mask) + + # The logits should be exactly the same (checkpoint loading worked correctly) + torch.testing.assert_close(logits_after_second_training, logits_after_reload_and_training, atol=0.0, rtol=0.0) + + finally: + # Clean up checkpoint directory + if checkpoint_dir and os.path.exists(checkpoint_dir): + print(f"Removing checkpoint directory: {checkpoint_dir}") + shutil.rmtree(checkpoint_dir) diff --git a/skyrl-train/tests/gpu/gpu_ci/test_skyrl_gym_generator.py b/skyrl-train/tests/gpu/gpu_ci/test_skyrl_gym_generator.py new file mode 100644 index 0000000000..74166d1c32 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_skyrl_gym_generator.py @@ -0,0 +1,510 @@ +""" +uv run --extra dev --extra vllm --isolated pytest tests/gpu/gpu_ci/test_skyrl_gym_generator.py +""" + +import os +import pytest +import ray +from transformers import AutoTokenizer +from skyrl_train.inference_engines.ray_wrapped_inference_engine import create_ray_wrapped_inference_engines +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.generators.skyrl_gym_generator import SkyRLGymGenerator +from skyrl_train.generators.base import GeneratorInput, GeneratorOutput +from tests.gpu.utils import Timer, get_test_generator_input +from omegaconf import DictConfig, OmegaConf +from skyrl_train.utils.utils import initialize_ray +from skyrl_gym.envs import register +from skyrl_gym.envs.base_text_env import BaseTextEnv, BaseTextEnvStepOutput +from typing import Any, Dict +from loguru import logger +from skyrl_train.config.utils import get_default_config + +OBSERVATION_PROMPT = "give me another solution" + + +def get_test_config( + max_generate_length, + max_input_length, + batched, + max_turns, + use_conversation_multi_turn, + max_env_workers, + model, + is_step_wise, + temperature, + get_logprobs, +): + # Create a mock generator config + cfg = get_default_config() + OmegaConf.update( + cfg, + "generator", + { + "sampling_params": { + "max_generate_length": max_generate_length, + # 0 -> get logprobs for the first/ chosen token + "logprobs": 0 if get_logprobs else None, + "temperature": temperature, + }, + "append_eos_token_after_stop_str_in_multi_turn": True, # for search + "max_input_length": max_input_length, + "batched": batched, + "max_turns": max_turns, + "zero_reward_on_non_stop": False, + "use_conversation_multi_turn": use_conversation_multi_turn, + "apply_overlong_filtering": False, + "backend": "vllm", + "enable_http_endpoint": False, + "http_endpoint_host": "127.0.0.1", + "http_endpoint_port": 8000, + "step_wise_trajectories": is_step_wise, + }, + ) + + OmegaConf.update( + cfg, + "environment.skyrl_gym", + { + "search": { + "log_requests": True, + "search_url": "http://127.0.0.1:8000/retrieve", + }, + "max_env_workers": max_env_workers, + }, + ) + + cfg.trainer.policy.model.path = model + cfg.generator = cfg.generator + return cfg + + +# Setup for formatting tests +class TestEnv(BaseTextEnv): + def __init__(self, env_config: DictConfig, extras: Dict[str, Any] = {}): + super().__init__() + self.max_turns = 3 + + def init(self, prompt): + return prompt, {} + + def step(self, action: str): + self.turns += 1 + done = self.turns >= self.max_turns + return BaseTextEnvStepOutput( + observations=[{"role": "user", "content": f"{OBSERVATION_PROMPT} {self.turns}"}] if not done else [], + reward=0, + done=done, + metadata={}, + ) + + +register( + id="test_env", + entry_point="tests.gpu.gpu_ci.test_skyrl_gym_generator:TestEnv", +) + +MODEL_TO_GENERATION_PROMPT = { + "Qwen/Qwen2.5-1.5B-Instruct": "<|im_start|>assistant\n", + "unsloth/Llama-3.2-1B-Instruct": "<|start_header_id|>assistant<|end_header_id|>\n\n", + "Qwen/Qwen3-0.6B": "<|im_start|>assistant\n", +} + + +async def run_generator_end_to_end( + use_async_engine, + batched, + n_samples_per_prompt, + num_inference_engines, + tensor_parallel_size, + model="Qwen/Qwen2.5-1.5B-Instruct", + max_prompt_length=512, + max_input_length=2048, + max_generate_length=1024, + data_path=os.path.expanduser("~/data/gsm8k/validation.parquet"), + env_class="gsm8k", + num_prompts=2, + max_turns=1, + use_conversation_multi_turn=True, + max_env_workers=10, + is_step_wise: bool = False, + temperature=1.0, + get_logprobs: bool = False, +): + """ + End to end generator test - requires minimum 2 GPUs + """ + tokenizer = AutoTokenizer.from_pretrained(model) + + inference_engines = create_ray_wrapped_inference_engines( + num_inference_engines=num_inference_engines, + tensor_parallel_size=tensor_parallel_size, + model_dtype="bfloat16", + pretrain=model, + seed=42, + vllm_v1_disable_multiproc=True, + enable_prefix_caching=True, + enforce_eager=True, + shared_pg=None, + gpu_memory_utilization=0.8, + inference_engine_enable_sleep=True, + async_engine=use_async_engine, + max_num_batched_tokens=32768, + max_num_seqs=1024, + tokenizer=tokenizer, + sleep_level=1, # in unit tests that do not explicitly sync weights, we do not discard weights + ) + + cfg = get_test_config( + max_generate_length, + max_input_length, + batched, + max_turns, + use_conversation_multi_turn, + max_env_workers, + model, + is_step_wise, + temperature, + get_logprobs, + ) + + env_cfg = cfg.environment.skyrl_gym + generator_cfg = cfg.generator + + inference_engine_client = InferenceEngineClient( + inference_engines, + tokenizer, + cfg, + ) + + await inference_engine_client.wake_up() + + generator = SkyRLGymGenerator( + generator_cfg=generator_cfg, + skyrl_gym_cfg=env_cfg, + inference_engine_client=inference_engine_client, + tokenizer=tokenizer, + model_name=model, + ) + + input_batch: GeneratorInput = get_test_generator_input( + model=model, + num_prompts=num_prompts, + n_samples_per_prompt=n_samples_per_prompt, + max_prompt_length=max_prompt_length, + data_path=data_path, + env_class=env_class, + ) + # Attach request-time sampling params into the generator input + input_batch["sampling_params"] = get_sampling_params_for_backend( + "vllm", + DictConfig( + { + "temperature": 1.0, + "top_p": 1.0, + "top_k": -1, + "max_generate_length": max_generate_length, + "min_p": 0.0, + "logprobs": 0 if get_logprobs else None, + "stop": ["", ""] if env_class == "search" else None, + } + ), + ) + + with Timer(f"generate_responses_async_engine_{use_async_engine}"): + generator_output = await generator.generate(input_batch) + + prompts_out = generator_output["prompt_token_ids"] + outputs = [ + { + "response": generator_output["response_ids"][i], + "loss_mask": generator_output["loss_masks"][i], + "rollout_logprobs": ( + generator_output["rollout_logprobs"][i] if generator_output["rollout_logprobs"] else None + ), + } + for i in range(len(generator_output["response_ids"])) + ] + + output_keys = [ + "prompt_token_ids", + "response_ids", + "rewards", + "loss_masks", + "stop_reasons", + "rollout_metrics", + "rollout_logprobs", + ] + for key in output_keys: + assert key in generator_output, f"Key {key} not found in generator output" + if max_turns == 1: + # make sure that the max number of tokens is less than the max generate length for single turn generation + assert "generate/max_num_tokens" in generator_output["rollout_metrics"] + assert generator_output["rollout_metrics"]["generate/max_num_tokens"] <= max_generate_length + + assert len(prompts_out) == len(outputs), "Mismatch between prompts and outputs" + assert isinstance(prompts_out[0], list), "Prompts output should be a list" + assert isinstance(prompts_out[0][0], int), "Prompts output should be a list of list of token ids" + assert isinstance(outputs[0]["response"][0], int), "Prompts output should be a list of list of token ids" + if not is_step_wise: + assert ( + len(outputs) == num_prompts * n_samples_per_prompt + ), "Mismatch between number of outputs and expected outputs" + + if get_logprobs: + assert generator_output["rollout_logprobs"] is not None, "expected `rollout_logprobs` to be computed" + + for i in range(len(outputs)): + response_length = len(outputs[i]["response"]) + # TODO (erictang000): make this more precise for multi-turn + assert response_length <= max_generate_length + max_input_length, f"Output {i} exceeds max length" + assert response_length == len(outputs[i]["loss_mask"]), f"Output {i} loss mask length mismatch" + if get_logprobs: + assert response_length == len( + outputs[i]["rollout_logprobs"] + ), f"Output {i} rollout logprobs lenght mismatch" + + # TODO (tgriggs): Extend this test to compare the outputs to HF generation with temperature 0 + return generator_output + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("use_async_engine", "batched", "n_samples_per_prompt", "num_inference_engines", "tensor_parallel_size"), + [ + (False, True, 5, 2, 1), # tests SkyRLGymGenerator.generate_batched for single-turn + (True, False, 5, 1, 2), # tests SkyRLGymGenerator.agent_loop for single-turn + # Add more combinations as needed + ], + ids=[ + "test_generator_single_turn_gsm8k_batched", + "test_generator_single_turn_gsm8k_async_engine", + ], +) +async def test_generator_single_turn_gsm8k( + use_async_engine, batched, n_samples_per_prompt, num_inference_engines, tensor_parallel_size +): + """ + Test the generator with a single turn of GSM8K + """ + initialize_ray(get_default_config()) + try: + await run_generator_end_to_end( + use_async_engine=use_async_engine, + batched=batched, + n_samples_per_prompt=n_samples_per_prompt, + num_inference_engines=num_inference_engines, + tensor_parallel_size=tensor_parallel_size, + # TODO (sumanthrh): Add tests for non-batched mode once supported + get_logprobs=batched, + ) + finally: + ray.shutdown() + + +@pytest.mark.asyncio +async def test_generator_multi_turn_search(): + """ + Test the generator with multiple turns of search + """ + initialize_ray(get_default_config()) + try: + await run_generator_end_to_end( + use_async_engine=True, + batched=False, + n_samples_per_prompt=5, + num_inference_engines=2, + tensor_parallel_size=2, + model="Qwen/Qwen2.5-1.5B-Instruct", + max_prompt_length=2048, + max_input_length=4096, + max_generate_length=1000, + data_path=os.path.expanduser("~/data/searchR1/validation.parquet"), + env_class="search", + num_prompts=2, + max_turns=2, + use_conversation_multi_turn=False, + max_env_workers=0, + ) + finally: + ray.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_name", ["unsloth/Llama-3.2-1B-Instruct", "Qwen/Qwen2.5-1.5B-Instruct", "Qwen/Qwen3-0.6B"] +) +async def test_generator_formatting_use_conversation_multi_turn(model_name): + """ + Test generator formatting when using conversation formatting for multi-turn + """ + initialize_ray(get_default_config()) + try: + tokenizer = AutoTokenizer.from_pretrained(model_name) + generator_output = await run_generator_end_to_end( + use_async_engine=True, + batched=False, + n_samples_per_prompt=1, + num_inference_engines=1, + tensor_parallel_size=1, + model=model_name, + max_prompt_length=3000, + max_input_length=10000, + max_generate_length=3000, + env_class="test_env", + num_prompts=2, + max_turns=3, + use_conversation_multi_turn=True, + ) + + for i, resp_ids in enumerate(generator_output["response_ids"]): + loss_mask = generator_output["loss_masks"][i] + prompt_token_ids = generator_output["prompt_token_ids"][i] + stop_reason = generator_output["stop_reasons"][i] + masked_out_resp_ids = [resp_ids[j] for j in range(len(resp_ids)) if loss_mask[j] == 0] + masked_in_resp_ids = [resp_ids[j] for j in range(len(resp_ids)) if loss_mask[j] == 1] + + masked_out_resp_str = tokenizer.decode(masked_out_resp_ids) + masked_in_resp_str = tokenizer.decode(masked_in_resp_ids) + + assert ( + MODEL_TO_GENERATION_PROMPT[model_name] in masked_out_resp_str + and MODEL_TO_GENERATION_PROMPT[model_name] not in masked_in_resp_str + ), "generation prompts should be loss masked out" + + # Observations and EOS expectations only strictly apply when the model finished turns + if stop_reason == "stop": + assert ( + f"{OBSERVATION_PROMPT} 1" in masked_out_resp_str + ), f'"{OBSERVATION_PROMPT} 1" observation should be loss masked out' + assert ( + f"{OBSERVATION_PROMPT} 2" in masked_out_resp_str + ), f'"{OBSERVATION_PROMPT} 2" observation should be loss masked out' + # TODO(Charlie): add more rigorous tests that is robust to stop_reason being length. + # Either make GeneratorOutput return stop reason for each turn, or change the way we manage + # max generation length. + num_resp_eos = sum(1 for _ in masked_in_resp_ids if _ == tokenizer.eos_token_id) + num_total_eos = sum(1 for _ in resp_ids if _ == tokenizer.eos_token_id) + common_msg = "Could be due to stop_reason is length in some of the turns." + # count number of eos tokens in masked_in_resp_ids: 1 eos per assistant response (3 turns) + if num_resp_eos != 3: + logger.warning(f"Got {num_resp_eos} eos tokens in masked_in_resp_ids, expected 3. {common_msg}") + # total eos in full response: 2 user eos + 3 assistant eos + if num_total_eos != 5: + logger.warning(f"Got {num_total_eos} eos tokens in resp_ids, expected 5. {common_msg}") + else: + # On length stops, the model may not produce EOS at the end of each assistant turn. + # Only check that generation prompts are masked out. + logger.warning(f"Got stop reason {stop_reason}, so we did not fully check the response") + if model_name == "Qwen/Qwen3-0.6B": + assert ( + sum(1 for _ in prompt_token_ids if _ == tokenizer.eos_token_id) == 1 + ) # 1 user eos (no system for Qwen3) + else: + assert sum(1 for _ in prompt_token_ids if _ == tokenizer.eos_token_id) == 2 # 1 system eos, 1 user eos + finally: + ray.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_name", ["unsloth/Llama-3.2-1B-Instruct", "Qwen/Qwen2.5-1.5B-Instruct", "Qwen/Qwen3-0.6B"] +) +async def test_generator_formatting_no_use_conversation_multi_turn(model_name): + """ + Test generator formatting when not using conversation formatting for multi-turn + """ + initialize_ray(get_default_config()) + try: + tokenizer = AutoTokenizer.from_pretrained(model_name) + generator_output = await run_generator_end_to_end( + use_async_engine=True, + batched=False, + n_samples_per_prompt=1, + num_inference_engines=1, + tensor_parallel_size=1, + model=model_name, + max_prompt_length=3000, + max_input_length=10000, + max_generate_length=3000, + env_class="test_env", + num_prompts=2, + max_turns=3, + use_conversation_multi_turn=False, + ) + + for i, resp_ids in enumerate(generator_output["response_ids"]): + loss_mask = generator_output["loss_masks"][i] + prompt_token_ids = generator_output["prompt_token_ids"][i] + masked_out_resp_ids = [resp_ids[j] for j in range(len(resp_ids)) if loss_mask[j] == 0] + masked_in_resp_ids = [resp_ids[j] for j in range(len(resp_ids)) if loss_mask[j] == 1] + + prompt_str = tokenizer.decode(prompt_token_ids) + resp_str = tokenizer.decode(resp_ids) + masked_out_resp_str = tokenizer.decode(masked_out_resp_ids) + masked_in_resp_str = tokenizer.decode(masked_in_resp_ids) + + assert ( + f"{OBSERVATION_PROMPT} 1" in masked_out_resp_str + ), f'"{OBSERVATION_PROMPT} 1" observation should be loss masked out' + assert ( + f"{OBSERVATION_PROMPT} 2" in masked_out_resp_str + ), f'"{OBSERVATION_PROMPT} 2" observation should be loss masked out' + assert ( + prompt_str.count(MODEL_TO_GENERATION_PROMPT[model_name]) + + resp_str.count(MODEL_TO_GENERATION_PROMPT[model_name]) + == 1 + ), "the single generation prompt should be included in the prompt" + assert ( + MODEL_TO_GENERATION_PROMPT[model_name] in prompt_str + and MODEL_TO_GENERATION_PROMPT[model_name] not in masked_in_resp_str + ), "the single generation prompt should be included in the prompt" + + # count number of eos tokens in masked_in_resp_ids + assert ( + sum(1 for _ in masked_in_resp_ids if _ == tokenizer.eos_token_id) == 1 + ) # 1 eos for each assistant response + if model_name == "Qwen/Qwen3-0.6B": + assert ( + sum(1 for _ in prompt_token_ids if _ == tokenizer.eos_token_id) == 1 + ) # 1 user eos (no system for Qwen3) + else: + assert sum(1 for _ in prompt_token_ids if _ == tokenizer.eos_token_id) == 2 # 1 system eos, 1 user eos + finally: + ray.shutdown() + + +@pytest.mark.asyncio +async def test_generator_multi_turn_gsm8k_step_wise(): + """ + Test the generator with the multi-turn GSM8K environment for step-wise training + """ + initialize_ray(get_default_config()) + try: + generator_output: GeneratorOutput = await run_generator_end_to_end( + use_async_engine=True, + batched=False, + n_samples_per_prompt=5, + num_inference_engines=2, + tensor_parallel_size=2, + model="Qwen/Qwen2.5-1.5B-Instruct", + max_prompt_length=2048, + max_input_length=4096, + max_generate_length=1000, + data_path=os.path.expanduser("~/data/gsm8k/validation.parquet"), + env_class="gsm8k_multi_turn", + num_prompts=2, + max_turns=2, + use_conversation_multi_turn=True, + max_env_workers=0, + is_step_wise=True, + temperature=0, + ) + + assert isinstance(generator_output["is_last_step"], list) and isinstance( + generator_output["is_last_step"][0], bool + ) + # Expect atleast one response with more than one turn + assert sum(generator_output["is_last_step"]) != len(generator_output["is_last_step"]) + finally: + ray.shutdown() diff --git a/skyrl-train/tests/gpu/gpu_ci/test_train_batch.py b/skyrl-train/tests/gpu/gpu_ci/test_train_batch.py new file mode 100644 index 0000000000..06e8bd0636 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_train_batch.py @@ -0,0 +1,15 @@ +""" +uv run --isolated --extra dev --extra sglang pytest tests/gpu/gpu_ci/test_train_batch.py +""" + +from skyrl_train.training_batch import TrainingInputBatch +import torch + + +def test_data_to_device(): + batch = {"sequences": torch.tensor([1, 2, 3]), "attention_mask": torch.tensor([4, 5, 6])} + data = TrainingInputBatch(batch) + # in-place + data.to(device="cuda") + assert data["sequences"].device.type == "cuda" + assert data["attention_mask"].device.type == "cuda" diff --git a/skyrl-train/tests/gpu/gpu_ci/test_trainer_full_checkpointing.py b/skyrl-train/tests/gpu/gpu_ci/test_trainer_full_checkpointing.py new file mode 100644 index 0000000000..17e5783ae4 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_trainer_full_checkpointing.py @@ -0,0 +1,260 @@ +""" +Integration test for full trainer checkpointing functionality. + +This test validates that the RayPPOTrainer can save and restore ALL training state, +ensuring that training can resume exactly where it left off. + +Run with: +For FSDP and FSDP2, run: +uv run --isolated --extra dev --extra vllm pytest tests/gpu/gpu_ci/test_trainer_full_checkpointing.py -m "not megatron" + +For Megatron, run: +uv run --isolated --extra dev --extra mcore pytest tests/gpu/gpu_ci/test_trainer_full_checkpointing.py -m "megatron" +""" + +import ray +import pytest +import hydra +import torch +import os +import shutil +import tempfile +from omegaconf import DictConfig +from torch.utils.data import Dataset +from unittest.mock import MagicMock +from transformers import AutoTokenizer + +from skyrl_train.utils.tracking import Tracking +from skyrl_train.trainer import RayPPOTrainer +from tests.gpu.utils import import_worker, ray_init_for_tests +from skyrl_train.entrypoints.main_base import config_dir + +MODEL_NAME = "Qwen/Qwen3-0.6B" +NUM_GPUS = 2 + + +class DummyDataset(Dataset): + """Minimal dataset for testing""" + + def __init__(self, size=10): + self.data = [([{"role": "user", "content": f"Question {i}"}], None) for i in range(size)] + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + return self.data[idx] + + def collate_fn(self, batch): + return batch + + +def get_test_trainer_config(strategy: str, fsdp2_cpu_offload: bool = False) -> DictConfig: + """Create minimal trainer config for testing""" + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = MODEL_NAME + cfg.trainer.critic.model.path = MODEL_NAME # Enable critic for testing + cfg.trainer.strategy = strategy + if strategy == "fsdp2": + cfg.trainer.policy.fsdp_config.cpu_offload = fsdp2_cpu_offload + + # Use minimal settings for faster testing + cfg.trainer.placement.policy_num_gpus_per_node = NUM_GPUS + cfg.trainer.placement.critic_num_gpus_per_node = NUM_GPUS + cfg.trainer.placement.policy_num_nodes = 1 + cfg.trainer.placement.critic_num_nodes = 1 + cfg.trainer.algorithm.use_kl_loss = ( + False # disable ref model so we just have policy and critic (NUM_GPUS total GPUs) + ) + cfg.trainer.placement.colocate_all = False # Disable colocation for simpler testing + cfg.trainer.train_batch_size = NUM_GPUS + cfg.trainer.policy_mini_batch_size = cfg.trainer.train_batch_size + cfg.trainer.micro_train_batch_size_per_gpu = 1 + cfg.trainer.update_epochs_per_batch = 1 + cfg.trainer.epochs = 1 + cfg.trainer.logger = "console" + cfg.generator.n_samples_per_prompt = 1 + cfg.generator.num_inference_engines = NUM_GPUS // 2 + cfg.generator.inference_engine_tensor_parallel_size = 2 + + # Megatron-specific + if strategy == "megatron": + cfg.trainer.policy.megatron_config.tensor_model_parallel_size = 2 + cfg.trainer.policy.megatron_config.pipeline_model_parallel_size = 2 + cfg.trainer.placement.policy_num_gpus_per_node = 4 + # Disable critic for megatron + cfg.trainer.critic.model.path = "" + + # Use temporary directories + cfg.trainer.export_path = tempfile.mkdtemp(prefix="trainer_ckpt_test_") + cfg.trainer.ckpt_path = cfg.trainer.export_path + + # Enable checkpointing with correct config names + cfg.trainer.ckpt_interval = 1 # Save every step + cfg.trainer.resume_mode = "none" # Initially false, will be set to True for resume + + return cfg + + +def create_minimal_trainer(cfg: DictConfig): + """Create a minimal trainer setup for testing""" + # Create minimal tokenizer + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + + # Create dummy dataset + train_dataset = DummyDataset(size=4) # Small dataset for quick testing + + # Create mock generator for testing + mock_generator = MagicMock() + + # Create tracker + tracker = Tracking( + project_name=cfg.trainer.project_name, + experiment_name=cfg.trainer.run_name, + backends=cfg.trainer.logger, + config=cfg, + ) + + # Create trainer (no inference engine needed for checkpointing tests) + trainer = RayPPOTrainer( + cfg=cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=train_dataset, + eval_dataset=None, + inference_engine_client=None, + generator=mock_generator, + ) + + return trainer + + +@pytest.mark.parametrize( + ("strategy, fsdp2_cpu_offload"), + [ + ("fsdp", False), + ("fsdp2", False), + ("fsdp2", True), + pytest.param("megatron", False, marks=pytest.mark.megatron), + ], +) +def test_trainer_full_checkpointing(ray_init_fixture, strategy, fsdp2_cpu_offload): + """ + Test full trainer checkpointing by: + 1. Creating trainer and setting it up + 2. Saving checkpoint + 3. Capturing training state + 4. Destroying trainer + 5. Creating new trainer with resume enabled + 6. Loading checkpoint + 7. Verifying all state matches + 8. Continuing training to ensure it works + """ + cfg = get_test_trainer_config(strategy, fsdp2_cpu_offload) + + checkpoint_dir = None + try: + # ============= PHASE 1: Initial Training and Save ============= + print("Phase 1: Initial training and checkpoint save") + + trainer1 = create_minimal_trainer(cfg) + + # Get worker classes + PolicyWorker = import_worker(strategy, "policy") + CriticWorker = import_worker(strategy, "critic") + RefWorker = import_worker(strategy, "ref") + + # Build models + trainer1.build_models(PolicyWorker, CriticWorker, RefWorker) + + # Set initial global step as if 2 steps were completed + trainer1.global_step = 2 + + # Save checkpoint + trainer1.save_checkpoints() + + # Capture state before teardown + saved_global_step = trainer1.global_step + checkpoint_dir = os.path.join(cfg.trainer.export_path, f"global_step_{trainer1.global_step}") + + # Verify checkpoint structure was created + expected_files = [ + os.path.join(checkpoint_dir, "policy"), + os.path.join(checkpoint_dir, "trainer_state.pt"), + os.path.join(checkpoint_dir, "data.pt"), + ] + # Only expect critic dir for non-megatron strategies + if strategy != "megatron": + expected_files.append(os.path.join(checkpoint_dir, "critic")) + for expected_file in expected_files: + assert os.path.exists(expected_file), f"Expected checkpoint file/dir not found: {expected_file}" + + # Verify atomic tracking file + latest_ckpt_file = os.path.join(cfg.trainer.ckpt_path, "latest_ckpt_global_step.txt") + assert os.path.exists(latest_ckpt_file) + with open(latest_ckpt_file, "r") as f: + latest_step = int(f.read()) + assert latest_step == trainer1.global_step, "Atomic tracking file has incorrect step after first save" + + # Verify trainer state content + print("Verifying checkpoint content...") + loaded_trainer_state = torch.load( + os.path.join(checkpoint_dir, "trainer_state.pt"), map_location="cpu", weights_only=False + ) + + # Check key configuration values are preserved + assert ( + loaded_trainer_state["config"]["trainer"]["train_batch_size"] == cfg.trainer.train_batch_size + ), "train_batch_size not preserved in checkpoint" + assert loaded_trainer_state["config"]["trainer"]["strategy"] == strategy, "strategy not preserved in checkpoint" + assert loaded_trainer_state["global_step"] == saved_global_step, "global_step not preserved in checkpoint" + + # Cleanup first trainer + del trainer1 + ray.shutdown() + + # ============= PHASE 2: Resume from Checkpoint ============= + print("Phase 2: Resume from checkpoint") + ray_init_for_tests() + # Create new config with resume enabled + cfg_resume = get_test_trainer_config(strategy, fsdp2_cpu_offload) + cfg_resume.trainer.resume_mode = "from_path" # Enable resume + cfg_resume.trainer.resume_path = checkpoint_dir # Set resume path + cfg_resume.trainer.export_path = cfg.trainer.export_path # Use same export path + cfg_resume.trainer.ckpt_path = cfg.trainer.ckpt_path + + trainer2 = create_minimal_trainer(cfg_resume) + + # Build models again + trainer2.build_models(PolicyWorker, CriticWorker, RefWorker) + + # Load checkpoints + loaded_global_step, loaded_checkpoint_dir = trainer2.load_checkpoints() + assert ( + loaded_global_step == saved_global_step + ), f"Expected global_step={saved_global_step}, got {loaded_global_step}" + assert loaded_checkpoint_dir == checkpoint_dir, "Checkpoint path mismatch" + + # ============= PHASE 3: Continue Training ============= + print("Phase 3: Second checkpoint save") + + # Try to save another checkpoint to test cleanup logic + trainer2.global_step = 3 + trainer2.save_checkpoints() + + next_checkpoint_dir = os.path.join(cfg.trainer.export_path, f"global_step_{trainer2.global_step}") + assert os.path.exists(next_checkpoint_dir), "Could not save checkpoint after resume" + + # Verify atomic tracking file is updated + latest_ckpt_file = os.path.join(cfg.trainer.ckpt_path, "latest_ckpt_global_step.txt") + assert os.path.exists(latest_ckpt_file) + with open(latest_ckpt_file, "r") as f: + latest_step = int(f.read()) + assert latest_step == trainer2.global_step, "Atomic tracking file was not updated after second save" + + finally: + if checkpoint_dir and os.path.exists(os.path.dirname(checkpoint_dir)): + print(f"Cleaning up checkpoint directory: {os.path.dirname(checkpoint_dir)}") + shutil.rmtree(os.path.dirname(checkpoint_dir)) diff --git a/skyrl-train/tests/gpu/gpu_ci/test_training_step.py b/skyrl-train/tests/gpu/gpu_ci/test_training_step.py new file mode 100644 index 0000000000..aea3784be8 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_training_step.py @@ -0,0 +1,129 @@ +""" +Run with: +uv run --isolated --extra dev -- pytest tests/gpu/gpu_ci/test_training_step.py +""" + +import ray +import pytest +import hydra +from omegaconf import DictConfig + +from tests.gpu.utils import init_worker_with_type, make_dummy_training_batch, validate_cfg +from skyrl_train.utils.utils import print_mem +from skyrl_train.entrypoints.main_base import config_dir + + +MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct" + + +def get_test_actor_config() -> DictConfig: + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = MODEL_NAME + cfg.trainer.placement.policy_num_gpus_per_node = 2 + cfg.trainer.logger = "console" + cfg.generator.inference_engine_tensor_parallel_size = 2 + + return cfg + + +@pytest.fixture +def cfg() -> DictConfig: + return get_test_actor_config() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("packed", "strategy"), + [(True, "fsdp"), (False, "fsdp"), (True, "fsdp2"), (False, "fsdp2")], + ids=[ + "packed-fsdp", + "unpacked-fsdp", + "packed-fsdp2", + "unpacked-fsdp2", + ], +) +async def test_policy_forward_backward_and_optim_step(ray_init_fixture, cfg, packed, strategy): + """ + Full test: initialize actor group, send dummy experience to forward_backward + optim_step, validate output. + """ + cfg.trainer.use_sample_packing = packed + cfg.trainer.strategy = strategy + validate_cfg(cfg) + + try: + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + # Create TrainingInputBatch - worker's forward_backward handles micro-batching internally + dp_size = actor_group.actor_infos[0].rank.dp_size + dummy_batch = make_dummy_training_batch(batch_size=dp_size) + + results = ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", data=dummy_batch)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + + memory = ray.get(actor_group.async_run_ray_method("pass_through", "get_cuda_memory")) + memory = memory[0] + print_mem("memory after forward_backward + optim_step", memory) + + for result in results: + assert isinstance(result, dict), "Result should be a dictionary of training stats" + assert "policy_loss" in result + assert "ppo_clip_ratio" in result + assert "policy_entropy" in result + for k, v in result.items(): + assert isinstance(v, (int, float)), f"{k} should be an int or float" + + finally: + ray.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("packed", "strategy"), + [(True, "fsdp"), (False, "fsdp"), (True, "fsdp2"), (False, "fsdp2")], + ids=[ + "packed-fsdp", + "unpacked-fsdp", + "packed-fsdp2", + "unpacked-fsdp2", + ], +) +async def test_critic_forward_backward_and_optim_step(ray_init_fixture, cfg, packed, strategy): + """ + Full test: initialize critic actor group, send dummy experience to forward_backward + optim_step, validate output. + """ + cfg.trainer.use_sample_packing = packed + cfg.trainer.strategy = strategy + validate_cfg(cfg) + try: + actor_group = init_worker_with_type( + "critic", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + # Create TrainingInputBatch - worker's forward_backward handles micro-batching internally + dp_size = actor_group.actor_infos[0].rank.dp_size + dummy_batch = make_dummy_training_batch(batch_size=dp_size) + + results = ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", data=dummy_batch)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + + for result in results: + assert isinstance(result, dict), "Result should be a dictionary of training stats" + assert "critic_loss" in result + assert "values_mean" in result + for k, v in result.items(): + assert isinstance(v, float), f"{k} should be a float" + + finally: + ray.shutdown() diff --git a/skyrl-train/tests/gpu/gpu_ci/test_transfer_strategies_e2e.py b/skyrl-train/tests/gpu/gpu_ci/test_transfer_strategies_e2e.py new file mode 100644 index 0000000000..e1ec565207 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_transfer_strategies_e2e.py @@ -0,0 +1,385 @@ +""" +GPU integration tests for weight transfer strategies without involving the actual training workers and inference engines. + +Tests the full execution flow with multiple training ranks: + 1. create_init_info: Extract config-derived args (master addr/port, dtype, etc.) + 2. create_sender/create_receiver: Initialize transfer components on both sides (multiple ranks) + 3. send_chunks/receive_weights: Transfer weight tensors between actors + +This test uses 2 sender actors (simulating 2 training worker ranks) and 2 receiver actors +(simulating 2 inference engines, each with 1 worker) to test rank-specific logic. + +GPU Requirements: + - CUDA IPC test: 2 GPUs (each sender-receiver pair colocated on 1 GPU) + - Broadcast test: 4 GPUs (each sender and receiver uses 1 GPU). + +Run with: + uv run --isolated --extra dev pytest tests/gpu/gpu_ci/test_transfer_strategy.py -v +""" + +import asyncio +import ray +import torch +import torch.distributed as dist +from skyrl_train.config.utils import get_default_config +from skyrl_train.weight_sync import ( + WeightChunk, + CudaIpcTransferStrategy, + BroadcastTransferStrategy, + WeightSyncInitInfo, +) +from skyrl_train.utils.utils import get_free_port, str_to_torch_dtype +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from ray.util.placement_group import placement_group + + +def make_cfg( + weight_sync_backend: str, + model_dtype: str, + num_inference_engines: int, + colocate_all: bool, +): + """Create a config object. + + Assumes no intra-engine parallelism (tp=pp=dp=1). + """ + cfg = get_default_config() + cfg.generator.weight_sync_backend = weight_sync_backend + cfg.generator.model_dtype = model_dtype + cfg.generator.num_inference_engines = num_inference_engines + cfg.generator.inference_engine_tensor_parallel_size = 1 + cfg.generator.inference_engine_pipeline_parallel_size = 1 + cfg.generator.inference_engine_data_parallel_size = 1 + cfg.trainer.placement.colocate_all = colocate_all + return cfg + + +@ray.remote +class SenderActor: + """Generic sender actor for transfer strategies.""" + + def __init__(self, rank: int, world_size: int): + """Initialize sender with its rank in the training worker group. + + Args: + rank: Rank in the training worker group. + world_size: Total number of training workers. + """ + self.rank = rank + self.world_size = world_size + + def get_master_addr_and_port(self): + """Get the node IP address and a free port (only valid on rank 0).""" + return ray._private.services.get_node_ip_address(), get_free_port() + + def init_process_group(self, master_addr: str, master_port: int): + """Initialize the training worker process group. + + Args: + master_addr: Master address for the process group. + master_port: Port for the training worker process group. + """ + if not dist.is_initialized(): + dist.init_process_group( + backend="nccl", + init_method=f"tcp://{master_addr}:{master_port}", + world_size=self.world_size, + rank=self.rank, + ) + + def create_init_info(self, strategy_cls, cfg): + """Create init info using the strategy (only rank 0).""" + if self.rank == 0: + return strategy_cls.create_init_info(cfg) + return None + + def create_sender(self, strategy_cls, init_info, receiver_handles): + """Create sender (must be called concurrently with receiver init for broadcast strategy).""" + + class MockInferenceClient: + def __init__(self, receiver_handles): + self.receiver_handles = receiver_handles + + async def update_named_weights(self, request): + # Start receive_weights on all receivers and await completion + # Ray ObjectRefs are awaitable, and the broadcast collective coordinates the communication + await asyncio.gather(*[r.receive_weights.remote(request) for r in self.receiver_handles]) + + mock_client = MockInferenceClient(receiver_handles) + self._sender = strategy_cls.create_sender(init_info, mock_client) + + async def send_weights( + self, + init_info, + names: list, + shapes: list, + send_individually: bool = False, + ): + """Send weights using the pre-created sender.""" + assert hasattr(self, "_sender"), "Sender not created. Call create_sender() first." + dtype_str = init_info.model_dtype_str + dtype = str_to_torch_dtype(dtype_str) + + # All ranks generate tensors because: + # - Broadcast strategy: All ranks iterate through chunks (simulating FSDP collective ops + # during weight extraction), but only rank 0's tensor values are actually broadcast + # - CUDA IPC strategy: All ranks create IPC handles from their tensors + # Use same seed on all ranks so they generate identical tensors for easier testing + torch.manual_seed(42) + tensors = [torch.randn(shape, device="cuda", dtype=dtype) for shape in shapes] + + if send_individually: + for name, tensor, shape in zip(names, tensors, shapes): + chunk = WeightChunk(names=[name], dtypes=[dtype_str], shapes=[shape], tensors=[tensor]) + await self._sender.send_chunks([chunk]) + else: + chunk = WeightChunk(names=names, dtypes=[dtype_str] * len(names), shapes=shapes, tensors=tensors) + await self._sender.send_chunks([chunk]) + + # Only rank 0 returns the tensors for comparison + if self.rank == 0: + return [t.cpu() for t in tensors] + return None + + def teardown_sender(self): + """Teardown the sender (tests cleanup logic).""" + if hasattr(self, "_sender"): + self._sender.teardown() + + +@ray.remote +class ReceiverActor: + """Generic receiver actor for transfer strategies. + + Each actor represents a complete inference engine with 1 worker (no intra-engine parallelism). + """ + + def __init__(self, strategy_cls, init_info): + """Initialize receiver for a single-worker inference engine. + + Args: + strategy_cls: Transfer strategy class. + init_info: Init info for creating the receiver. + """ + self.strategy_cls = strategy_cls + self.init_info = init_info + self.receiver = None + self.received_weights = [] + + def init_process_group(self): + """Initialize the engine's internal default process group. + + Gets a free port on this node and initializes the process group for a single-worker engine. + """ + # Get a free port on this receiver's node + master_port = get_free_port() + + # Initialize the engine's internal default process group (simulates TP/PP/DP group) + # Even with 1 worker, this is needed for torch.distributed.get_rank() calls + if not dist.is_initialized(): + dist.init_process_group( + backend="nccl", + init_method=f"tcp://localhost:{master_port}", + world_size=1, + rank=0, + ) + + def init_receiver(self): + """Initialize the receiver (must be called before receive_weights). + + For broadcast strategy, this must be called concurrently with sender creation + to avoid deadlock when joining the model_update_group (a named group). + + The model_update_group is separate from the engine's internal default group. + """ + self.receiver = self.strategy_cls.create_receiver(self.init_info) + + def receive_weights(self, request): + """Receive weights using the pre-created receiver.""" + assert self.receiver is not None, "Receiver not initialized. Call init_receiver() first." + received = list(self.receiver.receive_weights(request)) + self.received_weights.extend([(name, tensor.cpu()) for name, tensor in received]) + + def get_received_weights(self): + return self.received_weights + + def teardown_receiver(self): + """Teardown the receiver (tests cleanup logic).""" + if self.receiver is not None: + self.receiver.teardown() + + +def _run_weight_sync_e2e( + strategy_cls, + cfg, + num_training_ranks: int, + num_inference_engines: int, + send_individually: bool, + colocate: bool, +): + """Run end-to-end weight sync test for a given strategy. + + Args: + strategy_cls: Transfer strategy class to test. + cfg: Mock config object. + num_training_ranks: Number of training worker ranks to create. + num_inference_engines: Number of inference engines to create (each with 1 worker). + send_individually: Whether to send weights one at a time or batched. + colocate: Whether to colocate sender and receiver on the same GPU (required for CUDA IPC). + """ + + # Simplifying assumption: each receiver represents one complete engine with 1 rank + # (not testing intra-engine parallelism like TP/PP/DP) + assert num_inference_engines == cfg.generator.num_inference_engines, "Test assumes 1 rank per engine" + + # For CUDA IPC, each sender-receiver pair must share the same GPU (required for IPC handles) + pg = None + if colocate: + # Create placement group: one bundle per sender-receiver pair + # Bundle i contains: training worker rank i (0.5 GPU) + inference engine i (0.5 GPU) = 1 GPU total + pg = placement_group([{"GPU": 1}] * num_training_ranks, strategy="PACK") + ray.get(pg.ready()) + + # Create sender actors + senders = [] + for i in range(num_training_ranks): + if colocate: + # CUDA IPC: sender and receiver share GPU (0.5 each) + sender_options = { + "num_cpus": 0, + "num_gpus": 0.5, + "scheduling_strategy": PlacementGroupSchedulingStrategy( + placement_group=pg, placement_group_bundle_index=i + ), + } + else: + # Broadcast: each actor on separate GPU for NCCL + sender_options = {"num_gpus": 1} + + sender = SenderActor.options(**sender_options).remote(rank=i, world_size=num_training_ranks) + senders.append(sender) + + # Get master_addr and master_port from rank 0 (its node IP and a free port on its node) + master_addr, training_master_port = ray.get(senders[0].get_master_addr_and_port.remote()) + + # Initialize process groups on all senders + ray.get([sender.init_process_group.remote(master_addr, training_master_port) for sender in senders]) + + # Only rank 0 creates init_info + init_info: WeightSyncInitInfo = ray.get(senders[0].create_init_info.remote(strategy_cls, cfg)) + + # Create receiver actors + receivers = [] + for i in range(num_inference_engines): + if colocate: + # CUDA IPC: sender and receiver share GPU (0.5 each) + receiver_options = { + "num_cpus": 0, + "num_gpus": 0.5, + "scheduling_strategy": PlacementGroupSchedulingStrategy( + placement_group=pg, placement_group_bundle_index=i + ), + } + else: + # Broadcast: each actor on separate GPU for NCCL + receiver_options = {"num_gpus": 1} + + # Use for_engine() to get per-engine init_info + # Test assumes tp_size=pp_size=1 (single worker per engine) + receiver_init_info = init_info.for_engine(engine_index=i, tp_size=1, pp_size=1) + + receiver = ReceiverActor.options(**receiver_options).remote(strategy_cls, receiver_init_info) + receivers.append(receiver) + + # Initialize process groups on all receivers + ray.get([receiver.init_process_group.remote() for receiver in receivers]) + + # Create senders and receivers concurrently (required for broadcast strategy to avoid deadlock) + # Both need to join the model_update_group at the same time + init_tasks = [receiver.init_receiver.remote() for receiver in receivers] + create_sender_tasks = [sender.create_sender.remote(strategy_cls, init_info, receivers) for sender in senders] + ray.get(init_tasks + create_sender_tasks) + + names = ["layer1.weight", "layer1.bias", "layer2.weight"] + shapes = [[32, 64], [64], [16, 16]] + + # Now send weights (senders and receivers are fully initialized) + results = ray.get( + [ + sender.send_weights.remote(init_info, names, shapes, send_individually=send_individually) + for sender in senders + ] + ) + + # Only rank 0 returns tensors + src_tensors = results[0] + assert src_tensors is not None + assert results[1] is None # Non-rank-0 training worker returns None + + # All receivers should have received the weights + for receiver in receivers: + received = ray.get(receiver.get_received_weights.remote()) + + assert len(received) == len(names) + for i, (name, tensor) in enumerate(received): + assert name == names[i] + assert tensor.shape == tuple(shapes[i]) + assert torch.allclose(tensor, src_tensors[i]) + + # Test teardown on all actors (should not raise exceptions) + teardown_tasks = [] + for sender in senders: + teardown_tasks.append(sender.teardown_sender.remote()) + for receiver in receivers: + teardown_tasks.append(receiver.teardown_receiver.remote()) + ray.get(teardown_tasks) # Ensure all teardowns complete successfully + + +class TestCudaIpcTransferStrategy: + """Integration tests for CUDA IPC transfer strategy. + + Tests weight synchronization using CUDA IPC handles with colocated sender-receiver pairs. + Requires 2 GPUs (one GPU per sender-receiver pair). + """ + + def test_weight_sync_e2e(self, ray_init_fixture): + """Test CUDA IPC strategy end-to-end with 2 training ranks and 2 inference engines.""" + cfg = make_cfg( + weight_sync_backend="nccl", + model_dtype="bfloat16", + num_inference_engines=2, + colocate_all=True, + ) + _run_weight_sync_e2e( + CudaIpcTransferStrategy, + cfg, + num_training_ranks=2, + num_inference_engines=2, + send_individually=False, + colocate=True, + ) + + +class TestBroadcastTransferStrategy: + """Integration tests for Broadcast transfer strategy. + + Tests weight synchronization using torch.distributed.broadcast with NCCL backend. + Requires 4 GPUs. + """ + + def test_weight_sync_e2e(self, ray_init_fixture): + """Test Broadcast strategy end-to-end with 2 training ranks and 2 inference engines.""" + cfg = make_cfg( + weight_sync_backend="nccl", + model_dtype="bfloat16", + num_inference_engines=2, + colocate_all=False, + ) + _run_weight_sync_e2e( + BroadcastTransferStrategy, + cfg, + num_training_ranks=2, + num_inference_engines=2, + send_individually=True, + colocate=False, + ) diff --git a/skyrl-train/tests/gpu/gpu_ci/test_verifiers_generator.py b/skyrl-train/tests/gpu/gpu_ci/test_verifiers_generator.py new file mode 100644 index 0000000000..e2cb838cd3 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_verifiers_generator.py @@ -0,0 +1,226 @@ +""" +uv run --isolated --extra dev --extra vllm --with verifiers pytest tests/gpu/gpu_ci/test_verifiers_generator.py +""" + +import pytest +import ray +from omegaconf import DictConfig +from transformers import AutoTokenizer +import socket + +from tests.gpu.utils import get_test_actor_config, init_inference_engines +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend + +# Mark all tests in this file as "integrations" +# NOTE (sumanthrh): Skipping these tests until we make the verifiers integration compatible with verifiers repo `main` again. +# Error: cannot import name 'GenerateInputs' from 'verifiers.types' +pytestmark = [pytest.mark.integrations, pytest.mark.skip] + + +def _get_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture(scope="module") +def verifiers_runtime(): + model = "Qwen/Qwen2.5-1.5B-Instruct" + http_port = _get_free_port() + + cfg = get_test_actor_config() + cfg.trainer.policy.model.path = model + cfg.generator.max_input_length = 2048 + cfg.generator.enable_http_endpoint = True + cfg.generator.http_endpoint_host = "127.0.0.1" + cfg.generator.http_endpoint_port = http_port + cfg.generator.sampling_params.max_generate_length = 256 + + # Reuse shared initializer for local engines and client + client, _ = init_inference_engines( + cfg=cfg, + model=model, + use_local=True, + async_engine=True, + tp_size=1, + colocate_all=False, + backend="vllm", + gpu_memory_utilization=0.8, + sleep_level=1, # since we do not explicitly sync weights + ) + + tokenizer = AutoTokenizer.from_pretrained(model) + + try: + yield {"client": client, "tokenizer": tokenizer, "http_port": http_port, "model": model} + finally: + ray.shutdown() + + +async def _run_verifiers_end_to_end( + *, + existing_client: InferenceEngineClient, + model: str = "Qwen/Qwen2.5-1.5B-Instruct", + num_prompts: int = 2, + http_host: str = "127.0.0.1", + http_port: int | None = None, + max_input_length: int = 2048, + max_generate_length: int = 256, + sampling_overrides: dict | None = None, + prompt_text: str | None = None, + existing_tokenizer=None, +): + client = existing_client + tokenizer = existing_tokenizer + if http_port is None: + http_port = _get_free_port() + + await client.wake_up() + + generator_cfg = DictConfig( + { + "sampling_params": {"max_generate_length": max_generate_length, "logprobs": None}, + "max_input_length": max_input_length, + "backend": "vllm", + "enable_http_endpoint": True, + "http_endpoint_host": http_host, + "http_endpoint_port": http_port, + } + ) + + from integrations.verifiers.verifiers_generator import VerifiersGenerator + + generator = VerifiersGenerator( + generator_cfg=generator_cfg, + tokenizer=tokenizer, + model_name=model, + ) + + prompts = [ + [ + { + "role": "user", + "content": prompt_text + or "You are playing Wordle. Think step-by-step and propose the next guess based on previous feedback.", + } + ] + for _ in range(num_prompts) + ] + env_extras = [ + {"verifiers": {"environment": "wordle", "answer": "", "info": {}, "task": "default", "example_id": i}} + for i in range(num_prompts) + ] + + base_sampling = DictConfig( + { + "temperature": 0.7, + "top_p": 0.95, + "top_k": -1, + "max_generate_length": max_generate_length, + "min_p": 0.0, + "logprobs": None, + "stop": None, + } + ) + if sampling_overrides: + for k, v in sampling_overrides.items(): + base_sampling[k] = v + + sampling_params = get_sampling_params_for_backend("vllm", base_sampling) + + input_batch = { + "prompts": prompts, + "env_extras": env_extras, + "sampling_params": sampling_params, + } + + output = await generator.generate(input_batch) + return output + + +@pytest.mark.asyncio +async def test_verifiers_e2e_wordle_http(verifiers_runtime): + rt = verifiers_runtime + out = await _run_verifiers_end_to_end( + existing_client=rt["client"], + model=rt["model"], + num_prompts=2, + max_input_length=2048, + max_generate_length=256, + http_host="127.0.0.1", + http_port=rt["http_port"], + existing_tokenizer=rt["tokenizer"], + ) + + for key in [ + "prompt_token_ids", + "response_ids", + "rewards", + "loss_masks", + "rollout_logprobs", + "rollout_metrics", + ]: + assert key in out + + assert len(out["response_ids"]) == 2 + assert len(out["prompt_token_ids"]) == 2 + for resp, mask, logp in zip( + out["response_ids"], out["loss_masks"], out["rollout_logprobs"] or [[]] * len(out["response_ids"]) + ): + assert isinstance(resp, list) and all(isinstance(t, int) for t in resp) + assert len(resp) == len(mask) + if out["rollout_logprobs"] is not None: + assert len(logp) == len(resp) + + +@pytest.mark.asyncio +async def test_verifiers_e2e_sampling_toggles(verifiers_runtime): + rt = verifiers_runtime + out = await _run_verifiers_end_to_end( + existing_client=rt["client"], + model=rt["model"], + num_prompts=2, + max_input_length=2048, + max_generate_length=128, + sampling_overrides={ + "skip_special_tokens": True, + "include_stop_str_in_output": False, + "top_k": 20, + "min_p": 0.05, + "repetition_penalty": 1.05, + "min_tokens": 1, + }, + http_host="127.0.0.1", + http_port=rt["http_port"], + existing_tokenizer=rt["tokenizer"], + ) + + assert len(out["response_ids"]) == 2 + for resp, mask in zip(out["response_ids"], out["loss_masks"]): + assert len(resp) == len(mask) + + +@pytest.mark.asyncio +async def test_verifiers_length_constraints(verifiers_runtime): + rt = verifiers_runtime + max_input_length = 512 + max_generate_length = 64 + long_prompt = "Wordle analysis: " + ("A" * 400) + + out = await _run_verifiers_end_to_end( + existing_client=rt["client"], + model=rt["model"], + num_prompts=1, + max_input_length=max_input_length, + max_generate_length=max_generate_length, + prompt_text=long_prompt, + http_host="127.0.0.1", + http_port=rt["http_port"], + existing_tokenizer=rt["tokenizer"], + ) + + limit = max_input_length + max_generate_length + resp = out["response_ids"][0] + assert len(resp) <= limit + assert len(resp) == len(out["loss_masks"][0]) diff --git a/skyrl-train/tests/gpu/gpu_ci/test_worker_dispatch_offload.py b/skyrl-train/tests/gpu/gpu_ci/test_worker_dispatch_offload.py new file mode 100644 index 0000000000..7418d5c018 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_worker_dispatch_offload.py @@ -0,0 +1,293 @@ +""" +Test WorkerDispatch automatic offload/onload with colocation policies. + +Run with: +uv run --isolated --extra dev -- pytest tests/gpu/gpu_ci/test_worker_dispatch_offload.py -v + +These tests validate that WorkerDispatch correctly manages GPU memory when +multiple models share the same GPU (colocate_all=True or colocate_policy_ref=True). +""" + +import ray +import pytest +import hydra +from omegaconf import DictConfig +from ray.util.placement_group import placement_group + +from tests.gpu.utils import make_dummy_training_batch, get_rank_0_memory +from skyrl_train.utils.utils import validate_cfg +from skyrl_train.utils import get_ray_pg_ready_with_timeout +from skyrl_train.entrypoints.main_base import config_dir +from skyrl_train.workers.worker_dispatch import WorkerDispatch, GPUState +from skyrl_train.workers.fsdp.fsdp_worker import PolicyWorker, RefWorker, CriticWorker +from skyrl_train.workers.worker import PPORayActorGroup + +MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct" + + +def get_test_config() -> DictConfig: + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = MODEL_NAME + cfg.trainer.placement.policy_num_gpus_per_node = 1 + cfg.generator.inference_engine_tensor_parallel_size = 1 + cfg.trainer.placement.colocate_all = True + cfg.trainer.use_sample_packing = False + cfg.trainer.logger = "console" + cfg.trainer.strategy = "fsdp2" + cfg.trainer.ref.fsdp_config.cpu_offload = False + + validate_cfg(cfg) + return cfg + + +def init_colocated_actor_group( + worker_cls, + shared_pg, + cfg: DictConfig, +) -> PPORayActorGroup: + """Initialize an actor group that shares a placement group with others.""" + return PPORayActorGroup( + cfg, + num_nodes=1, + num_gpus_per_node=1, + ray_actor_type=worker_cls, + pg=shared_pg, + num_gpus_per_actor=0.4, # Share GPU + colocate_all=True, + sequence_parallel_size=cfg.trainer.policy.sequence_parallel_size, + record_memory=cfg.trainer.policy.record_memory, + ) + + +@pytest.mark.asyncio +async def test_colocate_all_only_one_model_on_gpu(ray_init_fixture): + """ + Test that with colocate_all=True, only one model is on GPU at a time. + + Scenario: + 1. Initialize policy and ref on shared GPU + 2. Call dispatch.forward("ref", ...) - ref should be on GPU, policy offloaded + 3. Call dispatch.forward_backward("policy", ...) - policy on GPU, ref offloaded + 4. Verify memory drops when switching (indicates offload happened) + """ + cfg = get_test_config() + + try: + # Create shared placement group + pg = placement_group([{"GPU": 1, "CPU": 2}], strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=30) + + # Initialize both actor groups on shared GPU + policy_group = init_colocated_actor_group(PolicyWorker, pg, cfg) + ref_group = init_colocated_actor_group(RefWorker, pg, cfg) + + # Init models - after init, models are on GPU + ray.get(policy_group.async_init_model(cfg.trainer.policy.model.path)) + ray.get(ref_group.async_init_model(cfg.trainer.policy.model.path)) + + # Create dispatch with colocate_all=True + dispatch = WorkerDispatch( + cfg, + policy_actor_group=policy_group, + ref_actor_group=ref_group, + ) + + # Mark both as on GPU after init + dispatch._gpu_state["policy"] = GPUState(model_on_gpu=True, optimizer_on_gpu=True) + dispatch._gpu_state["ref"] = GPUState(model_on_gpu=True, optimizer_on_gpu=False) + + # Manually offload both to start from clean state + policy_group.offload_to_cpu() + ref_group.offload_to_cpu() + dispatch.mark_all_offloaded() + + dp_size = policy_group.actor_infos[0].rank.dp_size + dummy_batch = make_dummy_training_batch(batch_size=dp_size) + + # === Test 1: Load ref model === + dispatch.forward("ref", dummy_batch) + + # Verify state tracking + assert dispatch._gpu_state["ref"].model_on_gpu, "ref should be marked on GPU" + assert not dispatch._gpu_state["policy"].model_on_gpu, "policy should be marked offloaded" + + # Verify ref memory increased (measure from ref_group since it's a separate actor) + ref_mem_after_load = get_rank_0_memory(ref_group, "After ref forward") + assert ref_mem_after_load > 1e8, f"Ref model should use significant memory: {ref_mem_after_load}" + + # === Test 2: Switch to policy (should offload ref) === + dispatch.forward_backward("policy", dummy_batch) + + # Verify state tracking + assert dispatch._gpu_state["policy"].model_on_gpu, "policy should be on GPU" + assert dispatch._gpu_state["policy"].optimizer_on_gpu, "policy optimizer should be on GPU" + assert not dispatch._gpu_state["ref"].model_on_gpu, "ref should be offloaded" + + # Verify policy is on GPU and ref was offloaded + policy_mem = get_rank_0_memory(policy_group, "After policy forward_backward") + ref_mem_after_offload = get_rank_0_memory(ref_group, "Ref after being offloaded") + assert policy_mem > 1e8, f"Policy model should use significant memory: {policy_mem}" + assert ( + ref_mem_after_offload < ref_mem_after_load + ), f"Ref memory should decrease after offload: {ref_mem_after_offload} < {ref_mem_after_load}" + + # === Test 3: Switch back to ref (should offload policy) === + dispatch.forward("ref", dummy_batch) + + # Verify state tracking + assert dispatch._gpu_state["ref"].model_on_gpu, "ref should be on GPU" + assert not dispatch._gpu_state["policy"].model_on_gpu, "policy should be offloaded" + assert not dispatch._gpu_state["policy"].optimizer_on_gpu, "policy optimizer should be offloaded" + + # Verify policy was offloaded + policy_mem_after_offload = get_rank_0_memory(policy_group, "Policy after being offloaded") + assert ( + policy_mem_after_offload < policy_mem + ), f"Policy memory should decrease after offload: {policy_mem_after_offload} < {policy_mem}" + + finally: + ray.shutdown() + + +@pytest.mark.asyncio +async def test_gpu_state_tracking_accuracy(ray_init_fixture): + """ + Test that _gpu_state accurately reflects what's actually on GPU. + + This verifies the internal state tracking matches the actual offload/onload operations. + """ + cfg = get_test_config() + + try: + pg = placement_group([{"GPU": 1, "CPU": 2}], strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=30) + + policy_group = init_colocated_actor_group(PolicyWorker, pg, cfg) + ref_group = init_colocated_actor_group(RefWorker, pg, cfg) + + ray.get(policy_group.async_init_model(cfg.trainer.policy.model.path)) + ray.get(ref_group.async_init_model(cfg.trainer.policy.model.path)) + + dispatch = WorkerDispatch( + cfg, + policy_actor_group=policy_group, + ref_actor_group=ref_group, + ) + + # Start from clean state + policy_group.offload_to_cpu() + ref_group.offload_to_cpu() + dispatch.mark_all_offloaded() + + # Verify initial state + assert dispatch._gpu_state["policy"] == GPUState(model_on_gpu=False, optimizer_on_gpu=False) + assert dispatch._gpu_state["ref"] == GPUState(model_on_gpu=False, optimizer_on_gpu=False) + + # Load policy for training (needs model + optimizer) + dp_size = policy_group.actor_infos[0].rank.dp_size + dummy_batch = make_dummy_training_batch(batch_size=dp_size) + dispatch.forward_backward("policy", dummy_batch) + + assert dispatch._gpu_state["policy"] == GPUState(model_on_gpu=True, optimizer_on_gpu=True) + assert dispatch._gpu_state["ref"] == GPUState(model_on_gpu=False, optimizer_on_gpu=False) + + # Load ref for inference (only needs model) + dispatch.forward("ref", dummy_batch) + + assert dispatch._gpu_state["ref"] == GPUState(model_on_gpu=True, optimizer_on_gpu=False) + assert dispatch._gpu_state["policy"] == GPUState(model_on_gpu=False, optimizer_on_gpu=False) + + finally: + ray.shutdown() + + +@pytest.mark.asyncio +async def test_colocate_policy_critic_training_switch(ray_init_fixture): + """ + Test switching between policy and critic training with colocate_all=True. + + This tests the common PPO training pattern where we alternate between + training policy and critic on the same GPU. + + Scenario: + 1. Train policy (forward_backward + optim_step) + 2. Train critic (forward_backward + optim_step) + 3. Train policy again + 4. Verify correct offload/onload at each switch + """ + cfg = get_test_config() + + try: + pg = placement_group([{"GPU": 1, "CPU": 2}], strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=30) + + policy_group = init_colocated_actor_group(PolicyWorker, pg, cfg) + critic_group = init_colocated_actor_group(CriticWorker, pg, cfg) + + ray.get(policy_group.async_init_model(cfg.trainer.policy.model.path)) + ray.get(critic_group.async_init_model(cfg.trainer.policy.model.path)) + + dispatch = WorkerDispatch( + cfg, + policy_actor_group=policy_group, + critic_actor_group=critic_group, + ) + + # Start from clean state + policy_group.offload_to_cpu() + critic_group.offload_to_cpu() + dispatch.mark_all_offloaded() + + dp_size = policy_group.actor_infos[0].rank.dp_size + dummy_batch = make_dummy_training_batch(batch_size=dp_size) + + # === Step 1: Train policy === + dispatch.forward_backward("policy", dummy_batch) + dispatch.optim_step("policy") + + assert dispatch._gpu_state["policy"].model_on_gpu + assert dispatch._gpu_state["policy"].optimizer_on_gpu + assert not dispatch._gpu_state["critic"].model_on_gpu + assert not dispatch._gpu_state["critic"].optimizer_on_gpu + + policy_mem = get_rank_0_memory(policy_group, "After policy training") + assert policy_mem > 1e8, f"Policy model should use significant memory: {policy_mem}" + + # === Step 2: Train critic (should offload policy) === + dispatch.forward_backward("critic", dummy_batch) + dispatch.optim_step("critic") + + assert dispatch._gpu_state["critic"].model_on_gpu + assert dispatch._gpu_state["critic"].optimizer_on_gpu + assert not dispatch._gpu_state["policy"].model_on_gpu + assert not dispatch._gpu_state["policy"].optimizer_on_gpu + + # Verify critic is loaded and policy was offloaded + critic_mem = get_rank_0_memory(critic_group, "After critic training") + policy_mem_after_offload = get_rank_0_memory(policy_group, "Policy after offload") + assert critic_mem > 1e8, f"Critic model should use significant memory: {critic_mem}" + assert ( + policy_mem_after_offload < policy_mem + ), f"Policy memory should decrease after offload: {policy_mem_after_offload} < {policy_mem}" + + # === Step 3: Train policy again (should offload critic) === + dispatch.forward_backward("policy", dummy_batch) + dispatch.optim_step("policy") + + assert dispatch._gpu_state["policy"].model_on_gpu + assert dispatch._gpu_state["policy"].optimizer_on_gpu + assert not dispatch._gpu_state["critic"].model_on_gpu + assert not dispatch._gpu_state["critic"].optimizer_on_gpu + + # Verify policy is loaded again and critic was offloaded + policy_mem_reloaded = get_rank_0_memory(policy_group, "Policy reloaded") + critic_mem_after_offload = get_rank_0_memory(critic_group, "Critic after offload") + assert policy_mem_reloaded > 1e8, f"Policy should be back on GPU: {policy_mem_reloaded}" + assert ( + critic_mem_after_offload < critic_mem + ), f"Critic memory should decrease after offload: {critic_mem_after_offload} < {critic_mem}" + + finally: + ray.shutdown() diff --git a/skyrl-train/tests/gpu/gpu_ci/test_worker_offload.py b/skyrl-train/tests/gpu/gpu_ci/test_worker_offload.py new file mode 100644 index 0000000000..450aa7511e --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_ci/test_worker_offload.py @@ -0,0 +1,363 @@ +""" +uv run --extra dev --isolated pytest tests/gpu/gpu_ci/test_worker_offload.py +""" + +import ray +import pytest +import hydra +from omegaconf import DictConfig +import os +import shutil + +from tests.gpu.utils import init_worker_with_type, make_dummy_training_batch, make_dummy_tensorbatch, get_rank_0_memory +from skyrl_train.utils.utils import validate_cfg +from skyrl_train.entrypoints.main_base import config_dir +from skyrl_train.training_batch import TrainingOutputBatch + +MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct" + + +def get_test_actor_config() -> DictConfig: + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = MODEL_NAME + cfg.trainer.placement.policy_num_gpus_per_node = 2 + cfg.trainer.use_sample_packing = False + cfg.trainer.logger = "console" + cfg.generator.inference_engine_tensor_parallel_size = 2 + + validate_cfg(cfg) + + return cfg + + +@pytest.fixture +def cfg() -> DictConfig: + return get_test_actor_config() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_type", "strategy"), + [ + ("policy", "fsdp"), + ("critic", "fsdp"), + ("policy", "fsdp2"), + ("critic", "fsdp2"), + ], + ids=[ + "fsdp_policy", + "fsdp_critic", + "fsdp2_policy", + "fsdp2_critic", + ], +) +async def test_critic_policy_offload_memory_and_correctness(ray_init_fixture, cfg, worker_type, strategy): + """ + Test that offloading model memory to cpu lowers memory usage and that correctness + is maintained after backloading and running a training step. + + steps: + 1. Initialize actor group with the specified worker class. + 2. Offload model to CPU and check memory usage. + 3. Backload model to GPU and check memory usage. + 4. Run a training step with dummy experience (with optimizer step) + 5. Offload model to CPU again and check memory usage. + 6. Backload model to GPU and check memory usage. + 7. Run another training step and ensure output consistency. + + Note for FSDP/FSDP2: optimizer is lazily initialized on the first step currently (see: https://github.com/volcengine/verl/pull/1349) + so memory after training step + offload might be higher than after initial offload. + """ + cfg.trainer.strategy = strategy + # 0 learning rate and wd so we can optimizer step to free gradients but still check results are the same + getattr(cfg.trainer, worker_type).optimizer_config.lr = 0 + getattr(cfg.trainer, worker_type).optimizer_config.weight_decay = 0 + try: + actor_group = init_worker_with_type( + worker_type, + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + get_rank_0_memory(actor_group, "After init") + # offload then backload first (no training step) + actor_group.offload_to_cpu() + + initial_offload_mem = get_rank_0_memory(actor_group, "After initial offload") + + # Backload to GPU + actor_group.backload_to_gpu() + get_rank_0_memory(actor_group, "Before training") + + dp_size = actor_group.actor_infos[0].rank.dp_size + dummy_batch = make_dummy_training_batch(batch_size=dp_size) + # Run first forward_backward + optim_step to get optimizer initialized and stepped + results = ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", data=dummy_batch)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + + after_training = get_rank_0_memory(actor_group, "After training") + + # Offload model to CPU + actor_group.offload_to_cpu(offload_optimizer=True, offload_model=False) + after_offload_optimizer = get_rank_0_memory(actor_group, "After optimizer offload") + + assert ( + after_offload_optimizer < after_training + ), f"Memory after offload optimizer should be less than after training: {after_offload_optimizer} bytes, after training: {after_training} bytes" + + actor_group.offload_to_cpu(offload_optimizer=False, offload_model=True) + after_offload = get_rank_0_memory(actor_group, "After model offload") + + assert ( + after_offload < after_offload_optimizer + ), f"Memory after offload model should be less than after offload optimizer: {after_offload} bytes, after offload optimizer: {after_offload_optimizer} bytes" + + # check that allocated memory is similar to initial offload memory + delta = abs(initial_offload_mem - after_offload) + assert ( + delta < 4e8 # 400MB (should be close to 0 diff) + ), f"Memory after training step + offload is not similar to initial offloaded memory: {delta} bytes. Initial offload mem: {initial_offload_mem}, after offload mem: {after_offload} bytes" + + # also check that allocated memory goes down after offloading + delta_forward = after_training - after_offload + assert ( + delta_forward > 0 + ), f"Memory after offloading should be less than after forward pass: {delta_forward} bytes" + + # Backload model to GPU + actor_group.backload_to_gpu(backload_optimizer=True, backload_model=False) + after_backload_optimizer = get_rank_0_memory(actor_group, "After backload optimizer") + assert ( + after_backload_optimizer > after_offload + ), f"Memory after backload optimizer should be greater than after offload: {after_backload_optimizer} bytes, after offload: {after_offload} bytes" + + actor_group.backload_to_gpu(backload_optimizer=False, backload_model=True) + after_backload = get_rank_0_memory(actor_group, "After backload model") + assert ( + after_backload > after_backload_optimizer + ), f"Memory after backload model should be greater than after backload optimizer: {after_backload} bytes, after backload optimizer: {after_backload_optimizer} bytes" + + # Run training again and ensure output consistency + results_backload = ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", data=dummy_batch)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + + for i, result in enumerate(results): + result_backload = results_backload[i] + for k, v in result.items(): + assert k in result_backload + assert v == result_backload[k], f"Results mismatch for {k}: {v} != {result_backload[k]}" + + finally: + ray.shutdown() # Clean up Ray resources after the test + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_type", "strategy"), + [ + # TODO (erictang000): fsdp 1 manual offloading is broken for single gpu ref worker + ("ref", "fsdp"), + ("ref", "fsdp2"), + # TODO (erictang000): Add support for reward worker. + ], + ids=[ + "fsdp_ref", + "fsdp2_ref", + ], +) +async def test_fsdp_ref_offload_memory_and_correctness(ray_init_fixture, cfg, worker_type, strategy): + """ + Test that offloading model memory to cpu lowers memory usage and that correctness + is maintained after backloading and running a forward pass. + + steps: + 1. Initialize actor group with the specified worker class. + 2. Offload model to CPU and check memory usage. + 3. Backload model to GPU and check memory usage. + 4. Run a forward pass with dummy experience. + 5. Offload model to CPU again and check memory usage. + 6. Backload model to GPU and check memory usage. + 7. Run another forward pass and ensure output consistency. + """ + cfg.trainer.strategy = strategy + # test that things work without any offloading setup by FSDP/FSDP2 + cfg.trainer.ref.fsdp_config.cpu_offload = False + try: + actor_group = init_worker_with_type( + worker_type, + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + get_rank_0_memory(actor_group, "After init") + # offload then backload first (no training step) + actor_group.offload_to_cpu() + initial_offload_mem = get_rank_0_memory(actor_group, "After initial offload") + + # should be close to 0 + assert ( + initial_offload_mem < 1e8 + ), f"Memory after offloading should be close to 0: instead {initial_offload_mem} bytes" + + # Backload to GPU + actor_group.backload_to_gpu() + get_rank_0_memory(actor_group, "Before forward") + + dummy_batch = make_dummy_tensorbatch() + # Run forward pass + results: TrainingOutputBatch = actor_group.run_method("pass_through", "forward", dummy_batch) + + after_forward = get_rank_0_memory(actor_group, "After forward") + + # Offload model to CPU + actor_group.offload_to_cpu() + + after_offload = get_rank_0_memory(actor_group, "After offload") + + # check that allocated memory is similar to initial offload memory + delta = abs(initial_offload_mem - after_offload) + assert ( + delta < 1e8 # 100MB (should be close to 0 diff) + ), f"Memory after training step + offload is not similar to initial offloaded memory: {delta} bytes" + + # also check that allocated memory goes down after offloading + delta_forward = after_forward - after_offload + assert ( + delta_forward > 0 + ), f"Memory after offloading should be less than after forward pass: {delta_forward} bytes" + + # Backload model to GPU + actor_group.backload_to_gpu() + + get_rank_0_memory(actor_group, "After backload") + + # Run forward again and ensure output consistency + results_backload: TrainingOutputBatch = actor_group.run_method("pass_through", "forward", dummy_batch) + + assert ( + results == results_backload + ), f"Results mismatch after backload. Results: {results}, Results backload: {results_backload}" + finally: + ray.shutdown() # Clean up Ray resources after the test + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("worker_type", "strategy"), + [ + ("policy", "fsdp2"), + ("critic", "fsdp2"), + ("ref", "fsdp2"), + ], + ids=[ + "fsdp2_policy", + "fsdp2_critic", + "fsdp2_ref", + ], +) +async def test_cpu_offload_correctness(ray_init_fixture, cfg, worker_type, strategy): + """ + Test that the cpu_offload is working correctly for different backends. + + steps: + 1. Initialize actor group with the specified worker class. + 2. Make sure that the model is fully offloaded to cpu + 3. Run a forward pass and make sure that the memory is still close to 0 + """ + cfg.trainer.strategy = strategy + # test that things work without any offloading setup by FSDP/FSDP2 + getattr(cfg.trainer, worker_type).fsdp_config.cpu_offload = True + try: + actor_group = init_worker_with_type( + worker_type, + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + after_init = get_rank_0_memory(actor_group, "After init") + + # should be close to 0 + assert after_init < 1e8, f"Memory after offloading should be close to 0: instead {after_init} bytes" + + dummy_batch = make_dummy_tensorbatch() + # Run forward pass + ray.get(actor_group.async_run_ray_method("pass_through", "forward", dummy_batch)) + + after_offload = get_rank_0_memory(actor_group, "After offload") + + # should still be relatively small + assert after_offload < 4e8, f"Memory after forward pass should be < 400MB: instead {after_offload} bytes" + + finally: + ray.shutdown() + + +@pytest.mark.parametrize( + "strategy", + [ + "fsdp", + "fsdp2", + ], +) +def test_offload_after_ckpt(ray_init_fixture, strategy): + """ + Test ckpt+offload logic by: + 1. Creating model and doing one training step + 2. Saving checkpoint + 3. Offload parameters and optimizer + 4. Ensure that memory was freed + """ + cfg = get_test_actor_config() + ckpt_path = "$HOME/ckpts/test/" + cfg.trainer.ckpt_path = ckpt_path + cfg.trainer.export_path = ckpt_path + cfg.trainer.strategy = strategy + + checkpoint_dir = None + try: + actor_group = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + get_rank_0_memory(actor_group, "After init") + + # Create dummy training batch for training steps + dp_size = actor_group.actor_infos[0].rank.dp_size + dummy_batch_1 = make_dummy_training_batch(batch_size=dp_size) + + # Step 1: Do initial forward_backward + optim_step + ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", data=dummy_batch_1)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + get_rank_0_memory(actor_group, "After training step 1") + + checkpoint_path = os.path.expandvars(os.path.join(cfg.trainer.ckpt_path, "global_step_1", "policy")) + checkpoint_dir = os.path.expandvars(os.path.join(cfg.trainer.ckpt_path, "global_step_1")) # Store for cleanup + + # Step 2: Save checkpoint + ray.get(actor_group.async_run_ray_method("pass_through", "save_checkpoint", ckpt_dir=checkpoint_path)) + after_training = get_rank_0_memory(actor_group, "After ckpt") + + # Step 3:Offload model to CPU + actor_group.offload_to_cpu() + after_offload = get_rank_0_memory(actor_group, "After offload") + + # Step 4: Check that memory is offloaded + offload_delta = after_training - after_offload + assert offload_delta > 2.5 * 1024**3, f"Offload memory is {offload_delta} bytes, should be > 2.5GB" + + finally: + # Clean up ray + ray.shutdown() + + # Clean up checkpoint directory + if checkpoint_dir and os.path.exists(checkpoint_dir): + print(f"Removing checkpoint directory: {checkpoint_dir}") + shutil.rmtree(checkpoint_dir) diff --git a/skyrl-train/tests/gpu/gpu_e2e_test/gsm8k_colocate.sh b/skyrl-train/tests/gpu/gpu_e2e_test/gsm8k_colocate.sh new file mode 100644 index 0000000000..4baa52b982 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_e2e_test/gsm8k_colocate.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The anyscale job's working_dir is the repo root, so we can use relative paths. +bash examples/gsm8k/run_gsm8k.sh \ + trainer.epochs=1 \ + trainer.eval_before_train=true \ + trainer.micro_forward_batch_size_per_gpu=16 \ + trainer.micro_train_batch_size_per_gpu=16 \ + trainer.project_name=\"gsm8k_ci\" \ + trainer.run_name=\"gsm8k_colocate\" diff --git a/skyrl-train/tests/gpu/gpu_e2e_test/gsm8k_colocate_megatron.sh b/skyrl-train/tests/gpu/gpu_e2e_test/gsm8k_colocate_megatron.sh new file mode 100755 index 0000000000..1a8320b919 --- /dev/null +++ b/skyrl-train/tests/gpu/gpu_e2e_test/gsm8k_colocate_megatron.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The anyscale job's working_dir is the repo root, so we can use relative paths. +bash examples/megatron/run_megatron.sh \ + trainer.epochs=1 \ + trainer.eval_before_train=true \ + trainer.micro_forward_batch_size_per_gpu=4 \ + trainer.micro_train_batch_size_per_gpu=4 \ + trainer.project_name=\"gsm8k_ci_megatron\" \ + trainer.run_name=\"gsm8k_megatron_colocate\" diff --git a/skyrl-train/tests/gpu/runner.py b/skyrl-train/tests/gpu/runner.py new file mode 100644 index 0000000000..a9114da4a1 --- /dev/null +++ b/skyrl-train/tests/gpu/runner.py @@ -0,0 +1,34 @@ +"""Helper script to run tests on GPU as a ray task. +Useful if working on a multi-node cluster. + +Example: + +``` +uv run --isolated tests/gpu/runner.py --num-gpus 4 --test-file tests/gpu/test_models.py +``` +or you can also do: + +``` +uv run --isolated tests/gpu/runner.py --num-gpus 4 --test-file tests/gpu/ +``` +""" + +import ray +import pytest +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("--num-gpus", type=int, default=4) +parser.add_argument("--test-file", type=str, default="tests/gpu/test_models.py") +args = parser.parse_args() + + +@ray.remote(num_gpus=args.num_gpus) +def run_tests(): + return pytest.main(["-s", "-vvv", args.test_file]) + + +if __name__ == "__main__": + ray.init() + ray.get(run_tests.remote()) + ray.shutdown() diff --git a/skyrl-train/tests/gpu/test_expert_parallel_inference.py b/skyrl-train/tests/gpu/test_expert_parallel_inference.py new file mode 100644 index 0000000000..899d1a6a0b --- /dev/null +++ b/skyrl-train/tests/gpu/test_expert_parallel_inference.py @@ -0,0 +1,220 @@ +""" +Tests for expert parallel (EP). + +uv run --isolated --extra dev --extra vllm pytest tests/gpu/test_expert_parallel_inference.py + +""" + +import asyncio +import pytest +import ray +from typing import Optional +from ray.util.placement_group import PlacementGroup +from omegaconf import DictConfig +from transformers import AutoTokenizer + +from tests.gpu.utils import ( + get_available_gpus, + get_test_prompts, + init_worker_with_type, + are_responses_similar, + get_test_actor_config, +) +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.base import InferenceEngineInput +from skyrl_train.utils import initialize_ray, get_ray_pg_ready_with_timeout +from skyrl_train.inference_engines.utils import get_sampling_params_for_backend +from skyrl_train.inference_engines.ray_wrapped_inference_engine import create_ray_wrapped_inference_engines +from ray.util.placement_group import placement_group + + +MODEL = "Qwen/Qwen1.5-MoE-A2.7B-Chat" +NUM_GPUS = 4 # Should be divisible by 2 + + +def _check_gpus(num_gpus: int): + available = get_available_gpus() + if len(available) < num_gpus: + pytest.skip(f"Expert parallel tests require >= {num_gpus} GPUs, found {len(available)}: {available}") + + +def _get_test_cfg() -> DictConfig: + cfg = get_test_actor_config() + + # Use MoE policy model + cfg.trainer.policy.model.path = MODEL + + # vLLM generator with EP enabled + cfg.generator.backend = "vllm" + cfg.generator.async_engine = True + cfg.generator.num_inference_engines = NUM_GPUS // 2 + cfg.generator.inference_engine_tensor_parallel_size = 2 + cfg.generator.inference_engine_expert_parallel_size = 2 + cfg.generator.inference_engine_data_parallel_size = 1 + cfg.generator.gpu_memory_utilization = 0.8 + + # Small lengths for faster tests + cfg.generator.max_input_length = 2048 + cfg.generator.sampling_params.max_generate_length = 512 + + # Training knobs for tests + cfg.trainer.strategy = "fsdp2" + cfg.trainer.train_batch_size = 128 + cfg.trainer.policy_mini_batch_size = 128 + cfg.trainer.micro_forward_batch_size_per_gpu = 8 + cfg.trainer.micro_train_batch_size_per_gpu = 8 + cfg.trainer.placement.policy_num_nodes = 1 + cfg.trainer.placement.policy_num_gpus_per_node = NUM_GPUS + # Small micro batches to fit the MoE in 2 GPUs during training. + cfg.trainer.micro_train_batch_size_per_gpu = 1 + cfg.trainer.micro_forward_batch_size_per_gpu = 1 + cfg.trainer.update_epochs_per_batch = 1 + + return cfg + + +async def _run_single_generation(client: InferenceEngineClient, prompts, sampling_params): + tasks = [client.generate(InferenceEngineInput(prompts=[p], sampling_params=sampling_params)) for p in prompts] + results = await asyncio.gather(*tasks) + responses, reasons = [], [] + for r in results: + responses.extend(r["responses"]) + reasons.extend(r["stop_reasons"]) + return responses, reasons + + +def init_ray_inference_engines( + backend: str, tp_size: int, shared_pg: Optional[PlacementGroup], config: DictConfig +) -> InferenceEngineClient: + """Initialize ray-wrapped inference engines for the specified backend""" + tokenizer = AutoTokenizer.from_pretrained(MODEL) + engine = create_ray_wrapped_inference_engines( + num_inference_engines=1, + tensor_parallel_size=tp_size, + expert_parallel_size=config.generator.inference_engine_expert_parallel_size, + model_dtype="bfloat16", + pretrain=MODEL, + seed=42, + vllm_v1_disable_multiproc=True, + enable_prefix_caching=True, + enforce_eager=True, + shared_pg=shared_pg, + gpu_memory_utilization=0.8, + inference_engine_enable_sleep=False, + async_engine=True, + max_num_batched_tokens=8192, + max_num_seqs=1024, + tokenizer=tokenizer, + backend=backend, + ) + client = InferenceEngineClient(engine, tokenizer, config) + return client + + +def test_ep_generation(): + """ + Ensure vLLM generation with expert parallel enabled (EP=2) runs without errors. + Validate that the number of outputs matches the number of inputs. + """ + _check_gpus(num_gpus=NUM_GPUS) + + try: + cfg = _get_test_cfg() + # Deterministic sampling for stable execution + cfg.generator.sampling_params.temperature = 0.0 + cfg.generator.sampling_params.top_p = 1.0 + cfg.generator.sampling_params.top_k = -1 + initialize_ray(cfg) + + client = init_ray_inference_engines( + backend=cfg.generator.backend, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + shared_pg=None, + config=cfg, + ) + + prompts = get_test_prompts(MODEL, num_samples=4) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + + responses, reasons = asyncio.run(_run_single_generation(client, prompts, sampling_params)) + assert len(responses) == len(prompts) + assert len(reasons) == len(prompts) + finally: + ray.shutdown() + + +def test_ep_weight_sync(): + """ + Ensure generation works after syncing weights from training policy worker. + """ + _check_gpus(num_gpus=NUM_GPUS) + + pg = None + try: + cfg = _get_test_cfg() + cfg.trainer.placement.colocate_all = True + # Deterministic sampling for robust comparisons + cfg.generator.sampling_params.temperature = 0.0 + cfg.generator.sampling_params.top_p = 1.0 + cfg.generator.sampling_params.top_k = -1 + + initialize_ray(cfg) + + # Create a shared PG with 2 bundles (sufficient for two engines with tp=2 and training) + pg = placement_group([{"GPU": 1, "CPU": 1}] * NUM_GPUS, strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=60) + + # Spin up two inference engines with EP enabled, colocated + client = init_ray_inference_engines( + backend=cfg.generator.backend, + tp_size=cfg.generator.inference_engine_tensor_parallel_size, + shared_pg=pg, + config=cfg, + ) + asyncio.run(client.wake_up()) + + # Generate before weight sync + prompts = get_test_prompts(MODEL, num_samples=4) + sampling_params = get_sampling_params_for_backend(cfg.generator.backend, cfg.generator.sampling_params) + out_before = asyncio.run( + client.generate(InferenceEngineInput(prompts=prompts, sampling_params=sampling_params)) + ) + assert len(out_before["responses"]) == len(prompts) + + asyncio.run(client.sleep()) + + # Initialize policy worker + policy = init_worker_with_type( + "policy", + shared_pg=pg, + colocate_all=True, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + # Sync weights to inference engines + ray.get(policy.async_run_ray_method("pass_through", "init_weight_sync_state", client)) + asyncio.run(client.wake_up(tags=["weights"])) + ray.get(policy.async_run_ray_method("pass_through", "broadcast_to_inference_engines", client)) + policy.offload_to_cpu() + asyncio.run(client.wake_up(tags=["kv_cache"])) + asyncio.run(client.reset_prefix_cache()) + + # Generate after weight sync + out_after = asyncio.run(client.generate(InferenceEngineInput(prompts=prompts, sampling_params=sampling_params))) + assert len(out_after["responses"]) == len(prompts) + assert len(out_after["stop_reasons"]) == len(prompts) + + # Check that weights are not corrupted: responses should be similar pre/post sync + for i in range(len(prompts)): + if not are_responses_similar([out_before["responses"][i]], [out_after["responses"][i]], tolerance=0.02): + print( + f"Response changed significantly after weight sync: before={out_before['responses'][i][:200]} ... after={out_after['responses'][i][:200]} ..." + ) + finally: + if pg is not None: + try: + ray.util.remove_placement_group(pg) + except Exception: + pass + ray.shutdown() diff --git a/skyrl-train/tests/gpu/test_grpo_sp_sanity.py b/skyrl-train/tests/gpu/test_grpo_sp_sanity.py new file mode 100644 index 0000000000..857dd27ba4 --- /dev/null +++ b/skyrl-train/tests/gpu/test_grpo_sp_sanity.py @@ -0,0 +1,200 @@ +""" +uv run --isolated --extra vllm --extra dev -- pytest -s -vvv tests/gpu/test_grpo_sp_sanity.py +""" + +import os +from hydra import initialize, compose +from omegaconf import DictConfig +from pathlib import Path +from loguru import logger +import numpy as np +from skyrl_train.entrypoints.main_base import BasePPOExp, config_dir +from skyrl_train.trainer import RayPPOTrainer +import ray +from tqdm import tqdm +from skyrl_train.utils import Timer +from skyrl_train.utils.ppo_utils import normalize_advantages_dict + + +import asyncio + + +class TestExp(BasePPOExp): + def get_trainer( + self, + cfg, + tracker, + tokenizer, + train_dataset, + eval_dataset, + inference_engine_client, + generator, + colocate_pg, + ): + return RayPPOTestTrainer( + cfg=cfg, + tracker=tracker, + tokenizer=tokenizer, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + inference_engine_client=inference_engine_client, + generator=generator, + colocate_pg=colocate_pg, + ) + + +class RayPPOTestTrainer(RayPPOTrainer): + def train(self): + """ + Main training loop for PPO + """ + self.all_metrics = {} + self.all_timings = {} + + # Initialize weight sync state + with Timer("init_weight_sync_state"): + self.init_weight_sync_state() + + # main training loop + consumed_samples = 0 + num_rollouts_per_episodes = ( + self.num_update_steps_per_episodes + // self.cfg.trainer.update_epochs_per_batch + // self.cfg.generator.n_samples_per_prompt + ) + + self.global_step = consumed_samples // self.cfg.trainer.train_batch_size + start_episode = consumed_samples // self.cfg.trainer.train_batch_size // num_rollouts_per_episodes + consumed_samples = consumed_samples % (num_rollouts_per_episodes * self.cfg.trainer.train_batch_size) + + # Run just one iteration for testing + for episode in range(start_episode, start_episode + 1): + pbar = tqdm( + range(self.train_dataloader.__len__()), + desc=f"Episode [{episode + 1}/{self.cfg.trainer.num_episodes}]", + ) + for iter, rand_prompts in enumerate(self.train_dataloader): + with Timer("step", self.all_timings): + all_prompts = sum( + [[prompt[0]] * self.cfg.generator.n_samples_per_prompt for prompt in rand_prompts], [] + ) + all_extras = sum( + [[prompt[1]] * self.cfg.generator.n_samples_per_prompt for prompt in rand_prompts], [] + ) + + # 1.1 generation phase + with Timer("generate", self.all_timings): + data = asyncio.run(self.generate(all_prompts, all_extras)) + + # 1.2 compute rewards + with Timer("compute_rewards", self.all_timings): + data = self.compute_rewards(data) + # keep only the keys needed later on + data = data.pop(non_tensor_batch_keys=["response_ids", "prompt_ids", "loss_mask", "rewards"]) + + # 2. print example just for debugging + vis = self.tokenizer.decode(data.non_tensor_batch["response_ids"][0]) + print("example: ", vis) + + with Timer("convert_to_batch_regular", self.all_timings): + data = self.convert_to_batch_regular(data) + + # sequences are the full input ids for the model. + data = data.select( + batch_keys=["sequences", "attention_mask", "rewards", "loss_mask", "response_mask"], + non_tensor_batch_keys=["response_ids"], + ) + + # 1.4 inference and calculate values, log probs, rewards, kl divergence + with Timer("fwd_logprobs_values_reward", self.all_timings): + data = self.fwd_logprobs_values_reward(data) + + # 1.5 calculate kl divergence and create experiences + with Timer("calc_values_logprobs_rewards_kl", self.all_timings): + data = self.calculate_kl(data) + logger.info(f"Number of sequences: {len(data['sequences'])}") + + # 3. calculate advantages and returns / along with tensorboard logging + with Timer("calc_advantages_and_returns", self.all_timings): + data = self.compute_advantages_and_returns(data) + # remove some unwanted keys + data.pop(batch_keys=["rewards"]) + + if self.cfg.trainer.algorithm.advantage_batch_normalize: + data = normalize_advantages_dict(data) + + # 4. train policy/critic model + with Timer("train_critic_and_policy", self.all_timings): + status = self.train_critic_and_policy(data) + + # 5. set logs + logger.info(status) + pbar.update() + # log epoch info + self.all_metrics.update({"trainer/epoch": episode, "step": self.global_step}) + self.tracker.log(self.all_metrics, step=self.global_step) + self.tracker.log({"timing/" + k: v for k, v in self.all_timings.items()}, step=self.global_step) + self.global_step += 1 + + # Return metrics after one iteration + return self.all_metrics + + +def run_exp_and_get_metrics(exp: BasePPOExp, cfg: DictConfig): + metrics = exp.run() + # ray shutdown will clear all state for the ray session + ray.shutdown() + return metrics + + +def run_with_hydra(func, config_name: str): + current_directory = Path(__file__).parent.absolute() + abs_config_dir = Path(config_dir).absolute() + relative_config_dir = os.path.relpath(abs_config_dir, current_directory) + print("relative_config_dir: ", relative_config_dir) + with initialize(version_base=None, config_path=relative_config_dir): + cfg = compose(config_name=config_name) + func(cfg) + + +def ppo_run(cfg: DictConfig) -> None: + # Configure test settings + cfg.trainer.train_batch_size = 8 + cfg.trainer.num_episodes = 1 + cfg.trainer.policy_mini_batch_size = 8 + # use zero temperature for consistency. + # We will anyways only check for log probability values so this is fine. + cfg.generator.sampling_params.temperature = 0.0 + cfg.trainer.placement.policy_num_gpus_per_node = 4 + cfg.trainer.placement.critic_num_gpus_per_node = 4 + cfg.trainer.placement.ref_num_gpus_per_node = 4 + cfg.generator.num_inference_engines = 2 + cfg.generator.inference_engine_tensor_parallel_size = 2 + cfg.generator.gpu_memory_utilization = 0.7 + + # Run baseline (no sequence parallel) + cfg.trainer.policy.sequence_parallel_size = 1 + exp_baseline = TestExp(cfg) + metrics_baseline = run_exp_and_get_metrics(exp_baseline, cfg) + print("Baseline metrics: ", metrics_baseline) + + # Run with sequence parallel + cfg.trainer.policy.sequence_parallel_size = 2 + exp_sp = TestExp(cfg) + metrics_sp = run_exp_and_get_metrics(exp_sp, cfg) + print("Metrics with sequence parallel: ", metrics_sp) + + # Compare policy entropy values + # NOTE: typical values are ~ 0.225 and ~ 0.228 + # Some difference can be due to ignoring attention mask with seq parallelism + np.testing.assert_allclose( + metrics_sp["policy/policy_entropy"], metrics_baseline["policy/policy_entropy"], atol=5e-3 + ) + + +def test_ppo_run(): + run_with_hydra(ppo_run, "ppo_base_config") + + +if __name__ == "__main__": + test_ppo_run() diff --git a/skyrl-train/tests/gpu/test_main_generate.py b/skyrl-train/tests/gpu/test_main_generate.py new file mode 100644 index 0000000000..cf99d38891 --- /dev/null +++ b/skyrl-train/tests/gpu/test_main_generate.py @@ -0,0 +1,59 @@ +""" +uv run --extra dev --extra vllm --isolated pytest tests/gpu/test_main_generate.py +""" + +import json +import asyncio +import ray + +from skyrl_train.entrypoints.main_generate import EvalOnlyEntrypoint +from skyrl_train.utils.utils import initialize_ray +from tests.gpu.utils import get_test_actor_config, get_test_generator_input + + +def create_dataset(tmp_path, model_name: str): + input_batch = get_test_generator_input(model=model_name, num_prompts=1, n_samples_per_prompt=1) + data = { + "prompt": input_batch["prompts"][0], + "env_class": input_batch["env_classes"][0], # defaults to "gsm8k" + "data_source": "test", + **(input_batch["env_extras"][0] or {}), + } + path = tmp_path / "data.json" + with open(path, "w") as f: + f.write(json.dumps(data) + "\n") + return str(path) + + +def test_main_generate(tmp_path): + cfg = get_test_actor_config() + + # Minimize resources and side effects + cfg.trainer.logger = "console" + cfg.trainer.placement.colocate_all = False + cfg.trainer.export_path = str(tmp_path) + cfg.trainer.ckpt_path = str(tmp_path) + cfg.trainer.dump_eval_results = False + + cfg.generator.num_inference_engines = 1 + cfg.generator.inference_engine_tensor_parallel_size = 1 + cfg.generator.sampling_params.max_generate_length = 4 + cfg.generator.eval_sampling_params.max_generate_length = 4 + cfg.generator.eval_n_samples_per_prompt = 1 + cfg.generator.async_engine = True + + cfg.environment.skyrl_gym.max_env_workers = 1 + + initialize_ray(cfg) + try: + data_path = create_dataset(tmp_path, cfg.trainer.policy.model.path) + cfg.data.val_data = [data_path] + cfg.trainer.train_batch_size = 1 + cfg.trainer.eval_batch_size = 1 + cfg.trainer.eval_interval = 1 + + exp = EvalOnlyEntrypoint(cfg) + metrics = asyncio.run(exp.run()) + assert isinstance(metrics, dict) and len(metrics) > 0, f"Eval results not correctly computed: {metrics}" + finally: + ray.shutdown() diff --git a/skyrl-train/tests/gpu/test_multi_node_pg.py b/skyrl-train/tests/gpu/test_multi_node_pg.py new file mode 100644 index 0000000000..e7ec7390c3 --- /dev/null +++ b/skyrl-train/tests/gpu/test_multi_node_pg.py @@ -0,0 +1,158 @@ +""" +Run with: +uv run --isolated --extra dev -- pytest tests/gpu/test_multi_node_pg.py +NOTE: Placement group bundle ordering across nodes only typically has race conditions when using >16 GPUs +so this test is best run with >16 GPUs to actually see that ordering is correct +""" + +import ray +import pytest +import hydra +from omegaconf import DictConfig +from ray.util.placement_group import placement_group + +from skyrl_train.utils.utils import get_ray_pg_ready_with_timeout +from skyrl_train.workers.fsdp.fsdp_worker import PolicyWorker +from skyrl_train.workers.worker import PPORayActorGroup +from skyrl_train.utils.utils import validate_cfg +from skyrl_train.entrypoints.main_base import config_dir + + +MODEL_NAME = "Qwen/Qwen3-0.6B" + + +def get_test_actor_config() -> DictConfig: + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + cfg.trainer.policy.model.path = MODEL_NAME + cfg.generator.weight_sync_backend = "nccl" + cfg.trainer.strategy = "fsdp2" + validate_cfg(cfg) + + return cfg + + +@pytest.fixture +def cfg() -> DictConfig: + return get_test_actor_config() + + +def get_pg(placement_group_type, num_gpus_per_node, num_nodes): + if placement_group_type == "single_gpu_per_bundle": + pg = placement_group( + [{"GPU": 1, "CPU": 1}] * num_gpus_per_node * num_nodes, + strategy="PACK", + ) + get_ray_pg_ready_with_timeout(pg, timeout=60) + return pg + elif placement_group_type == "whole_node_bundle": + pg = placement_group( + [{"GPU": num_gpus_per_node, "CPU": num_gpus_per_node}] * num_nodes, + strategy="PACK", + ) + get_ray_pg_ready_with_timeout(pg, timeout=60) + return pg + elif placement_group_type == "none": + return None + else: + raise ValueError(f"Invalid placement group type: {placement_group_type}") + + +def test_multi_node_pg_invalid_pg(ray_init_fixture, cfg): + colocate_all = True + num_nodes = 2 + num_gpus_per_node = 4 + pg = get_pg("whole_node_bundle", num_gpus_per_node, num_nodes) + with pytest.raises( + AssertionError, + match="if colocate_all is True, the number of bundles in the shared placement group must match the world size", + ): + PPORayActorGroup( + cfg, + num_nodes=num_nodes, + num_gpus_per_node=num_gpus_per_node, + ray_actor_type=PolicyWorker, + pg=pg, + num_gpus_per_actor=0.2, + colocate_all=colocate_all, + ) + + +def test_multi_node_pg_errors_no_pg(ray_init_fixture, cfg): + colocate_all = True + num_nodes = 2 + num_gpus_per_node = 4 + pg = None + with pytest.raises( + AssertionError, + match="if colocate_all is True, the shared placement group must be provided to PPORayActorGroup", + ): + PPORayActorGroup( + cfg, + num_nodes=num_nodes, + num_gpus_per_node=num_gpus_per_node, + ray_actor_type=PolicyWorker, + pg=pg, + num_gpus_per_actor=0.2, + colocate_all=colocate_all, + ) + + +@pytest.mark.parametrize( + ("colocate_all", "placement_group_type"), + [ + (True, "single_gpu_per_bundle"), + (False, "single_gpu_per_bundle"), # this is technically not used in practice, but testing for completeness + (False, "whole_node_bundle"), + (False, "none"), + # (True, "none"), and ("True", "whole_node_bundle") should fail and are tested above in test_multi_node_pg_init_error + ], + ids=[ + "colocate_all_single_gpu_per_bundle", + "not_colocate_all_single_gpu_per_bundle", + "not_colocate_all_whole_node_bundle", + "not_colocate_all_none", + ], +) +def test_multi_node_pg_init(ray_init_fixture, cfg, colocate_all, placement_group_type): + try: + cfg.trainer.placement.colocate_all = colocate_all + cfg.trainer.placement.policy_num_nodes = 2 + cfg.trainer.placement.policy_num_gpus_per_node = 4 + + pg = get_pg( + placement_group_type, cfg.trainer.placement.policy_num_gpus_per_node, cfg.trainer.placement.policy_num_nodes + ) + + policy = PPORayActorGroup( + cfg, + num_nodes=cfg.trainer.placement.policy_num_nodes, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + ray_actor_type=PolicyWorker, + pg=pg, + num_gpus_per_actor=0.2 if colocate_all else 0.75, + colocate_all=colocate_all, + ) + + # get info from policy workers + mesh_ranks = [ray.get(actor.get_mesh_rank.remote()) for actor in policy._actor_handlers] + gpu_ids = [ray.get(actor.get_gpu_id.remote()) for actor in policy._actor_handlers] + node_ids = [ray.get(actor.get_ray_node_id.remote()) for actor in policy._actor_handlers] + + # use dp rank in mesh rank as proxy for world rank to verify correct layout + for rank, mesh_rank in enumerate(mesh_ranks): + assert rank == mesh_rank.dp, f"Mesh rank {mesh_rank} has incorrect dp rank" + assert ( + rank % cfg.trainer.placement.policy_num_gpus_per_node == gpu_ids[rank] + ), f"Mesh rank {mesh_rank} has incorrect gpu id" + + num_nodes = len(set(node_ids)) + gpus_per_node = cfg.trainer.placement.policy_num_gpus_per_node + for i in range(num_nodes): + node_ids_for_group = node_ids[i * gpus_per_node : (i + 1) * gpus_per_node] + unique_node_ids = set(node_ids_for_group) + assert ( + len(unique_node_ids) == 1 + ), f"Node IDs are not consistent for node group {i}. Found: {unique_node_ids}" + finally: + ray.shutdown() diff --git a/skyrl-train/tests/gpu/test_save_load_model.py b/skyrl-train/tests/gpu/test_save_load_model.py new file mode 100644 index 0000000000..71593cb867 --- /dev/null +++ b/skyrl-train/tests/gpu/test_save_load_model.py @@ -0,0 +1,175 @@ +""" +Test save_hf_model and load_hf_model functionality for different strategies. + +For FSDP and FSDP2, run with: +uv run --isolated --extra dev -- pytest tests/gpu/test_save_load_model.py -m "not megatron" + +For Megatron, run with: +uv run --isolated --extra dev --extra mcore -- pytest tests/gpu/test_save_load_model.py -m "megatron" +""" + +import ray +import pytest +import hydra +import torch +import os +import shutil +import tempfile +from omegaconf import DictConfig +import json +from transformers import AutoTokenizer + +from tests.gpu.utils import ( + init_worker_with_type, + make_dummy_training_batch, + get_model_logits_from_actor, + ray_init_for_tests, + validate_cfg, +) +from skyrl_train.entrypoints.main_base import config_dir + +MODEL_NAME = "Qwen/Qwen3-0.6B" +MODEL_ARCH = "Qwen3ForCausalLM" +NUM_GPUS = 4 + + +def get_test_actor_config(strategy: str) -> DictConfig: + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = MODEL_NAME + cfg.trainer.placement.policy_num_gpus_per_node = NUM_GPUS + cfg.trainer.strategy = strategy + + # Use temporary directories for testing + cfg.trainer.ckpt_path = tempfile.mkdtemp(prefix="model_test_ckpt_") + cfg.trainer.export_path = tempfile.mkdtemp(prefix="model_test_save_") + cfg.trainer.logger = "console" + + validate_cfg(cfg) + + return cfg + + +def run_one_training_step( + actor_group, + strategy, + data=None, + megatron_batch=None, +): + """Run forward_backward + optim_step to perform one training step.""" + if strategy == "megatron": + assert megatron_batch is not None, "Megatron requires a TrainingInputBatch for ppo_train" + return ray.get(actor_group.async_run_ray_method("mesh", "ppo_train", megatron_batch)) + else: + assert data is not None, f"{strategy} requires a TrainingInputBatch for forward_backward" + ray.get(actor_group.async_run_ray_method("mesh", "forward_backward", data=data)) + ray.get(actor_group.async_run_ray_method("pass_through", "optim_step")) + + +@pytest.mark.parametrize( + "strategy", + [ + "fsdp", + "fsdp2", + pytest.param("megatron", marks=pytest.mark.megatron), + ], +) +def test_save_load_hf_model(ray_init_fixture, strategy): + """ + Test save_hf_model functionality by: + 1. Loading a pretrained model into an ActorGroup + 2. Running a forward pass to get some outputs + 3. Saving model in HuggingFace format using save_hf_model + 4. Loading model from saved HuggingFace format and comparing outputs + """ + cfg = get_test_actor_config(strategy) + + model_save_dir = None + try: + # ============= PHASE 1: Train and Save ============= + actor_group_1 = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg.trainer.placement.policy_num_gpus_per_node, + cfg=cfg, + ) + + # Prepare training input and run one training step + dp_size = actor_group_1.actor_infos[0].rank.dp_size + if "megatron" in strategy: + from tests.gpu.test_megatron_worker import get_test_training_batch + + train_batch_1 = get_test_training_batch(dp_size if dp_size % NUM_GPUS == 0 else NUM_GPUS) + run_one_training_step( + actor_group_1, + strategy, + data=None, + megatron_batch=train_batch_1, + ) + else: + dummy_batch = make_dummy_training_batch(batch_size=dp_size) + run_one_training_step( + actor_group_1, + strategy, + data=dummy_batch, + megatron_batch=None, + ) + + # Step 2: Create test input and compute logits from trained model + dp_size = actor_group_1.actor_infos[0].rank.dp_size + test_input = torch.randint(0, 1000, (dp_size, 20), device="cpu") # batch_size=dp_size, seq_len=20 + attention_mask = torch.ones_like(test_input) + + logits_from_trained_model = get_model_logits_from_actor(actor_group_1, test_input, attention_mask) + + # Step 3: Save model in HuggingFace format (include tokenizer so Megatron can reload it) + export_dir = os.path.join(cfg.trainer.export_path, "global_step_1", "policy") + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + ray.get( + actor_group_1.async_run_ray_method( + "pass_through", "save_hf_model", export_dir=export_dir, tokenizer=tokenizer + ) + ) + + # Verify that model files were saved + model_save_dir = export_dir + expected_files = ["config.json", "model.safetensors", "tokenizer.json"] # Basic HuggingFace model files + for expected_file in expected_files: + file_path = os.path.join(model_save_dir, expected_file) + assert os.path.exists(file_path), f"Expected model file not found: {file_path}" + + with open(os.path.join(model_save_dir, "config.json"), "r") as f: + config = json.load(f) + assert config["architectures"] == [MODEL_ARCH], f"Architecture should be {MODEL_ARCH}" + + # Step 4: Destroy first worker to ensure fresh weights. + ray.shutdown() + + # ============= PHASE 2: Fresh Worker Loading from Saved Path ============= + ray_init_for_tests() + # Create a new config that points to the saved model instead of the original model + cfg_fresh = get_test_actor_config(strategy) + # IMPT: Point to the saved model directory instead of original model + cfg_fresh.trainer.policy.model.path = model_save_dir + + actor_group_2 = init_worker_with_type( + "policy", + shared_pg=None, + colocate_all=False, + num_gpus_per_node=cfg_fresh.trainer.placement.policy_num_gpus_per_node, + cfg=cfg_fresh, + ) + + # Step 5: Compute logits from worker that loaded the saved model + logits_from_loaded_saved_model = get_model_logits_from_actor(actor_group_2, test_input, attention_mask) + + # Step 6: Compare logits - they should match the original trained model exactly + torch.testing.assert_close(logits_from_trained_model, logits_from_loaded_saved_model, atol=1e-8, rtol=1e-8) + + finally: + # Clean up temporary directories + for temp_dir in [cfg.trainer.ckpt_path, cfg.trainer.export_path, model_save_dir]: + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir) diff --git a/skyrl-train/tests/gpu/test_skyrl_gym_generator.py b/skyrl-train/tests/gpu/test_skyrl_gym_generator.py new file mode 100644 index 0000000000..787e8e28db --- /dev/null +++ b/skyrl-train/tests/gpu/test_skyrl_gym_generator.py @@ -0,0 +1,38 @@ +""" +uv run --extra dev --extra vllm --isolated pytest tests/gpu/test_skyrl_gym_generator.py +""" + +import os +import pytest +import ray +from skyrl_train.utils.utils import initialize_ray +from tests.gpu.gpu_ci.test_skyrl_gym_generator import run_generator_end_to_end +from skyrl_train.config.utils import get_default_config + + +# TODO: Make this test lightweight. It currently requires a ~20GB dataset download. Then, transfer the test to gpu_ci. +@pytest.mark.asyncio +async def test_generator_multi_turn_text2sql(): + """ + Test the generator with multiple turns of text2sql + """ + initialize_ray(get_default_config()) + try: + await run_generator_end_to_end( + use_async_engine=True, + batched=False, + n_samples_per_prompt=5, + num_inference_engines=2, + tensor_parallel_size=2, + model="Qwen/Qwen2.5-Coder-7B-Instruct", + max_prompt_length=6000, + max_input_length=29048, + max_generate_length=3000, + data_path=os.path.expanduser("~/data/sql/validation.parquet"), + env_class="text2sql", + num_prompts=2, + max_turns=6, + use_conversation_multi_turn=False, + ) + finally: + ray.shutdown() diff --git a/skyrl-train/tests/gpu/utils.py b/skyrl-train/tests/gpu/utils.py new file mode 100644 index 0000000000..e8f59d5c44 --- /dev/null +++ b/skyrl-train/tests/gpu/utils.py @@ -0,0 +1,510 @@ +import asyncio +import os +import ray +import torch +import time +import requests +import importlib +from loguru import logger +from ray.util.placement_group import placement_group +from omegaconf import DictConfig +import hydra +from typing import List, Tuple +from transformers import AutoTokenizer, PreTrainedTokenizerBase +from functools import lru_cache +import subprocess + +from skyrl_train.dataset.replay_buffer import Experience +from skyrl_train.workers.worker import PPORayActorGroup +from skyrl_train.dataset import PromptDataset +from skyrl_train.training_batch import TensorBatch, TrainingInputBatch, TrainingOutputBatch +from skyrl_train.entrypoints.main_base import config_dir +from skyrl_train.utils import get_ray_pg_ready_with_timeout +from skyrl_train.distributed.dispatch import concatenate_outputs_after_mesh_dispatch +from skyrl_train.generators.base import GeneratorInput, ConversationType, TrajectoryID +from skyrl_train.utils.utils import peer_access_supported, print_mem, initialize_ray, validate_cfg +from skyrl_train.inference_engines.ray_wrapped_inference_engine import create_ray_wrapped_inference_engines +from skyrl_train.inference_engines.inference_engine_client import InferenceEngineClient +from skyrl_train.inference_engines.base import InferenceEngineInput +from skyrl_train.inference_engines.remote_inference_engine import create_remote_inference_engines +from skyrl_train.env_vars import SKYRL_PYTHONPATH_EXPORT + +TEST_DATA_PATH = os.path.expanduser("~/data/gsm8k/validation.parquet") + + +def get_test_actor_config() -> DictConfig: + """Get base config with test-specific overrides.""" + with hydra.initialize_config_dir(config_dir=config_dir): + cfg = hydra.compose(config_name="ppo_base_config") + + cfg.trainer.policy.model.path = "Qwen/Qwen2.5-0.5B-Instruct" + cfg.trainer.logger = "console" + validate_cfg(cfg) + + return cfg + + +def get_rank_0_memory(actor_group, message: str): + mem = ray.get(actor_group.async_run_ray_method("pass_through", "get_cuda_memory"))[0] + print_mem(message, mem) + return mem["allocated"] + + +def make_dummy_tensorbatch(seq_len=10, num_actions=4) -> TensorBatch: + B, T = 2, seq_len + data = TensorBatch( + sequences=torch.ones(B, T, dtype=int, device="cpu"), + attention_mask=torch.ones(B, T, dtype=int, device="cpu"), + ) + data.metadata = {"response_length": num_actions} + return data + + +def make_dummy_training_batch(batch_size=2, seq_len=10, num_actions=4) -> TrainingInputBatch: + """Create a dummy TrainingInputBatch""" + + torch.manual_seed(42) + + # Add all the required fields for training + data = TrainingInputBatch( + { + "sequences": torch.randint(0, 100, (batch_size, seq_len), device="cpu"), + "attention_mask": torch.ones((batch_size, seq_len), dtype=int, device="cpu"), + "action_log_probs": 0.4 * torch.ones((batch_size, num_actions), device="cpu"), + "base_action_log_probs": 0.3 * torch.ones((batch_size, num_actions), device="cpu"), + "values": 0.5 * torch.ones((batch_size, num_actions), device="cpu"), + "returns": 0.5 * torch.ones((batch_size, num_actions), device="cpu"), + "advantages": 0.6 * torch.ones((batch_size, num_actions), device="cpu"), + "loss_mask": torch.ones((batch_size, num_actions), dtype=int, device="cpu"), + "response_mask": torch.ones((batch_size, num_actions), dtype=int, device="cpu"), + } + ) + data.metadata = {"response_length": num_actions} + return data + + +def make_dummy_experience(seq_len=10, num_actions=4) -> Experience: + torch.manual_seed(42) + B, T = 2, seq_len + num_actions = num_actions + + return Experience( + sequences=torch.randint(0, 100, (B, T), device="cpu"), + action_log_probs=0.4 * torch.ones((B, num_actions), device="cpu"), + base_action_log_probs=0.3 * torch.ones((B, num_actions), device="cpu"), + rollout_logprobs=0.2 * torch.ones((B, num_actions), device="cpu"), + values=0.5 * torch.ones((B, num_actions), device="cpu"), + returns=0.5 * torch.ones((B, num_actions), device="cpu"), + advantages=0.6 * torch.ones((B, num_actions), device="cpu"), + attention_mask=torch.ones((B, T), dtype=int, device="cpu"), + loss_mask=torch.ones((B, num_actions), dtype=int, device="cpu"), + action_mask=torch.ones((B, num_actions), dtype=int, device="cpu"), + num_actions=num_actions, + info={}, + ) + + +def import_worker(strategy: str, worker_type: str): + if strategy in ("fsdp", "fsdp2"): + module_path = "skyrl_train.workers.fsdp.fsdp_worker" + elif strategy == "megatron": + module_path = "skyrl_train.workers.megatron.megatron_worker" + else: + raise ValueError(f"Unknown strategy type for {worker_type}: {strategy}") + + module = importlib.import_module(module_path) + return getattr(module, f"{worker_type.capitalize()}Worker") + + +def init_worker_with_type( + worker_type: str, shared_pg=None, colocate_all=False, num_gpus_per_node=1, num_nodes=1, cfg=None +) -> PPORayActorGroup: + if cfg is None: + cfg = get_test_actor_config() + + if shared_pg is not None: + pg = shared_pg + num_gpus_per_actor = 0.2 + else: + bundles = [{"GPU": num_gpus_per_node, "CPU": num_gpus_per_node} for _ in range(num_nodes)] + pg = placement_group(bundles, strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=30) + num_gpus_per_actor = 0.75 + + worker_cls = import_worker(cfg.trainer.strategy, worker_type) + model = PPORayActorGroup( + cfg, + num_nodes=num_nodes, + num_gpus_per_node=num_gpus_per_node, + ray_actor_type=worker_cls, + pg=pg, + num_gpus_per_actor=num_gpus_per_actor, + colocate_all=colocate_all, + sequence_parallel_size=cfg.trainer.policy.sequence_parallel_size, + record_memory=cfg.trainer.policy.record_memory, + ) + # we use policy model path for all tests (regardless of actor type) + ray.get(model.async_init_model(cfg.trainer.policy.model.path)) + return model + + +class Timer: + def __init__(self, message): + self.message = message + + def __enter__(self): + self.start_time = time.time() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + logger.opt(depth=1).info(f"{self.message}, time cost: {time.time() - self.start_time:.2f}s") + + +def get_available_gpus(): + """Get list of available GPU IDs from CUDA_VISIBLE_DEVICES or all available GPUs""" + cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES") + if cuda_visible: + # Parse CUDA_VISIBLE_DEVICES (can be comma-separated list) + gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip().isdigit()] + return gpu_ids + else: + # If not set, warn user but proceed with all GPUs + try: + import torch + + if torch.cuda.is_available(): + gpu_count = torch.cuda.device_count() + gpu_ids = list(range(gpu_count)) + print(f"CUDA_VISIBLE_DEVICES not set. Using all {gpu_count} GPUs: {gpu_ids}") + print("This might conflict with other processes. Consider setting CUDA_VISIBLE_DEVICES explicitly.") + return gpu_ids + else: + return [] + except Exception as e: + print(f"Error getting available GPUs: {e}") + return [] + + +def wait_for_server(url: str, health_path: str, timeout: int = 60, interval: float = 1.0): + start_time = time.time() + while True: + try: + response = requests.get(f"http://{url}/{health_path}") + if response.ok: + return + except requests.exceptions.ConnectionError: + if time.time() - start_time > timeout: + raise TimeoutError(f"Server at {url} did not come online within {timeout} seconds") + time.sleep(interval) + + +def levenshtein(s1, s2): + m, n = len(s1), len(s2) + # Initialize matrix of zeros + dp = [[0] * (n + 1) for _ in range(m + 1)] + # Initialize first column and first row of the matrix + for i in range(m + 1): + dp[i][0] = i # Deletion from s1 to empty string + for j in range(n + 1): + dp[0][j] = j # Insertion to s1 from empty string + # Compute the Levenshtein distance matrix + for i in range(1, m + 1): + for j in range(1, n + 1): + cost = 0 if s1[i - 1] == s2[j - 1] else 1 # No cost if characters match + dp[i][j] = min( + dp[i - 1][j] + 1, # Deletion + dp[i][j - 1] + 1, # Insertion + dp[i - 1][j - 1] + cost, # Substitution + ) + return dp[m][n] + + +def are_responses_similar(responses_a: List[str], responses_b: List[str], tolerance: float = 0.01) -> float: + if len(responses_a) != len(responses_b): + return False + + total_length = 0 + total_diff = 0 + + for s1, s2 in zip(responses_a, responses_b): + max_len = max(len(s1), len(s2)) + total_length += max_len + diff = levenshtein(s1, s2) + total_diff += diff + + difference = float(total_diff / total_length) + return difference <= tolerance + + +def get_test_prompts(model: str, num_samples: int = 20) -> List[ConversationType]: + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True) + # Ensure pad_token is set correctly + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + dataset = PromptDataset( + datasets=[TEST_DATA_PATH], + tokenizer=tokenizer, + max_prompt_length=512, + ) + + # Extract the actual prompts from the dataset + prompts = [] + for i in range(min(num_samples, len(dataset))): + prompt_data, _, _, _ = dataset[i] # dataset returns (messages, env_class, extra, uid) + prompts.append(prompt_data) + + return prompts + + +def get_test_generator_input( + model: str, + num_prompts: int = 20, + n_samples_per_prompt: int = 1, + max_prompt_length: int = 512, + data_path: str = TEST_DATA_PATH, + env_class: str = "gsm8k", +): + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True) + # Ensure pad_token is set correctly + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + dataset = PromptDataset( + datasets=[data_path], + tokenizer=tokenizer, + max_prompt_length=max_prompt_length, + ) + + prompts = [] + env_extras = [] + for i in range(min(num_prompts, len(dataset))): + prompt_data, _, env_extra, _ = dataset[i] # dataset returns (messages, env_class, extra, uid) + prompts.extend([prompt_data] * n_samples_per_prompt) + env_extras.extend([env_extra] * n_samples_per_prompt) + + env_classes = [env_class] * len(prompts) + + input_batch: GeneratorInput = { + "prompts": prompts, + "env_classes": env_classes, + "env_extras": env_extras, + "trajectory_ids": [TrajectoryID(instance_id=f"{i}", repetition_id=0) for i in range(len(prompts))], + } + + return input_batch + + +def get_model_logits_from_actor(actor_group: PPORayActorGroup, input_sequences, attention_mask): + """Helper function to get model logits for comparison""" + + seq_len = input_sequences.shape[1] + num_actions_val = seq_len - 5 # Leave some tokens for response + + data = TrainingInputBatch( + { + "sequences": input_sequences, + "attention_mask": attention_mask, + } + ) + data.metadata = {"response_length": num_actions_val} + + results_refs = actor_group.async_run_ray_method("mesh", "forward", data) + results = ray.get(results_refs) + ret_databatch: TrainingOutputBatch = concatenate_outputs_after_mesh_dispatch(actor_group.actor_infos, results) + logits = ret_databatch["output"] + + return logits + + +@lru_cache(5) +def log_once(msg): + logger.info(msg) + return None + + +def ray_init_for_tests(): + env_vars = {} + if not peer_access_supported(max_num_gpus_per_node=4): + log_once("Disabling NCCL P2P for test environment") + env_vars = {"NCCL_P2P_DISABLE": "1", "NCCL_SHM_DISABLE": "1"} + # TODO (erictang000): refactor this to use the same prepare_runtime_environment function as in utils.py for tests + # to remove duplicate code + if SKYRL_PYTHONPATH_EXPORT: + env_vars["PYTHONPATH"] = os.environ.get("PYTHONPATH") + env_vars["CUDA_DEVICE_MAX_CONNECTIONS"] = "1" + env_vars["NVTE_FUSED_ATTN"] = "0" + env_vars["LD_LIBRARY_PATH"] = os.environ.get("LD_LIBRARY_PATH") + ray.init(runtime_env={"env_vars": env_vars}) + + +async def run_inference(client, prompts, sampling_params): + engine_input = InferenceEngineInput(prompts=prompts, sampling_params=sampling_params) + return await client.generate(engine_input) + + +# TODO: this is kind of messy. All these information are inside cfg but we are passing them in +# again. Make a global get_test_config function that is parametrized. +def init_inference_engines( + cfg, + model, + use_local, + async_engine, + tp_size, + colocate_all, + backend, + gpu_memory_utilization=0.6, + num_inference_engines=1, + sleep_level=2, # use level 1 in unit tests that do not explicitly sync weights or for LoRA + enable_lora=False, + max_num_seqs=1024, + engine_init_kwargs={}, +): + assert use_local, "This test does not yet support remote engines." + assert backend in ["vllm", "sglang"] + if not ray.is_initialized(): + initialize_ray(cfg) + if colocate_all: + pg = placement_group([{"GPU": 1, "CPU": 1}] * tp_size * num_inference_engines, strategy="PACK") + get_ray_pg_ready_with_timeout(pg, timeout=30) + sleep = True + else: + pg, sleep = None, False + + tokenizer = AutoTokenizer.from_pretrained(model) + eps = create_ray_wrapped_inference_engines( + num_inference_engines=num_inference_engines, + tensor_parallel_size=tp_size, + model_dtype="bfloat16", + pretrain=model, + seed=42, + vllm_v1_disable_multiproc=True, + enable_prefix_caching=True, + enforce_eager=True, + shared_pg=pg, + gpu_memory_utilization=gpu_memory_utilization, + inference_engine_enable_sleep=sleep, + async_engine=async_engine, + max_num_batched_tokens=8192, + max_num_seqs=max_num_seqs, + tokenizer=tokenizer, + backend=backend, + sleep_level=sleep_level, + enable_lora=enable_lora, + engine_init_kwargs=engine_init_kwargs, + ) + client = InferenceEngineClient(eps, tokenizer, cfg) + if sleep: + asyncio.run(client.wake_up()) + return client, pg + + +def init_remote_inference_servers( + tp_size: int, + backend: str, + tokenizer: PreTrainedTokenizerBase, + config: DictConfig, + model: str, +) -> Tuple[InferenceEngineClient, subprocess.Popen]: + available_gpus = get_available_gpus() + assert ( + len(available_gpus) >= tp_size + ), f"Not enough GPUs available. Need {tp_size}, but only {len(available_gpus)} available: {available_gpus}" + + selected_gpus = available_gpus[:tp_size] + gpu_ids_str = ",".join(map(str, selected_gpus)) + print(f"Using GPUs {gpu_ids_str} for vLLM server (tensor_parallel_size={tp_size})") + + def get_free_port(): + import socket + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", 0)) + port = s.getsockname()[1] + s.close() + return port + + engine_port = get_free_port() + + # Launch vLLM server using subprocess + if backend == "vllm": + remote_server_command = [ + "uv", + "run", + "--isolated", + "--extra", + "vllm", + "-m", + "skyrl_train.inference_engines.vllm.vllm_server", + "--model", + model, + "--enforce-eager", + "--gpu-memory-utilization", + "0.8", + "--tensor-parallel-size", + str(tp_size), + # TODO (erictang000): for 0.13+ vllm, the MP backend runs into issues with CUDA_VISIBLE_DEVICES + # when we refactor the inference backend to use remote inference engines as a default, revisit this + "--distributed-executor-backend", + "ray", + # vLLM 0.13+ V1 engine spawns worker processes that can't inherit CUDA context + # when CUDA_VISIBLE_DEVICES is set. Disable frontend multiprocessing to fix this. + "--disable-frontend-multiprocessing", + "--dtype", + "bfloat16", + "--host", + "127.0.0.1", + "--port", + str(engine_port), + "--worker-extension-cls", + "skyrl_train.inference_engines.vllm.vllm_engine.WorkerWrap", + ] + elif backend == "sglang": + remote_server_command = [ + "uv", + "run", + "--isolated", + "--extra", + "sglang", + "-m", + "skyrl_train.inference_engines.sglang.sglang_server", + "--model-path", + model, + "--tp-size", + str(tp_size), + "--dtype", + "bfloat16", + "--host", + "127.0.0.1", + "--port", + str(engine_port), + "--mm-attention-backend", + "fa3", + "--attention-backend", + "fa3", + ] + else: + raise ValueError(f"Unsupported backend: {backend}") + + # Set CUDA_VISIBLE_DEVICES environment variable for the subprocess + env = os.environ.copy() + env["CUDA_VISIBLE_DEVICES"] = gpu_ids_str + + # Start the vLLM server process + server_process = subprocess.Popen(remote_server_command, env=env) + + wait_for_server(url=f"localhost:{engine_port}", health_path="health", timeout=120) + print(f"Server at localhost:{engine_port} is online") + + engines = create_remote_inference_engines( + urls=[f"localhost:{engine_port}"], + model_name=model, + tokenizer=tokenizer, + engine_backend=backend, + tensor_parallel_size=tp_size, + data_parallel_size=1, + expert_parallel_size=1, + ) + + client = InferenceEngineClient(engines, tokenizer, config) + return client, server_process diff --git a/skyrl-train/uv.lock b/skyrl-train/uv.lock new file mode 100644 index 0000000000..3fad416f20 --- /dev/null +++ b/skyrl-train/uv.lock @@ -0,0 +1,6503 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +conflicts = [[ + { package = "skyrl-train", extra = "sglang" }, + { package = "skyrl-train", extra = "vllm" }, +], [ + { package = "skyrl-train", extra = "flashrl" }, + { package = "skyrl-train", extra = "sglang" }, + { package = "skyrl-train", extra = "vllm" }, +], [ + { package = "skyrl-train", extra = "flashrl" }, + { package = "skyrl-train", extra = "miniswe" }, +], [ + { package = "skyrl-train", extra = "flashrl" }, + { package = "skyrl-train", extra = "mcore" }, + { package = "skyrl-train", extra = "sglang" }, + { package = "skyrl-train", extra = "vllm" }, +]] + +[manifest] +overrides = [ + { name = "causal-conv1d", marker = "sys_platform == 'never'" }, + { name = "mamba-ssm", marker = "sys_platform == 'never'" }, + { name = "megatron-core", specifier = "==0.15.0" }, + { name = "nvidia-resiliency-ext", marker = "sys_platform == 'never'" }, + { name = "transformer-engine", extras = ["pytorch"], specifier = "==2.10.0" }, +] + +[[package]] +name = "absl-py" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" }, +] + +[[package]] +name = "accelerate" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/8e/ac2a9566747a93f8be36ee08532eb0160558b07630a081a6056a9f89bf1d/accelerate-1.12.0.tar.gz", hash = "sha256:70988c352feb481887077d2ab845125024b2a137a5090d6d7a32b57d03a45df6", size = 398399, upload-time = "2025-11-21T11:27:46.973Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/d2/c581486aa6c4fbd7394c23c47b83fa1a919d34194e16944241daf9e762dd/accelerate-1.12.0-py3-none-any.whl", hash = "sha256:3e2091cd341423207e2f084a6654b1efcd250dc326f2a37d6dde446e07cabb11", size = 380935, upload-time = "2025-11-21T11:27:44.522Z" }, +] + +[[package]] +name = "aiobotocore" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aioitertools" }, + { name = "botocore" }, + { name = "jmespath" }, + { name = "multidict" }, + { name = "python-dateutil" }, + { name = "urllib3" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/0a/1db39b01304fbfc5a47e3ea45294a43b2e2de276e95ddec8c724fd51826b/aiobotocore-2.20.0.tar.gz", hash = "sha256:8902d8a96e3a389c99a68d23e2ca07c8d5059439c2a55acf28a8af19e65fe293", size = 108347, upload-time = "2025-02-20T08:26:29.329Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/99/e7cb7c26c7109bf2b2a5b05a74575fada7c178718d2f3a57c7f621c44fd9/aiobotocore-2.20.0-py3-none-any.whl", hash = "sha256:02d9d727355a03f83ee1a4fc91f2fd14966862144b4d2cc3434181e82dfafd83", size = 78060, upload-time = "2025-02-20T08:26:25.777Z" }, +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/ce/3b83ebba6b3207a7135e5fcaba49706f8a4b6008153b4e30540c982fae26/aiohttp-3.13.2.tar.gz", hash = "sha256:40176a52c186aefef6eb3cad2cdd30cd06e3afbe88fe8ab2af9c0b90f228daca", size = 7837994, upload-time = "2025-10-28T20:59:39.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/9b/01f00e9856d0a73260e86dd8ed0c2234a466c5c1712ce1c281548df39777/aiohttp-3.13.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b1e56bab2e12b2b9ed300218c351ee2a3d8c8fdab5b1ec6193e11a817767e47b", size = 737623, upload-time = "2025-10-28T20:56:30.797Z" }, + { url = "https://files.pythonhosted.org/packages/5a/1b/4be39c445e2b2bd0aab4ba736deb649fabf14f6757f405f0c9685019b9e9/aiohttp-3.13.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:364e25edaabd3d37b1db1f0cbcee8c73c9a3727bfa262b83e5e4cf3489a2a9dc", size = 492664, upload-time = "2025-10-28T20:56:32.708Z" }, + { url = "https://files.pythonhosted.org/packages/28/66/d35dcfea8050e131cdd731dff36434390479b4045a8d0b9d7111b0a968f1/aiohttp-3.13.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c5c94825f744694c4b8db20b71dba9a257cd2ba8e010a803042123f3a25d50d7", size = 491808, upload-time = "2025-10-28T20:56:34.57Z" }, + { url = "https://files.pythonhosted.org/packages/00/29/8e4609b93e10a853b65f8291e64985de66d4f5848c5637cddc70e98f01f8/aiohttp-3.13.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba2715d842ffa787be87cbfce150d5e88c87a98e0b62e0f5aa489169a393dbbb", size = 1738863, upload-time = "2025-10-28T20:56:36.377Z" }, + { url = "https://files.pythonhosted.org/packages/9d/fa/4ebdf4adcc0def75ced1a0d2d227577cd7b1b85beb7edad85fcc87693c75/aiohttp-3.13.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:585542825c4bc662221fb257889e011a5aa00f1ae4d75d1d246a5225289183e3", size = 1700586, upload-time = "2025-10-28T20:56:38.034Z" }, + { url = "https://files.pythonhosted.org/packages/da/04/73f5f02ff348a3558763ff6abe99c223381b0bace05cd4530a0258e52597/aiohttp-3.13.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:39d02cb6025fe1aabca329c5632f48c9532a3dabccd859e7e2f110668972331f", size = 1768625, upload-time = "2025-10-28T20:56:39.75Z" }, + { url = "https://files.pythonhosted.org/packages/f8/49/a825b79ffec124317265ca7d2344a86bcffeb960743487cb11988ffb3494/aiohttp-3.13.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e67446b19e014d37342f7195f592a2a948141d15a312fe0e700c2fd2f03124f6", size = 1867281, upload-time = "2025-10-28T20:56:41.471Z" }, + { url = "https://files.pythonhosted.org/packages/b9/48/adf56e05f81eac31edcfae45c90928f4ad50ef2e3ea72cb8376162a368f8/aiohttp-3.13.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4356474ad6333e41ccefd39eae869ba15a6c5299c9c01dfdcfdd5c107be4363e", size = 1752431, upload-time = "2025-10-28T20:56:43.162Z" }, + { url = "https://files.pythonhosted.org/packages/30/ab/593855356eead019a74e862f21523db09c27f12fd24af72dbc3555b9bfd9/aiohttp-3.13.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeacf451c99b4525f700f078becff32c32ec327b10dcf31306a8a52d78166de7", size = 1562846, upload-time = "2025-10-28T20:56:44.85Z" }, + { url = "https://files.pythonhosted.org/packages/39/0f/9f3d32271aa8dc35036e9668e31870a9d3b9542dd6b3e2c8a30931cb27ae/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8a9b889aeabd7a4e9af0b7f4ab5ad94d42e7ff679aaec6d0db21e3b639ad58d", size = 1699606, upload-time = "2025-10-28T20:56:46.519Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3c/52d2658c5699b6ef7692a3f7128b2d2d4d9775f2a68093f74bca06cf01e1/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fa89cb11bc71a63b69568d5b8a25c3ca25b6d54c15f907ca1c130d72f320b76b", size = 1720663, upload-time = "2025-10-28T20:56:48.528Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d4/8f8f3ff1fb7fb9e3f04fcad4e89d8a1cd8fc7d05de67e3de5b15b33008ff/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8aa7c807df234f693fed0ecd507192fc97692e61fee5702cdc11155d2e5cadc8", size = 1737939, upload-time = "2025-10-28T20:56:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/03/d3/ddd348f8a27a634daae39a1b8e291ff19c77867af438af844bf8b7e3231b/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9eb3e33fdbe43f88c3c75fa608c25e7c47bbd80f48d012763cb67c47f39a7e16", size = 1555132, upload-time = "2025-10-28T20:56:52.568Z" }, + { url = "https://files.pythonhosted.org/packages/39/b8/46790692dc46218406f94374903ba47552f2f9f90dad554eed61bfb7b64c/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9434bc0d80076138ea986833156c5a48c9c7a8abb0c96039ddbb4afc93184169", size = 1764802, upload-time = "2025-10-28T20:56:54.292Z" }, + { url = "https://files.pythonhosted.org/packages/ba/e4/19ce547b58ab2a385e5f0b8aa3db38674785085abcf79b6e0edd1632b12f/aiohttp-3.13.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff15c147b2ad66da1f2cbb0622313f2242d8e6e8f9b79b5206c84523a4473248", size = 1719512, upload-time = "2025-10-28T20:56:56.428Z" }, + { url = "https://files.pythonhosted.org/packages/70/30/6355a737fed29dcb6dfdd48682d5790cb5eab050f7b4e01f49b121d3acad/aiohttp-3.13.2-cp312-cp312-win32.whl", hash = "sha256:27e569eb9d9e95dbd55c0fc3ec3a9335defbf1d8bc1d20171a49f3c4c607b93e", size = 426690, upload-time = "2025-10-28T20:56:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0d/b10ac09069973d112de6ef980c1f6bb31cb7dcd0bc363acbdad58f927873/aiohttp-3.13.2-cp312-cp312-win_amd64.whl", hash = "sha256:8709a0f05d59a71f33fd05c17fc11fcb8c30140506e13c2f5e8ee1b8964e1b45", size = 453465, upload-time = "2025-10-28T20:57:00.795Z" }, +] + +[[package]] +name = "aioitertools" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322, upload-time = "2025-11-06T22:17:07.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "airportsdata" +version = "20250909" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/c6/17ae8a65f7fa5bbbeee166f8070063eb8b70c89501a65c2e6885db61fc08/airportsdata-20250909.tar.gz", hash = "sha256:f39974fe1101817ced4ccf7c6ed336408469e5e778395d0a3e7a5112ec298f90", size = 907204, upload-time = "2025-09-09T01:07:31.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/87/59b570b9c4b014532777dc3baffc9bea10cf0cc8b232cf3c17e4bd0754a6/airportsdata-20250909-py3-none-any.whl", hash = "sha256:ce7dc6e1485afe3915e708212c7024ad158470c1c934e6a6cb217cf28b798ac7", size = 914391, upload-time = "2025-09-09T01:07:29.364Z" }, +] + +[[package]] +name = "alabaster" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anthropic" +version = "0.71.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/4f/70682b068d897841f43223df82d96ec1d617435a8b759c4a2d901a50158b/anthropic-0.71.0.tar.gz", hash = "sha256:eb8e6fa86d049061b3ef26eb4cbae0174ebbff21affa6de7b3098da857d8de6a", size = 489102, upload-time = "2025-10-16T15:54:40.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/77/073e8ac488f335aec7001952825275582fb8f433737e90f24eeef9d878f6/anthropic-0.71.0-py3-none-any.whl", hash = "sha256:85c5015fcdbdc728390f11b17642a65a4365d03b12b799b18b6cc57e71fdb327", size = 355035, upload-time = "2025-10-16T15:54:38.238Z" }, +] + +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } + +[[package]] +name = "anyio" +version = "4.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/8a777047513153587e5434fd752e89334ac33e379aa3497db860eeb60377/anyio-4.12.0.tar.gz", hash = "sha256:73c693b567b0c55130c104d0b43a9baf3aa6a31fc6110116509f27bf75e21ec0", size = 228266, upload-time = "2025-11-28T23:37:38.911Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl", hash = "sha256:dad2376a628f98eeca4881fc56cd06affd18f659b17a747d3ff0307ced94b1bb", size = 113362, upload-time = "2025-11-28T23:36:57.897Z" }, +] + +[[package]] +name = "apache-tvm-ffi" +version = "0.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/45/20/8da071821b2142bdeed757d2859dede4817e0b82a96e9a4d8cfbffd49006/apache_tvm_ffi-0.1.6.tar.gz", hash = "sha256:53088126f7fce11823ddf0fb101e968a90298d79fd68829c0a981f25467a574c", size = 2387987, upload-time = "2025-12-16T19:00:33.523Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/de/4ae5dd4d493b1cea755a25d59088895486432c053cff5a3287b75e36ce54/apache_tvm_ffi-0.1.6-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:5f4c0678854dbf3bfaa37795465f570d79c68759896b04b3d31774af0a03bcb8", size = 1779381, upload-time = "2025-12-16T18:59:59.593Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/2e943cbda764c3266a6966a34e582d3f0ac6046ab6aaa756631df9afd7bf/apache_tvm_ffi-0.1.6-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:653f1d4c8ffd6bca5300fd1825a81373a5be82f31dc79353d1c476fa31cf377a", size = 1936756, upload-time = "2025-12-16T19:00:00.844Z" }, + { url = "https://files.pythonhosted.org/packages/a3/91/fc43f155b4d4363e61707655c1f4bee75af1d6dd4a76680f4956dd9846fe/apache_tvm_ffi-0.1.6-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6a2cdfa90860a80e3cfb2364ce3b66a559fa5748de8d593a203b2e5992d92bc1", size = 2013641, upload-time = "2025-12-16T19:00:02.479Z" }, + { url = "https://files.pythonhosted.org/packages/14/9b/45208f2a9c70a88fd8e65668c0628f3917625d64668800ff55a2390d7fe0/apache_tvm_ffi-0.1.6-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223ac7ac08b34a6dbabe7085f23939b4aaa70666e72ddad41015659034e095af", size = 1881149, upload-time = "2025-12-16T19:00:03.776Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c5/e3ba08379127578bb3417605b61e9cd5e513184a6947ec7f3fac93d16355/apache_tvm_ffi-0.1.6-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05cedb3ba7600dc9ae35c17b7325d44ecf02c56c3ba1b62668dca8390da7ec28", size = 1992886, upload-time = "2025-12-16T19:00:05.047Z" }, + { url = "https://files.pythonhosted.org/packages/d6/7b/4df1e523ae4bcbfbe65a3e7ef3c8810cb76e9ae44fa9b44c9fac152ecc2b/apache_tvm_ffi-0.1.6-cp312-abi3-win_amd64.whl", hash = "sha256:a6c29ba9dbc6273f4534bfc0e8a52a784f264724eb62df62daedc2b349dabe85", size = 1758454, upload-time = "2025-12-16T19:00:06.498Z" }, +] + +[[package]] +name = "apscheduler" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/81/192db4f8471de5bc1f0d098783decffb1e6e69c4f8b4bc6711094691950b/apscheduler-3.11.1.tar.gz", hash = "sha256:0db77af6400c84d1747fe98a04b8b58f0080c77d11d338c4f507a9752880f221", size = 108044, upload-time = "2025-10-31T18:55:42.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/9f/d3c76f76c73fcc959d28e9def45b8b1cc3d7722660c5003b19c1022fd7f4/apscheduler-3.11.1-py3-none-any.whl", hash = "sha256:6162cb5683cb09923654fa9bdd3130c4be4bfda6ad8990971c9597ecd52965d2", size = 64278, upload-time = "2025-10-31T18:55:41.186Z" }, +] + +[[package]] +name = "astor" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, +] + +[[package]] +name = "asttokens" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/a5/8e3f9b6771b0b408517c82d97aed8f2036509bc247d46114925e32fe33f0/asttokens-3.0.1.tar.gz", hash = "sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7", size = 62308, upload-time = "2025-11-15T16:43:48.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, +] + +[[package]] +name = "attrs" +version = "25.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/685e6633917e101e5dcb62b9dd76946cbb57c26e133bae9e0cd36033c0a9/attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", size = 934251, upload-time = "2025-10-06T13:54:44.725Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, +] + +[[package]] +name = "av" +version = "16.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/c3/fd72a0315bc6c943ced1105aaac6e0ec1be57c70d8a616bd05acaa21ffee/av-16.0.1.tar.gz", hash = "sha256:dd2ce779fa0b5f5889a6d9e00fbbbc39f58e247e52d31044272648fe16ff1dbf", size = 3904030, upload-time = "2025-10-13T12:28:51.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/78/12a11d7a44fdd8b26a65e2efa1d8a5826733c8887a989a78306ec4785956/av-16.0.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:e41a8fef85dfb2c717349f9ff74f92f9560122a9f1a94b1c6c9a8a9c9462ba71", size = 27206375, upload-time = "2025-10-13T12:25:44.423Z" }, + { url = "https://files.pythonhosted.org/packages/27/19/3a4d3882852a0ee136121979ce46f6d2867b974eb217a2c9a070939f55ad/av-16.0.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:6352a64b25c9f985d4f279c2902db9a92424e6f2c972161e67119616f0796cb9", size = 21752603, upload-time = "2025-10-13T12:25:49.122Z" }, + { url = "https://files.pythonhosted.org/packages/cb/6e/f7abefba6e008e2f69bebb9a17ba38ce1df240c79b36a5b5fcacf8c8fcfd/av-16.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5201f7b4b5ed2128118cb90c2a6d64feedb0586ca7c783176896c78ffb4bbd5c", size = 38931978, upload-time = "2025-10-13T12:25:55.021Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7a/1305243ab47f724fdd99ddef7309a594e669af7f0e655e11bdd2c325dfae/av-16.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:daecc2072b82b6a942acbdaa9a2e00c05234c61fef976b22713983c020b07992", size = 40549383, upload-time = "2025-10-13T12:26:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/32/b2/357cc063185043eb757b4a48782bff780826103bcad1eb40c3ddfc050b7e/av-16.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6573da96e8bebc3536860a7def108d7dbe1875c86517072431ced702447e6aea", size = 40241993, upload-time = "2025-10-13T12:26:06.993Z" }, + { url = "https://files.pythonhosted.org/packages/20/bb/ced42a4588ba168bf0ef1e9d016982e3ba09fde6992f1dda586fd20dcf71/av-16.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4bc064e48a8de6c087b97dd27cf4ef8c13073f0793108fbce3ecd721201b2502", size = 41532235, upload-time = "2025-10-13T12:26:12.488Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/c7811eca0f318d5fd3212f7e8c3d8335f75a54907c97a89213dc580b8056/av-16.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0c669b6b6668c8ae74451c15ec6d6d8a36e4c3803dc5d9910f607a174dd18f17", size = 32296912, upload-time = "2025-10-13T12:26:19.187Z" }, +] + +[[package]] +name = "azure-core" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/83/41c9371c8298999c67b007e308a0a3c4d6a59c6908fa9c62101f031f886f/azure_core-1.37.0.tar.gz", hash = "sha256:7064f2c11e4b97f340e8e8c6d923b822978be3016e46b7bc4aa4b337cfb48aee", size = 357620, upload-time = "2025-12-11T20:05:13.518Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/34/a9914e676971a13d6cc671b1ed172f9804b50a3a80a143ff196e52f4c7ee/azure_core-1.37.0-py3-none-any.whl", hash = "sha256:b3abe2c59e7d6bb18b38c275a5029ff80f98990e7c90a5e646249a56630fcc19", size = 214006, upload-time = "2025-12-11T20:05:14.96Z" }, +] + +[[package]] +name = "azure-identity" +version = "1.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "msal" }, + { name = "msal-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/8d/1a6c41c28a37eab26dc85ab6c86992c700cd3f4a597d9ed174b0e9c69489/azure_identity-1.25.1.tar.gz", hash = "sha256:87ca8328883de6036443e1c37b40e8dc8fb74898240f61071e09d2e369361456", size = 279826, upload-time = "2025-10-06T20:30:02.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/7b/5652771e24fff12da9dde4c20ecf4682e606b104f26419d139758cc935a6/azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651", size = 191317, upload-time = "2025-10-06T20:30:04.251Z" }, +] + +[[package]] +name = "azure-storage-blob" +version = "12.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "isodate" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/7c/2fd872e11a88163f208b9c92de273bf64bb22d0eef9048cc6284d128a77a/azure_storage_blob-12.27.1.tar.gz", hash = "sha256:a1596cc4daf5dac9be115fcb5db67245eae894cf40e4248243754261f7b674a6", size = 597579, upload-time = "2025-10-29T12:27:16.185Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9e/1c90a122ea6180e8c72eb7294adc92531b0e08eb3d2324c2ba70d37f4802/azure_storage_blob-12.27.1-py3-none-any.whl", hash = "sha256:65d1e25a4628b7b6acd20ff7902d8da5b4fde8e46e19c8f6d213a3abc3ece272", size = 428954, upload-time = "2025-10-29T12:27:18.072Z" }, +] + +[[package]] +name = "babel" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, +] + +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + +[[package]] +name = "black" +version = "24.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/0d/cc2fb42b8c50d80143221515dd7e4766995bd07c56c9a3ed30baf080b6dc/black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875", size = 645813, upload-time = "2024-10-07T19:20:50.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/04/bf74c71f592bcd761610bbf67e23e6a3cff824780761f536512437f1e655/black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3", size = 1644256, upload-time = "2024-10-07T19:27:53.355Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ea/a77bab4cf1887f4b2e0bce5516ea0b3ff7d04ba96af21d65024629afedb6/black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65", size = 1448534, upload-time = "2024-10-07T19:26:44.953Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3e/443ef8bc1fbda78e61f79157f303893f3fddf19ca3c8989b163eb3469a12/black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f", size = 1761892, upload-time = "2024-10-07T19:24:10.264Z" }, + { url = "https://files.pythonhosted.org/packages/52/93/eac95ff229049a6901bc84fec6908a5124b8a0b7c26ea766b3b8a5debd22/black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8", size = 1434796, upload-time = "2024-10-07T19:25:06.239Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a7/4b27c50537ebca8bec139b872861f9d2bf501c5ec51fcf897cb924d9e264/black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d", size = 206898, upload-time = "2024-10-07T19:20:48.317Z" }, +] + +[[package]] +name = "blake3" +version = "1.0.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/aa/abcd75e9600987a0bc6cfe9b6b2ff3f0e2cb08c170addc6e76035b5c4cb3/blake3-1.0.8.tar.gz", hash = "sha256:513cc7f0f5a7c035812604c2c852a0c1468311345573de647e310aca4ab165ba", size = 117308, upload-time = "2025-10-14T06:47:48.83Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/a0/b7b6dff04012cfd6e665c09ee446f749bd8ea161b00f730fe1bdecd0f033/blake3-1.0.8-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:d8da4233984d51471bd4e4366feda1d90d781e712e0a504ea54b1f2b3577557b", size = 347983, upload-time = "2025-10-14T06:45:47.214Z" }, + { url = "https://files.pythonhosted.org/packages/5b/a2/264091cac31d7ae913f1f296abc20b8da578b958ffb86100a7ce80e8bf5c/blake3-1.0.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1257be19f2d381c868a34cc822fc7f12f817ddc49681b6d1a2790bfbda1a9865", size = 325415, upload-time = "2025-10-14T06:45:48.482Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/85a4c0782f613de23d114a7a78fcce270f75b193b3ff3493a0de24ba104a/blake3-1.0.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:269f255b110840e52b6ce9db02217e39660ebad3e34ddd5bca8b8d378a77e4e1", size = 371296, upload-time = "2025-10-14T06:45:49.674Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/488475254976ed93fab57c67aa80d3b40df77f7d9db6528c9274bff53e08/blake3-1.0.8-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66ca28a673025c40db3eba21a9cac52f559f83637efa675b3f6bd8683f0415f3", size = 374516, upload-time = "2025-10-14T06:45:51.23Z" }, + { url = "https://files.pythonhosted.org/packages/7b/21/2a1c47fedb77fb396512677ec6d46caf42ac6e9a897db77edd0a2a46f7bb/blake3-1.0.8-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcb04966537777af56c1f399b35525aa70a1225816e121ff95071c33c0f7abca", size = 447911, upload-time = "2025-10-14T06:45:52.637Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7d/db0626df16029713e7e61b67314c4835e85c296d82bd907c21c6ea271da2/blake3-1.0.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5b5da177d62cc4b7edf0cea08fe4dec960c9ac27f916131efa890a01f747b93", size = 505420, upload-time = "2025-10-14T06:45:54.445Z" }, + { url = "https://files.pythonhosted.org/packages/5b/55/6e737850c2d58a6d9de8a76dad2ae0f75b852a23eb4ecb07a0b165e6e436/blake3-1.0.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:38209b10482c97e151681ea3e91cc7141f56adbbf4820a7d701a923124b41e6a", size = 394189, upload-time = "2025-10-14T06:45:55.719Z" }, + { url = "https://files.pythonhosted.org/packages/5b/94/eafaa5cdddadc0c9c603a6a6d8339433475e1a9f60c8bb9c2eed2d8736b6/blake3-1.0.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504d1399b7fb91dfe5c25722d2807990493185faa1917456455480c36867adb5", size = 388001, upload-time = "2025-10-14T06:45:57.067Z" }, + { url = "https://files.pythonhosted.org/packages/17/81/735fa00d13de7f68b25e1b9cb36ff08c6f165e688d85d8ec2cbfcdedccc5/blake3-1.0.8-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c84af132aa09abeadf9a0118c8fb26f4528f3f42c10ef8be0fcf31c478774ec4", size = 550302, upload-time = "2025-10-14T06:45:58.657Z" }, + { url = "https://files.pythonhosted.org/packages/0e/c6/d1fe8bdea4a6088bd54b5a58bc40aed89a4e784cd796af7722a06f74bae7/blake3-1.0.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a25db3d36b55f5ed6a86470155cc749fc9c5b91c949b8d14f48658f9d960d9ec", size = 554211, upload-time = "2025-10-14T06:46:00.269Z" }, + { url = "https://files.pythonhosted.org/packages/55/d1/ca74aa450cbe10e396e061f26f7a043891ffa1485537d6b30d3757e20995/blake3-1.0.8-cp312-cp312-win32.whl", hash = "sha256:e0fee93d5adcd44378b008c147e84f181f23715307a64f7b3db432394bbfce8b", size = 228343, upload-time = "2025-10-14T06:46:01.533Z" }, + { url = "https://files.pythonhosted.org/packages/4d/42/bbd02647169e3fbed27558555653ac2578c6f17ccacf7d1956c58ef1d214/blake3-1.0.8-cp312-cp312-win_amd64.whl", hash = "sha256:6a6eafc29e4f478d365a87d2f25782a521870c8514bb43734ac85ae9be71caf7", size = 215704, upload-time = "2025-10-14T06:46:02.79Z" }, +] + +[[package]] +name = "blobfile" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "lxml" }, + { name = "pycryptodomex" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/a9/a34e8153b0203d9060ff7aa5dfcd175e161117949697a83c4cc003b523ff/blobfile-3.0.0.tar.gz", hash = "sha256:32ec777414de7bb2a76ca812a838f0d33327ca28ae844a253503cde625cdf2f1", size = 77863, upload-time = "2024-08-27T00:02:53.092Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/4d/1392562369b1139e741b30d624f09fe7091d17dd5579fae5732f044b12bb/blobfile-3.0.0-py3-none-any.whl", hash = "sha256:48ecc3307e622804bd8fe13bf6f40e6463c4439eba7a1f9ad49fd78aa63cc658", size = 75413, upload-time = "2024-08-27T00:02:51.518Z" }, +] + +[[package]] +name = "boto3" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/c6/ec86c6eafc942dbddffcaa4eb623373bf94ecf38fab0ab3e7f9fe7051e62/boto3-1.36.0.tar.gz", hash = "sha256:159898f51c2997a12541c0e02d6e5a8fe2993ddb307b9478fd9a339f98b57e00", size = 111035, upload-time = "2025-01-15T21:37:38.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/36/b91f560a0ed11f7f90ac59554cbc52340158ce24db879a7c8faa68ff1cef/boto3-1.36.0-py3-none-any.whl", hash = "sha256:d0ca7a58ce25701a52232cc8df9d87854824f1f2964b929305722ebc7959d5a9", size = 139165, upload-time = "2025-01-15T21:37:35.343Z" }, +] + +[[package]] +name = "botocore" +version = "1.36.23" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/d4/e93bcf4f825faa25c2ff27b653cd9e976226df527445f8c414198020c967/botocore-1.36.23.tar.gz", hash = "sha256:9feaa2d876f487e718a5fd80a35fa401042b518c0c75117d3e1ea39a567439e7", size = 13571101, upload-time = "2025-02-18T20:27:59.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/93/a566604998182dd354aa6bab3c5b0393077a2ce4b97e18713d94c362da06/botocore-1.36.23-py3-none-any.whl", hash = "sha256:886730e79495a2e153842725ebdf85185c8277cdf255b3b5879cd097ddc7fcc3", size = 13355279, upload-time = "2025-02-18T20:27:56.088Z" }, +] + +[[package]] +name = "cachetools" +version = "6.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/1d/ede8680603f6016887c062a2cf4fc8fdba905866a3ab8831aa8aa651320c/cachetools-6.2.4.tar.gz", hash = "sha256:82c5c05585e70b6ba2d3ae09ea60b79548872185d2f24ae1f2709d37299fd607", size = 31731, upload-time = "2025-12-15T18:24:53.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/fc/1d7b80d0eb7b714984ce40efc78859c022cd930e402f599d8ca9e39c78a4/cachetools-6.2.4-py3-none-any.whl", hash = "sha256:69a7a52634fed8b8bf6e24a050fb60bff1c9bd8f6d24572b99c32d4e71e62a51", size = 11551, upload-time = "2025-12-15T18:24:52.332Z" }, +] + +[[package]] +name = "causal-conv1d" +version = "1.5.3.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ninja", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "packaging", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/cb/104778c728dc3d5ea3bf65a484e3a4cdbe894bdaa2586320e2f61d007b8c/causal_conv1d-1.5.3.post1.tar.gz", hash = "sha256:aba1b717484472d0b2f2e40520a1c03f35fe5155555bd753d1c324afc56ba468", size = 24198, upload-time = "2025-10-10T10:16:23.921Z" } + +[[package]] +name = "cbor2" +version = "5.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/b8/c0f6a7d46f816cb18b1fda61a2fe648abe16039f1ff93ea720a6e9fb3cee/cbor2-5.7.1.tar.gz", hash = "sha256:7a405a1d7c8230ee9acf240aad48ae947ef584e8af05f169f3c1bde8f01f8b71", size = 102467, upload-time = "2025-10-24T09:23:06.569Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/54/48426472f0c051982c647331441aed09b271a0500356ae0b7054c813d174/cbor2-5.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bd5ca44891c06f6b85d440836c967187dc1d30b15f86f315d55c675d3a841078", size = 69031, upload-time = "2025-10-24T09:22:25.438Z" }, + { url = "https://files.pythonhosted.org/packages/d3/68/1dd58c7706e9752188358223db58c83f3c48e07f728aa84221ffd244652f/cbor2-5.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:537d73ef930ccc1a7b6a2e8d2cbf81407d270deb18e40cda5eb511bd70f71078", size = 68825, upload-time = "2025-10-24T09:22:26.497Z" }, + { url = "https://files.pythonhosted.org/packages/09/4e/380562fe9f9995a1875fb5ec26fd041e19d61f4630cb690a98c5195945fc/cbor2-5.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:edbf814dd7763b6eda27a5770199f6ccd55bd78be8f4367092460261bfbf19d0", size = 286222, upload-time = "2025-10-24T09:22:27.546Z" }, + { url = "https://files.pythonhosted.org/packages/7c/bb/9eccdc1ea3c4d5c7cdb2e49b9de49534039616be5455ce69bd64c0b2efe2/cbor2-5.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fc81da8c0e09beb42923e455e477b36ff14a03b9ca18a8a2e9b462de9a953e8", size = 285688, upload-time = "2025-10-24T09:22:28.651Z" }, + { url = "https://files.pythonhosted.org/packages/59/8c/4696d82f5bd04b3d45d9a64ec037fa242630c134e3218d6c252b4f59b909/cbor2-5.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e4a7d660d428911a3aadb7105e94438d7671ab977356fdf647a91aab751033bd", size = 277063, upload-time = "2025-10-24T09:22:29.775Z" }, + { url = "https://files.pythonhosted.org/packages/95/50/6538e44ca970caaad2fa376b81701d073d84bf597aac07a59d0a253b1a7f/cbor2-5.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:228e0af9c0a9ddf6375b6ae010eaa1942a1901d403f134ac9ee6a76a322483f9", size = 278334, upload-time = "2025-10-24T09:22:30.904Z" }, + { url = "https://files.pythonhosted.org/packages/64/a9/156ccd2207fb26b5b61d23728b4dbdc595d1600125aa79683a4a8ddc9313/cbor2-5.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:2d08a6c0d9ed778448e185508d870f4160ba74f59bb17a966abd0d14d0ff4dd3", size = 68404, upload-time = "2025-10-24T09:22:32.108Z" }, + { url = "https://files.pythonhosted.org/packages/4f/49/adc53615e9dd32c4421f6935dfa2235013532c6e6b28ee515bbdd92618be/cbor2-5.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:752506cfe72da0f4014b468b30191470ee8919a64a0772bd3b36a4fccf5fcefc", size = 64047, upload-time = "2025-10-24T09:22:33.147Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/383bafeabb54c17fe5b6d5aca4e863e6b7df10bcc833b34aa169e9dfce1a/cbor2-5.7.1-py3-none-any.whl", hash = "sha256:68834e4eff2f56629ce6422b0634bc3f74c5a4269de5363f5265fe452c706ba7", size = 23829, upload-time = "2025-10-24T09:23:05.54Z" }, +] + +[[package]] +name = "certifi" +version = "2025.11.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/8c/58f469717fa48465e4a50c014a0400602d3c437d7c0c468e17ada824da3a/certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316", size = 160538, upload-time = "2025-11-12T02:54:51.517Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/7d/9bc192684cea499815ff478dfcdc13835ddf401365057044fb721ec6bddb/certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b", size = 159438, upload-time = "2025-11-12T02:54:49.735Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, +] + +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "compressed-tensors" +version = "0.12.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "loguru" }, + { name = "pydantic" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.52.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/79/4c5c1cd14266f8cf2650bdb940f986ce7fcaeb56aad8cfa9e9afedf14e2f/compressed_tensors-0.12.2.tar.gz", hash = "sha256:5bb40856dd17f128ab73557ecc73799f80db4dd82fab6de875f1e6899b9ea0c4", size = 190409, upload-time = "2025-10-07T14:30:59.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/c0/1695b87d369e6652ec0d650912e02eca2151c5e9c29244f94d2afccfe970/compressed_tensors-0.12.2-py3-none-any.whl", hash = "sha256:e554ea761710ca2b0c0ea49276a4ef8e08658624f1591e6a7368817106b48fbe", size = 183049, upload-time = "2025-10-07T14:30:56.523Z" }, +] + +[[package]] +name = "croniter" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/2f/44d1ae153a0e27be56be43465e5cb39b9650c781e001e7864389deb25090/croniter-6.0.0.tar.gz", hash = "sha256:37c504b313956114a983ece2c2b07790b1f1094fe9d81cc94739214748255577", size = 64481, upload-time = "2024-12-17T17:17:47.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/4b/290b4c3efd6417a8b0c284896de19b1d5855e6dbdb97d2a35e68fa42de85/croniter-6.0.0-py2.py3-none-any.whl", hash = "sha256:2f878c3856f17896979b2a4379ba1f09c83e374931ea15cc835c5dd2eee9b368", size = 25468, upload-time = "2024-12-17T17:17:45.359Z" }, +] + +[[package]] +name = "cryptography" +version = "46.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/33/c00162f49c0e2fe8064a62cb92b93e50c74a72bc370ab92f86112b33ff62/cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1", size = 749258, upload-time = "2025-10-15T23:18:31.74Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/42/9c391dd801d6cf0d561b5890549d4b27bafcc53b39c31a817e69d87c625b/cryptography-46.0.3-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:109d4ddfadf17e8e7779c39f9b18111a09efb969a301a31e987416a0191ed93a", size = 7225004, upload-time = "2025-10-15T23:16:52.239Z" }, + { url = "https://files.pythonhosted.org/packages/1c/67/38769ca6b65f07461eb200e85fc1639b438bdc667be02cf7f2cd6a64601c/cryptography-46.0.3-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:09859af8466b69bc3c27bdf4f5d84a665e0f7ab5088412e9e2ec49758eca5cbc", size = 4296667, upload-time = "2025-10-15T23:16:54.369Z" }, + { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" }, + { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" }, + { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" }, + { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" }, + { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" }, + { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" }, + { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" }, + { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" }, + { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" }, + { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/8a6a9525893325fc057a01f654d7efc2c64b9de90413adcf605a85744ff4/cryptography-46.0.3-cp311-abi3-win32.whl", hash = "sha256:f260d0d41e9b4da1ed1e0f1ce571f97fe370b152ab18778e9e8f67d6af432018", size = 3055988, upload-time = "2025-10-15T23:17:14.65Z" }, + { url = "https://files.pythonhosted.org/packages/7e/bf/80fbf45253ea585a1e492a6a17efcb93467701fa79e71550a430c5e60df0/cryptography-46.0.3-cp311-abi3-win_amd64.whl", hash = "sha256:a9a3008438615669153eb86b26b61e09993921ebdd75385ddd748702c5adfddb", size = 3514451, upload-time = "2025-10-15T23:17:16.142Z" }, + { url = "https://files.pythonhosted.org/packages/2e/af/9b302da4c87b0beb9db4e756386a7c6c5b8003cd0e742277888d352ae91d/cryptography-46.0.3-cp311-abi3-win_arm64.whl", hash = "sha256:5d7f93296ee28f68447397bf5198428c9aeeab45705a55d53a6343455dcb2c3c", size = 2928007, upload-time = "2025-10-15T23:17:18.04Z" }, + { url = "https://files.pythonhosted.org/packages/fd/23/45fe7f376a7df8daf6da3556603b36f53475a99ce4faacb6ba2cf3d82021/cryptography-46.0.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:cb3d760a6117f621261d662bccc8ef5bc32ca673e037c83fbe565324f5c46936", size = 7218248, upload-time = "2025-10-15T23:17:46.294Z" }, + { url = "https://files.pythonhosted.org/packages/27/32/b68d27471372737054cbd34c84981f9edbc24fe67ca225d389799614e27f/cryptography-46.0.3-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4b7387121ac7d15e550f5cb4a43aef2559ed759c35df7336c402bb8275ac9683", size = 4294089, upload-time = "2025-10-15T23:17:48.269Z" }, + { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" }, + { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" }, + { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" }, + { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" }, + { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" }, + { url = "https://files.pythonhosted.org/packages/0a/6e/1c8331ddf91ca4730ab3086a0f1be19c65510a33b5a441cb334e7a2d2560/cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df", size = 3036695, upload-time = "2025-10-15T23:18:08.672Z" }, + { url = "https://files.pythonhosted.org/packages/90/45/b0d691df20633eff80955a0fc7695ff9051ffce8b69741444bd9ed7bd0db/cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f", size = 3501720, upload-time = "2025-10-15T23:18:10.632Z" }, + { url = "https://files.pythonhosted.org/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" }, +] + +[[package]] +name = "cuda-bindings" +version = "13.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/3d/c8ed9d169843091f3f0d6b8218e826fd59520a37e0434c204feada597988/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e75ad0cb863330df784236d289612d71ca855c013d19ae00e5693574abd6915", size = 15530160, upload-time = "2025-12-09T22:05:55.386Z" }, + { url = "https://files.pythonhosted.org/packages/4a/8e/368295623ee43fba622909d780fbb6863efc1638dff55f67a0f04eac6470/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25785d1a3cdcd98f151240fd5efd025609319a6720a217dee2a929241749d488", size = 16110386, upload-time = "2025-12-09T22:05:57.71Z" }, + { url = "https://files.pythonhosted.org/packages/60/1f/ecc4701ade3e85f091c625a920574527b9daf7fb354189fbfbc5516af6cd/cuda_bindings-13.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:ccde9c95c0e953b31fe7731bb08da9d0a34b1770498df9a3c156fdfdbe3951ad", size = 15250028, upload-time = "2025-12-09T22:06:00.346Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/02/4dbe7568a42e46582248942f54dc64ad094769532adbe21e525e4edf7bc4/cuda_pathfinder-1.3.3-py3-none-any.whl", hash = "sha256:9984b664e404f7c134954a771be8775dfd6180ea1e1aef4a5a37d4be05d9bbb1", size = 27154, upload-time = "2025-12-04T22:35:08.996Z" }, +] + +[[package]] +name = "cuda-python" +version = "13.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings" }, + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/08/b5e3b9822662d72d540d830531e3ab6a7cabbda3dd56175696aabccfeb76/cuda_python-13.1.1-py3-none-any.whl", hash = "sha256:944cc4fe6482673d28dd545797a28840945a1668739328fa2ad1e9be4f7050d9", size = 8038, upload-time = "2025-12-09T22:13:10.719Z" }, +] + +[[package]] +name = "cupy-cuda12x" +version = "13.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastrlock", marker = "(sys_platform != 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/c5/7e7fc4816d0de0154e5d9053242c3a08a0ca8b43ee656a6f7b3b95055a7b/cupy_cuda12x-13.6.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:a6970ceefe40f9acbede41d7fe17416bd277b1bd2093adcde457b23b578c5a59", size = 127334633, upload-time = "2025-08-18T08:24:43.065Z" }, + { url = "https://files.pythonhosted.org/packages/e0/95/d7e1295141e7d530674a3cc567e13ed0eb6b81524cb122d797ed996b5bea/cupy_cuda12x-13.6.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:79b0cacb5e8b190ef409f9e03f06ac8de1b021b0c0dda47674d446f5557e0eb1", size = 112886268, upload-time = "2025-08-18T08:24:49.294Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8c/14555b63fd78cfac7b88af0094cea0a3cb845d243661ec7da69f7b3ea0de/cupy_cuda12x-13.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca06fede7b8b83ca9ad80062544ef2e5bb8d4762d1c4fc3ac8349376de9c8a5e", size = 89785108, upload-time = "2025-08-18T08:24:54.527Z" }, +] + +[[package]] +name = "datasets" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"] }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/9d/348ed92110ba5f9b70b51ca1078d4809767a835aa2b7ce7e74ad2b98323d/datasets-4.0.0.tar.gz", hash = "sha256:9657e7140a9050db13443ba21cb5de185af8af944479b00e7ff1e00a61c8dbf1", size = 569566, upload-time = "2025-07-09T14:35:52.431Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/62/eb8157afb21bd229c864521c1ab4fa8e9b4f1b06bafdd8c4668a7a31b5dd/datasets-4.0.0-py3-none-any.whl", hash = "sha256:7ef95e62025fd122882dbce6cb904c8cd3fbc829de6669a5eb939c77d50e203d", size = 494825, upload-time = "2025-07-09T14:35:50.658Z" }, +] + +[[package]] +name = "debugpy" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/fe/0486b90b9ac0d9afced236fdfe6e54c2f45b7ef09225210090f23dc6e48a/debugpy-1.8.0.zip", hash = "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0", size = 4824504, upload-time = "2023-09-12T20:32:17.726Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/59/b9c5dbe60c3a3e4e4bb12b1968aa75e98b99ff8b65273a5e7de75cceef75/debugpy-1.8.0-py2.py3-none-any.whl", hash = "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4", size = 5036334, upload-time = "2023-09-12T20:33:17.263Z" }, +] + +[[package]] +name = "decorator" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "depyf" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astor" }, + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/35/83fb0178212279aa0af031031905804c6de5618435d229f41ed21bb9ad2c/depyf-0.20.0.tar.gz", hash = "sha256:fb7683bd72c44f67b56029df2c47721e9a02ffa4d7b19095f1c54c4ebf797a98", size = 6168761, upload-time = "2025-10-13T12:33:38.589Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/65/4df6936130b56e1429114e663e7c1576cf845f3aef1b2dd200c0a5d19dba/depyf-0.20.0-py3-none-any.whl", hash = "sha256:d31effad4261cebecb58955d832e448ace88f432328f95f82fd99c30fd9308d4", size = 39381, upload-time = "2025-10-13T12:33:33.647Z" }, +] + +[[package]] +name = "dill" +version = "0.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/17/4d/ac7ffa80c69ea1df30a8aa11b3578692a5118e7cd1aa157e3ef73b092d15/dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", size = 184847, upload-time = "2024-01-27T23:42:16.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, +] + +[[package]] +name = "diskcache" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + +[[package]] +name = "docstring-parser" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/9d/c3b43da9515bd270df0f80548d9944e389870713cc1fe2b8fb35fe2bcefd/docstring_parser-0.17.0.tar.gz", hash = "sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912", size = 27442, upload-time = "2025-07-21T07:35:01.868Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/e2/2537ebcff11c1ee1ff17d8d0b6f4db75873e3b0fb32c2d4a2ee31ecb310a/docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708", size = 36896, upload-time = "2025-07-21T07:35:00.684Z" }, +] + +[[package]] +name = "docutils" +version = "0.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444, upload-time = "2024-04-23T18:57:18.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, +] + +[[package]] +name = "einops" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/81/df4fbe24dff8ba3934af99044188e20a98ed441ad17a274539b74e82e126/einops-0.8.1.tar.gz", hash = "sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84", size = 54805, upload-time = "2025-02-09T03:17:00.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/62/9773de14fe6c45c23649e98b83231fffd7b9892b6cf863251dc2afa73643/einops-0.8.1-py3-none-any.whl", hash = "sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737", size = 64359, upload-time = "2025-02-09T03:17:01.998Z" }, +] + +[[package]] +name = "email-validator" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, +] + +[[package]] +name = "executing" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, +] + +[[package]] +name = "fastapi" +version = "0.126.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/6c/28773e19bc203a2f3cf1d54a8e96ca7d05b58157a350aa4d8d37f2a5ba07/fastapi-0.126.0.tar.gz", hash = "sha256:f099fceb2a6d56dd21c59c4543d00be123dedacff869e76ae31ba3c0f963e2cd", size = 367455, upload-time = "2025-12-20T16:16:44.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/0b/d5f999f27cb90152a6aadf094205b4d0eeab6a6b03e3e60346cde988c1bd/fastapi-0.126.0-py3-none-any.whl", hash = "sha256:c9330b9731e3bd2caae0a00e76353f86adbf592c5a25649a1682f3a92aeaff41", size = 111758, upload-time = "2025-12-20T16:16:42.349Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "email-validator" }, + { name = "fastapi-cli", extra = ["standard"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "pydantic-extra-types" }, + { name = "pydantic-settings" }, + { name = "python-multipart" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, extra = ["standard"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] + +[[package]] +name = "fastapi-cli" +version = "0.0.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rich-toolkit" }, + { name = "typer" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, extra = ["standard"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/75/9407a6b452be4c988feacec9c9d2f58d8f315162a6c7258d5a649d933ebe/fastapi_cli-0.0.16.tar.gz", hash = "sha256:e8a2a1ecf7a4e062e3b2eec63ae34387d1e142d4849181d936b23c4bdfe29073", size = 19447, upload-time = "2025-11-10T19:01:07.856Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/43/678528c19318394320ee43757648d5e0a8070cf391b31f69d931e5c840d2/fastapi_cli-0.0.16-py3-none-any.whl", hash = "sha256:addcb6d130b5b9c91adbbf3f2947fe115991495fdb442fe3e51b5fc6327df9f4", size = 12312, upload-time = "2025-11-10T19:01:06.728Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "fastapi-cloud-cli" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, extra = ["standard"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] + +[[package]] +name = "fastapi-cloud-cli" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastar" }, + { name = "httpx" }, + { name = "pydantic", extra = ["email"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "rich-toolkit" }, + { name = "rignore" }, + { name = "sentry-sdk" }, + { name = "typer" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, extra = ["standard"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/0d/3b0d2991f481c122c552b4ae38a8b400a75ab0edbc85536f2a6224f72da2/fastapi_cloud_cli-0.7.0.tar.gz", hash = "sha256:8b025944475c3d53262105886dfe051f46383e4f287787a46892b524922ac0b6", size = 30906, upload-time = "2025-12-16T12:51:49.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/aa/a2319f008e123b178a5c4cba34935e8459e72d24d8c31214b244ab7266a6/fastapi_cloud_cli-0.7.0-py3-none-any.whl", hash = "sha256:9a152e80d08d465d4a6c4f5c75aa8871d1e8919f69ff131b0567f04e95d5f36c", size = 23376, upload-time = "2025-12-16T12:51:47.847Z" }, +] + +[[package]] +name = "fastapi-sso" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastapi" }, + { name = "httpx" }, + { name = "oauthlib" }, + { name = "pydantic", extra = ["email"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/9b/25c43c928b46ec919cb8941d3de53dd2e12bab12e1c0182646425dbefd60/fastapi_sso-0.16.0.tar.gz", hash = "sha256:f3941f986347566b7d3747c710cf474a907f581bfb6697ff3bb3e44eb76b438c", size = 16555, upload-time = "2024-11-04T11:54:38.579Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/84/df15745ff06c1b44e478b72759d5cf48e4583e221389d4cdea76c472dd1c/fastapi_sso-0.16.0-py3-none-any.whl", hash = "sha256:3a66a942474ef9756d3a9d8b945d55bd9faf99781facdb9b87a40b73d6d6b0c3", size = 23942, upload-time = "2024-11-04T11:54:37.189Z" }, +] + +[[package]] +name = "fastar" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/e7/f89d54fb04104114dd0552836dc2b47914f416cc0e200b409dd04a33de5e/fastar-0.8.0.tar.gz", hash = "sha256:f4d4d68dbf1c4c2808f0e730fac5843493fc849f70fe3ad3af60dfbaf68b9a12", size = 68524, upload-time = "2025-11-26T02:36:00.72Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/f1/5b2ff898abac7f1a418284aad285e3a4f68d189c572ab2db0f6c9079dd16/fastar-0.8.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f10d2adfe40f47ff228f4efaa32d409d732ded98580e03ed37c9535b5fc923d", size = 706369, upload-time = "2025-11-26T02:34:37.783Z" }, + { url = "https://files.pythonhosted.org/packages/23/60/8046a386dca39154f80c927cbbeeb4b1c1267a3271bffe61552eb9995757/fastar-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b930da9d598e3bc69513d131f397e6d6be4643926ef3de5d33d1e826631eb036", size = 629097, upload-time = "2025-11-26T02:34:21.888Z" }, + { url = "https://files.pythonhosted.org/packages/22/7e/1ae005addc789924a9268da2394d3bb5c6f96836f7e37b7e3d23c2362675/fastar-0.8.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9d210da2de733ca801de83e931012349d209f38b92d9630ccaa94bd445bdc9b8", size = 868938, upload-time = "2025-11-26T02:33:51.119Z" }, + { url = "https://files.pythonhosted.org/packages/a6/77/290a892b073b84bf82e6b2259708dfe79c54f356e252c2dd40180b16fe07/fastar-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa02270721517078a5bd61a38719070ac2537a4aa6b6c48cf369cf2abc59174a", size = 765204, upload-time = "2025-11-26T02:32:47.02Z" }, + { url = "https://files.pythonhosted.org/packages/d0/00/c3155171b976003af3281f5258189f1935b15d1221bfc7467b478c631216/fastar-0.8.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:83c391e5b789a720e4d0029b9559f5d6dee3226693c5b39c0eab8eaece997e0f", size = 764717, upload-time = "2025-11-26T02:33:02.453Z" }, + { url = "https://files.pythonhosted.org/packages/b7/43/405b7ad76207b2c11b7b59335b70eac19e4a2653977f5588a1ac8fed54f4/fastar-0.8.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3258d7a78a72793cdd081545da61cabe85b1f37634a1d0b97ffee0ff11d105ef", size = 931502, upload-time = "2025-11-26T02:33:18.619Z" }, + { url = "https://files.pythonhosted.org/packages/da/8a/a3dde6d37cc3da4453f2845cdf16675b5686b73b164f37e2cc579b057c2c/fastar-0.8.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6eab95dd985cdb6a50666cbeb9e4814676e59cfe52039c880b69d67cfd44767", size = 821454, upload-time = "2025-11-26T02:33:33.427Z" }, + { url = "https://files.pythonhosted.org/packages/da/c1/904fe2468609c8990dce9fe654df3fbc7324a8d8e80d8240ae2c89757064/fastar-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:829b1854166141860887273c116c94e31357213fa8e9fe8baeb18bd6c38aa8d9", size = 821647, upload-time = "2025-11-26T02:34:07Z" }, + { url = "https://files.pythonhosted.org/packages/c8/73/a0642ab7a400bc07528091785e868ace598fde06fcd139b8f865ec1b6f3c/fastar-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b1667eae13f9457a3c737f4376d68e8c3e548353538b28f7e4273a30cb3965cd", size = 986342, upload-time = "2025-11-26T02:34:53.371Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/60c1bfa6edab72366461a95f053d0f5f7ab1825fe65ca2ca367432cd8629/fastar-0.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b864a95229a7db0814cd9ef7987cb713fd43dce1b0d809dd17d9cd6f02fdde3e", size = 1040207, upload-time = "2025-11-26T02:35:10.65Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a0/0d624290dec622e7fa084b6881f456809f68777d54a314f5dde932714506/fastar-0.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c05fbc5618ce17675a42576fa49858d79734627f0a0c74c0875ab45ee8de340c", size = 1045031, upload-time = "2025-11-26T02:35:28.108Z" }, + { url = "https://files.pythonhosted.org/packages/a7/74/cf663af53c4706ba88e6b4af44a6b0c3bd7d7ca09f079dc40647a8f06585/fastar-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7f41c51ee96f338662ee3c3df4840511ba3f9969606840f1b10b7cb633a3c716", size = 994877, upload-time = "2025-11-26T02:35:45.797Z" }, + { url = "https://files.pythonhosted.org/packages/52/17/444c8be6e77206050e350da7c338102b6cab384be937fa0b1d6d1f9ede73/fastar-0.8.0-cp312-cp312-win32.whl", hash = "sha256:d949a1a2ea7968b734632c009df0571c94636a5e1622c87a6e2bf712a7334f47", size = 455996, upload-time = "2025-11-26T02:36:26.938Z" }, + { url = "https://files.pythonhosted.org/packages/dc/34/fc3b5e56d71a17b1904800003d9251716e8fd65f662e1b10a26881698a74/fastar-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:fc645994d5b927d769121094e8a649b09923b3c13a8b0b98696d8f853f23c532", size = 490429, upload-time = "2025-11-26T02:36:12.707Z" }, + { url = "https://files.pythonhosted.org/packages/35/a8/5608cc837417107c594e2e7be850b9365bcb05e99645966a5d6a156285fe/fastar-0.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:d81ee82e8dc78a0adb81728383bd39611177d642a8fa2d601d4ad5ad59e5f3bd", size = 461297, upload-time = "2025-11-26T02:36:03.546Z" }, +] + +[[package]] +name = "fastrlock" +version = "0.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/b1/1c3d635d955f2b4bf34d45abf8f35492e04dbd7804e94ce65d9f928ef3ec/fastrlock-0.8.3.tar.gz", hash = "sha256:4af6734d92eaa3ab4373e6c9a1dd0d5ad1304e172b1521733c6c3b3d73c8fa5d", size = 79327, upload-time = "2024-12-17T11:03:39.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/df/56270f2e10c1428855c990e7a7e5baafa9e1262b8e789200bd1d047eb501/fastrlock-0.8.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:8cb2cf04352ea8575d496f31b3b88c42c7976e8e58cdd7d1550dfba80ca039da", size = 55727, upload-time = "2024-12-17T11:02:17.26Z" }, + { url = "https://files.pythonhosted.org/packages/57/21/ea1511b0ef0d5457efca3bf1823effb9c5cad4fc9dca86ce08e4d65330ce/fastrlock-0.8.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:85a49a1f1e020097d087e1963e42cea6f307897d5ebe2cb6daf4af47ffdd3eed", size = 52201, upload-time = "2024-12-17T11:02:19.512Z" }, + { url = "https://files.pythonhosted.org/packages/80/07/cdecb7aa976f34328372f1c4efd6c9dc1b039b3cc8d3f38787d640009a25/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670", size = 53924, upload-time = "2024-12-17T11:02:20.85Z" }, + { url = "https://files.pythonhosted.org/packages/88/6d/59c497f8db9a125066dd3a7442fab6aecbe90d6fec344c54645eaf311666/fastrlock-0.8.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0ea4e53a04980d646def0f5e4b5e8bd8c7884288464acab0b37ca0c65c482bfe", size = 52140, upload-time = "2024-12-17T11:02:22.263Z" }, + { url = "https://files.pythonhosted.org/packages/62/04/9138943c2ee803d62a48a3c17b69de2f6fa27677a6896c300369e839a550/fastrlock-0.8.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:38340f6635bd4ee2a4fb02a3a725759fe921f2ca846cb9ca44531ba739cc17b4", size = 53261, upload-time = "2024-12-17T11:02:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4b/db35a52589764c7745a613b6943bbd018f128d42177ab92ee7dde88444f6/fastrlock-0.8.3-cp312-cp312-win_amd64.whl", hash = "sha256:da06d43e1625e2ffddd303edcd6d2cd068e1c486f5fd0102b3f079c44eb13e2c", size = 31235, upload-time = "2024-12-17T11:02:25.708Z" }, +] + +[[package]] +name = "fastuuid" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232, upload-time = "2025-10-19T22:19:22.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/a2/e78fcc5df65467f0d207661b7ef86c5b7ac62eea337c0c0fcedbeee6fb13/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a", size = 510164, upload-time = "2025-10-19T22:31:45.635Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b3/c846f933f22f581f558ee63f81f29fa924acd971ce903dab1a9b6701816e/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d", size = 261837, upload-time = "2025-10-19T22:38:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/54/ea/682551030f8c4fa9a769d9825570ad28c0c71e30cf34020b85c1f7ee7382/fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070", size = 251370, upload-time = "2025-10-19T22:40:26.07Z" }, + { url = "https://files.pythonhosted.org/packages/14/dd/5927f0a523d8e6a76b70968e6004966ee7df30322f5fc9b6cdfb0276646a/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796", size = 277766, upload-time = "2025-10-19T22:37:23.779Z" }, + { url = "https://files.pythonhosted.org/packages/16/6e/c0fb547eef61293153348f12e0f75a06abb322664b34a1573a7760501336/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09", size = 278105, upload-time = "2025-10-19T22:26:56.821Z" }, + { url = "https://files.pythonhosted.org/packages/2d/b1/b9c75e03b768f61cf2e84ee193dc18601aeaf89a4684b20f2f0e9f52b62c/fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8", size = 301564, upload-time = "2025-10-19T22:30:31.604Z" }, + { url = "https://files.pythonhosted.org/packages/fc/fa/f7395fdac07c7a54f18f801744573707321ca0cee082e638e36452355a9d/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741", size = 459659, upload-time = "2025-10-19T22:31:32.341Z" }, + { url = "https://files.pythonhosted.org/packages/66/49/c9fd06a4a0b1f0f048aacb6599e7d96e5d6bc6fa680ed0d46bf111929d1b/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057", size = 478430, upload-time = "2025-10-19T22:26:22.962Z" }, + { url = "https://files.pythonhosted.org/packages/be/9c/909e8c95b494e8e140e8be6165d5fc3f61fdc46198c1554df7b3e1764471/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8", size = 450894, upload-time = "2025-10-19T22:27:01.647Z" }, + { url = "https://files.pythonhosted.org/packages/90/eb/d29d17521976e673c55ef7f210d4cdd72091a9ec6755d0fd4710d9b3c871/fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176", size = 154374, upload-time = "2025-10-19T22:29:19.879Z" }, + { url = "https://files.pythonhosted.org/packages/cc/fc/f5c799a6ea6d877faec0472d0b27c079b47c86b1cdc577720a5386483b36/fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397", size = 156550, upload-time = "2025-10-19T22:27:49.658Z" }, +] + +[[package]] +name = "filelock" +version = "3.20.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/23/ce7a1126827cedeb958fc043d61745754464eb56c5937c35bbf2b8e26f34/filelock-3.20.1.tar.gz", hash = "sha256:b8360948b351b80f420878d8516519a2204b07aefcdcfd24912a5d33127f188c", size = 19476, upload-time = "2025-12-15T23:54:28.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/7f/a1a97644e39e7316d850784c642093c99df1290a460df4ede27659056834/filelock-3.20.1-py3-none-any.whl", hash = "sha256:15d9e9a67306188a44baa72f569d2bfd803076269365fdea0934385da4dc361a", size = 16666, upload-time = "2025-12-15T23:54:26.874Z" }, +] + +[[package]] +name = "fla-core" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "einops" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/3d/79a9d5c8cd973c86f35403931031787dfc6cc97d838a42d4c62e8cbbb66f/fla_core-0.4.0.tar.gz", hash = "sha256:d975022b074e97bfd086dc6b767dccb35e27a9fe36f26f3b26b1c2b68b36a1c8", size = 316316, upload-time = "2025-10-27T08:18:51.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/0c/d52ab65e9c163631895052d70d4111f8530ca52f45beb0895378d1a2a8b5/fla_core-0.4.0-py3-none-any.whl", hash = "sha256:5396f36a9838c99f9e45c70e88e2e0b26688f719d07d2ddd61be16d29327f4ea", size = 438519, upload-time = "2025-10-27T08:18:49.561Z" }, +] + +[[package]] +name = "flash-attn" +version = "2.8.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "einops", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/6d/7066d160bdffa2f9da29a8c3957f266b17a03ca0b3bdc8fdae86d9881fe7/flash_attn-2.8.1.tar.gz", hash = "sha256:0ff003899fcb244f357905b04f622d5c9736887126dd6675f8f4bc52954e3923", size = 8166563, upload-time = "2025-07-10T05:16:39.729Z" } + +[[package]] +name = "flash-attn" +version = "2.8.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "einops", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/b2/8d76c41ad7974ee264754709c22963447f7f8134613fd9ce80984ed0dab7/flash_attn-2.8.3.tar.gz", hash = "sha256:1e71dd64a9e0280e0447b8a0c2541bad4bf6ac65bdeaa2f90e51a9e57de0370d", size = 8447812, upload-time = "2025-08-15T08:28:12.911Z" } + +[[package]] +name = "flash-linear-attention" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fla-core" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/9a/e546815da2bf149e0af58449ff1ca10074165af4384febead438ad46f74c/flash_linear_attention-0.4.0.tar.gz", hash = "sha256:c5d2bf6e1a766af3a4426f07f710b0b87809f7218de21eb313314be6ff1b0dba", size = 157646, upload-time = "2025-10-27T08:18:52.445Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/76/4f716c953608204c970de7cd4045db1af02643d7f19c94a49254834b7563/flash_linear_attention-0.4.0-py3-none-any.whl", hash = "sha256:50c97163f7cb64dc53585194ef36af44d2a6bc545227c4f73bb3ba9062630f1a", size = 290439, upload-time = "2025-10-27T08:18:50.589Z" }, +] + +[[package]] +name = "flashinfer-jit-cache" +version = "0.5.3+cu128" +source = { registry = "https://flashinfer.ai/whl/cu128" } +wheels = [ + { url = "https://github.com/flashinfer-ai/flashinfer/releases/download/v0.5.3/flashinfer_jit_cache-0.5.3+cu128-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5c432dfc3f30f8297a185853586dd3eaa582dd93ab2885ee686155bb402b992c" }, + { url = "https://github.com/flashinfer-ai/flashinfer/releases/download/v0.5.3/flashinfer_jit_cache-0.5.3+cu128-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b0c9c3791bcd46ce523bf65e83829b035081f97950c124d3f9eb8a4241f2ccf4" }, +] + +[[package]] +name = "flashinfer-python" +version = "0.2.6.post1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "ninja" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "requests" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/26/3c6f12ffaefbfa0c453030d6e15941269b3a4ffcd267daec32d1a10dda96/flashinfer_python-0.2.6.post1.tar.gz", hash = "sha256:0cb5b346024b75deb08bd09a1cc6273ceb23f1f5f0d7e40dbe594abc6a67be69", size = 3137122, upload-time = "2025-06-07T04:11:25.672Z" } + +[[package]] +name = "flashinfer-python" +version = "0.5.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "click" }, + { name = "einops" }, + { name = "ninja" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-vllm' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang')" }, + { name = "nvidia-cudnn-frontend" }, + { name = "nvidia-cutlass-dsl" }, + { name = "nvidia-ml-py" }, + { name = "packaging" }, + { name = "requests" }, + { name = "tabulate" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/91/cca69baeff24bb3efd12c7479a026432c8717ee47193694010494c528b22/flashinfer_python-0.5.3.tar.gz", hash = "sha256:100d59b0ede47878d2808cd3a1b9039d7a952d66338bc9f68dac192ae1b2e3f1", size = 4682367, upload-time = "2025-11-20T21:22:46.976Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/78/6dc7e7da8cb87c9965644ea0d2439457a1bc9256c45ceda0044595be4143/flashinfer_python-0.5.3-py3-none-any.whl", hash = "sha256:b601293b72f9138bad173edc28df84b9f239a013be974e2e79d4ba98aeb38cf5", size = 6998069, upload-time = "2025-11-20T21:22:45.104Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "fsspec" +version = "2025.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/f4/5721faf47b8c499e776bc34c6a8fc17efdf7fdef0b00f398128bc5dcb4ac/fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972", size = 298491, upload-time = "2025-03-07T21:47:56.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/53/eb690efa8513166adef3e0669afd31e95ffde69fb3c52ec2ac7223ed6018/fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3", size = 193615, upload-time = "2025-03-07T21:47:54.809Z" }, +] + +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, +] + +[[package]] +name = "ftfy" +version = "6.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/d3/8650919bc3c7c6e90ee3fa7fd618bf373cbbe55dff043bd67353dbb20cd8/ftfy-6.3.1.tar.gz", hash = "sha256:9b3c3d90f84fb267fe64d375a07b7f8912d817cf86009ae134aa03e1819506ec", size = 308927, upload-time = "2024-10-26T00:50:35.149Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821, upload-time = "2024-10-26T00:50:33.425Z" }, +] + +[[package]] +name = "func-timeout" +version = "4.3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/0d/bf0567477f7281d9a3926c582bfef21bff7498fc0ffd3e9de21811896a0b/func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd", size = 44264, upload-time = "2019-08-19T21:32:07.43Z" } + +[[package]] +name = "gguf" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyyaml" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/08/7de1ca4b71e7bf33b547f82bb22505e221b5fa42f67d635e200e0ad22ad6/gguf-0.17.1.tar.gz", hash = "sha256:36ad71aad900a3e75fc94ebe96ea6029f03a4e44be7627ef7ad3d03e8c7bcb53", size = 89338, upload-time = "2025-06-19T14:00:33.705Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/31/6a93a887617ee7deeaa602ca3d02d1c12a6cb8a742a695de5d128f5fa46a/gguf-0.17.1-py3-none-any.whl", hash = "sha256:7bc5aa7eeb1931f7d39b48fdc5b38fda6b294b9dca75cf607ac69557840a3943", size = 96224, upload-time = "2025-06-19T14:00:32.88Z" }, +] + +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, +] + +[[package]] +name = "grpcio" +version = "1.67.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/53/d9282a66a5db45981499190b77790570617a604a38f3d103d0400974aeb5/grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732", size = 12580022, upload-time = "2024-10-29T06:30:07.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/25/6f95bd18d5f506364379eabc0d5874873cc7dbdaf0757df8d1e82bc07a88/grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953", size = 5089809, upload-time = "2024-10-29T06:24:31.24Z" }, + { url = "https://files.pythonhosted.org/packages/10/3f/d79e32e5d0354be33a12db2267c66d3cfeff700dd5ccdd09fd44a3ff4fb6/grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb", size = 10981985, upload-time = "2024-10-29T06:24:34.942Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/36fbc14b3542e3a1c20fb98bd60c4732c55a44e374a4eb68f91f28f14aab/grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0", size = 5588770, upload-time = "2024-10-29T06:24:38.145Z" }, + { url = "https://files.pythonhosted.org/packages/0d/af/bbc1305df60c4e65de8c12820a942b5e37f9cf684ef5e49a63fbb1476a73/grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af", size = 6214476, upload-time = "2024-10-29T06:24:41.006Z" }, + { url = "https://files.pythonhosted.org/packages/92/cf/1d4c3e93efa93223e06a5c83ac27e32935f998bc368e276ef858b8883154/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e", size = 5850129, upload-time = "2024-10-29T06:24:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ca/26195b66cb253ac4d5ef59846e354d335c9581dba891624011da0e95d67b/grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75", size = 6568489, upload-time = "2024-10-29T06:24:46.453Z" }, + { url = "https://files.pythonhosted.org/packages/d1/94/16550ad6b3f13b96f0856ee5dfc2554efac28539ee84a51d7b14526da985/grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38", size = 6149369, upload-time = "2024-10-29T06:24:49.112Z" }, + { url = "https://files.pythonhosted.org/packages/33/0d/4c3b2587e8ad7f121b597329e6c2620374fccbc2e4e1aa3c73ccc670fde4/grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78", size = 3599176, upload-time = "2024-10-29T06:24:51.443Z" }, + { url = "https://files.pythonhosted.org/packages/7d/36/0c03e2d80db69e2472cf81c6123aa7d14741de7cf790117291a703ae6ae1/grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc", size = 4346574, upload-time = "2024-10-29T06:24:54.587Z" }, +] + +[[package]] +name = "gunicorn" +version = "23.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-transfer" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/eb/8fc64f40388c29ce8ce3b2b180a089d4d6b25b1d0d232d016704cb852104/hf_transfer-0.1.9.tar.gz", hash = "sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf", size = 25201, upload-time = "2025-01-07T10:05:12.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/f5/461d2e5f307e5048289b1168d5c642ae3bb2504e88dff1a38b92ed990a21/hf_transfer-0.1.9-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e66acf91df4a8b72f60223059df3003062a5ae111757187ed1a06750a30e911b", size = 1393046, upload-time = "2025-01-07T10:04:51.003Z" }, + { url = "https://files.pythonhosted.org/packages/41/ba/8d9fd9f1083525edfcb389c93738c802f3559cb749324090d7109c8bf4c2/hf_transfer-0.1.9-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:8669dbcc7a3e2e8d61d42cd24da9c50d57770bd74b445c65123291ca842a7e7a", size = 1348126, upload-time = "2025-01-07T10:04:45.712Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a2/cd7885bc9959421065a6fae0fe67b6c55becdeda4e69b873e52976f9a9f0/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fd0167c4407a3bc4cdd0307e65ada2294ec04f1813d8a69a5243e379b22e9d8", size = 3728604, upload-time = "2025-01-07T10:04:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/f6/2e/a072cf196edfeda3310c9a5ade0a0fdd785e6154b3ce24fc738c818da2a7/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f", size = 3064995, upload-time = "2025-01-07T10:04:18.663Z" }, + { url = "https://files.pythonhosted.org/packages/c2/84/aec9ef4c0fab93c1ea2b1badff38c78b4b2f86f0555b26d2051dbc920cde/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5828057e313de59300dd1abb489444bc452efe3f479d3c55b31a8f680936ba42", size = 3580908, upload-time = "2025-01-07T10:04:32.834Z" }, + { url = "https://files.pythonhosted.org/packages/29/63/b560d39651a56603d64f1a0212d0472a44cbd965db2fa62b99d99cb981bf/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d", size = 3400839, upload-time = "2025-01-07T10:04:26.122Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d8/f87ea6f42456254b48915970ed98e993110521e9263472840174d32c880d/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdca9bfb89e6f8f281890cc61a8aff2d3cecaff7e1a4d275574d96ca70098557", size = 3552664, upload-time = "2025-01-07T10:04:40.123Z" }, + { url = "https://files.pythonhosted.org/packages/d6/56/1267c39b65fc8f4e2113b36297320f102718bf5799b544a6cbe22013aa1d/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:89a23f58b7b7effbc047b8ca286f131b17728c99a9f972723323003ffd1bb916", size = 4073732, upload-time = "2025-01-07T10:04:55.624Z" }, + { url = "https://files.pythonhosted.org/packages/82/1a/9c748befbe3decf7cb415e34f8a0c3789a0a9c55910dea73d581e48c0ce5/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:dc7fff1345980d6c0ebb92c811d24afa4b98b3e07ed070c8e38cc91fd80478c5", size = 3390096, upload-time = "2025-01-07T10:04:59.98Z" }, + { url = "https://files.pythonhosted.org/packages/72/85/4c03da147b6b4b7cb12e074d3d44eee28604a387ed0eaf7eaaead5069c57/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:1a6bd16c667ebe89a069ca163060127a794fa3a3525292c900b8c8cc47985b0d", size = 3664743, upload-time = "2025-01-07T10:05:05.416Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6e/e597b04f753f1b09e6893075d53a82a30c13855cbaa791402695b01e369f/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d2fde99d502093ade3ab1b53f80da18480e9902aa960dab7f74fb1b9e5bc5746", size = 3695243, upload-time = "2025-01-07T10:05:11.411Z" }, + { url = "https://files.pythonhosted.org/packages/09/89/d4e234727a26b2546c8fb70a276cd924260d60135f2165bf8b9ed67bb9a4/hf_transfer-0.1.9-cp38-abi3-win32.whl", hash = "sha256:435cc3cdc8524ce57b074032b8fd76eed70a4224d2091232fa6a8cef8fd6803e", size = 1086605, upload-time = "2025-01-07T10:05:18.873Z" }, + { url = "https://files.pythonhosted.org/packages/a1/14/f1e15b851d1c2af5b0b1a82bf8eb10bda2da62d98180220ba6fd8879bb5b/hf_transfer-0.1.9-cp38-abi3-win_amd64.whl", hash = "sha256:16f208fc678911c37e11aa7b586bc66a37d02e636208f18b6bc53d29b5df40ad", size = 1160240, upload-time = "2025-01-07T10:05:14.324Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/6e/0f11bacf08a67f7fb5ee09740f2ca54163863b07b70d579356e9222ce5d8/hf_xet-1.2.0.tar.gz", hash = "sha256:a8c27070ca547293b6890c4bf389f713f80e8c478631432962bb7f4bc0bd7d7f", size = 506020, upload-time = "2025-10-24T19:04:32.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/2d/22338486473df5923a9ab7107d375dbef9173c338ebef5098ef593d2b560/hf_xet-1.2.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:46740d4ac024a7ca9b22bebf77460ff43332868b661186a8e46c227fdae01848", size = 2866099, upload-time = "2025-10-24T19:04:15.366Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/c5becfa53234299bc2210ba314eaaae36c2875e0045809b82e40a9544f0c/hf_xet-1.2.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:27df617a076420d8845bea087f59303da8be17ed7ec0cd7ee3b9b9f579dff0e4", size = 2722178, upload-time = "2025-10-24T19:04:13.695Z" }, + { url = "https://files.pythonhosted.org/packages/9a/92/cf3ab0b652b082e66876d08da57fcc6fa2f0e6c70dfbbafbd470bb73eb47/hf_xet-1.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3651fd5bfe0281951b988c0facbe726aa5e347b103a675f49a3fa8144c7968fd", size = 3320214, upload-time = "2025-10-24T19:04:03.596Z" }, + { url = "https://files.pythonhosted.org/packages/46/92/3f7ec4a1b6a65bf45b059b6d4a5d38988f63e193056de2f420137e3c3244/hf_xet-1.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d06fa97c8562fb3ee7a378dd9b51e343bc5bc8190254202c9771029152f5e08c", size = 3229054, upload-time = "2025-10-24T19:04:01.949Z" }, + { url = "https://files.pythonhosted.org/packages/0b/dd/7ac658d54b9fb7999a0ccb07ad863b413cbaf5cf172f48ebcd9497ec7263/hf_xet-1.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4c1428c9ae73ec0939410ec73023c4f842927f39db09b063b9482dac5a3bb737", size = 3413812, upload-time = "2025-10-24T19:04:24.585Z" }, + { url = "https://files.pythonhosted.org/packages/92/68/89ac4e5b12a9ff6286a12174c8538a5930e2ed662091dd2572bbe0a18c8a/hf_xet-1.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a55558084c16b09b5ed32ab9ed38421e2d87cf3f1f89815764d1177081b99865", size = 3508920, upload-time = "2025-10-24T19:04:26.927Z" }, + { url = "https://files.pythonhosted.org/packages/cb/44/870d44b30e1dcfb6a65932e3e1506c103a8a5aea9103c337e7a53180322c/hf_xet-1.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:e6584a52253f72c9f52f9e549d5895ca7a471608495c4ecaa6cc73dba2b24d69", size = 2905735, upload-time = "2025-10-24T19:04:35.928Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httptools" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/46/120a669232c7bdedb9d52d4aeae7e6c7dfe151e99dc70802e2fc7a5e1993/httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9", size = 258961, upload-time = "2025-10-10T03:55:08.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/7f/403e5d787dc4942316e515e949b0c8a013d84078a915910e9f391ba9b3ed/httptools-0.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:38e0c83a2ea9746ebbd643bdfb521b9aa4a91703e2cd705c20443405d2fd16a5", size = 206280, upload-time = "2025-10-10T03:54:39.274Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0d/7f3fd28e2ce311ccc998c388dd1c53b18120fda3b70ebb022b135dc9839b/httptools-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f25bbaf1235e27704f1a7b86cd3304eabc04f569c828101d94a0e605ef7205a5", size = 110004, upload-time = "2025-10-10T03:54:40.403Z" }, + { url = "https://files.pythonhosted.org/packages/84/a6/b3965e1e146ef5762870bbe76117876ceba51a201e18cc31f5703e454596/httptools-0.7.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c15f37ef679ab9ecc06bfc4e6e8628c32a8e4b305459de7cf6785acd57e4d03", size = 517655, upload-time = "2025-10-10T03:54:41.347Z" }, + { url = "https://files.pythonhosted.org/packages/11/7d/71fee6f1844e6fa378f2eddde6c3e41ce3a1fb4b2d81118dd544e3441ec0/httptools-0.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fe6e96090df46b36ccfaf746f03034e5ab723162bc51b0a4cf58305324036f2", size = 511440, upload-time = "2025-10-10T03:54:42.452Z" }, + { url = "https://files.pythonhosted.org/packages/22/a5/079d216712a4f3ffa24af4a0381b108aa9c45b7a5cc6eb141f81726b1823/httptools-0.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f72fdbae2dbc6e68b8239defb48e6a5937b12218e6ffc2c7846cc37befa84362", size = 495186, upload-time = "2025-10-10T03:54:43.937Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9e/025ad7b65278745dee3bd0ebf9314934c4592560878308a6121f7f812084/httptools-0.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e99c7b90a29fd82fea9ef57943d501a16f3404d7b9ee81799d41639bdaae412c", size = 499192, upload-time = "2025-10-10T03:54:45.003Z" }, + { url = "https://files.pythonhosted.org/packages/6d/de/40a8f202b987d43afc4d54689600ff03ce65680ede2f31df348d7f368b8f/httptools-0.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3e14f530fefa7499334a79b0cf7e7cd2992870eb893526fb097d51b4f2d0f321", size = 86694, upload-time = "2025-10-10T03:54:45.923Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "0.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/63/4910c5fa9128fdadf6a9c5ac138e8b1b6cee4ca44bf7915bbfbce4e355ee/huggingface_hub-0.36.0.tar.gz", hash = "sha256:47b3f0e2539c39bf5cde015d63b72ec49baff67b6931c3d97f3f84532e2b8d25", size = 463358, upload-time = "2025-10-23T12:12:01.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/bd/1a875e0d592d447cbc02805fd3fe0f497714d6a2583f59d14fa9ebad96eb/huggingface_hub-0.36.0-py3-none-any.whl", hash = "sha256:7bcc9ad17d5b3f07b57c78e79d527102d08313caa278a641993acddcb894548d", size = 566094, upload-time = "2025-10-23T12:11:59.557Z" }, +] + +[[package]] +name = "hydra-core" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "omegaconf" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/8e/07e42bc434a847154083b315779b0a81d567154504624e181caf2c71cd98/hydra-core-1.3.2.tar.gz", hash = "sha256:8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824", size = 3263494, upload-time = "2023-02-23T18:33:43.03Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/50/e0edd38dcd63fb26a8547f13d28f7a008bc4a3fd4eb4ff030673f22ad41a/hydra_core-1.3.2-py3-none-any.whl", hash = "sha256:fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b", size = 154547, upload-time = "2023-02-23T18:33:40.801Z" }, +] + +[[package]] +name = "identify" +version = "2.6.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/e7/685de97986c916a6d93b3876139e00eef26ad5bbbd61925d670ae8013449/identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf", size = 99311, upload-time = "2025-10-02T17:43:40.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757", size = 99183, upload-time = "2025-10-02T17:43:39.137Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "ijson" +version = "3.4.0.post0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/30/7ab4b9e88e7946f6beef419f74edcc541df3ea562c7882257b4eaa82417d/ijson-3.4.0.post0.tar.gz", hash = "sha256:9aa02dc70bb245670a6ca7fba737b992aeeb4895360980622f7e568dbf23e41e", size = 67216, upload-time = "2025-10-10T05:29:25.62Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/fe/3b6af0025288e769dbfa30485dae1b3bd3f33f00390f3ee532cbb1c33e9b/ijson-3.4.0.post0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b607a500fca26101be47d2baf7cddb457b819ab60a75ce51ed1092a40da8b2f9", size = 87847, upload-time = "2025-10-10T05:28:07.229Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a5/95ee2ca82f3b1a57892452f6e5087607d56c620beb8ce625475194568698/ijson-3.4.0.post0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4827d9874a6a81625412c59f7ca979a84d01f7f6bfb3c6d4dc4c46d0382b14e0", size = 59815, upload-time = "2025-10-10T05:28:08.448Z" }, + { url = "https://files.pythonhosted.org/packages/51/8d/5a704ab3c17c55c21c86423458db8610626ca99cc9086a74dfeb7ee9054c/ijson-3.4.0.post0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d4d4afec780881edb2a0d2dd40b1cdbe246e630022d5192f266172a0307986a7", size = 59648, upload-time = "2025-10-10T05:28:09.307Z" }, + { url = "https://files.pythonhosted.org/packages/25/56/ca5d6ca145d007f30b44e747f3c163bc08710ce004af0deaad4a2301339b/ijson-3.4.0.post0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432fb60ffb952926f9438e0539011e2dfcd108f8426ee826ccc6173308c3ff2c", size = 138279, upload-time = "2025-10-10T05:28:10.489Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d3/22e3cc806fcdda7ad4c8482ed74db7a017d4a1d49b4300c7bc07052fb561/ijson-3.4.0.post0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54a0e3e05d9a0c95ecba73d9579f146cf6d5c5874116c849dba2d39a5f30380e", size = 149110, upload-time = "2025-10-10T05:28:12.263Z" }, + { url = "https://files.pythonhosted.org/packages/3e/04/efb30f413648b9267f5a33920ac124d7ebef3bc4063af8f6ffc8ca11ddcb/ijson-3.4.0.post0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05807edc0bcbd222dc6ea32a2b897f0c81dc7f12c8580148bc82f6d7f5e7ec7b", size = 149026, upload-time = "2025-10-10T05:28:13.557Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cf/481165f7046ade32488719300a3994a437020bc41cfbb54334356348f513/ijson-3.4.0.post0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a5269af16f715855d9864937f9dd5c348ca1ac49cee6a2c7a1b7091c159e874f", size = 150012, upload-time = "2025-10-10T05:28:14.859Z" }, + { url = "https://files.pythonhosted.org/packages/0f/24/642e3289917ecf860386e26dfde775f9962d26ab7f6c2e364ed3ca3c25d8/ijson-3.4.0.post0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b200df83c901f5bfa416d069ac71077aa1608f854a4c50df1b84ced560e9c9ec", size = 142193, upload-time = "2025-10-10T05:28:16.131Z" }, + { url = "https://files.pythonhosted.org/packages/0f/f5/fd2f038abe95e553e1c3ee207cda19db9196eb416e63c7c89699a8cf0db7/ijson-3.4.0.post0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6458bd8e679cdff459a0a5e555b107c3bbacb1f382da3fe0f40e392871eb518d", size = 150904, upload-time = "2025-10-10T05:28:17.401Z" }, + { url = "https://files.pythonhosted.org/packages/49/35/24259d22519987928164e6cb8fe3486e1df0899b2999ada4b0498639b463/ijson-3.4.0.post0-cp312-cp312-win32.whl", hash = "sha256:55f7f656b5986326c978cbb3a9eea9e33f3ef6ecc4535b38f1d452c731da39ab", size = 52358, upload-time = "2025-10-10T05:28:18.315Z" }, + { url = "https://files.pythonhosted.org/packages/a1/2b/6f7ade27a8ff5758fc41006dadd2de01730def84fe3e60553b329c59e0d4/ijson-3.4.0.post0-cp312-cp312-win_amd64.whl", hash = "sha256:e15833dcf6f6d188fdc624a31cd0520c3ba21b6855dc304bc7c1a8aeca02d4ac", size = 54789, upload-time = "2025-10-10T05:28:19.552Z" }, +] + +[[package]] +name = "imagesize" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a", size = 1280026, upload-time = "2022-07-01T12:21:05.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769, upload-time = "2022-07-01T12:21:02.467Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "interegular" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/9d/8b6dde58a028a3962ce17e84d5fe73758df61378e00ef8ac3d85da34b0ff/interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600", size = 24705, upload-time = "2024-01-06T23:01:22.372Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/01/72d6472f80651673716d1deda2a5bbb633e563ecf94f4479da5519d69d25/interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c", size = 23635, upload-time = "2024-01-06T23:01:20.829Z" }, +] + +[[package]] +name = "ipython" +version = "9.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/51/a703c030f4928646d390b4971af4938a1b10c9dfce694f0d99a0bb073cb2/ipython-9.8.0.tar.gz", hash = "sha256:8e4ce129a627eb9dd221c41b1d2cdaed4ef7c9da8c17c63f6f578fe231141f83", size = 4424940, upload-time = "2025-12-03T10:18:24.353Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/df/8ee1c5dd1e3308b5d5b2f2dfea323bb2f3827da8d654abb6642051199049/ipython-9.8.0-py3-none-any.whl", hash = "sha256:ebe6d1d58d7d988fbf23ff8ff6d8e1622cfdb194daf4b7b73b792c4ec3b85385", size = 621374, upload-time = "2025-12-03T10:18:22.335Z" }, +] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, +] + +[[package]] +name = "isodate" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, +] + +[[package]] +name = "jaxtyping" +version = "0.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wadler-lindig" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/fe/90f8884647073ea77a0fc5ec27e1605288591c89e60e53a6dd1d849ca6fc/jaxtyping-0.3.4.tar.gz", hash = "sha256:b4aac576a1b6c62a363f76f543f21c7cd4c7bb8714816c2c875f28b7abcdb770", size = 45665, upload-time = "2025-12-15T19:26:06.056Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/92/f30138ffa65d51791f85ec057eb76ae8eab125ed20f8f337918df3f1c775/jaxtyping-0.3.4-py3-none-any.whl", hash = "sha256:70e438db2f361575d04cccea50f77f9c8fe92f8b2086dc0ce89e5f1658bebaab", size = 56017, upload-time = "2025-12-15T19:26:04.835Z" }, +] + +[[package]] +name = "jedi" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jiter" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/9d/e0660989c1370e25848bb4c52d061c71837239738ad937e83edca174c273/jiter-0.12.0.tar.gz", hash = "sha256:64dfcd7d5c168b38d3f9f8bba7fc639edb3418abcc74f22fdbe6b8938293f30b", size = 168294, upload-time = "2025-11-09T20:49:23.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/c9/5b9f7b4983f1b542c64e84165075335e8a236fa9e2ea03a0c79780062be8/jiter-0.12.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:305e061fa82f4680607a775b2e8e0bcb071cd2205ac38e6ef48c8dd5ebe1cf37", size = 314449, upload-time = "2025-11-09T20:47:22.999Z" }, + { url = "https://files.pythonhosted.org/packages/98/6e/e8efa0e78de00db0aee82c0cf9e8b3f2027efd7f8a71f859d8f4be8e98ef/jiter-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c1860627048e302a528333c9307c818c547f214d8659b0705d2195e1a94b274", size = 319855, upload-time = "2025-11-09T20:47:24.779Z" }, + { url = "https://files.pythonhosted.org/packages/20/26/894cd88e60b5d58af53bec5c6759d1292bd0b37a8b5f60f07abf7a63ae5f/jiter-0.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df37577a4f8408f7e0ec3205d2a8f87672af8f17008358063a4d6425b6081ce3", size = 350171, upload-time = "2025-11-09T20:47:26.469Z" }, + { url = "https://files.pythonhosted.org/packages/f5/27/a7b818b9979ac31b3763d25f3653ec3a954044d5e9f5d87f2f247d679fd1/jiter-0.12.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:75fdd787356c1c13a4f40b43c2156276ef7a71eb487d98472476476d803fb2cf", size = 365590, upload-time = "2025-11-09T20:47:27.918Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7e/e46195801a97673a83746170b17984aa8ac4a455746354516d02ca5541b4/jiter-0.12.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1eb5db8d9c65b112aacf14fcd0faae9913d07a8afea5ed06ccdd12b724e966a1", size = 479462, upload-time = "2025-11-09T20:47:29.654Z" }, + { url = "https://files.pythonhosted.org/packages/ca/75/f833bfb009ab4bd11b1c9406d333e3b4357709ed0570bb48c7c06d78c7dd/jiter-0.12.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73c568cc27c473f82480abc15d1301adf333a7ea4f2e813d6a2c7d8b6ba8d0df", size = 378983, upload-time = "2025-11-09T20:47:31.026Z" }, + { url = "https://files.pythonhosted.org/packages/71/b3/7a69d77943cc837d30165643db753471aff5df39692d598da880a6e51c24/jiter-0.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4321e8a3d868919bcb1abb1db550d41f2b5b326f72df29e53b2df8b006eb9403", size = 361328, upload-time = "2025-11-09T20:47:33.286Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ac/a78f90caf48d65ba70d8c6efc6f23150bc39dc3389d65bbec2a95c7bc628/jiter-0.12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a51bad79f8cc9cac2b4b705039f814049142e0050f30d91695a2d9a6611f126", size = 386740, upload-time = "2025-11-09T20:47:34.703Z" }, + { url = "https://files.pythonhosted.org/packages/39/b6/5d31c2cc8e1b6a6bcf3c5721e4ca0a3633d1ab4754b09bc7084f6c4f5327/jiter-0.12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2a67b678f6a5f1dd6c36d642d7db83e456bc8b104788262aaefc11a22339f5a9", size = 520875, upload-time = "2025-11-09T20:47:36.058Z" }, + { url = "https://files.pythonhosted.org/packages/30/b5/4df540fae4e9f68c54b8dab004bd8c943a752f0b00efd6e7d64aa3850339/jiter-0.12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efe1a211fe1fd14762adea941e3cfd6c611a136e28da6c39272dbb7a1bbe6a86", size = 511457, upload-time = "2025-11-09T20:47:37.932Z" }, + { url = "https://files.pythonhosted.org/packages/07/65/86b74010e450a1a77b2c1aabb91d4a91dd3cd5afce99f34d75fd1ac64b19/jiter-0.12.0-cp312-cp312-win32.whl", hash = "sha256:d779d97c834b4278276ec703dc3fc1735fca50af63eb7262f05bdb4e62203d44", size = 204546, upload-time = "2025-11-09T20:47:40.47Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c7/6659f537f9562d963488e3e55573498a442503ced01f7e169e96a6110383/jiter-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:e8269062060212b373316fe69236096aaf4c49022d267c6736eebd66bbbc60bb", size = 205196, upload-time = "2025-11-09T20:47:41.794Z" }, + { url = "https://files.pythonhosted.org/packages/21/f4/935304f5169edadfec7f9c01eacbce4c90bb9a82035ac1de1f3bd2d40be6/jiter-0.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:06cb970936c65de926d648af0ed3d21857f026b1cf5525cb2947aa5e01e05789", size = 186100, upload-time = "2025-11-09T20:47:43.007Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f5/12efb8ada5f5c9edc1d4555fe383c1fb2eac05ac5859258a72d61981d999/jiter-0.12.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:e8547883d7b96ef2e5fe22b88f8a4c8725a56e7f4abafff20fd5272d634c7ecb", size = 309974, upload-time = "2025-11-09T20:49:17.187Z" }, + { url = "https://files.pythonhosted.org/packages/85/15/d6eb3b770f6a0d332675141ab3962fd4a7c270ede3515d9f3583e1d28276/jiter-0.12.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:89163163c0934854a668ed783a2546a0617f71706a2551a4a0666d91ab365d6b", size = 304233, upload-time = "2025-11-09T20:49:18.734Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3e/e7e06743294eea2cf02ced6aa0ff2ad237367394e37a0e2b4a1108c67a36/jiter-0.12.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d96b264ab7d34bbb2312dedc47ce07cd53f06835eacbc16dde3761f47c3a9e7f", size = 338537, upload-time = "2025-11-09T20:49:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/2f/9c/6753e6522b8d0ef07d3a3d239426669e984fb0eba15a315cdbc1253904e4/jiter-0.12.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24e864cb30ab82311c6425655b0cdab0a98c5d973b065c66a3f020740c2324c", size = 346110, upload-time = "2025-11-09T20:49:21.817Z" }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/69/f7185de793a29082a9f3c7728268ffb31cb5095131a9c139a74078e27336/jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85", size = 357342, upload-time = "2025-08-18T17:03:50.038Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63", size = 90040, upload-time = "2025-08-18T17:03:48.373Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "lark" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, +] + +[[package]] +name = "linkify-it-py" +version = "2.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "uc-micro-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/ae/bb56c6828e4797ba5a4821eec7c43b8bf40f69cda4d4f5f8c8a2810ec96a/linkify-it-py-2.0.3.tar.gz", hash = "sha256:68cda27e162e9215c17d786649d1da0021a451bdc436ef9e0fa0ba5234b9b048", size = 27946, upload-time = "2024-02-04T14:48:04.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/1e/b832de447dee8b582cac175871d2f6c3d5077cc56d5575cadba1fd1cccfa/linkify_it_py-2.0.3-py3-none-any.whl", hash = "sha256:6bcbc417b0ac14323382aef5c5192c0075bf8a9d6b41820a2b66371eac6b6d79", size = 19820, upload-time = "2024-02-04T14:48:02.496Z" }, +] + +[[package]] +name = "litellm" +version = "1.79.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "aiohttp", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "click", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fastuuid", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "httpx", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "importlib-metadata", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jinja2", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jsonschema", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "openai", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pydantic", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "python-dotenv", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tiktoken", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tokenizers", version = "0.21.4", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/fe/c498e9da7155895c3f2db508da3b4d60fba2e2108887bba1a64c965a0229/litellm-1.79.2.tar.gz", hash = "sha256:af0127ca6ff4ec1397b5aa203cbf63d4e2a660899cab56d332061a11dc38436e", size = 11303434, upload-time = "2025-11-08T23:59:08.315Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/b8/2a429e05bb709fc297201b37c6e039a4a1dfbcfd04a7e032a75864ed12b8/litellm-1.79.2-py3-none-any.whl", hash = "sha256:0d4e0962cbc72be1e13f759b539294a56837f8ff81d3ad9ffa743d9b8b6461bd", size = 10396310, upload-time = "2025-11-08T23:59:05.165Z" }, +] + +[package.optional-dependencies] +proxy = [ + { name = "apscheduler", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "azure-identity", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "azure-storage-blob", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "backoff", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "boto3", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "cryptography", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fastapi", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fastapi-sso", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "gunicorn", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm-enterprise", version = "0.1.20", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm-proxy-extras", version = "0.4.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "mcp", version = "1.12.4", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "orjson", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "polars", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyjwt", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pynacl", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "python-multipart", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyyaml", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "rich", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "rq", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvicorn", version = "0.29.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvloop", marker = "(sys_platform != 'win32' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "websockets", version = "13.1", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] + +[[package]] +name = "litellm" +version = "1.80.10" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "aiohttp", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "click", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fastuuid", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "grpcio", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "httpx", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "importlib-metadata", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jinja2", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jsonschema", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "openai", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pydantic", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "python-dotenv", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tiktoken", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tokenizers", version = "0.22.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/44/0aaa7449e7c4aa05668ec03f1f68a01b1e476591071d9659a68db19371a2/litellm-1.80.10.tar.gz", hash = "sha256:4a4aff7558945c2f7e5c6523e67c1b5525a46b10b0e1ad6b8f847cb13b16779e", size = 12764777, upload-time = "2025-12-14T02:07:05.362Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/a9/4814b6aa58f6705df2831eaadeb5bc8240684c8c9d5964245212f85049d1/litellm-1.80.10-py3-none-any.whl", hash = "sha256:9b3e561efaba0eb1291cb1555d3dcb7283cf7f3cb65aadbcdb42e2a8765898c8", size = 11264240, upload-time = "2025-12-14T02:07:02.414Z" }, +] + +[package.optional-dependencies] +proxy = [ + { name = "apscheduler", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "azure-identity", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "azure-storage-blob", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "backoff", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "boto3", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "cryptography", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fastapi", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fastapi-sso", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "gunicorn", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm-enterprise", version = "0.1.25", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm-proxy-extras", version = "0.4.14", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "mcp", version = "1.25.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "orjson", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "polars", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyjwt", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pynacl", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "python-multipart", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyyaml", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "rich", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "rq", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "soundfile", version = "0.12.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvloop", marker = "(sys_platform != 'win32' and extra == 'extra-11-skyrl-train-flashrl') or (sys_platform != 'win32' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'win32' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] + +[[package]] +name = "litellm-enterprise" +version = "0.1.20" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/ae/5478c23dad4e63b8fb01387a1d17b05c8b62a0182f76101a86eca74104c1/litellm_enterprise-0.1.20.tar.gz", hash = "sha256:f6b8dd75b53bd835c68caf6402a8bae744a150db7bb6b0e617178c6056ac6c01", size = 61379, upload-time = "2025-09-19T02:13:03.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/e2/3d113fe5262577d92ca2ace2454ad042be9c01c62bf1e44be4e2d12b2cc5/litellm_enterprise-0.1.20-py3-none-any.whl", hash = "sha256:744a79956a8cd7748ef4c3f40d5a564c61519834e706beafbc0b931162773ae8", size = 120792, upload-time = "2025-09-19T02:13:02.159Z" }, +] + +[[package]] +name = "litellm-enterprise" +version = "0.1.25" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/0a/2a166a3e598c581fcd295ec74f0706718051d0aeefccc636914d18c7fc86/litellm_enterprise-0.1.25.tar.gz", hash = "sha256:1c82178b8e2c85f47b31910fd103a322b46d6caea44cd7a8c80b00fdcfeacd22", size = 43420, upload-time = "2025-12-11T04:33:10.694Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/b2/31ac8e551216a6a92e6d47b6d9e3e49d8957ee6da15abf73eeb9aaeec1f3/litellm_enterprise-0.1.25-py3-none-any.whl", hash = "sha256:80c8f1996846453ad309e74cd6d2659d9508320370df5d462d34326b06401c4d", size = 104440, upload-time = "2025-12-11T04:33:06.103Z" }, +] + +[[package]] +name = "litellm-proxy-extras" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/59/6a9963955f6499c9ec0e65a9445add92f4854d3b03c920127a406ae61dd3/litellm_proxy_extras-0.4.3.tar.gz", hash = "sha256:420400d0db186319695526f6765d3d481206fe025b70bc74a1ce895a7d720bee", size = 16920, upload-time = "2025-11-08T18:22:16.296Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/d6/954a8ce8810d7daee5ec1334d74bbc3da889d89cba9825a4750fab7f1187/litellm_proxy_extras-0.4.3-py3-none-any.whl", hash = "sha256:e7ab09aa78d04d02dc48975620defa36784e1a0baa6d04a078b98b5717fcae24", size = 35518, upload-time = "2025-11-08T18:22:15.143Z" }, +] + +[[package]] +name = "litellm-proxy-extras" +version = "0.4.14" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/3f/8beffc57faae37a3c47a532cdfb2ac3bf6397905a8c0f4b50366bc0f0dc5/litellm_proxy_extras-0.4.14.tar.gz", hash = "sha256:518680192aac39c8c4f96ee0f3a87e2905250d3edaea8e0c4ff54ee598e775b0", size = 19525, upload-time = "2025-12-14T00:20:52.111Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/61/67f9d39afda151e3f37d2f1426fd797675a2af0d2c1045764b598476a4a9/litellm_proxy_extras-0.4.14-py3-none-any.whl", hash = "sha256:6943e19abb696e080b5a2a01472b99b1d78603ecb9df24604eba428f54440e7d", size = 43086, upload-time = "2025-12-14T00:20:50.298Z" }, +] + +[[package]] +name = "llguidance" +version = "0.7.30" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/38/d1ef3ae08d8d857e5e0690c5b1e07bf7eb4a1cae5881d87215826dc6cadb/llguidance-0.7.30.tar.gz", hash = "sha256:e93bf75f2b6e48afb86a5cee23038746975e1654672bf5ba0ae75f7d4d4a2248", size = 1055528, upload-time = "2025-06-23T00:23:49.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/e1/694c89986fcae7777184fc8b22baa0976eba15a6847221763f6ad211fc1f/llguidance-0.7.30-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c80af02c118d2b0526bcecaab389af2ed094537a069b0fc724cd2a2f2ba3990f", size = 3327974, upload-time = "2025-06-23T00:23:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/ab7a548ae189dc23900fdd37803c115c2339b1223af9e8eb1f4329b5935a/llguidance-0.7.30-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:00a256d532911d2cf5ba4ef63e182944e767dd2402f38d63002016bc37755958", size = 3210709, upload-time = "2025-06-23T00:23:45.872Z" }, + { url = "https://files.pythonhosted.org/packages/9c/5b/6a166564b14f9f805f0ea01ec233a84f55789cb7eeffe1d6224ccd0e6cdd/llguidance-0.7.30-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af8741c867e4bc7e42f7cdc68350c076b4edd0ca10ecefbde75f15a9f6bc25d0", size = 14867038, upload-time = "2025-06-23T00:23:39.571Z" }, + { url = "https://files.pythonhosted.org/packages/17/ec/69507bdb36767f9b6ff2e290660a9b5afdda0fb8a7903faa37f37c6c2a72/llguidance-0.7.30-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4a327a30dd37d86dd6347861ac8de3521fc1dbef9475296c06744e5b40ffc54", size = 15142936, upload-time = "2025-06-23T00:23:41.944Z" }, + { url = "https://files.pythonhosted.org/packages/af/80/5a40b9689f17612434b820854cba9b8cabd5142072c491b5280fe5f7a35e/llguidance-0.7.30-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9edc409b9decd6cffba5f5bf3b4fbd7541f95daa8cbc9510cbf96c6ab1ffc153", size = 15004926, upload-time = "2025-06-23T00:23:43.965Z" }, + { url = "https://files.pythonhosted.org/packages/bb/bc/2d2f9b446bb3e51e4dd4db290590afee03ae29163f417168569f0361204c/llguidance-0.7.30-cp39-abi3-win32.whl", hash = "sha256:a0d52b8d1b2d3b0e661e3f953ecccfa16644f302026b3067a4815c1baa2ae643", size = 2585627, upload-time = "2025-06-23T00:23:52.39Z" }, + { url = "https://files.pythonhosted.org/packages/99/47/58e49a118b514855b245f8a962c6aaf9a5cc95a0f61eac7e230e691c7b7e/llguidance-0.7.30-cp39-abi3-win_amd64.whl", hash = "sha256:05234ecceea7c9c6ff13b9739112043173a3bcb88cae860249b20335a07b3075", size = 2796878, upload-time = "2025-06-23T00:23:51Z" }, +] + +[[package]] +name = "llguidance" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/95/48/3f7a9d3ff1b36bba92b5107a3a21286821227afe9ea464736133994d61fb/llguidance-1.3.0.tar.gz", hash = "sha256:861249afd51dc325646834462ea827e57a5c2b2042e108e6aae7059fdad9104d", size = 1070460, upload-time = "2025-10-20T19:58:44.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/33/be5acb85cd8cdc4afde33d9c234eece9f318e087920255af3c05864cd3e7/llguidance-1.3.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f7685222660a762e481ac633d49cc559c64980fe2ee59c8f932a5bb5cbc0c2c2", size = 3220647, upload-time = "2025-10-20T19:58:42.542Z" }, + { url = "https://files.pythonhosted.org/packages/82/e6/b48bda5b15efeaeb62bd0dba8fc6a01d4ae5457a85dbb5d18632385fe15c/llguidance-1.3.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:098030ff0687261a3f1bd54cf21fe951fc861d56d37a0671250dd36677eaf224", size = 3099830, upload-time = "2025-10-20T19:58:40.826Z" }, + { url = "https://files.pythonhosted.org/packages/aa/11/44389d3d1526d7a5c38ffd587a5ebc61d7bee443ac1dea95f2089ad58f5f/llguidance-1.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f6caca5d78db7f76e1fbb0fff8607b861c32d47fa3d5dee2fc49de27ee269df", size = 2835242, upload-time = "2025-10-20T19:58:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ca/53ea256396405e4dee70d5a4a35e18543408e18bb16b251d6ca6b5d80310/llguidance-1.3.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0612bb3f034d2487b6e8f9561f02a94a6039d88273bf0c5c539a3bd3895e47d2", size = 3297480, upload-time = "2025-10-20T19:58:37.033Z" }, + { url = "https://files.pythonhosted.org/packages/83/a8/1ff2bedb8f9acb46a2d2d603415d272bb622c142ea86f5b95445cc6e366c/llguidance-1.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc17e9dd602c3879bf91664a64bf72f54c74dbfbeb24ccfab6a5fe435b12f7aa", size = 3033133, upload-time = "2025-10-20T19:58:38.721Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a7/9b8086c0cfdddf3f6d47b173a404fa7ac46272f7affbee082c36740f4f1c/llguidance-1.3.0-cp39-abi3-win32.whl", hash = "sha256:2f6f558485a43e273fc5c6c974a9a3ace5d5e170076db9b40e0560e41c3ff18f", size = 2598109, upload-time = "2025-10-20T19:58:47.656Z" }, + { url = "https://files.pythonhosted.org/packages/5a/7e/809349638231f469b9056c0e1bfd924d5ef5558b3b3ec72d093b6fad33b1/llguidance-1.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:1d1cd1c8618d1a13605d3e057c978651e551c8c469b481ee4041f1d6c436002d", size = 2789946, upload-time = "2025-10-20T19:58:45.958Z" }, +] + +[[package]] +name = "llvmlite" +version = "0.44.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/6a/95a3d3610d5c75293d5dbbb2a76480d5d4eeba641557b69fe90af6c5b84e/llvmlite-0.44.0.tar.gz", hash = "sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4", size = 171880, upload-time = "2025-01-20T11:14:41.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/86/e3c3195b92e6e492458f16d233e58a1a812aa2bfbef9bdd0fbafcec85c60/llvmlite-0.44.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:1d671a56acf725bf1b531d5ef76b86660a5ab8ef19bb6a46064a705c6ca80aad", size = 28132297, upload-time = "2025-01-20T11:13:32.57Z" }, + { url = "https://files.pythonhosted.org/packages/d6/53/373b6b8be67b9221d12b24125fd0ec56b1078b660eeae266ec388a6ac9a0/llvmlite-0.44.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f79a728e0435493611c9f405168682bb75ffd1fbe6fc360733b850c80a026db", size = 26201105, upload-time = "2025-01-20T11:13:38.744Z" }, + { url = "https://files.pythonhosted.org/packages/cb/da/8341fd3056419441286c8e26bf436923021005ece0bff5f41906476ae514/llvmlite-0.44.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0143a5ef336da14deaa8ec26c5449ad5b6a2b564df82fcef4be040b9cacfea9", size = 42361901, upload-time = "2025-01-20T11:13:46.711Z" }, + { url = "https://files.pythonhosted.org/packages/53/ad/d79349dc07b8a395a99153d7ce8b01d6fcdc9f8231355a5df55ded649b61/llvmlite-0.44.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d752f89e31b66db6f8da06df8b39f9b91e78c5feea1bf9e8c1fba1d1c24c065d", size = 41184247, upload-time = "2025-01-20T11:13:56.159Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3b/a9a17366af80127bd09decbe2a54d8974b6d8b274b39bf47fbaedeec6307/llvmlite-0.44.0-cp312-cp312-win_amd64.whl", hash = "sha256:eae7e2d4ca8f88f89d315b48c6b741dcb925d6a1042da694aa16ab3dd4cbd3a1", size = 30332380, upload-time = "2025-01-20T11:14:02.442Z" }, +] + +[[package]] +name = "lm-format-enforcer" +version = "0.11.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "interegular" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/ef/11292bb0b85cf4c93447cab5a29f64576ed14d3ab4280e35ddd23486594a/lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f", size = 45418, upload-time = "2025-08-24T19:37:46.325Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "win32-setctime", marker = "sys_platform == 'win32' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + +[[package]] +name = "lxml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887, upload-time = "2025-09-22T04:01:17.265Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818, upload-time = "2025-09-22T04:01:19.688Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807, upload-time = "2025-09-22T04:01:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179, upload-time = "2025-09-22T04:01:23.32Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044, upload-time = "2025-09-22T04:01:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685, upload-time = "2025-09-22T04:01:27.398Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127, upload-time = "2025-09-22T04:01:29.629Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958, upload-time = "2025-09-22T04:01:31.535Z" }, + { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541, upload-time = "2025-09-22T04:01:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426, upload-time = "2025-09-22T04:01:35.639Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917, upload-time = "2025-09-22T04:01:37.448Z" }, + { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795, upload-time = "2025-09-22T04:01:39.165Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759, upload-time = "2025-09-22T04:01:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666, upload-time = "2025-09-22T04:01:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989, upload-time = "2025-09-22T04:01:45.215Z" }, + { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456, upload-time = "2025-09-22T04:01:48.243Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793, upload-time = "2025-09-22T04:01:50.042Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" }, +] + +[[package]] +name = "mamba-ssm" +version = "2.2.6.post3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "einops", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "ninja", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "packaging", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "setuptools", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "triton", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/0c/9373a469ff7a33bdd0644e55fa45165ba3900274dcf7fe9f10ccc232aef9/mamba_ssm-2.2.6.post3.tar.gz", hash = "sha256:826a3cdb651959f191dac64502f8a29627d9116fe6bb7c57e4f562da1aea7bf3", size = 113913, upload-time = "2025-10-10T06:00:44.939Z" } + +[[package]] +name = "markdown" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/7dd27d9d863b3376fcf23a5a13cb5d024aed1db46f963f1b5735ae43b3be/markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e", size = 364931, upload-time = "2025-11-03T19:51:15.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c", size = 107678, upload-time = "2025-11-03T19:51:13.887Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, +] + +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py" }, +] +plugins = [ + { name = "mdit-py-plugins" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/74/97e72a36efd4ae2bccb3463284300f8953f199b5ffbc04cbbb0ec78f74b1/matplotlib_inline-0.2.1.tar.gz", hash = "sha256:e1ee949c340d771fc39e241ea75683deb94762c8fa5f2927ec57c83c4dffa9fe", size = 8110, upload-time = "2025-10-23T09:00:22.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, +] + +[[package]] +name = "mcp" +version = "1.12.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "anyio", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "httpx", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "httpx-sse", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jsonschema", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pydantic", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pydantic-settings", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "python-multipart", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pywin32", marker = "(sys_platform == 'win32' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "sse-starlette", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "starlette", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvicorn", version = "0.29.0", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'emscripten' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/88/f6cb7e7c260cd4b4ce375f2b1614b33ce401f63af0f49f7141a2e9bf0a45/mcp-1.12.4.tar.gz", hash = "sha256:0765585e9a3a5916a3c3ab8659330e493adc7bd8b2ca6120c2d7a0c43e034ca5", size = 431148, upload-time = "2025-08-07T20:31:18.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/68/316cbc54b7163fa22571dcf42c9cc46562aae0a021b974e0a8141e897200/mcp-1.12.4-py3-none-any.whl", hash = "sha256:7aa884648969fab8e78b89399d59a683202972e12e6bc9a1c88ce7eda7743789", size = 160145, upload-time = "2025-08-07T20:31:15.69Z" }, +] + +[[package]] +name = "mcp" +version = "1.25.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "anyio", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "httpx", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "httpx-sse", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jsonschema", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pydantic", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pydantic-settings", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyjwt", extra = ["crypto"], marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "python-multipart", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pywin32", marker = "(sys_platform == 'win32' and extra == 'extra-11-skyrl-train-flashrl') or (sys_platform == 'win32' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'win32' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "sse-starlette", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "starlette", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "typing-extensions", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "typing-inspection", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'emscripten' and extra == 'extra-11-skyrl-train-flashrl') or (sys_platform != 'emscripten' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'emscripten' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/2d/649d80a0ecf6a1f82632ca44bec21c0461a9d9fc8934d38cb5b319f2db5e/mcp-1.25.0.tar.gz", hash = "sha256:56310361ebf0364e2d438e5b45f7668cbb124e158bb358333cd06e49e83a6802", size = 605387, upload-time = "2025-12-19T10:19:56.985Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/fc/6dc7659c2ae5ddf280477011f4213a74f806862856b796ef08f028e664bf/mcp-1.25.0-py3-none-any.whl", hash = "sha256:b37c38144a666add0862614cc79ec276e97d72aa8ca26d622818d4e278b9721a", size = 233076, upload-time = "2025-12-19T10:19:55.416Z" }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "megatron-bridge" +version = "0.3.0rc0" +source = { git = "https://github.com/NVIDIA-NeMo/Megatron-Bridge?rev=04e370eedf8cc44a812189a19f2171d90555c07a#04e370eedf8cc44a812189a19f2171d90555c07a" } +dependencies = [ + { name = "accelerate" }, + { name = "causal-conv1d", marker = "sys_platform == 'never'" }, + { name = "datasets" }, + { name = "flash-linear-attention" }, + { name = "hydra-core" }, + { name = "mamba-ssm", marker = "sys_platform == 'never'" }, + { name = "megatron-core" }, + { name = "nvidia-resiliency-ext", marker = "sys_platform == 'never'" }, + { name = "omegaconf" }, + { name = "open-clip-torch" }, + { name = "pyyaml" }, + { name = "qwen-vl-utils" }, + { name = "regex" }, + { name = "rich" }, + { name = "six" }, + { name = "tensorboard" }, + { name = "timm" }, + { name = "tqdm" }, + { name = "transformer-engine", extra = ["pytorch"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, + { name = "wandb" }, +] + +[[package]] +name = "megatron-core" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/bd/6bdffdf3ce9d2bcdd8d32c437c04712ce630c71359b3f29b2b15ab83a5e7/megatron_core-0.15.0.tar.gz", hash = "sha256:58f0d98a4ea2041190a40520aaac1564048ed396bc97a285e54da55830b8ac03", size = 882299, upload-time = "2025-12-18T01:36:02.074Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/8e/6f52874c759aa10f6f5542fb7ae26f1ccdf23a115807715aa1a623df6cc1/megatron_core-0.15.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1baefc30bb53840c5a30197f0dedba3236f4236ef883098e89450253c5eecb2f", size = 2243667, upload-time = "2025-12-18T01:35:53.383Z" }, + { url = "https://files.pythonhosted.org/packages/4d/50/c12390df40f5b25a8a247f92ab1404efddf444e3910b7c0d07382a3c8d28/megatron_core-0.15.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e4f01ea4d28f4e7a0450701164cedd23463021bed49b831ebfcf16c54be8232", size = 2268089, upload-time = "2025-12-18T01:35:54.715Z" }, +] + +[[package]] +name = "mini-swe-agent" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "litellm", version = "1.79.2", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm", version = "1.80.10", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "openai" }, + { name = "platformdirs" }, + { name = "prompt-toolkit" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rich" }, + { name = "tenacity" }, + { name = "textual" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/24/c974167db6ede57094d4e56e9a47cdac6d179e318292ab597ce04d03c5a9/mini_swe_agent-1.17.3.tar.gz", hash = "sha256:9e3c9193f7720a29f22b553a9eaba85ed49bdd1aaec7b7a6a4695f0c77c6293e", size = 50250, upload-time = "2025-12-15T00:03:06.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/0e/ce0b5c0ba90fb718755d33f6060b277baac346ad4b3414ecfa15c13ba0e6/mini_swe_agent-1.17.3-py3-none-any.whl", hash = "sha256:c5ece02463db5edbf8e5d45b71f8b04aa802f9da8e38627931856fec11052aa1", size = 80887, upload-time = "2025-12-15T00:03:05.065Z" }, +] + +[[package]] +name = "mistral-common" +version = "1.8.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "pydantic-extra-types", extra = ["pycountry"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "requests" }, + { name = "tiktoken" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/3d/2ce630343f4574ac5150e355dc15bf95e8ce18954aa0f5949e195ffde382/mistral_common-1.8.6.tar.gz", hash = "sha256:c61702720093f7a06508e81923917b04e35062b9ff396b8512b9c4d1139767ee", size = 6332366, upload-time = "2025-11-30T22:03:37.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/eb/56ba11528da93b890fe985cbbaa17955064ff4b7019dd18566e3c3559b9e/mistral_common-1.8.6-py3-none-any.whl", hash = "sha256:dd8c0e55b397e8167751eb3da147cf23fd970824673ca0e260aa58c888be1b0a", size = 6515009, upload-time = "2025-11-30T22:03:35.108Z" }, +] + +[package.optional-dependencies] +image = [ + { name = "opencv-python-headless", version = "4.11.0.86", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "opencv-python-headless", version = "4.12.0.88", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] + +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, +] + +[[package]] +name = "mlx" +version = "0.30.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mlx-metal", marker = "sys_platform == 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/4b/ad57b2f0ede3f0d009c0e3e1270c219bd18f9025388855ee149680cffa20/mlx-0.30.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:deaef3ecd2f99930867a29de748e3bffa9cc7e4dfa834f2501c37ed29aece1cc", size = 593397, upload-time = "2025-12-18T01:55:41.814Z" }, + { url = "https://files.pythonhosted.org/packages/ef/14/7fa03a0f66ac3cfb2fd6752178a1488f13c7233fff26eed0f832d961db35/mlx-0.30.1-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:86ccdcda0b5ea4768b87da25beae5b83ac7cc802506116b6845cea6f450e2377", size = 593397, upload-time = "2025-12-18T01:55:43Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c8/9f1343dbe2381f9653df4e0a62dc8bf38f575a2553dc2aa6916de32d2a85/mlx-0.30.1-cp312-cp312-macosx_26_0_arm64.whl", hash = "sha256:a625cb434b2acc5674fe10683374641dab9671fb354ae7c2c67a1fb0405eeb37", size = 567576, upload-time = "2025-12-18T00:15:55.114Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/485ed9c99c18ef89ac987178c0a526cb4148ba38b14838d315311d9d76a8/mlx-0.30.1-cp312-cp312-manylinux_2_35_aarch64.whl", hash = "sha256:ccc1ff3aca8fb1073c7dcd1274cebe48ae75f852d14b16c7db8228fbbad594dd", size = 643654, upload-time = "2025-12-18T01:55:44.165Z" }, + { url = "https://files.pythonhosted.org/packages/8a/d3/54d3bf5e404c3b6424b49c505dc8b3c06c6bb498fe720195b1fafbd69b5e/mlx-0.30.1-cp312-cp312-manylinux_2_35_x86_64.whl", hash = "sha256:55ed7fc4b389d6e49dac6d34a97b41e61cbe3662ac29c3d29cf612e6b2ed9827", size = 687305, upload-time = "2025-12-18T01:55:45.526Z" }, +] + +[[package]] +name = "mlx-lm" +version = "0.29.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2", marker = "(sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "protobuf", marker = "(sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyyaml", marker = "(sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "sentencepiece", marker = "(sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.52.3", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'darwin' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'linux' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/62/f46e1355256a114808517947f8e83ad6be310c7288c551db0fa678f47923/mlx_lm-0.29.1.tar.gz", hash = "sha256:b99180d8f33d33a077b814e550bfb2d8a59ae003d668fd1f4b3fff62a381d34b", size = 232302, upload-time = "2025-12-16T16:58:27.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/53/913099c91d384e115ea078325efd9a0bc1ea3eb3458c694b4596cbd267f2/mlx_lm-0.29.1-py3-none-any.whl", hash = "sha256:440941b3054c2a2216e97615de584cc90fa1ea874782e20699b9895721fad8dc", size = 324884, upload-time = "2025-12-16T16:58:26.36Z" }, +] + +[[package]] +name = "mlx-metal" +version = "0.30.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/3f/0be35ddad7e13d8ecd33a9185895f9739bb00b96ef0cce36cf0405d4aec0/mlx_metal-0.30.1-py3-none-macosx_14_0_arm64.whl", hash = "sha256:e7e92c6bdbd7ac8083f528a4c6640552ae106a57bb3d99856ac10a32e93a4b5e", size = 36864966, upload-time = "2025-12-18T01:55:31.473Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1f/c0bddd0d5bf3871411aabe32121e09e1b7cdbece8917a49d5a442310e3e5/mlx_metal-0.30.1-py3-none-macosx_15_0_arm64.whl", hash = "sha256:bb50f57418af7fc3c42a2da2c4bde0e7ab7ac0b997de1f6f642a6680ac65d626", size = 36859011, upload-time = "2025-12-18T01:55:34.541Z" }, + { url = "https://files.pythonhosted.org/packages/67/b3/73cc2f584ac612a476096d35a61eed75ee7ed8b4e320b0c36cf60a14d4eb/mlx_metal-0.30.1-py3-none-macosx_26_0_arm64.whl", hash = "sha256:e0b151a0053ac00b4226710bfb6dbf54b87283fb01e10fb3877f9ea969f680aa", size = 44981160, upload-time = "2025-12-18T00:15:47.518Z" }, +] + +[[package]] +name = "model-hosting-container-standards" +version = "0.1.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastapi" }, + { name = "httpx" }, + { name = "jmespath" }, + { name = "pydantic" }, + { name = "setuptools" }, + { name = "starlette" }, + { name = "supervisor" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d7/b7/a6a31b4dfd30d14b1019dc358f09c9d88ca38e555ba7c976e7d3e6b593fe/model_hosting_container_standards-0.1.13.tar.gz", hash = "sha256:27a1333410dde2719286a300a2803e24fdde407baa91894eb845c0f268aa194d", size = 79116, upload-time = "2026-01-09T21:45:20.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/37/6dc61971ba31450bbed460b5f40543f0915e352680534e3bcaf57116d8d7/model_hosting_container_standards-0.1.13-py3-none-any.whl", hash = "sha256:be307d4a988cc660df4e6bd8bdedb7917844bac940e332f9fd001cb385d7994c", size = 105738, upload-time = "2026-01-09T21:45:18.959Z" }, +] + +[[package]] +name = "modelscope" +version = "1.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "requests" }, + { name = "setuptools" }, + { name = "tqdm" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/02/db35ce865e66fd212fcf0cb5b43db3a3474cf82fae8d835b56ce7dba9247/modelscope-1.33.0.tar.gz", hash = "sha256:5d9ca8eb934cabea236104ed774b3ddf352f96c705272876108aaa25a3bb0b38", size = 4558673, upload-time = "2025-12-10T03:50:01.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/05/63f01821681b2be5d1739b4aad7b186c28d4ead2c5c99a9fc4aa53c13c19/modelscope-1.33.0-py3-none-any.whl", hash = "sha256:d9bdd566303f813d762e133410007eaf1b78f065c871228ab38640919b707489", size = 6050040, upload-time = "2025-12-10T03:49:58.428Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "msal" +version = "1.34.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/0e/c857c46d653e104019a84f22d4494f2119b4fe9f896c92b4b864b3b045cc/msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f", size = 153961, upload-time = "2025-09-22T23:05:48.989Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/dc/18d48843499e278538890dc709e9ee3dea8375f8be8e82682851df1b48b5/msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1", size = 116987, upload-time = "2025-09-22T23:05:47.294Z" }, +] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, +] + +[[package]] +name = "msgpack" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/bd/8b0d01c756203fbab65d265859749860682ccd2a59594609aeec3a144efa/msgpack-1.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:70a0dff9d1f8da25179ffcf880e10cf1aad55fdb63cd59c9a49a1b82290062aa", size = 81939, upload-time = "2025-10-08T09:15:01.472Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/ba4f155f793a74c1483d4bdef136e1023f7bcba557f0db4ef3db3c665cf1/msgpack-1.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:446abdd8b94b55c800ac34b102dffd2f6aa0ce643c55dfc017ad89347db3dbdb", size = 85064, upload-time = "2025-10-08T09:15:03.764Z" }, + { url = "https://files.pythonhosted.org/packages/f2/60/a064b0345fc36c4c3d2c743c82d9100c40388d77f0b48b2f04d6041dbec1/msgpack-1.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c63eea553c69ab05b6747901b97d620bb2a690633c77f23feb0c6a947a8a7b8f", size = 417131, upload-time = "2025-10-08T09:15:05.136Z" }, + { url = "https://files.pythonhosted.org/packages/65/92/a5100f7185a800a5d29f8d14041f61475b9de465ffcc0f3b9fba606e4505/msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:372839311ccf6bdaf39b00b61288e0557916c3729529b301c52c2d88842add42", size = 427556, upload-time = "2025-10-08T09:15:06.837Z" }, + { url = "https://files.pythonhosted.org/packages/f5/87/ffe21d1bf7d9991354ad93949286f643b2bb6ddbeab66373922b44c3b8cc/msgpack-1.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2929af52106ca73fcb28576218476ffbb531a036c2adbcf54a3664de124303e9", size = 404920, upload-time = "2025-10-08T09:15:08.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/8543ed2b8604f7c0d89ce066f42007faac1eaa7d79a81555f206a5cdb889/msgpack-1.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:be52a8fc79e45b0364210eef5234a7cf8d330836d0a64dfbb878efa903d84620", size = 415013, upload-time = "2025-10-08T09:15:09.83Z" }, + { url = "https://files.pythonhosted.org/packages/41/0d/2ddfaa8b7e1cee6c490d46cb0a39742b19e2481600a7a0e96537e9c22f43/msgpack-1.1.2-cp312-cp312-win32.whl", hash = "sha256:1fff3d825d7859ac888b0fbda39a42d59193543920eda9d9bea44d958a878029", size = 65096, upload-time = "2025-10-08T09:15:11.11Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ec/d431eb7941fb55a31dd6ca3404d41fbb52d99172df2e7707754488390910/msgpack-1.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1de460f0403172cff81169a30b9a92b260cb809c4cb7e2fc79ae8d0510c78b6b", size = 72708, upload-time = "2025-10-08T09:15:12.554Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/5b1a1f70eb0e87d1678e9624908f86317787b536060641d6798e3cf70ace/msgpack-1.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:be5980f3ee0e6bd44f3a9e9dea01054f175b50c3e6cdb692bc9424c0bbb8bf69", size = 64119, upload-time = "2025-10-08T09:15:13.589Z" }, +] + +[[package]] +name = "msgspec" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/9c/bfbd12955a49180cbd234c5d29ec6f74fe641698f0cd9df154a854fc8a15/msgspec-0.20.0.tar.gz", hash = "sha256:692349e588fde322875f8d3025ac01689fead5901e7fb18d6870a44519d62a29", size = 317862, upload-time = "2025-11-24T03:56:28.934Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/6f/1e25eee957e58e3afb2a44b94fa95e06cebc4c236193ed0de3012fff1e19/msgspec-0.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2aba22e2e302e9231e85edc24f27ba1f524d43c223ef5765bd8624c7df9ec0a5", size = 196391, upload-time = "2025-11-24T03:55:32.677Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ee/af51d090ada641d4b264992a486435ba3ef5b5634bc27e6eb002f71cef7d/msgspec-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:716284f898ab2547fedd72a93bb940375de9fbfe77538f05779632dc34afdfde", size = 188644, upload-time = "2025-11-24T03:55:33.934Z" }, + { url = "https://files.pythonhosted.org/packages/49/d6/9709ee093b7742362c2934bfb1bbe791a1e09bed3ea5d8a18ce552fbfd73/msgspec-0.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:558ed73315efa51b1538fa8f1d3b22c8c5ff6d9a2a62eff87d25829b94fc5054", size = 218852, upload-time = "2025-11-24T03:55:35.575Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a2/488517a43ccf5a4b6b6eca6dd4ede0bd82b043d1539dd6bb908a19f8efd3/msgspec-0.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:509ac1362a1d53aa66798c9b9fd76872d7faa30fcf89b2fba3bcbfd559d56eb0", size = 224937, upload-time = "2025-11-24T03:55:36.859Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e8/49b832808aa23b85d4f090d1d2e48a4e3834871415031ed7c5fe48723156/msgspec-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1353c2c93423602e7dea1aa4c92f3391fdfc25ff40e0bacf81d34dbc68adb870", size = 222858, upload-time = "2025-11-24T03:55:38.187Z" }, + { url = "https://files.pythonhosted.org/packages/9f/56/1dc2fa53685dca9c3f243a6cbecd34e856858354e455b77f47ebd76cf5bf/msgspec-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cb33b5eb5adb3c33d749684471c6a165468395d7aa02d8867c15103b81e1da3e", size = 227248, upload-time = "2025-11-24T03:55:39.496Z" }, + { url = "https://files.pythonhosted.org/packages/5a/51/aba940212c23b32eedce752896205912c2668472ed5b205fc33da28a6509/msgspec-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:fb1d934e435dd3a2b8cf4bbf47a8757100b4a1cfdc2afdf227541199885cdacb", size = 190024, upload-time = "2025-11-24T03:55:40.829Z" }, + { url = "https://files.pythonhosted.org/packages/41/ad/3b9f259d94f183daa9764fef33fdc7010f7ecffc29af977044fa47440a83/msgspec-0.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:00648b1e19cf01b2be45444ba9dc961bd4c056ffb15706651e64e5d6ec6197b7", size = 175390, upload-time = "2025-11-24T03:55:42.05Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/80/1e/5492c365f222f907de1039b91f922b93fa4f764c713ee858d235495d8f50/multidict-6.7.0.tar.gz", hash = "sha256:c6e99d9a65ca282e578dfea819cfa9c0a62b2499d8677392e09feaf305e9e6f5", size = 101834, upload-time = "2025-10-06T14:52:30.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/9e/9f61ac18d9c8b475889f32ccfa91c9f59363480613fc807b6e3023d6f60b/multidict-6.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8a3862568a36d26e650a19bb5cbbba14b71789032aebc0423f8cc5f150730184", size = 76877, upload-time = "2025-10-06T14:49:20.884Z" }, + { url = "https://files.pythonhosted.org/packages/38/6f/614f09a04e6184f8824268fce4bc925e9849edfa654ddd59f0b64508c595/multidict-6.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:960c60b5849b9b4f9dcc9bea6e3626143c252c74113df2c1540aebce70209b45", size = 45467, upload-time = "2025-10-06T14:49:22.054Z" }, + { url = "https://files.pythonhosted.org/packages/b3/93/c4f67a436dd026f2e780c433277fff72be79152894d9fc36f44569cab1a6/multidict-6.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2049be98fb57a31b4ccf870bf377af2504d4ae35646a19037ec271e4c07998aa", size = 43834, upload-time = "2025-10-06T14:49:23.566Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f5/013798161ca665e4a422afbc5e2d9e4070142a9ff8905e482139cd09e4d0/multidict-6.7.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0934f3843a1860dd465d38895c17fce1f1cb37295149ab05cd1b9a03afacb2a7", size = 250545, upload-time = "2025-10-06T14:49:24.882Z" }, + { url = "https://files.pythonhosted.org/packages/71/2f/91dbac13e0ba94669ea5119ba267c9a832f0cb65419aca75549fcf09a3dc/multidict-6.7.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3e34f3a1b8131ba06f1a73adab24f30934d148afcd5f5de9a73565a4404384e", size = 258305, upload-time = "2025-10-06T14:49:26.778Z" }, + { url = "https://files.pythonhosted.org/packages/ef/b0/754038b26f6e04488b48ac621f779c341338d78503fb45403755af2df477/multidict-6.7.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:efbb54e98446892590dc2458c19c10344ee9a883a79b5cec4bc34d6656e8d546", size = 242363, upload-time = "2025-10-06T14:49:28.562Z" }, + { url = "https://files.pythonhosted.org/packages/87/15/9da40b9336a7c9fa606c4cf2ed80a649dffeb42b905d4f63a1d7eb17d746/multidict-6.7.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a35c5fc61d4f51eb045061e7967cfe3123d622cd500e8868e7c0c592a09fedc4", size = 268375, upload-time = "2025-10-06T14:49:29.96Z" }, + { url = "https://files.pythonhosted.org/packages/82/72/c53fcade0cc94dfaad583105fd92b3a783af2091eddcb41a6d5a52474000/multidict-6.7.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29fe6740ebccba4175af1b9b87bf553e9c15cd5868ee967e010efcf94e4fd0f1", size = 269346, upload-time = "2025-10-06T14:49:31.404Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e2/9baffdae21a76f77ef8447f1a05a96ec4bc0a24dae08767abc0a2fe680b8/multidict-6.7.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:123e2a72e20537add2f33a79e605f6191fba2afda4cbb876e35c1a7074298a7d", size = 256107, upload-time = "2025-10-06T14:49:32.974Z" }, + { url = "https://files.pythonhosted.org/packages/3c/06/3f06f611087dc60d65ef775f1fb5aca7c6d61c6db4990e7cda0cef9b1651/multidict-6.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b284e319754366c1aee2267a2036248b24eeb17ecd5dc16022095e747f2f4304", size = 253592, upload-time = "2025-10-06T14:49:34.52Z" }, + { url = "https://files.pythonhosted.org/packages/20/24/54e804ec7945b6023b340c412ce9c3f81e91b3bf5fa5ce65558740141bee/multidict-6.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:803d685de7be4303b5a657b76e2f6d1240e7e0a8aa2968ad5811fa2285553a12", size = 251024, upload-time = "2025-10-06T14:49:35.956Z" }, + { url = "https://files.pythonhosted.org/packages/14/48/011cba467ea0b17ceb938315d219391d3e421dfd35928e5dbdc3f4ae76ef/multidict-6.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c04a328260dfd5db8c39538f999f02779012268f54614902d0afc775d44e0a62", size = 251484, upload-time = "2025-10-06T14:49:37.631Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2f/919258b43bb35b99fa127435cfb2d91798eb3a943396631ef43e3720dcf4/multidict-6.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8a19cdb57cd3df4cd865849d93ee14920fb97224300c88501f16ecfa2604b4e0", size = 263579, upload-time = "2025-10-06T14:49:39.502Z" }, + { url = "https://files.pythonhosted.org/packages/31/22/a0e884d86b5242b5a74cf08e876bdf299e413016b66e55511f7a804a366e/multidict-6.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b2fd74c52accced7e75de26023b7dccee62511a600e62311b918ec5c168fc2a", size = 259654, upload-time = "2025-10-06T14:49:41.32Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/17e10e1b5c5f5a40f2fcbb45953c9b215f8a4098003915e46a93f5fcaa8f/multidict-6.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3e8bfdd0e487acf992407a140d2589fe598238eaeffa3da8448d63a63cd363f8", size = 251511, upload-time = "2025-10-06T14:49:46.021Z" }, + { url = "https://files.pythonhosted.org/packages/e3/9a/201bb1e17e7af53139597069c375e7b0dcbd47594604f65c2d5359508566/multidict-6.7.0-cp312-cp312-win32.whl", hash = "sha256:dd32a49400a2c3d52088e120ee00c1e3576cbff7e10b98467962c74fdb762ed4", size = 41895, upload-time = "2025-10-06T14:49:48.718Z" }, + { url = "https://files.pythonhosted.org/packages/46/e2/348cd32faad84eaf1d20cce80e2bb0ef8d312c55bca1f7fa9865e7770aaf/multidict-6.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:92abb658ef2d7ef22ac9f8bb88e8b6c3e571671534e029359b6d9e845923eb1b", size = 46073, upload-time = "2025-10-06T14:49:50.28Z" }, + { url = "https://files.pythonhosted.org/packages/25/ec/aad2613c1910dce907480e0c3aa306905830f25df2e54ccc9dea450cb5aa/multidict-6.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:490dab541a6a642ce1a9d61a4781656b346a55c13038f0b1244653828e3a83ec", size = 43226, upload-time = "2025-10-06T14:49:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/b7/da/7d22601b625e241d4f23ef1ebff8acfc60da633c9e7e7922e24d10f592b3/multidict-6.7.0-py3-none-any.whl", hash = "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", size = 12317, upload-time = "2025-10-06T14:52:29.272Z" }, +] + +[[package]] +name = "multiprocess" +version = "0.70.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" }, + { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "myst-parser" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "jinja2" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "pyyaml" }, + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a5/9626ba4f73555b3735ad86247a8077d4603aa8628537687c839ab08bfe44/myst_parser-4.0.1.tar.gz", hash = "sha256:5cfea715e4f3574138aecbf7d54132296bfd72bb614d31168f48c477a830a7c4", size = 93985, upload-time = "2025-02-12T10:53:03.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/df/76d0321c3797b54b60fef9ec3bd6f4cfd124b9e422182156a1dd418722cf/myst_parser-4.0.1-py3-none-any.whl", hash = "sha256:9134e88959ec3b5780aedf8a99680ea242869d012e8821db3126d427edc9c95d", size = 84579, upload-time = "2025-02-12T10:53:02.078Z" }, +] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "ninja" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/73/79a0b22fc731989c708068427579e840a6cf4e937fe7ae5c5d0b7356ac22/ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978", size = 242558, upload-time = "2025-08-11T15:10:19.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/74/d02409ed2aa865e051b7edda22ad416a39d81a84980f544f8de717cab133/ninja-1.13.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:fa2a8bfc62e31b08f83127d1613d10821775a0eb334197154c4d6067b7068ff1", size = 310125, upload-time = "2025-08-11T15:09:50.971Z" }, + { url = "https://files.pythonhosted.org/packages/8e/de/6e1cd6b84b412ac1ef327b76f0641aeb5dcc01e9d3f9eee0286d0c34fd93/ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630", size = 177467, upload-time = "2025-08-11T15:09:52.767Z" }, + { url = "https://files.pythonhosted.org/packages/c8/83/49320fb6e58ae3c079381e333575fdbcf1cca3506ee160a2dcce775046fa/ninja-1.13.0-py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:be7f478ff9f96a128b599a964fc60a6a87b9fa332ee1bd44fa243ac88d50291c", size = 187834, upload-time = "2025-08-11T15:09:54.115Z" }, + { url = "https://files.pythonhosted.org/packages/56/c7/ba22748fb59f7f896b609cd3e568d28a0a367a6d953c24c461fe04fc4433/ninja-1.13.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:60056592cf495e9a6a4bea3cd178903056ecb0943e4de45a2ea825edb6dc8d3e", size = 202736, upload-time = "2025-08-11T15:09:55.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/22/d1de07632b78ac8e6b785f41fa9aad7a978ec8c0a1bf15772def36d77aac/ninja-1.13.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1c97223cdda0417f414bf864cfb73b72d8777e57ebb279c5f6de368de0062988", size = 179034, upload-time = "2025-08-11T15:09:57.394Z" }, + { url = "https://files.pythonhosted.org/packages/ed/de/0e6edf44d6a04dabd0318a519125ed0415ce437ad5a1ec9b9be03d9048cf/ninja-1.13.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa", size = 180716, upload-time = "2025-08-11T15:09:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/54/28/938b562f9057aaa4d6bfbeaa05e81899a47aebb3ba6751e36c027a7f5ff7/ninja-1.13.0-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4be9c1b082d244b1ad7ef41eb8ab088aae8c109a9f3f0b3e56a252d3e00f42c1", size = 146843, upload-time = "2025-08-11T15:10:00.046Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fb/d06a3838de4f8ab866e44ee52a797b5491df823901c54943b2adb0389fbb/ninja-1.13.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6739d3352073341ad284246f81339a384eec091d9851a886dfa5b00a6d48b3e2", size = 154402, upload-time = "2025-08-11T15:10:01.657Z" }, + { url = "https://files.pythonhosted.org/packages/31/bf/0d7808af695ceddc763cf251b84a9892cd7f51622dc8b4c89d5012779f06/ninja-1.13.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11be2d22027bde06f14c343f01d31446747dbb51e72d00decca2eb99be911e2f", size = 552388, upload-time = "2025-08-11T15:10:03.349Z" }, + { url = "https://files.pythonhosted.org/packages/9d/70/c99d0c2c809f992752453cce312848abb3b1607e56d4cd1b6cded317351a/ninja-1.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aa45b4037b313c2f698bc13306239b8b93b4680eb47e287773156ac9e9304714", size = 472501, upload-time = "2025-08-11T15:10:04.735Z" }, + { url = "https://files.pythonhosted.org/packages/9f/43/c217b1153f0e499652f5e0766da8523ce3480f0a951039c7af115e224d55/ninja-1.13.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f8e1e8a1a30835eeb51db05cf5a67151ad37542f5a4af2a438e9490915e5b72", size = 638280, upload-time = "2025-08-11T15:10:06.512Z" }, + { url = "https://files.pythonhosted.org/packages/8c/45/9151bba2c8d0ae2b6260f71696330590de5850e5574b7b5694dce6023e20/ninja-1.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:3d7d7779d12cb20c6d054c61b702139fd23a7a964ec8f2c823f1ab1b084150db", size = 642420, upload-time = "2025-08-11T15:10:08.35Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/95752eb635bb8ad27d101d71bef15bc63049de23f299e312878fc21cb2da/ninja-1.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:d741a5e6754e0bda767e3274a0f0deeef4807f1fec6c0d7921a0244018926ae5", size = 585106, upload-time = "2025-08-11T15:10:09.818Z" }, + { url = "https://files.pythonhosted.org/packages/c1/31/aa56a1a286703800c0cbe39fb4e82811c277772dc8cd084f442dd8e2938a/ninja-1.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:e8bad11f8a00b64137e9b315b137d8bb6cbf3086fbdc43bf1f90fd33324d2e96", size = 707138, upload-time = "2025-08-11T15:10:11.366Z" }, + { url = "https://files.pythonhosted.org/packages/34/6f/5f5a54a1041af945130abdb2b8529cbef0cdcbbf9bcf3f4195378319d29a/ninja-1.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4f2a072db3c0f944c32793e91532d8948d20d9ab83da9c0c7c15b5768072200", size = 581758, upload-time = "2025-08-11T15:10:13.295Z" }, + { url = "https://files.pythonhosted.org/packages/95/97/51359c77527d45943fe7a94d00a3843b81162e6c4244b3579fe8fc54cb9c/ninja-1.13.0-py3-none-win32.whl", hash = "sha256:8cfbb80b4a53456ae8a39f90ae3d7a2129f45ea164f43fadfa15dc38c4aef1c9", size = 267201, upload-time = "2025-08-11T15:10:15.158Z" }, + { url = "https://files.pythonhosted.org/packages/29/45/c0adfbfb0b5895aa18cec400c535b4f7ff3e52536e0403602fc1a23f7de9/ninja-1.13.0-py3-none-win_amd64.whl", hash = "sha256:fb8ee8719f8af47fed145cced4a85f0755dd55d45b2bddaf7431fa89803c5f3e", size = 309975, upload-time = "2025-08-11T15:10:16.697Z" }, + { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + +[[package]] +name = "numba" +version = "0.61.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/a0/e21f57604304aa03ebb8e098429222722ad99176a4f979d34af1d1ee80da/numba-0.61.2.tar.gz", hash = "sha256:8750ee147940a6637b80ecf7f95062185ad8726c8c28a2295b8ec1160a196f7d", size = 2820615, upload-time = "2025-04-09T02:58:07.659Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/a0/c6b7b9c615cfa3b98c4c63f4316e3f6b3bbe2387740277006551784218cd/numba-0.61.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2", size = 2776626, upload-time = "2025-04-09T02:57:51.857Z" }, + { url = "https://files.pythonhosted.org/packages/92/4a/fe4e3c2ecad72d88f5f8cd04e7f7cff49e718398a2fac02d2947480a00ca/numba-0.61.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ddce10009bc097b080fc96876d14c051cc0c7679e99de3e0af59014dab7dfe8", size = 2779287, upload-time = "2025-04-09T02:57:53.658Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2d/e518df036feab381c23a624dac47f8445ac55686ec7f11083655eb707da3/numba-0.61.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b1bb509d01f23d70325d3a5a0e237cbc9544dd50e50588bc581ba860c213546", size = 3885928, upload-time = "2025-04-09T02:57:55.206Z" }, + { url = "https://files.pythonhosted.org/packages/10/0f/23cced68ead67b75d77cfcca3df4991d1855c897ee0ff3fe25a56ed82108/numba-0.61.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:48a53a3de8f8793526cbe330f2a39fe9a6638efcbf11bd63f3d2f9757ae345cd", size = 3577115, upload-time = "2025-04-09T02:57:56.818Z" }, + { url = "https://files.pythonhosted.org/packages/68/1d/ddb3e704c5a8fb90142bf9dc195c27db02a08a99f037395503bfbc1d14b3/numba-0.61.2-cp312-cp312-win_amd64.whl", hash = "sha256:97cf4f12c728cf77c9c1d7c23707e4d8fb4632b46275f8f3397de33e5877af18", size = 2831929, upload-time = "2025-04-09T02:57:58.45Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, + { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, + { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, +] + +[[package]] +name = "nv-one-logger-core" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "overrides", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "pydantic", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "strenum", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "toml", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "typing-extensions", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/37/963095797035f371e0db6ea761f5aaccb624fc786af217115b423baeb0e2/nv_one_logger_core-2.3.1.tar.gz", hash = "sha256:cbb2f87604c78b96a302f32d87199902129d76153a73a20f8455a250b3246c1d", size = 52640, upload-time = "2025-10-29T21:11:55.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/c4/ea91554c4fcbff66057f667690101d7a4b965605741350ac661b03fa6c46/nv_one_logger_core-2.3.1-py3-none-any.whl", hash = "sha256:0c8b77bcdac4daa1ea913bf8d4afd2a057bd5526e3654ac39f67caba157341a6", size = 63066, upload-time = "2025-10-29T21:11:52.753Z" }, +] + +[[package]] +name = "nv-one-logger-training-telemetry" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nv-one-logger-core", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "strenum", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "typing-extensions", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/21/016fa067967734d52f1ccf5a2a37a1a65216f2d7053bc2b85872cce956ca/nv_one_logger_training_telemetry-2.3.1.tar.gz", hash = "sha256:8c67940ea71799afaf1f46df3ba2f52f93aea26321c6f1c1d54aae02efc2a4af", size = 44435, upload-time = "2025-10-29T21:21:42.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/15/97e6e4ddfe5fc35bcee74a45b7c33fb73abb83713c7dfa26420b971a86c3/nv_one_logger_training_telemetry-2.3.1-py3-none-any.whl", hash = "sha256:5319443829b59378a498c3c62ac98973e14f31be675c229ff2b14e2fe109aa0b", size = 44140, upload-time = "2025-10-29T21:21:40.72Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.3.14" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/63/684a6f72f52671ea222c12ecde9bdf748a0ba025e2ad3ec374e466c26eb6/nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:93a4e0e386cc7f6e56c822531396de8170ed17068a1e18f987574895044cd8c3", size = 604900717, upload-time = "2025-01-23T17:52:55.486Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/4b01f10069e23c641f116c62fc31e31e8dc361a153175d81561d15c8143b/nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:3f0e05e7293598cf61933258b73e66a160c27d59c4422670bf0b79348c04be44", size = 609620630, upload-time = "2025-01-23T17:55:00.753Z" }, + { url = "https://files.pythonhosted.org/packages/6c/54/fbfa3315b936d3358517f7da5f9f2557c279bf210e5261f0cf66cc0f9832/nvidia_cublas_cu12-12.8.3.14-py3-none-win_amd64.whl", hash = "sha256:9ae5eae500aead01fc4bdfc458209df638b1a3551557ce11a78eea9ece602ae9", size = 578387959, upload-time = "2025-01-23T18:08:00.662Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/99/db44d685f0e257ff0e213ade1964fc459b4a690a73293220e98feb3307cf/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0", size = 590537124, upload-time = "2025-03-07T01:43:53.556Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, + { url = "https://files.pythonhosted.org/packages/70/61/7d7b3c70186fb651d0fbd35b01dbfc8e755f69fd58f817f3d0f642df20c3/nvidia_cublas_cu12-12.8.4.1-py3-none-win_amd64.whl", hash = "sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af", size = 567544208, upload-time = "2025-03-07T01:53:30.535Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.57" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/53/458956a65283c55c22ba40a65745bbe9ff20c10b68ea241bc575e20c0465/nvidia_cuda_cupti_cu12-12.8.57-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff154211724fd824e758ce176b66007b558eea19c9a5135fc991827ee147e317", size = 9526469, upload-time = "2025-01-23T17:47:33.104Z" }, + { url = "https://files.pythonhosted.org/packages/39/6f/3683ecf4e38931971946777d231c2df00dd5c1c4c2c914c42ad8f9f4dca6/nvidia_cuda_cupti_cu12-12.8.57-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e0b2eb847de260739bee4a3f66fac31378f4ff49538ff527a38a01a9a39f950", size = 10237547, upload-time = "2025-01-23T17:47:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2a/cabe033045427beb042b70b394ac3fd7cfefe157c965268824011b16af67/nvidia_cuda_cupti_cu12-12.8.57-py3-none-win_amd64.whl", hash = "sha256:bbed719c52a476958a74cfc42f2b95a3fd6b3fd94eb40134acc4601feb4acac3", size = 7002337, upload-time = "2025-01-23T18:04:35.34Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/1f/b3bd73445e5cb342727fd24fe1f7b748f690b460acadc27ea22f904502c8/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed", size = 9533318, upload-time = "2025-03-07T01:40:10.421Z" }, + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, + { url = "https://files.pythonhosted.org/packages/41/bc/83f5426095d93694ae39fe1311431b5d5a9bb82e48bf0dd8e19be2765942/nvidia_cuda_cupti_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e", size = 7015759, upload-time = "2025-03-07T01:51:11.355Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.61" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/22/32029d4583f7b19cfe75c84399cbcfd23f2aaf41c66fc8db4da460104fff/nvidia_cuda_nvrtc_cu12-12.8.61-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a0fa9c2a21583105550ebd871bd76e2037205d56f33f128e69f6d2a55e0af9ed", size = 88024585, upload-time = "2025-01-23T17:50:10.722Z" }, + { url = "https://files.pythonhosted.org/packages/f1/98/29f98d57fc40d6646337e942d37509c6d5f8abe29012671f7a6eb9978ebe/nvidia_cuda_nvrtc_cu12-12.8.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1f376bf58111ca73dde4fd4df89a462b164602e074a76a2c29c121ca478dcd4", size = 43097015, upload-time = "2025-01-23T17:49:44.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5b/052d05aa068e4752415ad03bac58e852ea8bc17c9321e08546b3f261e47e/nvidia_cuda_nvrtc_cu12-12.8.61-py3-none-win_amd64.whl", hash = "sha256:9c8887bf5e5dffc441018ba8c5dc59952372a6f4806819e8c1f03d62637dbeea", size = 73567440, upload-time = "2025-01-23T18:05:51.036Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d1/e50d0acaab360482034b84b6e27ee83c6738f7d32182b987f9c7a4e32962/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8", size = 43106076, upload-time = "2025-03-07T01:41:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/45/51/52a3d84baa2136cc8df15500ad731d74d3a1114d4c123e043cb608d4a32b/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909", size = 73586838, upload-time = "2025-03-07T01:52:13.483Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.57" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/9d/e77ec4227e70c6006195bdf410370f2d0e5abfa2dc0d1d315cacd57c5c88/nvidia_cuda_runtime_cu12-12.8.57-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:534ccebd967b6a44292678fa5da4f00666029cb2ed07a79515ea41ef31fe3ec7", size = 965264, upload-time = "2025-01-23T17:47:11.759Z" }, + { url = "https://files.pythonhosted.org/packages/16/f6/0e1ef31f4753a44084310ba1a7f0abaf977ccd810a604035abb43421c057/nvidia_cuda_runtime_cu12-12.8.57-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75342e28567340b7428ce79a5d6bb6ca5ff9d07b69e7ce00d2c7b4dc23eff0be", size = 954762, upload-time = "2025-01-23T17:47:22.21Z" }, + { url = "https://files.pythonhosted.org/packages/16/ee/52508c74bee2a3de8d59c6fd9af4ca2f216052fa2bc916da3a6a7bb998af/nvidia_cuda_runtime_cu12-12.8.57-py3-none-win_amd64.whl", hash = "sha256:89be637e3ee967323865b85e0f147d75f9a5bd98360befa37481b02dd57af8f5", size = 944309, upload-time = "2025-01-23T18:04:23.143Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/75/f865a3b236e4647605ea34cc450900854ba123834a5f1598e160b9530c3a/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d", size = 965265, upload-time = "2025-03-07T01:39:43.533Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, + { url = "https://files.pythonhosted.org/packages/30/a5/a515b7600ad361ea14bfa13fb4d6687abf500adc270f19e89849c0590492/nvidia_cuda_runtime_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8", size = 944318, upload-time = "2025-03-07T01:51:01.794Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.7.1.26" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.3.14", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/2e/ec5dda717eeb1de3afbbbb611ca556f9d6d057470759c6abd36d72f0063b/nvidia_cudnn_cu12-9.7.1.26-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:848a61d40ef3b32bd4e1fadb599f0cf04a4b942fbe5fb3be572ad75f9b8c53ef", size = 725862213, upload-time = "2025-02-06T22:14:57.169Z" }, + { url = "https://files.pythonhosted.org/packages/25/dc/dc825c4b1c83b538e207e34f48f86063c88deaa35d46c651c7c181364ba2/nvidia_cudnn_cu12-9.7.1.26-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:6d011159a158f3cfc47bf851aea79e31bcff60d530b70ef70474c84cac484d07", size = 726851421, upload-time = "2025-02-06T22:18:29.812Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ea/636cda41b3865caa0d43c34f558167304acde3d2c5f6c54c00a550e69ecd/nvidia_cudnn_cu12-9.7.1.26-py3-none-win_amd64.whl", hash = "sha256:7b805b9a4cf9f3da7c5f4ea4a9dff7baf62d1a612d6154a7e0d2ea51ed296241", size = 715962100, upload-time = "2025-02-06T22:21:32.431Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/41/e79269ce215c857c935fd86bcfe91a451a584dfc27f1e068f568b9ad1ab7/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8", size = 705026878, upload-time = "2025-06-06T21:52:51.348Z" }, + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, + { url = "https://files.pythonhosted.org/packages/3d/90/0bd6e586701b3a890fd38aa71c387dab4883d619d6e5ad912ccbd05bfd67/nvidia_cudnn_cu12-9.10.2.21-py3-none-win_amd64.whl", hash = "sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e", size = 692992268, upload-time = "2025-06-06T21:55:18.114Z" }, +] + +[[package]] +name = "nvidia-cudnn-frontend" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d9/f58ed6292c9396f7422812a0a2d9f80cc5a623ea6c758bcb3d34d4795bb8/nvidia_cudnn_frontend-1.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de0c473f32d705abcf14f351615f7ffbeed7320e3499cf2195ae5689652a2592", size = 1917620, upload-time = "2025-12-20T00:27:46.179Z" }, + { url = "https://files.pythonhosted.org/packages/db/eb/c641135632bd2afc21339aadee96af4c5db1460dfa07ca74836de75a590f/nvidia_cudnn_frontend-1.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c913c87fca691a91385287f2587575531933acfebc85c33dbcecb191886c7a53", size = 2038994, upload-time = "2025-12-20T00:25:18.9Z" }, + { url = "https://files.pythonhosted.org/packages/82/49/a92da03eb43bde90be770a43666c5ab26b4f8b15f6e46c4b0b0e84f37994/nvidia_cudnn_frontend-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0d4cfd03961592108abd1ba246e43c8bb7540aed984df860256d0bff181de98", size = 1441271, upload-time = "2025-12-20T00:29:52.056Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.41" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/95/6157cb45a49f5090a470de42353a22a0ed5b13077886dca891b4b0e350fe/nvidia_cufft_cu12-11.3.3.41-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68509dcd7e3306e69d0e2d8a6d21c8b25ed62e6df8aac192ce752f17677398b5", size = 193108626, upload-time = "2025-01-23T17:55:49.192Z" }, + { url = "https://files.pythonhosted.org/packages/ac/26/b53c493c38dccb1f1a42e1a21dc12cba2a77fbe36c652f7726d9ec4aba28/nvidia_cufft_cu12-11.3.3.41-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:da650080ab79fcdf7a4b06aa1b460e99860646b176a43f6208099bdc17836b6a", size = 193118795, upload-time = "2025-01-23T17:56:30.536Z" }, + { url = "https://files.pythonhosted.org/packages/32/f3/f6248aa119c2726b1bdd02d472332cae274133bd32ca5fa8822efb0c308c/nvidia_cufft_cu12-11.3.3.41-py3-none-win_amd64.whl", hash = "sha256:f9760612886786601d27a0993bb29ce1f757e6b8b173499d0ecfa850d31b50f8", size = 192216738, upload-time = "2025-01-23T18:08:51.102Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/bc/7771846d3a0272026c416fbb7e5f4c1f146d6d80704534d0b187dd6f4800/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a", size = 193109211, upload-time = "2025-03-07T01:44:56.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ec/ce1629f1e478bb5ccd208986b5f9e0316a78538dd6ab1d0484f012f8e2a1/nvidia_cufft_cu12-11.3.3.83-py3-none-win_amd64.whl", hash = "sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7", size = 192216559, upload-time = "2025-03-07T01:53:57.106Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.0.11" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/9c/1f3264d0a84c8a031487fb7f59780fc78fa6f1c97776233956780e3dc3ac/nvidia_cufile_cu12-1.13.0.11-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:483f434c541806936b98366f6d33caef5440572de8ddf38d453213729da3e7d4", size = 1197801, upload-time = "2025-01-23T17:57:07.247Z" }, + { url = "https://files.pythonhosted.org/packages/35/80/f6a0fc90ab6fa4ac916f3643e5b620fd19724626c59ae83b74f5efef0349/nvidia_cufile_cu12-1.13.0.11-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:2acbee65dc2eaf58331f0798c5e6bcdd790c4acb26347530297e63528c9eba5d", size = 1120660, upload-time = "2025-01-23T17:56:56.608Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f5/5607710447a6fe9fd9b3283956fceeee8a06cda1d2f56ce31371f595db2a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a", size = 1120705, upload-time = "2025-03-07T01:45:41.434Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.55" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/13/bbcf48e2f8a6a9adef58f130bc968810528a4e66bbbe62fad335241e699f/nvidia_curand_cu12-10.3.9.55-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b6bb90c044fa9b07cedae2ef29077c4cf851fb6fdd6d862102321f359dca81e9", size = 63623836, upload-time = "2025-01-23T17:57:22.319Z" }, + { url = "https://files.pythonhosted.org/packages/bd/fc/7be5d0082507269bb04ac07cc614c84b78749efb96e8cf4100a8a1178e98/nvidia_curand_cu12-10.3.9.55-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8387d974240c91f6a60b761b83d4b2f9b938b7e0b9617bae0f0dafe4f5c36b86", size = 63618038, upload-time = "2025-01-23T17:57:41.838Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f0/91252f3cffe3f3c233a8e17262c21b41534652edfe783c1e58ea1c92c115/nvidia_curand_cu12-10.3.9.55-py3-none-win_amd64.whl", hash = "sha256:570d82475fe7f3d8ed01ffbe3b71796301e0e24c98762ca018ff8ce4f5418e1f", size = 62761446, upload-time = "2025-01-23T18:09:21.663Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/5e/92aa15eca622a388b80fbf8375d4760738df6285b1e92c43d37390a33a9a/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd", size = 63625754, upload-time = "2025-03-07T01:46:10.735Z" }, + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, + { url = "https://files.pythonhosted.org/packages/b9/75/70c05b2f3ed5be3bb30b7102b6eb78e100da4bbf6944fd6725c012831cab/nvidia_curand_cu12-10.3.9.90-py3-none-win_amd64.whl", hash = "sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec", size = 62765309, upload-time = "2025-03-07T01:54:20.478Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.2.55" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.3.14", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.7.53", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/ce/4214a892e804b20bf66d04f04a473006fc2d3dac158160ef85f1bc906639/nvidia_cusolver_cu12-11.7.2.55-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:0fd9e98246f43c15bee5561147ad235dfdf2d037f5d07c9d41af3f7f72feb7cc", size = 260094827, upload-time = "2025-01-23T17:58:17.586Z" }, + { url = "https://files.pythonhosted.org/packages/c2/08/953675873a136d96bb12f93b49ba045d1107bc94d2551c52b12fa6c7dec3/nvidia_cusolver_cu12-11.7.2.55-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4d1354102f1e922cee9db51920dba9e2559877cf6ff5ad03a00d853adafb191b", size = 260373342, upload-time = "2025-01-23T17:58:56.406Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/e0e6f8b7aecd13e0f9e937d116fb3211329a0a92b9bea9624b1368de307a/nvidia_cusolver_cu12-11.7.2.55-py3-none-win_amd64.whl", hash = "sha256:a5a516c55da5c5aba98420d9bc9bcab18245f21ec87338cc1f930eb18dd411ac", size = 249600787, upload-time = "2025-01-23T18:10:07.641Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/32/f7cd6ce8a7690544d084ea21c26e910a97e077c9b7f07bf5de623ee19981/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0", size = 267229841, upload-time = "2025-03-07T01:46:54.356Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, + { url = "https://files.pythonhosted.org/packages/13/c0/76ca8551b8a84146ffa189fec81c26d04adba4bc0dbe09cd6e6fd9b7de04/nvidia_cusolver_cu12-11.7.3.90-py3-none-win_amd64.whl", hash = "sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34", size = 256720438, upload-time = "2025-03-07T01:54:39.898Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.7.53" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/a2/313db0453087f5324a5900380ca2e57e050c8de76f407b5e11383dc762ae/nvidia_cusparse_cu12-12.5.7.53-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d869c6146ca80f4305b62e02d924b4aaced936f8173e3cef536a67eed2a91af1", size = 291963692, upload-time = "2025-01-23T17:59:40.325Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ab/31e8149c66213b846c082a3b41b1365b831f41191f9f40c6ddbc8a7d550e/nvidia_cusparse_cu12-12.5.7.53-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3c1b61eb8c85257ea07e9354606b26397612627fdcd327bfd91ccf6155e7c86d", size = 292064180, upload-time = "2025-01-23T18:00:23.233Z" }, + { url = "https://files.pythonhosted.org/packages/7c/48/64b01653919a3d1d9b5117c156806ab0db8312c7496ff646477a5c1545bf/nvidia_cusparse_cu12-12.5.7.53-py3-none-win_amd64.whl", hash = "sha256:82c201d6781bacf6bb7c654f0446728d0fe596dfdd82ef4a04c204ce3e107441", size = 288767123, upload-time = "2025-01-23T18:11:01.543Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/cd777c4109681367721b00a106f491e0d0d15cfa1fd59672ce580ce42a97/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc", size = 288117129, upload-time = "2025-03-07T01:47:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, + { url = "https://files.pythonhosted.org/packages/62/07/f3b2ad63f8e3d257a599f422ae34eb565e70c41031aecefa3d18b62cabd1/nvidia_cusparse_cu12-12.5.8.93-py3-none-win_amd64.whl", hash = "sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd", size = 284937404, upload-time = "2025-03-07T01:55:07.742Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/da/4de092c61c6dea1fc9c936e69308a02531d122e12f1f649825934ad651b5/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1", size = 156402859, upload-time = "2024-10-16T02:23:17.184Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9a/72ef35b399b0e183bc2e8f6f558036922d453c4d8237dab26c666a04244b/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46", size = 156785796, upload-time = "2024-10-15T21:29:17.709Z" }, + { url = "https://files.pythonhosted.org/packages/46/3e/9e1e394a02a06f694be2c97bbe47288bb7c90ea84c7e9cf88f7b28afe165/nvidia_cusparselt_cu12-0.6.3-py3-none-win_amd64.whl", hash = "sha256:3b325bcbd9b754ba43df5a311488fca11a6b5dc3d11df4d190c000cf1a0765c7", size = 155595972, upload-time = "2024-10-15T22:58:35.426Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/b9/598f6ff36faaece4b3c50d26f50e38661499ff34346f00e057760b35cc9d/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5", size = 283835557, upload-time = "2025-02-26T00:16:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d8/a6b0d0d0c2435e9310f3e2bb0d9c9dd4c33daef86aa5f30b3681defd37ea/nvidia_cusparselt_cu12-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075", size = 271020911, upload-time = "2025-02-26T00:14:47.204Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl" +version = "4.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-vllm' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang')" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/d4/7c5ef53ccf75d7f99a9ea29cae9f9c0233229b75b3b22f85a4ef4f52e6ab/nvidia_cutlass_dsl-4.3.3-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3278526f54bddd920d8e539771e5820c6166c549a1e67813375025f39417dec6", size = 58734009, upload-time = "2025-12-10T09:23:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/88/a8/a27562194cc4182c67793cd21c5dbf9468cd5a49c775a487153c6f28364c/nvidia_cutlass_dsl-4.3.3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f2b25816b8bb8bc332bcbf6fc341347b5d728344cf185c65af0dd73e8503d5c7", size = 58596724, upload-time = "2025-12-10T11:01:07.228Z" }, +] + +[[package]] +name = "nvidia-ml-py" +version = "13.590.44" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/23/3871537f204aee823c574ba25cbeb08cae779979d4d43c01adddda00bab9/nvidia_ml_py-13.590.44.tar.gz", hash = "sha256:b358c7614b0fdeea4b95f046f1c90123bfe25d148ab93bb1c00248b834703373", size = 49737, upload-time = "2025-12-08T14:41:10.872Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/47/4c822bd37a008e72fd5a0eae33524ae3ac97b13f7030f63bae1728b8957e/nvidia_ml_py-13.590.44-py3-none-any.whl", hash = "sha256:18feb54eca7d0e3cdc8d1a040a771eda72d9ec3148e5443087970dbfd7377ecc", size = 50683, upload-time = "2025-12-08T14:41:09.597Z" }, +] + +[[package]] +name = "nvidia-modelopt" +version = "0.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ninja" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "nvidia-ml-py" }, + { name = "packaging" }, + { name = "pulp" }, + { name = "pydantic" }, + { name = "regex" }, + { name = "rich" }, + { name = "safetensors" }, + { name = "scipy" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchprofile" }, + { name = "tqdm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/4a/4b4c339637fdbd54bc98b92c87c8b22f5efee05ca9e31e40a8d49ee66187/nvidia_modelopt-0.40.0-py3-none-any.whl", hash = "sha256:0315f53aef014b902866e427038db5803e3c6787a8e1f09c3650031550885051", size = 901421, upload-time = "2025-12-12T10:35:28.506Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.26.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/5b/ca2f213f637305633814ae8c36b153220e40a07ea001966dcd87391f3acb/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522", size = 291671495, upload-time = "2025-03-13T00:30:07.805Z" }, + { url = "https://files.pythonhosted.org/packages/67/ca/f42388aed0fddd64ade7493dbba36e1f534d4e6fdbdd355c6a90030ae028/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6", size = 201319755, upload-time = "2025-03-13T00:29:55.296Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/1c/857979db0ef194ca5e21478a0612bcdbbe59458d7694361882279947b349/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31432ad4d1fb1004eb0c56203dc9bc2178a1ba69d1d9e02d64a6938ab5e40e7a", size = 322400625, upload-time = "2025-06-26T04:11:04.496Z" }, + { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.61" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/f8/9d85593582bd99b8d7c65634d2304780aefade049b2b94d96e44084be90b/nvidia_nvjitlink_cu12-12.8.61-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:45fd79f2ae20bd67e8bc411055939049873bfd8fac70ff13bd4865e0b9bdab17", size = 39243473, upload-time = "2025-01-23T18:03:03.509Z" }, + { url = "https://files.pythonhosted.org/packages/af/53/698f3758f48c5fcb1112721e40cc6714da3980d3c7e93bae5b29dafa9857/nvidia_nvjitlink_cu12-12.8.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b80ecab31085dda3ce3b41d043be0ec739216c3fc633b8abe212d5a30026df0", size = 38374634, upload-time = "2025-01-23T18:02:35.812Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c6/0d1b2bfeb2ef42c06db0570c4d081e5cde4450b54c09e43165126cfe6ff6/nvidia_nvjitlink_cu12-12.8.61-py3-none-win_amd64.whl", hash = "sha256:1166a964d25fdc0eae497574d38824305195a5283324a21ccb0ce0c802cbf41c", size = 268514099, upload-time = "2025-01-23T18:12:33.874Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a2/8cee5da30d13430e87bf99bb33455d2724d0a4a9cb5d7926d80ccb96d008/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7", size = 38386204, upload-time = "2025-03-07T01:49:43.612Z" }, + { url = "https://files.pythonhosted.org/packages/ed/d7/34f02dad2e30c31b10a51f6b04e025e5dd60e5f936af9045a9b858a05383/nvidia_nvjitlink_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f", size = 268553710, upload-time = "2025-03-07T01:56:24.13Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu12" +version = "3.3.20" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/3dd98852568fb845ec1f7902c90a22b240fe1cbabda411ccedf2fd737b7b/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b0b960da3842212758e4fa4696b94f129090b30e5122fea3c5345916545cff0", size = 124484616, upload-time = "2025-08-04T20:24:59.172Z" }, + { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.55" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/e8/ae6ecbdade8bb9174d75db2b302c57c1c27d9277d6531c62aafde5fb32a3/nvidia_nvtx_cu12-12.8.55-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c38405335fbc0f0bf363eaeaeb476e8dfa8bae82fada41d25ace458b9ba9f3db", size = 91103, upload-time = "2025-01-23T17:50:24.664Z" }, + { url = "https://files.pythonhosted.org/packages/8d/cd/0e8c51b2ae3a58f054f2e7fe91b82d201abfb30167f2431e9bd92d532f42/nvidia_nvtx_cu12-12.8.55-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dd0780f1a55c21d8e06a743de5bd95653de630decfff40621dbde78cc307102", size = 89896, upload-time = "2025-01-23T17:50:44.487Z" }, + { url = "https://files.pythonhosted.org/packages/e5/14/84d46e62bfde46dd20cfb041e0bb5c2ec454fd6a384696e7fa3463c5bb59/nvidia_nvtx_cu12-12.8.55-py3-none-win_amd64.whl", hash = "sha256:9022681677aef1313458f88353ad9c0d2fbbe6402d6b07c9f00ba0e3ca8774d3", size = 56435, upload-time = "2025-01-23T18:06:06.268Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/c0/1b303feea90d296f6176f32a2a70b5ef230f9bdeb3a72bddb0dc922dc137/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615", size = 91161, upload-time = "2025-03-07T01:42:23.922Z" }, + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/9f/99/4c9c0c329bf9fc125008c3b54c7c94c0023518d06fc025ae36431375e1fe/nvidia_nvtx_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e", size = 56492, upload-time = "2025-03-07T01:52:24.69Z" }, +] + +[[package]] +name = "nvidia-resiliency-ext" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "nv-one-logger-core", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "nv-one-logger-training-telemetry", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "nvidia-ml-py", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "packaging", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "psutil", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "pyyaml", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/73/232d9f25558f3c6165ff1d15c980a434b47c13e8f527f999cd265859abcf/nvidia_resiliency_ext-0.5.0-cp312-cp312-manylinux_2_39_aarch64.whl", hash = "sha256:81e3d827885e90bed369e67f76dda6709dd4073c2e5fa1228df85d6987cee495", size = 403317, upload-time = "2025-11-13T21:31:24.603Z" }, + { url = "https://files.pythonhosted.org/packages/44/89/4d7f39416aa3be72ee9f1260a7af56af40f2570f5add1e039d96279a8764/nvidia_resiliency_ext-0.5.0-cp312-cp312-manylinux_2_39_x86_64.whl", hash = "sha256:eb720cd25feabef07f971d4051c7bcac2f9ec73642a9031953d2663307950cb9", size = 407963, upload-time = "2025-11-13T21:30:28.998Z" }, +] + +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + +[[package]] +name = "omegaconf" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, +] + +[[package]] +name = "onnx" +version = "1.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/bf/824b13b7ea14c2d374b48a296cfa412442e5559326fbab5441a4fcb68924/onnx-1.20.0.tar.gz", hash = "sha256:1a93ec69996b4556062d552ed1aa0671978cfd3c17a40bf4c89a1ae169c6a4ad", size = 12049527, upload-time = "2025-12-01T18:14:34.679Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/19/2caa972a31014a8cb4525f715f2a75d93caef9d4b9da2809cc05d0489e43/onnx-1.20.0-cp312-abi3-macosx_12_0_universal2.whl", hash = "sha256:31efe37d7d1d659091f34ddd6a31780334acf7c624176832db9a0a8ececa8fb5", size = 18340913, upload-time = "2025-12-01T18:14:00.477Z" }, + { url = "https://files.pythonhosted.org/packages/78/bb/b98732309f2f6beb4cdcf7b955d7bbfd75a191185370ee21233373db381e/onnx-1.20.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d75da05e743eb9a11ff155a775cae5745e71f1cd0ca26402881b8f20e8d6e449", size = 17896118, upload-time = "2025-12-01T18:14:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/84/a7/38aa564871d062c11538d65c575af9c7e057be880c09ecbd899dd1abfa83/onnx-1.20.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02e0d72ab09a983fce46686b155a5049898558d9f3bc6e8515120d6c40666318", size = 18115415, upload-time = "2025-12-01T18:14:06.261Z" }, + { url = "https://files.pythonhosted.org/packages/3b/17/a600b62cf4ad72976c66f83ce9e324205af434706ad5ec0e35129e125aef/onnx-1.20.0-cp312-abi3-win32.whl", hash = "sha256:392ca68b34b97e172d33b507e1e7bfdf2eea96603e6e7ff109895b82ff009dc7", size = 16363019, upload-time = "2025-12-01T18:14:09.16Z" }, + { url = "https://files.pythonhosted.org/packages/9c/3b/5146ba0a89f73c026bb468c49612bab8d005aa28155ebf06cf5f2eb8d36c/onnx-1.20.0-cp312-abi3-win_amd64.whl", hash = "sha256:259b05758d41645f5545c09f887187662b350d40db8d707c33c94a4f398e1733", size = 16485934, upload-time = "2025-12-01T18:14:13.046Z" }, + { url = "https://files.pythonhosted.org/packages/f3/bc/d251b97395e721b3034e9578d4d4d9fb33aac4197ae16ce8c7ed79a26dce/onnx-1.20.0-cp312-abi3-win_arm64.whl", hash = "sha256:2d25a9e1fde44bc69988e50e2211f62d6afcd01b0fd6dfd23429fd978a35d32f", size = 16444946, upload-time = "2025-12-01T18:14:15.801Z" }, +] + +[[package]] +name = "onnx-ir" +version = "0.1.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "onnx" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/c2/6db31dc3132e540076f15ed0cdf4a8db7ab75557f4d6c19eda655cac666e/onnx_ir-0.1.13.tar.gz", hash = "sha256:e08f00d30579bdbff2152692a6f1bc1f0523d3321ac6348aadcd40595e56231e", size = 115872, upload-time = "2025-12-17T18:03:13.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/b6/f60fd79ff5bc617d49db1378eb7c4c315b21b786502674e4a2d48e64491a/onnx_ir-0.1.13-py3-none-any.whl", hash = "sha256:2791493d1529fdbea60c257dc7bc0933dc812e6d68f4976d8b59aa7b4c2de8cf", size = 133063, upload-time = "2025-12-17T18:03:12.268Z" }, +] + +[[package]] +name = "onnxscript" +version = "0.5.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "onnx" }, + { name = "onnx-ir" }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/f8/358a7d982ea51bc1b0c264f29c08adf096c62ba9f258ba13c954b41c46f5/onnxscript-0.5.7.tar.gz", hash = "sha256:480d572451bc233ed7f742b5005cb0c899594b2fdc28e15167dab26f7fd777ad", size = 596306, upload-time = "2025-12-16T20:47:15.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/ec/1656ea93be1e50baf429c20603dce249fa3571f3a180407cee79b1afa013/onnxscript-0.5.7-py3-none-any.whl", hash = "sha256:f94a66059c56d13b44908e9b7fd9dae4b4faa6681c784f3fd4c29cfa863e454e", size = 693353, upload-time = "2025-12-16T20:47:17.897Z" }, +] + +[[package]] +name = "open-clip-torch" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ftfy" }, + { name = "huggingface-hub" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "timm" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.24.0", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.24.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/46/fb8be250fa7fcfc56fbeb41583645e18d868268f67fbbbeb8ed62a8ff18a/open_clip_torch-3.2.0.tar.gz", hash = "sha256:62b7743012ccc40fb7c64819fa762fba0a13dd74585ac733babe58c2974c2506", size = 1502853, upload-time = "2025-09-21T17:32:08.289Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/91/397327cc1597fa317942cc15bef414175eee4b3c2263b34407c57f3521f9/open_clip_torch-3.2.0-py3-none-any.whl", hash = "sha256:e1f5b3ecbadb6d8ea64b1f887db23efee9739e7c0d0075a8a2a3cabae8fed8d1", size = 1546677, upload-time = "2025-09-21T17:32:06.269Z" }, +] + +[[package]] +name = "openai" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/b1/12fe1c196bea326261718eb037307c1c1fe1dedc2d2d4de777df822e6238/openai-2.14.0.tar.gz", hash = "sha256:419357bedde9402d23bf8f2ee372fca1985a73348debba94bddff06f19459952", size = 626938, upload-time = "2025-12-19T03:28:45.742Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/4b/7c1a00c2c3fbd004253937f7520f692a9650767aa73894d7a34f0d65d3f4/openai-2.14.0-py3-none-any.whl", hash = "sha256:7ea40aca4ffc4c4a776e77679021b47eec1160e341f42ae086ba949c9dcc9183", size = 1067558, upload-time = "2025-12-19T03:28:43.727Z" }, +] + +[[package]] +name = "openai-harmony" +version = "0.0.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/92/2d038d096f29179c7c9571b431f9e739f87a487121901725e23fe338dd9d/openai_harmony-0.0.8.tar.gz", hash = "sha256:6e43f98e6c242fa2de6f8ea12eab24af63fa2ed3e89c06341fb9d92632c5cbdf", size = 284777, upload-time = "2025-11-05T19:07:06.727Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/c6/2502f416d46be3ec08bb66d696cccffb57781a499e3ff2e4d7c174af4e8f/openai_harmony-0.0.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:029ec25ca74abe48fdb58eb9fdd2a8c1618581fc33ce8e5653f8a1ffbfbd9326", size = 2627806, upload-time = "2025-11-05T19:06:57.063Z" }, + { url = "https://files.pythonhosted.org/packages/d3/d2/ce6953ca87db9cae3e775024184da7d1c5cb88cead19a2d75b42f00a959c/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4f709815924ec325b9a890e6ab2bbb0ceec8e319a4e257328eb752cf36b2efc", size = 2948463, upload-time = "2025-11-05T19:06:48.17Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4c/b553c9651662d6ce102ca7f3629d268b23df1abe5841e24bed81e8a8e949/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5cfcfd963b50a41fc656c84d3440ca6eecdccd6c552158ce790b8f2e33dfb5a9", size = 2704083, upload-time = "2025-11-05T19:06:50.205Z" }, + { url = "https://files.pythonhosted.org/packages/9b/af/4eec8f9ab9c27bcdb444460c72cf43011d176fc44c79d6e113094ca1e152/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a3a16972aa1cee38ea958470cd04ac9a2d5ac38fdcf77ab686611246220c158", size = 2959765, upload-time = "2025-11-05T19:06:53.62Z" }, + { url = "https://files.pythonhosted.org/packages/11/3c/33f3374e4624e0e776f6b13b73c45a7ead7f9c4529f8369ed5bfcaa30cac/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4d5cfa168e74d08f8ba6d58a7e49bc7daef4d58951ec69b66b0d56f4927a68d", size = 3427031, upload-time = "2025-11-05T19:06:51.829Z" }, + { url = "https://files.pythonhosted.org/packages/25/3f/1a192b93bb47c6b44cd98ba8cc1d3d2a9308f1bb700c3017e6352da11bda/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c007d277218a50db8839e599ed78e0fffe5130f614c3f6d93ae257f282071a29", size = 2953260, upload-time = "2025-11-05T19:06:55.406Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f8/93b582cad3531797c3db7c2db5400fd841538ccddfd9f5e3df61be99a630/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8565d4f5a0638da1bffde29832ed63c9e695c558611053add3b2dc0b56c92dbc", size = 3127044, upload-time = "2025-11-05T19:06:59.553Z" }, + { url = "https://files.pythonhosted.org/packages/1d/10/4327dbf87f75ae813405fd9a9b4a5cde63d506ffed0a096a440a4cabd89c/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:cbaa3bda75ef0d8836e1f8cc84af62f971b1d756d740efc95c38c3e04c0bfde2", size = 2932931, upload-time = "2025-11-05T19:07:01.437Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c8/1774eec4f6f360ef57618fb8f52e3d3af245b2491bd0297513aa09eec04b/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:772922a9bd24e133950fad71eb1550836f415a88e8c77870e12d0c3bd688ddc2", size = 2996140, upload-time = "2025-11-05T19:07:03.438Z" }, + { url = "https://files.pythonhosted.org/packages/60/c3/3d1e01e2dba517a91760e4a03e4f20ffc75039a6fe584d0e6f9b5c78fd15/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:007b0476a1f331f8130783f901f1da6f5a7057af1a4891f1b6a31dec364189b5", size = 3205080, upload-time = "2025-11-05T19:07:05.078Z" }, + { url = "https://files.pythonhosted.org/packages/14/63/119de431572d7c70a7bf1037034a9be6ed0a7502a7498ba7302bca5b3242/openai_harmony-0.0.8-cp38-abi3-win32.whl", hash = "sha256:a9b5f893326b28d9e935ade14b4f655f5a840942473bc89b201c25f7a15af9cf", size = 2082457, upload-time = "2025-11-05T19:07:09.631Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/c83cf5a206c263ee70448a5ae4264682555f4d0b5bed0d2cc6ca1108103d/openai_harmony-0.0.8-cp38-abi3-win_amd64.whl", hash = "sha256:39d44f0d8f466bd56698e7ead708bead3141e27b9b87e3ab7d5a6d0e4a869ee5", size = 2438369, upload-time = "2025-11-05T19:07:08.1Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "4.11.0.86" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929, upload-time = "2025-01-16T13:53:40.22Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/53/2c50afa0b1e05ecdb4603818e85f7d174e683d874ef63a6abe3ac92220c8/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:48128188ade4a7e517237c8e1e11a9cdf5c282761473383e77beb875bb1e61ca", size = 37326460, upload-time = "2025-01-16T13:52:57.015Z" }, + { url = "https://files.pythonhosted.org/packages/3b/43/68555327df94bb9b59a1fd645f63fafb0762515344d2046698762fc19d58/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81", size = 56723330, upload-time = "2025-01-16T13:55:45.731Z" }, + { url = "https://files.pythonhosted.org/packages/45/be/1438ce43ebe65317344a87e4b150865c5585f4c0db880a34cdae5ac46881/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6efabcaa9df731f29e5ea9051776715b1bdd1845d7c9530065c7951d2a2899eb", size = 29487060, upload-time = "2025-01-16T13:51:59.625Z" }, + { url = "https://files.pythonhosted.org/packages/dd/5c/c139a7876099916879609372bfa513b7f1257f7f1a908b0bdc1c2328241b/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b", size = 49969856, upload-time = "2025-01-16T13:53:29.654Z" }, + { url = "https://files.pythonhosted.org/packages/95/dd/ed1191c9dc91abcc9f752b499b7928aacabf10567bb2c2535944d848af18/opencv_python_headless-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b", size = 29324425, upload-time = "2025-01-16T13:52:49.048Z" }, + { url = "https://files.pythonhosted.org/packages/86/8a/69176a64335aed183529207ba8bc3d329c2999d852b4f3818027203f50e6/opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca", size = 39402386, upload-time = "2025-01-16T13:52:56.418Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "4.12.0.88" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/63/6861102ec149c3cd298f4d1ea7ce9d6adbc7529221606ff1dab991a19adb/opencv-python-headless-4.12.0.88.tar.gz", hash = "sha256:cfdc017ddf2e59b6c2f53bc12d74b6b0be7ded4ec59083ea70763921af2b6c09", size = 95379675, upload-time = "2025-07-07T09:21:06.815Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/7d/414e243c5c8216a5277afd104a319cc1291c5e23f5eeef512db5629ee7f4/opencv_python_headless-4.12.0.88-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:1e58d664809b3350c1123484dd441e1667cd7bed3086db1b9ea1b6f6cb20b50e", size = 37877864, upload-time = "2025-07-07T09:14:41.693Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/7e162714beed1cd5e7b5eb66fcbcba2f065c51b1d9da2463024c84d2f7c0/opencv_python_headless-4.12.0.88-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:365bb2e486b50feffc2d07a405b953a8f3e8eaa63865bc650034e5c71e7a5154", size = 57326608, upload-time = "2025-07-07T09:14:51.885Z" }, + { url = "https://files.pythonhosted.org/packages/69/4e/116720df7f1f7f3b59abc608ca30fbec9d2b3ae810afe4e4d26483d9dfa0/opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:aeb4b13ecb8b4a0beb2668ea07928160ea7c2cd2d9b5ef571bbee6bafe9cc8d0", size = 33145800, upload-time = "2025-07-07T09:15:00.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/53/e19c21e0c4eb1275c3e2c97b081103b6dfb3938172264d283a519bf728b9/opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:236c8df54a90f4d02076e6f9c1cc763d794542e886c576a6fee46ec8ff75a7a9", size = 54023419, upload-time = "2025-07-07T09:15:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9c/a76fd5414de6ec9f21f763a600058a0c3e290053cea87e0275692b1375c0/opencv_python_headless-4.12.0.88-cp37-abi3-win32.whl", hash = "sha256:fde2cf5c51e4def5f2132d78e0c08f9c14783cd67356922182c6845b9af87dbd", size = 30225230, upload-time = "2025-07-07T09:15:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/f2/35/0858e9e71b36948eafbc5e835874b63e515179dc3b742cbe3d76bc683439/opencv_python_headless-4.12.0.88-cp37-abi3-win_amd64.whl", hash = "sha256:86b413bdd6c6bf497832e346cd5371995de148e579b9774f8eba686dee3f5528", size = 38923559, upload-time = "2025-07-07T09:15:25.229Z" }, +] + +[[package]] +name = "orjson" +version = "3.11.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/b8/333fdb27840f3bf04022d21b654a35f58e15407183aeb16f3b41aa053446/orjson-3.11.5.tar.gz", hash = "sha256:82393ab47b4fe44ffd0a7659fa9cfaacc717eb617c93cde83795f14af5c2e9d5", size = 5972347, upload-time = "2025-12-06T15:55:39.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a4/8052a029029b096a78955eadd68ab594ce2197e24ec50e6b6d2ab3f4e33b/orjson-3.11.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:334e5b4bff9ad101237c2d799d9fd45737752929753bf4faf4b207335a416b7d", size = 245347, upload-time = "2025-12-06T15:54:22.061Z" }, + { url = "https://files.pythonhosted.org/packages/64/67/574a7732bd9d9d79ac620c8790b4cfe0717a3d5a6eb2b539e6e8995e24a0/orjson-3.11.5-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:ff770589960a86eae279f5d8aa536196ebda8273a2a07db2a54e82b93bc86626", size = 129435, upload-time = "2025-12-06T15:54:23.615Z" }, + { url = "https://files.pythonhosted.org/packages/52/8d/544e77d7a29d90cf4d9eecd0ae801c688e7f3d1adfa2ebae5e1e94d38ab9/orjson-3.11.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed24250e55efbcb0b35bed7caaec8cedf858ab2f9f2201f17b8938c618c8ca6f", size = 132074, upload-time = "2025-12-06T15:54:24.694Z" }, + { url = "https://files.pythonhosted.org/packages/6e/57/b9f5b5b6fbff9c26f77e785baf56ae8460ef74acdb3eae4931c25b8f5ba9/orjson-3.11.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a66d7769e98a08a12a139049aac2f0ca3adae989817f8c43337455fbc7669b85", size = 130520, upload-time = "2025-12-06T15:54:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6d/d34970bf9eb33f9ec7c979a262cad86076814859e54eb9a059a52f6dc13d/orjson-3.11.5-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:86cfc555bfd5794d24c6a1903e558b50644e5e68e6471d66502ce5cb5fdef3f9", size = 136209, upload-time = "2025-12-06T15:54:27.264Z" }, + { url = "https://files.pythonhosted.org/packages/e7/39/bc373b63cc0e117a105ea12e57280f83ae52fdee426890d57412432d63b3/orjson-3.11.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a230065027bc2a025e944f9d4714976a81e7ecfa940923283bca7bbc1f10f626", size = 139837, upload-time = "2025-12-06T15:54:28.75Z" }, + { url = "https://files.pythonhosted.org/packages/cb/aa/7c4818c8d7d324da220f4f1af55c343956003aa4d1ce1857bdc1d396ba69/orjson-3.11.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b29d36b60e606df01959c4b982729c8845c69d1963f88686608be9ced96dbfaa", size = 137307, upload-time = "2025-12-06T15:54:29.856Z" }, + { url = "https://files.pythonhosted.org/packages/46/bf/0993b5a056759ba65145effe3a79dd5a939d4a070eaa5da2ee3180fbb13f/orjson-3.11.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c74099c6b230d4261fdc3169d50efc09abf38ace1a42ea2f9994b1d79153d477", size = 139020, upload-time = "2025-12-06T15:54:31.024Z" }, + { url = "https://files.pythonhosted.org/packages/65/e8/83a6c95db3039e504eda60fc388f9faedbb4f6472f5aba7084e06552d9aa/orjson-3.11.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e697d06ad57dd0c7a737771d470eedc18e68dfdefcdd3b7de7f33dfda5b6212e", size = 141099, upload-time = "2025-12-06T15:54:32.196Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b4/24fdc024abfce31c2f6812973b0a693688037ece5dc64b7a60c1ce69e2f2/orjson-3.11.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e08ca8a6c851e95aaecc32bc44a5aa75d0ad26af8cdac7c77e4ed93acf3d5b69", size = 413540, upload-time = "2025-12-06T15:54:33.361Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/01c0ec95d55ed0c11e4cae3e10427e479bba40c77312b63e1f9665e0737d/orjson-3.11.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e8b5f96c05fce7d0218df3fdfeb962d6b8cfff7e3e20264306b46dd8b217c0f3", size = 151530, upload-time = "2025-12-06T15:54:34.6Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d4/f9ebc57182705bb4bbe63f5bbe14af43722a2533135e1d2fb7affa0c355d/orjson-3.11.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ddbfdb5099b3e6ba6d6ea818f61997bb66de14b411357d24c4612cf1ebad08ca", size = 141863, upload-time = "2025-12-06T15:54:35.801Z" }, + { url = "https://files.pythonhosted.org/packages/0d/04/02102b8d19fdcb009d72d622bb5781e8f3fae1646bf3e18c53d1bc8115b5/orjson-3.11.5-cp312-cp312-win32.whl", hash = "sha256:9172578c4eb09dbfcf1657d43198de59b6cef4054de385365060ed50c458ac98", size = 135255, upload-time = "2025-12-06T15:54:37.209Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fb/f05646c43d5450492cb387de5549f6de90a71001682c17882d9f66476af5/orjson-3.11.5-cp312-cp312-win_amd64.whl", hash = "sha256:2b91126e7b470ff2e75746f6f6ee32b9ab67b7a93c8ba1d15d3a0caaf16ec875", size = 133252, upload-time = "2025-12-06T15:54:38.401Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a6/7b8c0b26ba18c793533ac1cd145e131e46fcf43952aa94c109b5b913c1f0/orjson-3.11.5-cp312-cp312-win_arm64.whl", hash = "sha256:acbc5fac7e06777555b0722b8ad5f574739e99ffe99467ed63da98f97f9ca0fe", size = 126777, upload-time = "2025-12-06T15:54:39.515Z" }, +] + +[[package]] +name = "outlines" +version = "0.1.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "airportsdata" }, + { name = "cloudpickle" }, + { name = "diskcache" }, + { name = "interegular" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "lark" }, + { name = "nest-asyncio" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "outlines-core", version = "0.1.26", source = { registry = "https://pypi.org/simple" } }, + { name = "pycountry" }, + { name = "pydantic" }, + { name = "referencing" }, + { name = "requests" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/d0/d59ae830bf7026425942899e3d48e77b58a713cff946a695e5405808da1b/outlines-0.1.11.tar.gz", hash = "sha256:0997bd9da1cc050e430bd08995dc7d4bd855918bafa4531e49d3f37110a23aba", size = 2488858, upload-time = "2024-12-13T07:24:08.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/b4/99ea4a122bef60e3fd6402d19665aff1f928e0daf8fac3044d0b73f72003/outlines-0.1.11-py3-none-any.whl", hash = "sha256:f5a5f2242ed9802d3aab7a92789bf4008d734c576be9258cc0a297f690124727", size = 87623, upload-time = "2024-12-13T07:24:05.817Z" }, +] + +[[package]] +name = "outlines-core" +version = "0.1.26" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "interegular" }, + { name = "jsonschema" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/f3/274d07f4702728b43581235a77e545ec602b25f9b0098b288a0f3052521d/outlines_core-0.1.26.tar.gz", hash = "sha256:481c4301341e77cc8f1832d616784adb4d461b4fec65878e7c0d2cba7163a189", size = 75139, upload-time = "2024-12-12T23:38:50.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/86/0fb40746e579db38d89f127122a3900d9e0350f76aae8cb61adeaff44cc2/outlines_core-0.1.26-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f54633bca50055d42ea4d94ae06dcbe52d3d76a9b621b75723b1177d0d952953", size = 321874, upload-time = "2024-12-12T23:38:26.834Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0c/b91f7bc03843796c1d643ee030b6cd8fd5a8ba2cd4856c855f140c878976/outlines_core-0.1.26-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9525321b48700dcaaabf60bcdc951e45f9357ba3fb3e1bfc81b662d7d4170e7c", size = 301995, upload-time = "2024-12-12T23:38:29.625Z" }, + { url = "https://files.pythonhosted.org/packages/ad/db/fa91a2d54288b900de82d86eda3adb2417b3b5b2db6256854a5e8bc85c32/outlines_core-0.1.26-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f409f72c11f6ffadb57066950dd384d5388015028c1a1a615c9a64988dae3e", size = 321050, upload-time = "2024-12-12T23:38:32.274Z" }, + { url = "https://files.pythonhosted.org/packages/e2/1d/a36292b6198986bd9c3ff8c24355deb82ed5475403379ee40b5b5473e2e3/outlines_core-0.1.26-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86a1bb46adc5cbf6dfd7a7fe4105e0e2a4c6e041732a053126b41c521a1f223", size = 343201, upload-time = "2024-12-12T23:38:34.631Z" }, + { url = "https://files.pythonhosted.org/packages/08/63/5dd2b5a364412f674b6edcb59b0c21513bdb07cdcc7613b064c1a0660d01/outlines_core-0.1.26-cp312-cp312-win32.whl", hash = "sha256:19f462f6b00935708677ad27cb4df55e0e17f6ffe713ab750f5f2683b090f95d", size = 233970, upload-time = "2024-12-12T23:38:37.318Z" }, + { url = "https://files.pythonhosted.org/packages/a5/56/8adf0b7446d1e975c2314454813c59eb7b195889908a2932ed34148c113c/outlines_core-0.1.26-cp312-cp312-win_amd64.whl", hash = "sha256:9b36bff12779e58883747116893a17b3551bbd10865878b951b03a44d112229a", size = 243578, upload-time = "2024-12-12T23:38:39.964Z" }, +] + +[[package]] +name = "outlines-core" +version = "0.2.11" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/d3/e04e9145f8f806723dec9b9e5227ad695a3efcd3ced7794cf7c22b15df5e/outlines_core-0.2.11.tar.gz", hash = "sha256:dfce56f717ff5083e54cbcfdb66cad243365437fccbb5509adaa7e31e030f1d8", size = 197263, upload-time = "2025-05-19T10:12:51.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/2c/c7636823244c70e2960060bf9bd978248dffb55c5e7c91c46d18354b2a24/outlines_core-0.2.11-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4a9db4872bae083631d720994f4cee603bce0536b33d5a988814576863b657cf", size = 1957668, upload-time = "2025-05-19T10:12:18.29Z" }, + { url = "https://files.pythonhosted.org/packages/c7/09/5c62047da139d722317a444a4d01cd5f11943a8c2eaecce784341dd0844a/outlines_core-0.2.11-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:8359a45c59f6a8f2eb717245806501a59044c75f6ea8bd08faaa131cc8cdec45", size = 2130493, upload-time = "2025-05-19T10:12:19.537Z" }, + { url = "https://files.pythonhosted.org/packages/89/7a/d6a2810f90e37d550168e0c0a9a915086ea721444727e3ca2c630898d1ef/outlines_core-0.2.11-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:5d26a46591377340e0b870b8a96ea8341058341a62ee0bded9098e0c88dd24f4", size = 1956804, upload-time = "2025-05-19T10:12:20.755Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ea/339e6c273b5581128c3b7ca27d428d8993c3085912af1a467aa32ef0e9d1/outlines_core-0.2.11-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:ae460a34675fb11d92a5c605a480fbae4cd6c1b2d11b3698da64a7fcaba64dcf", size = 2127085, upload-time = "2025-05-19T10:12:22.02Z" }, + { url = "https://files.pythonhosted.org/packages/92/c7/a65d1fddf49830ebc41422294eacde35286d9f68994a8aa905cb14f5aade/outlines_core-0.2.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86df9740368866295077346440d911df4972da2b3f1f54b8125e6f329e8a8891", size = 2287677, upload-time = "2025-05-19T10:12:24.24Z" }, + { url = "https://files.pythonhosted.org/packages/23/79/8795aed8be9b77dd69d78e7cfbfcf28c179e6b08da6e56bbbf48a09fe55f/outlines_core-0.2.11-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:96ce4dd78f106799be4a0a5795cefd1352806162973756a4b6fce4bb6eddd7e4", size = 2113000, upload-time = "2025-05-19T10:12:25.446Z" }, + { url = "https://files.pythonhosted.org/packages/59/e3/cbe9294b06d92ee1892dbb6f2125d833d68e8629d45d080d6daba54eec2d/outlines_core-0.2.11-cp312-cp312-win32.whl", hash = "sha256:358db161cce3650ba822e118dcf0a1efa571c7deb4864ab9d64ca2c9cca7425d", size = 1765703, upload-time = "2025-05-19T10:12:26.693Z" }, + { url = "https://files.pythonhosted.org/packages/1d/c9/ed3cf362515fac16e313368b9b2f2497051f4ded88679205830b6f889f54/outlines_core-0.2.11-cp312-cp312-win_amd64.whl", hash = "sha256:231f9d20d2630c70665345821780d7808b29539620a75c99f65113b518c51032", size = 2060945, upload-time = "2025-05-19T10:12:28.294Z" }, +] + +[[package]] +name = "overrides" +version = "7.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/86/b585f53236dec60aba864e050778b25045f857e17f6e5ea0ae95fe80edd2/overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a", size = 22812, upload-time = "2024-01-27T21:01:33.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/ab/fc8290c6a4c722e5514d80f62b2dc4c4df1a68a41d1364e625c35990fcf3/overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49", size = 17832, upload-time = "2024-01-27T21:01:31.393Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, + { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, + { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, + { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, + { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, +] + +[[package]] +name = "parso" +version = "0.8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/de/53e0bcf53d13e005bd8c92e7855142494f41171b34c2536b86187474184d/parso-0.8.5.tar.gz", hash = "sha256:034d7354a9a018bdce352f48b2a8a450f05e9d6ee85db84764e9b6bd96dafe5a", size = 401205, upload-time = "2025-08-23T15:15:28.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/32/f8e3c85d1d5250232a5d3477a2a28cc291968ff175caeadaf3cc19ce0e4a/parso-0.8.5-py2.py3-none-any.whl", hash = "sha256:646204b5ee239c396d040b90f9e272e9a8017c630092bf59980beb62fd033887", size = 106668, upload-time = "2025-08-23T15:15:25.663Z" }, +] + +[[package]] +name = "partial-json-parser" +version = "0.2.1.1.post7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/6d/eed37d7ebc1e0bcd27b831c0cf1fe94881934316187c4b30d23f29ea0bd4/partial_json_parser-0.2.1.1.post7.tar.gz", hash = "sha256:86590e1ba6bcb6739a2dfc17d2323f028cb5884f4c6ce23db376999132c9a922", size = 10296, upload-time = "2025-11-17T07:27:41.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/32/658973117bf0fd82a24abbfb94fe73a5e86216e49342985e10acce54775a/partial_json_parser-0.2.1.1.post7-py3-none-any.whl", hash = "sha256:145119e5eabcf80cbb13844a6b50a85c68bf99d376f8ed771e2a3c3b03e653ae", size = 10877, upload-time = "2025-11-17T07:27:40.457Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "peft" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "accelerate" }, + { name = "huggingface-hub" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tqdm" }, + { name = "transformers", version = "4.52.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/0c/f2938db546ac7fc961ab5917cd50fcf5d0d70b406de93e3faccaa504e152/peft-0.18.0.tar.gz", hash = "sha256:c81c80b2056ab40c23d58ef25f74daab417ac653970718589a11a8af28218588", size = 634141, upload-time = "2025-11-13T11:13:06.603Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/55/481bf25613d40ef53534f664deba7b138fe566356b6ca10304e2b3b2529c/peft-0.18.0-py3-none-any.whl", hash = "sha256:624f69ca6393b765ccc6734adda7ca57d80b238f0900a42c357d8b67a03d62ff", size = 556427, upload-time = "2025-11-13T11:13:03.664Z" }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + +[[package]] +name = "pillow" +version = "12.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/cace85a1b0c9775a9f8f5d5423c8261c858760e2466c79b2dd184638b056/pillow-12.0.0.tar.gz", hash = "sha256:87d4f8125c9988bfbed67af47dd7a953e2fc7b0cc1e7800ec6d2080d490bb353", size = 47008828, upload-time = "2025-10-15T18:24:14.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/90/4fcce2c22caf044e660a198d740e7fbc14395619e3cb1abad12192c0826c/pillow-12.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:53561a4ddc36facb432fae7a9d8afbfaf94795414f5cdc5fc52f28c1dca90371", size = 5249377, upload-time = "2025-10-15T18:22:05.993Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e0/ed960067543d080691d47d6938ebccbf3976a931c9567ab2fbfab983a5dd/pillow-12.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71db6b4c1653045dacc1585c1b0d184004f0d7e694c7b34ac165ca70c0838082", size = 4650343, upload-time = "2025-10-15T18:22:07.718Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a1/f81fdeddcb99c044bf7d6faa47e12850f13cee0849537a7d27eeab5534d4/pillow-12.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2fa5f0b6716fc88f11380b88b31fe591a06c6315e955c096c35715788b339e3f", size = 6232981, upload-time = "2025-10-15T18:22:09.287Z" }, + { url = "https://files.pythonhosted.org/packages/88/e1/9098d3ce341a8750b55b0e00c03f1630d6178f38ac191c81c97a3b047b44/pillow-12.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:82240051c6ca513c616f7f9da06e871f61bfd7805f566275841af15015b8f98d", size = 8041399, upload-time = "2025-10-15T18:22:10.872Z" }, + { url = "https://files.pythonhosted.org/packages/a7/62/a22e8d3b602ae8cc01446d0c57a54e982737f44b6f2e1e019a925143771d/pillow-12.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55f818bd74fe2f11d4d7cbc65880a843c4075e0ac7226bc1a23261dbea531953", size = 6347740, upload-time = "2025-10-15T18:22:12.769Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/424511bdcd02c8d7acf9f65caa09f291a519b16bd83c3fb3374b3d4ae951/pillow-12.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b87843e225e74576437fd5b6a4c2205d422754f84a06942cfaf1dc32243e45a8", size = 7040201, upload-time = "2025-10-15T18:22:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4d/435c8ac688c54d11755aedfdd9f29c9eeddf68d150fe42d1d3dbd2365149/pillow-12.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c607c90ba67533e1b2355b821fef6764d1dd2cbe26b8c1005ae84f7aea25ff79", size = 6462334, upload-time = "2025-10-15T18:22:16.375Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f2/ad34167a8059a59b8ad10bc5c72d4d9b35acc6b7c0877af8ac885b5f2044/pillow-12.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:21f241bdd5080a15bc86d3466a9f6074a9c2c2b314100dd896ac81ee6db2f1ba", size = 7134162, upload-time = "2025-10-15T18:22:17.996Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b1/a7391df6adacf0a5c2cf6ac1cf1fcc1369e7d439d28f637a847f8803beb3/pillow-12.0.0-cp312-cp312-win32.whl", hash = "sha256:dd333073e0cacdc3089525c7df7d39b211bcdf31fc2824e49d01c6b6187b07d0", size = 6298769, upload-time = "2025-10-15T18:22:19.923Z" }, + { url = "https://files.pythonhosted.org/packages/a2/0b/d87733741526541c909bbf159e338dcace4f982daac6e5a8d6be225ca32d/pillow-12.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe611163f6303d1619bbcb653540a4d60f9e55e622d60a3108be0d5b441017a", size = 7001107, upload-time = "2025-10-15T18:22:21.644Z" }, + { url = "https://files.pythonhosted.org/packages/bc/96/aaa61ce33cc98421fb6088af2a03be4157b1e7e0e87087c888e2370a7f45/pillow-12.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:7dfb439562f234f7d57b1ac6bc8fe7f838a4bd49c79230e0f6a1da93e82f1fad", size = 2436012, upload-time = "2025-10-15T18:22:23.621Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cf/86/0248f086a84f01b37aaec0fa567b397df1a119f73c16f6c7a9aac73ea309/platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", size = 21715, upload-time = "2025-12-05T13:52:58.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "polars" +version = "1.36.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/dc/56f2a90c79a2cb13f9e956eab6385effe54216ae7a2068b3a6406bae4345/polars-1.36.1.tar.gz", hash = "sha256:12c7616a2305559144711ab73eaa18814f7aa898c522e7645014b68f1432d54c", size = 711993, upload-time = "2025-12-10T01:14:53.033Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/c6/36a1b874036b49893ecae0ac44a2f63d1a76e6212631a5b2f50a86e0e8af/polars-1.36.1-py3-none-any.whl", hash = "sha256:853c1bbb237add6a5f6d133c15094a9b727d66dd6a4eb91dbb07cdb056b2b8ef", size = 802429, upload-time = "2025-12-10T01:13:53.838Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.36.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/df/597c0ef5eb8d761a16d72327846599b57c5d40d7f9e74306fc154aba8c37/polars_runtime_32-1.36.1.tar.gz", hash = "sha256:201c2cfd80ceb5d5cd7b63085b5fd08d6ae6554f922bcb941035e39638528a09", size = 2788751, upload-time = "2025-12-10T01:14:54.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/ea/871129a2d296966c0925b078a9a93c6c5e7facb1c5eebfcd3d5811aeddc1/polars_runtime_32-1.36.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:327b621ca82594f277751f7e23d4b939ebd1be18d54b4cdf7a2f8406cecc18b2", size = 43494311, upload-time = "2025-12-10T01:13:56.096Z" }, + { url = "https://files.pythonhosted.org/packages/d8/76/0038210ad1e526ce5bb2933b13760d6b986b3045eccc1338e661bd656f77/polars_runtime_32-1.36.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ab0d1f23084afee2b97de8c37aa3e02ec3569749ae39571bd89e7a8b11ae9e83", size = 39300602, upload-time = "2025-12-10T01:13:59.366Z" }, + { url = "https://files.pythonhosted.org/packages/54/1e/2707bee75a780a953a77a2c59829ee90ef55708f02fc4add761c579bf76e/polars_runtime_32-1.36.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:899b9ad2e47ceb31eb157f27a09dbc2047efbf4969a923a6b1ba7f0412c3e64c", size = 44511780, upload-time = "2025-12-10T01:14:02.285Z" }, + { url = "https://files.pythonhosted.org/packages/11/b2/3fede95feee441be64b4bcb32444679a8fbb7a453a10251583053f6efe52/polars_runtime_32-1.36.1-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:d9d077bb9df711bc635a86540df48242bb91975b353e53ef261c6fae6cb0948f", size = 40688448, upload-time = "2025-12-10T01:14:05.131Z" }, + { url = "https://files.pythonhosted.org/packages/05/0f/e629713a72999939b7b4bfdbf030a32794db588b04fdf3dc977dd8ea6c53/polars_runtime_32-1.36.1-cp39-abi3-win_amd64.whl", hash = "sha256:cc17101f28c9a169ff8b5b8d4977a3683cd403621841623825525f440b564cf0", size = 44464898, upload-time = "2025-12-10T01:14:08.296Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d8/a12e6aa14f63784cead437083319ec7cece0d5bb9a5bfe7678cc6578b52a/polars_runtime_32-1.36.1-cp39-abi3-win_arm64.whl", hash = "sha256:809e73857be71250141225ddd5d2b30c97e6340aeaa0d445f930e01bef6888dc", size = 39798896, upload-time = "2025-12-10T01:14:11.568Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f1/6d86a29246dfd2e9b6237f0b5823717f60cad94d47ddc26afa916d21f525/pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61", size = 198232, upload-time = "2025-12-16T21:14:33.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, +] + +[[package]] +name = "prometheus-client" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, +] + +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prometheus-client" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/6d/24d53033cf93826aa7857699a4450c1c67e5b9c710e925b1ed2b320c04df/prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e", size = 20220, upload-time = "2025-03-19T19:35:05.351Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/72/0824c18f3bc75810f55dacc2dd933f6ec829771180245ae3cc976195dec0/prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9", size = 19296, upload-time = "2025-03-19T19:35:04.323Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth", marker = "extra == 'extra-11-skyrl-train-miniswe' or extra == 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + +[[package]] +name = "propcache" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/da/e9fc233cf63743258bff22b3dfa7ea5baef7b5bc324af47a0ad89b8ffc6f/propcache-0.4.1.tar.gz", hash = "sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d", size = 46442, upload-time = "2025-10-08T19:49:02.291Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/0f/f17b1b2b221d5ca28b4b876e8bb046ac40466513960646bda8e1853cdfa2/propcache-0.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2", size = 80061, upload-time = "2025-10-08T19:46:46.075Z" }, + { url = "https://files.pythonhosted.org/packages/76/47/8ccf75935f51448ba9a16a71b783eb7ef6b9ee60f5d14c7f8a8a79fbeed7/propcache-0.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403", size = 46037, upload-time = "2025-10-08T19:46:47.23Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b6/5c9a0e42df4d00bfb4a3cbbe5cf9f54260300c88a0e9af1f47ca5ce17ac0/propcache-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207", size = 47324, upload-time = "2025-10-08T19:46:48.384Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d3/6c7ee328b39a81ee877c962469f1e795f9db87f925251efeb0545e0020d0/propcache-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72", size = 225505, upload-time = "2025-10-08T19:46:50.055Z" }, + { url = "https://files.pythonhosted.org/packages/01/5d/1c53f4563490b1d06a684742cc6076ef944bc6457df6051b7d1a877c057b/propcache-0.4.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367", size = 230242, upload-time = "2025-10-08T19:46:51.815Z" }, + { url = "https://files.pythonhosted.org/packages/20/e1/ce4620633b0e2422207c3cb774a0ee61cac13abc6217763a7b9e2e3f4a12/propcache-0.4.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4", size = 238474, upload-time = "2025-10-08T19:46:53.208Z" }, + { url = "https://files.pythonhosted.org/packages/46/4b/3aae6835b8e5f44ea6a68348ad90f78134047b503765087be2f9912140ea/propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf", size = 221575, upload-time = "2025-10-08T19:46:54.511Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a5/8a5e8678bcc9d3a1a15b9a29165640d64762d424a16af543f00629c87338/propcache-0.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3", size = 216736, upload-time = "2025-10-08T19:46:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/f1/63/b7b215eddeac83ca1c6b934f89d09a625aa9ee4ba158338854c87210cc36/propcache-0.4.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778", size = 213019, upload-time = "2025-10-08T19:46:57.595Z" }, + { url = "https://files.pythonhosted.org/packages/57/74/f580099a58c8af587cac7ba19ee7cb418506342fbbe2d4a4401661cca886/propcache-0.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6", size = 220376, upload-time = "2025-10-08T19:46:59.067Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ee/542f1313aff7eaf19c2bb758c5d0560d2683dac001a1c96d0774af799843/propcache-0.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9", size = 226988, upload-time = "2025-10-08T19:47:00.544Z" }, + { url = "https://files.pythonhosted.org/packages/8f/18/9c6b015dd9c6930f6ce2229e1f02fb35298b847f2087ea2b436a5bfa7287/propcache-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75", size = 215615, upload-time = "2025-10-08T19:47:01.968Z" }, + { url = "https://files.pythonhosted.org/packages/80/9e/e7b85720b98c45a45e1fca6a177024934dc9bc5f4d5dd04207f216fc33ed/propcache-0.4.1-cp312-cp312-win32.whl", hash = "sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8", size = 38066, upload-time = "2025-10-08T19:47:03.503Z" }, + { url = "https://files.pythonhosted.org/packages/54/09/d19cff2a5aaac632ec8fc03737b223597b1e347416934c1b3a7df079784c/propcache-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db", size = 41655, upload-time = "2025-10-08T19:47:04.973Z" }, + { url = "https://files.pythonhosted.org/packages/68/ab/6b5c191bb5de08036a8c697b265d4ca76148efb10fa162f14af14fb5f076/propcache-0.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1", size = 37789, upload-time = "2025-10-08T19:47:06.077Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/44/e49ecff446afeec9d1a66d6bbf9adc21e3c7cea7803a920ca3773379d4f6/protobuf-6.33.2.tar.gz", hash = "sha256:56dc370c91fbb8ac85bc13582c9e373569668a290aa2e66a590c2a0d35ddb9e4", size = 444296, upload-time = "2025-12-06T00:17:53.311Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/91/1e3a34881a88697a7354ffd177e8746e97a722e5e8db101544b47e84afb1/protobuf-6.33.2-cp310-abi3-win32.whl", hash = "sha256:87eb388bd2d0f78febd8f4c8779c79247b26a5befad525008e49a6955787ff3d", size = 425603, upload-time = "2025-12-06T00:17:41.114Z" }, + { url = "https://files.pythonhosted.org/packages/64/20/4d50191997e917ae13ad0a235c8b42d8c1ab9c3e6fd455ca16d416944355/protobuf-6.33.2-cp310-abi3-win_amd64.whl", hash = "sha256:fc2a0e8b05b180e5fc0dd1559fe8ebdae21a27e81ac77728fb6c42b12c7419b4", size = 436930, upload-time = "2025-12-06T00:17:43.278Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ca/7e485da88ba45c920fb3f50ae78de29ab925d9e54ef0de678306abfbb497/protobuf-6.33.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d9b19771ca75935b3a4422957bc518b0cecb978b31d1dd12037b088f6bcc0e43", size = 427621, upload-time = "2025-12-06T00:17:44.445Z" }, + { url = "https://files.pythonhosted.org/packages/7d/4f/f743761e41d3b2b2566748eb76bbff2b43e14d5fcab694f494a16458b05f/protobuf-6.33.2-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:b5d3b5625192214066d99b2b605f5783483575656784de223f00a8d00754fc0e", size = 324460, upload-time = "2025-12-06T00:17:45.678Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fa/26468d00a92824020f6f2090d827078c09c9c587e34cbfd2d0c7911221f8/protobuf-6.33.2-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:8cd7640aee0b7828b6d03ae518b5b4806fdfc1afe8de82f79c3454f8aef29872", size = 339168, upload-time = "2025-12-06T00:17:46.813Z" }, + { url = "https://files.pythonhosted.org/packages/56/13/333b8f421738f149d4fe5e49553bc2a2ab75235486259f689b4b91f96cec/protobuf-6.33.2-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:1f8017c48c07ec5859106533b682260ba3d7c5567b1ca1f24297ce03384d1b4f", size = 323270, upload-time = "2025-12-06T00:17:48.253Z" }, + { url = "https://files.pythonhosted.org/packages/0e/15/4f02896cc3df04fc465010a4c6a0cd89810f54617a32a70ef531ed75d61c/protobuf-6.33.2-py3-none-any.whl", hash = "sha256:7636aad9bb01768870266de5dc009de2d1b936771b38a793f73cbbf279c91c5c", size = 170501, upload-time = "2025-12-06T00:17:52.211Z" }, +] + +[[package]] +name = "psutil" +version = "7.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/88/bdd0a41e5857d5d703287598cbf08dad90aed56774ea52ae071bae9071b6/psutil-7.1.3.tar.gz", hash = "sha256:6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74", size = 489059, upload-time = "2025-11-02T12:25:54.619Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/94/46b9154a800253e7ecff5aaacdf8ebf43db99de4a2dfa18575b02548654e/psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab", size = 238359, upload-time = "2025-11-02T12:26:25.284Z" }, + { url = "https://files.pythonhosted.org/packages/68/3a/9f93cff5c025029a36d9a92fef47220ab4692ee7f2be0fba9f92813d0cb8/psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880", size = 239171, upload-time = "2025-11-02T12:26:27.23Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b1/5f49af514f76431ba4eea935b8ad3725cdeb397e9245ab919dbc1d1dc20f/psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3", size = 263261, upload-time = "2025-11-02T12:26:29.48Z" }, + { url = "https://files.pythonhosted.org/packages/e0/95/992c8816a74016eb095e73585d747e0a8ea21a061ed3689474fabb29a395/psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b", size = 264635, upload-time = "2025-11-02T12:26:31.74Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/c3ed1a622b6ae2fd3c945a366e64eb35247a31e4db16cf5095e269e8eb3c/psutil-7.1.3-cp37-abi3-win_amd64.whl", hash = "sha256:f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd", size = 247633, upload-time = "2025-11-02T12:26:33.887Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ad/33b2ccec09bf96c2b2ef3f9a6f66baac8253d7565d8839e024a6b905d45d/psutil-7.1.3-cp37-abi3-win_arm64.whl", hash = "sha256:bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1", size = 244608, upload-time = "2025-11-02T12:26:36.136Z" }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + +[[package]] +name = "pulp" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/1c/d880b739b841a8aa81143091c9bdda5e72e226a660aa13178cb312d4b27f/pulp-3.3.0.tar.gz", hash = "sha256:7eb99b9ce7beeb8bbb7ea9d1c919f02f003ab7867e0d1e322f2f2c26dd31c8ba", size = 16301847, upload-time = "2025-09-18T08:14:57.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/6c/64cafaceea3f99927e84b38a362ec6a8f24f33061c90bda77dfe1cd4c3c6/pulp-3.3.0-py3-none-any.whl", hash = "sha256:dd6ad2d63f196d1254eddf9dcff5cd224912c1f046120cb7c143c5b0eda63fae", size = 16387700, upload-time = "2025-09-18T08:14:53.368Z" }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + +[[package]] +name = "pyarrow" +version = "22.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/53/04a7fdc63e6056116c9ddc8b43bc28c12cdd181b85cbeadb79278475f3ae/pyarrow-22.0.0.tar.gz", hash = "sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9", size = 1151151, upload-time = "2025-10-24T12:30:00.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/63/ba23862d69652f85b615ca14ad14f3bcfc5bf1b99ef3f0cd04ff93fdad5a/pyarrow-22.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d", size = 34211578, upload-time = "2025-10-24T10:05:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d0/f9ad86fe809efd2bcc8be32032fa72e8b0d112b01ae56a053006376c5930/pyarrow-22.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8", size = 35989906, upload-time = "2025-10-24T10:05:29.485Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a8/f910afcb14630e64d673f15904ec27dd31f1e009b77033c365c84e8c1e1d/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5", size = 45021677, upload-time = "2025-10-24T10:05:38.274Z" }, + { url = "https://files.pythonhosted.org/packages/13/95/aec81f781c75cd10554dc17a25849c720d54feafb6f7847690478dcf5ef8/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe", size = 47726315, upload-time = "2025-10-24T10:05:47.314Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d4/74ac9f7a54cfde12ee42734ea25d5a3c9a45db78f9def949307a92720d37/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e", size = 47990906, upload-time = "2025-10-24T10:05:58.254Z" }, + { url = "https://files.pythonhosted.org/packages/2e/71/fedf2499bf7a95062eafc989ace56572f3343432570e1c54e6599d5b88da/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9", size = 50306783, upload-time = "2025-10-24T10:06:08.08Z" }, + { url = "https://files.pythonhosted.org/packages/68/ed/b202abd5a5b78f519722f3d29063dda03c114711093c1995a33b8e2e0f4b/pyarrow-22.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d", size = 27972883, upload-time = "2025-10-24T10:06:14.204Z" }, +] + +[[package]] +name = "pybase64" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/b8/4ed5c7ad5ec15b08d35cc79ace6145d5c1ae426e46435f4987379439dfea/pybase64-1.4.3.tar.gz", hash = "sha256:c2ed274c9e0ba9c8f9c4083cfe265e66dd679126cd9c2027965d807352f3f053", size = 137272, upload-time = "2025-12-06T13:27:04.013Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/a7/efcaa564f091a2af7f18a83c1c4875b1437db56ba39540451dc85d56f653/pybase64-1.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:18d85e5ab8b986bb32d8446aca6258ed80d1bafe3603c437690b352c648f5967", size = 38167, upload-time = "2025-12-06T13:23:16.821Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/c7ad35adff2d272bf2930132db2b3eea8c44bb1b1f64eb9b2b8e57cde7b4/pybase64-1.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3f5791a3491d116d0deaf4d83268f48792998519698f8751efb191eac84320e9", size = 31673, upload-time = "2025-12-06T13:23:17.835Z" }, + { url = "https://files.pythonhosted.org/packages/43/1b/9a8cab0042b464e9a876d5c65fe5127445a2436da36fda64899b119b1a1b/pybase64-1.4.3-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f0b3f200c3e06316f6bebabd458b4e4bcd4c2ca26af7c0c766614d91968dee27", size = 68210, upload-time = "2025-12-06T13:23:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/62/f7/965b79ff391ad208b50e412b5d3205ccce372a2d27b7218ae86d5295b105/pybase64-1.4.3-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb632edfd132b3eaf90c39c89aa314beec4e946e210099b57d40311f704e11d4", size = 71599, upload-time = "2025-12-06T13:23:20.195Z" }, + { url = "https://files.pythonhosted.org/packages/03/4b/a3b5175130b3810bbb8ccfa1edaadbd3afddb9992d877c8a1e2f274b476e/pybase64-1.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:356ef1d74648ce997f5a777cf8f1aefecc1c0b4fe6201e0ef3ec8a08170e1b54", size = 59922, upload-time = "2025-12-06T13:23:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/da/5d/c38d1572027fc601b62d7a407721688b04b4d065d60ca489912d6893e6cf/pybase64-1.4.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:c48361f90db32bacaa5518419d4eb9066ba558013aaf0c7781620279ecddaeb9", size = 56712, upload-time = "2025-12-06T13:23:22.77Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d4/4e04472fef485caa8f561d904d4d69210a8f8fc1608ea15ebd9012b92655/pybase64-1.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:702bcaa16ae02139d881aeaef5b1c8ffb4a3fae062fe601d1e3835e10310a517", size = 59300, upload-time = "2025-12-06T13:23:24.543Z" }, + { url = "https://files.pythonhosted.org/packages/86/e7/16e29721b86734b881d09b7e23dfd7c8408ad01a4f4c7525f3b1088e25ec/pybase64-1.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:53d0ffe1847b16b647c6413d34d1de08942b7724273dd57e67dcbdb10c574045", size = 60278, upload-time = "2025-12-06T13:23:25.608Z" }, + { url = "https://files.pythonhosted.org/packages/b1/02/18515f211d7c046be32070709a8efeeef8a0203de4fd7521e6b56404731b/pybase64-1.4.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9a1792e8b830a92736dae58f0c386062eb038dfe8004fb03ba33b6083d89cd43", size = 54817, upload-time = "2025-12-06T13:23:26.633Z" }, + { url = "https://files.pythonhosted.org/packages/e7/be/14e29d8e1a481dbff151324c96dd7b5d2688194bb65dc8a00ca0e1ad1e86/pybase64-1.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d468b1b1ac5ad84875a46eaa458663c3721e8be5f155ade356406848d3701f6", size = 58611, upload-time = "2025-12-06T13:23:27.684Z" }, + { url = "https://files.pythonhosted.org/packages/b4/8a/a2588dfe24e1bbd742a554553778ab0d65fdf3d1c9a06d10b77047d142aa/pybase64-1.4.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e97b7bdbd62e71898cd542a6a9e320d9da754ff3ebd02cb802d69087ee94d468", size = 52404, upload-time = "2025-12-06T13:23:28.714Z" }, + { url = "https://files.pythonhosted.org/packages/27/fc/afcda7445bebe0cbc38cafdd7813234cdd4fc5573ff067f1abf317bb0cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b33aeaa780caaa08ffda87fc584d5eab61e3d3bbb5d86ead02161dc0c20d04bc", size = 68817, upload-time = "2025-12-06T13:23:30.079Z" }, + { url = "https://files.pythonhosted.org/packages/d3/3a/87c3201e555ed71f73e961a787241a2438c2bbb2ca8809c29ddf938a3157/pybase64-1.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c0efcf78f11cf866bed49caa7b97552bc4855a892f9cc2372abcd3ed0056f0d", size = 57854, upload-time = "2025-12-06T13:23:31.17Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7d/931c2539b31a7b375e7d595b88401eeb5bd6c5ce1059c9123f9b608aaa14/pybase64-1.4.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66e3791f2ed725a46593f8bd2761ff37d01e2cdad065b1dceb89066f476e50c6", size = 54333, upload-time = "2025-12-06T13:23:32.422Z" }, + { url = "https://files.pythonhosted.org/packages/de/5e/537601e02cc01f27e9d75f440f1a6095b8df44fc28b1eef2cd739aea8cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:72bb0b6bddadab26e1b069bb78e83092711a111a80a0d6b9edcb08199ad7299b", size = 56492, upload-time = "2025-12-06T13:23:33.515Z" }, + { url = "https://files.pythonhosted.org/packages/96/97/2a2e57acf8f5c9258d22aba52e71f8050e167b29ed2ee1113677c1b600c1/pybase64-1.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5b3365dbcbcdb0a294f0f50af0c0a16b27a232eddeeb0bceeefd844ef30d2a23", size = 70974, upload-time = "2025-12-06T13:23:36.27Z" }, + { url = "https://files.pythonhosted.org/packages/75/2e/a9e28941c6dab6f06e6d3f6783d3373044be9b0f9a9d3492c3d8d2260ac0/pybase64-1.4.3-cp312-cp312-win32.whl", hash = "sha256:7bca1ed3a5df53305c629ca94276966272eda33c0d71f862d2d3d043f1e1b91a", size = 33686, upload-time = "2025-12-06T13:23:37.848Z" }, + { url = "https://files.pythonhosted.org/packages/83/e3/507ab649d8c3512c258819c51d25c45d6e29d9ca33992593059e7b646a33/pybase64-1.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:9f2da8f56d9b891b18b4daf463a0640eae45a80af548ce435be86aa6eff3603b", size = 35833, upload-time = "2025-12-06T13:23:38.877Z" }, + { url = "https://files.pythonhosted.org/packages/bc/8a/6eba66cd549a2fc74bb4425fd61b839ba0ab3022d3c401b8a8dc2cc00c7a/pybase64-1.4.3-cp312-cp312-win_arm64.whl", hash = "sha256:0631d8a2d035de03aa9bded029b9513e1fee8ed80b7ddef6b8e9389ffc445da0", size = 31185, upload-time = "2025-12-06T13:23:39.908Z" }, + { url = "https://files.pythonhosted.org/packages/17/45/92322aec1b6979e789b5710f73c59f2172bc37c8ce835305434796824b7b/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:2baaa092f3475f3a9c87ac5198023918ea8b6c125f4c930752ab2cbe3cd1d520", size = 38746, upload-time = "2025-12-06T13:26:25.869Z" }, + { url = "https://files.pythonhosted.org/packages/11/94/f1a07402870388fdfc2ecec0c718111189732f7d0f2d7fe1386e19e8fad0/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:cde13c0764b1af07a631729f26df019070dad759981d6975527b7e8ecb465b6c", size = 32573, upload-time = "2025-12-06T13:26:27.792Z" }, + { url = "https://files.pythonhosted.org/packages/fa/8f/43c3bb11ca9bacf81cb0b7a71500bb65b2eda6d5fe07433c09b543de97f3/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5c29a582b0ea3936d02bd6fe9bf674ab6059e6e45ab71c78404ab2c913224414", size = 43461, upload-time = "2025-12-06T13:26:28.906Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4c/2a5258329200be57497d3972b5308558c6de42e3749c6cc2aa1cbe34b25a/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6b664758c804fa919b4f1257aa8cf68e95db76fc331de5f70bfc3a34655afe1", size = 36058, upload-time = "2025-12-06T13:26:30.092Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6d/41faa414cde66ec023b0ca8402a8f11cb61731c3dc27c082909cbbd1f929/pybase64-1.4.3-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:f7537fa22ae56a0bf51e4b0ffc075926ad91c618e1416330939f7ef366b58e3b", size = 36231, upload-time = "2025-12-06T13:26:31.656Z" }, +] + +[[package]] +name = "pybind11" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/7b/a6d8dcb83c457e24a9df1e4d8fd5fb8034d4bbc62f3c324681e8a9ba57c2/pybind11-3.0.1.tar.gz", hash = "sha256:9c0f40056a016da59bab516efb523089139fcc6f2ba7e4930854c61efb932051", size = 546914, upload-time = "2025-08-22T20:09:27.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/8a/37362fc2b949d5f733a8b0f2ff51ba423914cabefe69f1d1b6aab710f5fe/pybind11-3.0.1-py3-none-any.whl", hash = "sha256:aa8f0aa6e0a94d3b64adfc38f560f33f15e589be2175e103c0a33c6bce55ee89", size = 293611, upload-time = "2025-08-22T20:09:25.235Z" }, +] + +[[package]] +name = "pycountry" +version = "24.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/57/c389fa68c50590881a75b7883eeb3dc15e9e73a0fdc001cdd45c13290c92/pycountry-24.6.1.tar.gz", hash = "sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221", size = 6043910, upload-time = "2024-06-01T04:12:15.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/ec/1fb891d8a2660716aadb2143235481d15ed1cbfe3ad669194690b0604492/pycountry-24.6.1-py3-none-any.whl", hash = "sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f", size = 6335189, upload-time = "2024-06-01T04:11:49.711Z" }, +] + +[[package]] +name = "pycparser" +version = "2.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, +] + +[[package]] +name = "pycryptodomex" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/85/e24bf90972a30b0fcd16c73009add1d7d7cd9140c2498a68252028899e41/pycryptodomex-3.23.0.tar.gz", hash = "sha256:71909758f010c82bc99b0abf4ea12012c98962fbf0583c2164f8b84533c2e4da", size = 4922157, upload-time = "2025-05-17T17:23:41.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/9c/1a8f35daa39784ed8adf93a694e7e5dc15c23c741bbda06e1d45f8979e9e/pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:06698f957fe1ab229a99ba2defeeae1c09af185baa909a31a5d1f9d42b1aaed6", size = 2499240, upload-time = "2025-05-17T17:22:46.953Z" }, + { url = "https://files.pythonhosted.org/packages/7a/62/f5221a191a97157d240cf6643747558759126c76ee92f29a3f4aee3197a5/pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2c2537863eccef2d41061e82a881dcabb04944c5c06c5aa7110b577cc487545", size = 1644042, upload-time = "2025-05-17T17:22:49.098Z" }, + { url = "https://files.pythonhosted.org/packages/8c/fd/5a054543c8988d4ed7b612721d7e78a4b9bf36bc3c5ad45ef45c22d0060e/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43c446e2ba8df8889e0e16f02211c25b4934898384c1ec1ec04d7889c0333587", size = 2186227, upload-time = "2025-05-17T17:22:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/c8/a9/8862616a85cf450d2822dbd4fff1fcaba90877907a6ff5bc2672cafe42f8/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f489c4765093fb60e2edafdf223397bc716491b2b69fe74367b70d6999257a5c", size = 2272578, upload-time = "2025-05-17T17:22:53.676Z" }, + { url = "https://files.pythonhosted.org/packages/46/9f/bda9c49a7c1842820de674ab36c79f4fbeeee03f8ff0e4f3546c3889076b/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdc69d0d3d989a1029df0eed67cc5e8e5d968f3724f4519bd03e0ec68df7543c", size = 2312166, upload-time = "2025-05-17T17:22:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/03/cc/870b9bf8ca92866ca0186534801cf8d20554ad2a76ca959538041b7a7cf4/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bbcb1dd0f646484939e142462d9e532482bc74475cecf9c4903d4e1cd21f003", size = 2185467, upload-time = "2025-05-17T17:22:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/96/e3/ce9348236d8e669fea5dd82a90e86be48b9c341210f44e25443162aba187/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:8a4fcd42ccb04c31268d1efeecfccfd1249612b4de6374205376b8f280321744", size = 2346104, upload-time = "2025-05-17T17:23:02.112Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e869bcee87beb89040263c416a8a50204f7f7a83ac11897646c9e71e0daf/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd", size = 2271038, upload-time = "2025-05-17T17:23:04.872Z" }, + { url = "https://files.pythonhosted.org/packages/8d/67/09ee8500dd22614af5fbaa51a4aee6e342b5fa8aecf0a6cb9cbf52fa6d45/pycryptodomex-3.23.0-cp37-abi3-win32.whl", hash = "sha256:189afbc87f0b9f158386bf051f720e20fa6145975f1e76369303d0f31d1a8d7c", size = 1771969, upload-time = "2025-05-17T17:23:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/69/96/11f36f71a865dd6df03716d33bd07a67e9d20f6b8d39820470b766af323c/pycryptodomex-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:52e5ca58c3a0b0bd5e100a9fbc8015059b05cffc6c66ce9d98b4b45e023443b9", size = 1803124, upload-time = "2025-05-17T17:23:09.267Z" }, + { url = "https://files.pythonhosted.org/packages/f9/93/45c1cdcbeb182ccd2e144c693eaa097763b08b38cded279f0053ed53c553/pycryptodomex-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:02d87b80778c171445d67e23d1caef279bf4b25c3597050ccd2e13970b57fd51", size = 1707161, upload-time = "2025-05-17T17:23:11.414Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[package.optional-dependencies] +email = [ + { name = "email-validator" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, +] + +[[package]] +name = "pydantic-extra-types" +version = "2.10.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/10/fb64987804cde41bcc39d9cd757cd5f2bb5d97b389d81aa70238b14b8a7e/pydantic_extra_types-2.10.6.tar.gz", hash = "sha256:c63d70bf684366e6bbe1f4ee3957952ebe6973d41e7802aea0b770d06b116aeb", size = 141858, upload-time = "2025-10-08T13:47:49.483Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/04/5c918669096da8d1c9ec7bb716bd72e755526103a61bc5e76a3e4fb23b53/pydantic_extra_types-2.10.6-py3-none-any.whl", hash = "sha256:6106c448316d30abf721b5b9fecc65e983ef2614399a24142d689c7546cc246a", size = 40949, upload-time = "2025-10-08T13:47:48.268Z" }, +] + +[package.optional-dependencies] +pycountry = [ + { name = "pycountry" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/ac7e0aae12027748076d72a8764ff1c9d82ca75a7a52622e67ed3f765c54/pydantic_settings-2.12.0.tar.gz", hash = "sha256:005538ef951e3c2a68e1c08b292b5f2e71490def8589d4221b95dab00dafcfd0", size = 194184, upload-time = "2025-11-10T14:25:47.013Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785, upload-time = "2024-11-28T03:43:29.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[[package]] +name = "pynacl" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/46/aeca065d227e2265125aea590c9c47fbf5786128c9400ee0eb7c88931f06/pynacl-1.6.1.tar.gz", hash = "sha256:8d361dac0309f2b6ad33b349a56cd163c98430d409fa503b10b70b3ad66eaa1d", size = 3506616, upload-time = "2025-11-10T16:02:13.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/41/3cfb3b4f3519f6ff62bf71bf1722547644bcfb1b05b8fdbdc300249ba113/pynacl-1.6.1-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:a6f9fd6d6639b1e81115c7f8ff16b8dedba1e8098d2756275d63d208b0e32021", size = 387591, upload-time = "2025-11-10T16:01:49.1Z" }, + { url = "https://files.pythonhosted.org/packages/18/21/b8a6563637799f617a3960f659513eccb3fcc655d5fc2be6e9dc6416826f/pynacl-1.6.1-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e49a3f3d0da9f79c1bec2aa013261ab9fa651c7da045d376bd306cf7c1792993", size = 798866, upload-time = "2025-11-10T16:01:55.688Z" }, + { url = "https://files.pythonhosted.org/packages/e8/6c/dc38033bc3ea461e05ae8f15a81e0e67ab9a01861d352ae971c99de23e7c/pynacl-1.6.1-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7713f8977b5d25f54a811ec9efa2738ac592e846dd6e8a4d3f7578346a841078", size = 1398001, upload-time = "2025-11-10T16:01:57.101Z" }, + { url = "https://files.pythonhosted.org/packages/9f/05/3ec0796a9917100a62c5073b20c4bce7bf0fea49e99b7906d1699cc7b61b/pynacl-1.6.1-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a3becafc1ee2e5ea7f9abc642f56b82dcf5be69b961e782a96ea52b55d8a9fc", size = 834024, upload-time = "2025-11-10T16:01:50.228Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b7/ae9982be0f344f58d9c64a1c25d1f0125c79201634efe3c87305ac7cb3e3/pynacl-1.6.1-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ce50d19f1566c391fedc8dc2f2f5be265ae214112ebe55315e41d1f36a7f0a9", size = 1436766, upload-time = "2025-11-10T16:01:51.886Z" }, + { url = "https://files.pythonhosted.org/packages/b4/51/b2ccbf89cf3025a02e044dd68a365cad593ebf70f532299f2c047d2b7714/pynacl-1.6.1-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:543f869140f67d42b9b8d47f922552d7a967e6c116aad028c9bfc5f3f3b3a7b7", size = 817275, upload-time = "2025-11-10T16:01:53.351Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6c/dd9ee8214edf63ac563b08a9b30f98d116942b621d39a751ac3256694536/pynacl-1.6.1-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a2bb472458c7ca959aeeff8401b8efef329b0fc44a89d3775cffe8fad3398ad8", size = 1401891, upload-time = "2025-11-10T16:01:54.587Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c1/97d3e1c83772d78ee1db3053fd674bc6c524afbace2bfe8d419fd55d7ed1/pynacl-1.6.1-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3206fa98737fdc66d59b8782cecc3d37d30aeec4593d1c8c145825a345bba0f0", size = 772291, upload-time = "2025-11-10T16:01:58.111Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ca/691ff2fe12f3bb3e43e8e8df4b806f6384593d427f635104d337b8e00291/pynacl-1.6.1-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:53543b4f3d8acb344f75fd4d49f75e6572fce139f4bfb4815a9282296ff9f4c0", size = 1370839, upload-time = "2025-11-10T16:01:59.252Z" }, + { url = "https://files.pythonhosted.org/packages/30/27/06fe5389d30391fce006442246062cc35773c84fbcad0209fbbf5e173734/pynacl-1.6.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:319de653ef84c4f04e045eb250e6101d23132372b0a61a7acf91bac0fda8e58c", size = 791371, upload-time = "2025-11-10T16:02:01.075Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7a/e2bde8c9d39074a5aa046c7d7953401608d1f16f71e237f4bef3fb9d7e49/pynacl-1.6.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:262a8de6bba4aee8a66f5edf62c214b06647461c9b6b641f8cd0cb1e3b3196fe", size = 1363031, upload-time = "2025-11-10T16:02:02.656Z" }, + { url = "https://files.pythonhosted.org/packages/dd/b6/63fd77264dae1087770a1bb414bc604470f58fbc21d83822fc9c76248076/pynacl-1.6.1-cp38-abi3-win32.whl", hash = "sha256:9fd1a4eb03caf8a2fe27b515a998d26923adb9ddb68db78e35ca2875a3830dde", size = 226585, upload-time = "2025-11-10T16:02:07.116Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/b419180f3fdb72ab4d45e1d88580761c267c7ca6eda9a20dcbcba254efe6/pynacl-1.6.1-cp38-abi3-win_amd64.whl", hash = "sha256:a569a4069a7855f963940040f35e87d8bc084cb2d6347428d5ad20550a0a1a21", size = 238923, upload-time = "2025-11-10T16:02:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/35/76/c34426d532e4dce7ff36e4d92cb20f4cbbd94b619964b93d24e8f5b5510f/pynacl-1.6.1-cp38-abi3-win_arm64.whl", hash = "sha256:5953e8b8cfadb10889a6e7bd0f53041a745d1b3d30111386a1bb37af171e6daf", size = 183970, upload-time = "2025-11-10T16:02:05.786Z" }, +] + +[[package]] +name = "pynvml" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-ml-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/57/da7dc63a79f59e082e26a66ac02d87d69ea316b35b35b7a00d82f3ce3d2f/pynvml-13.0.1.tar.gz", hash = "sha256:1245991d9db786b4d2f277ce66869bd58f38ac654e38c9397d18f243c8f6e48f", size = 35226, upload-time = "2025-09-05T20:33:25.377Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/4a/cac76c174bb439a0c46c9a4413fcbea5c6cabfb01879f7bbdb9fdfaed76c/pynvml-13.0.1-py3-none-any.whl", hash = "sha256:e2b20e0a501eeec951e2455b7ab444759cf048e0e13a57b08049fa2775266aa8", size = 28810, upload-time = "2025-09-05T20:33:24.13Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/26/19cadc79a718c5edbec86fd4919a6b6d3f681039a2f6d66d14be94e75fb9/python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6", size = 44221, upload-time = "2025-10-26T15:12:10.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, +] + +[[package]] +name = "python-json-logger" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/bf/eca6a3d43db1dae7070f70e160ab20b807627ba953663ba07928cdd3dc58/python_json_logger-4.0.0.tar.gz", hash = "sha256:f58e68eb46e1faed27e0f574a55a0455eecd7b8a5b88b85a784519ba3cff047f", size = 17683, upload-time = "2025-10-06T04:15:18.984Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/e5/fecf13f06e5e5f67e8837d777d1bc43fac0ed2b77a676804df5c34744727/python_json_logger-4.0.0-py3-none-any.whl", hash = "sha256:af09c9daf6a813aa4cc7180395f50f2a9e5fa056034c9953aec92e381c5ba1e2", size = 15548, upload-time = "2025-10-06T04:15:17.553Z" }, +] + +[[package]] +name = "python-multipart" +version = "0.0.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/86/b6b38677dec2e2e7898fc5b6f7e42c2d011919a92d25339451892f27b89c/python_multipart-0.0.18.tar.gz", hash = "sha256:7a68db60c8bfb82e460637fa4750727b45af1d5e2ed215593f917f64694d34fe", size = 36622, upload-time = "2024-11-28T19:16:02.383Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/6b/b60f47101ba2cac66b4a83246630e68ae9bbe2e614cbae5f4465f46dee13/python_multipart-0.0.18-py3-none-any.whl", hash = "sha256:efe91480f485f6a361427a541db4796f9e1591afc0fb8e7a4ba06bfbc6708996", size = 24389, upload-time = "2024-11-28T19:16:00.947Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "pyvers" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/39/c5432f541e6ea1d616dfd6ef42ce02792f7eb42dd44f5ed4439dbe17a58b/pyvers-0.1.0-py3-none-any.whl", hash = "sha256:065249805ae537ddf9a2d1a8dffc6d0a12474a347d2eaa2f35ebdae92c0c8199", size = 10092, upload-time = "2025-06-08T23:46:46.219Z" }, +] + +[[package]] +name = "pywin32" +version = "311" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543, upload-time = "2025-07-14T20:13:20.765Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040, upload-time = "2025-07-14T20:13:22.543Z" }, + { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102, upload-time = "2025-07-14T20:13:24.682Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "pyzmq" +version = "27.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, + { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" }, + { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" }, +] + +[[package]] +name = "qwen-vl-utils" +version = "0.0.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "av" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/b1/ad4fc2260a3badd278b38d642f3b987412f1f6682f0ef2b31b0572d5caa8/qwen_vl_utils-0.0.14.tar.gz", hash = "sha256:9c7cad5ae803b3a10f8bb7194deb12aeacdd032f92f4224e880c73587a7346ad", size = 8453, upload-time = "2025-09-23T09:38:57.532Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/43/80f67e0336cb2fc725f8e06f7fe35c1d0fe946f4d2b8b2175e797e07349e/qwen_vl_utils-0.0.14-py3-none-any.whl", hash = "sha256:5e28657bfd031e56bd447c5901b58ddfc3835285ed100f4c56580e0ade054e96", size = 8120, upload-time = "2025-09-23T09:38:56.297Z" }, +] + +[[package]] +name = "ray" +version = "2.51.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "jsonschema" }, + { name = "msgpack" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "pyyaml" }, + { name = "requests" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/95/51e44ce79e42f02ca1c4d4c5501e6dd49f3a384c5f6324aceb4e0015988a/ray-2.51.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ef847b025ca758baee4571a1ca001d973897cad772f8e95d7f303d24c38b649e", size = 68029226, upload-time = "2025-11-01T03:24:21.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/b5/a93e39e131067edb7cba3385a609f61aaaf7aa54728cd3a7474bfbf3b0fc/ray-2.51.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:0bed9408712bad1511e65683a455302f88d94e5e5cb6a58cc4a154b61d8a0b4a", size = 70502423, upload-time = "2025-11-01T03:24:27.398Z" }, + { url = "https://files.pythonhosted.org/packages/ee/59/69b7a653ed8176fc7fd894d462ed34bb1477e7fa71700324de99179b5b7e/ray-2.51.1-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:4e786da7862cf73664977d0212a505d6d5a585beadf63e7dc1e1c129259bee20", size = 71353730, upload-time = "2025-11-01T03:24:33.495Z" }, + { url = "https://files.pythonhosted.org/packages/38/91/0c4fe7aed34baa14d9c050c88f39ff16083d555bd6dcd6c4ffb4332a6f8a/ray-2.51.1-cp312-cp312-win_amd64.whl", hash = "sha256:198fda93074a6863555f4003e9013bb2ba0cd50b59b18c02affdc294b28a2eef", size = 26674921, upload-time = "2025-11-01T03:24:38.394Z" }, +] + +[package.optional-dependencies] +cgraph = [ + { name = "cupy-cuda12x", marker = "sys_platform != 'darwin'" }, +] + +[[package]] +name = "redis" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/c8/983d5c6579a411d8a99bc5823cc5712768859b5ce2c8afe1a65b37832c81/redis-7.1.0.tar.gz", hash = "sha256:b1cc3cfa5a2cb9c2ab3ba700864fb0ad75617b41f01352ce5779dabf6d5f9c3c", size = 4796669, upload-time = "2025-11-19T15:54:39.961Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/f0/8956f8a86b20d7bb9d6ac0187cf4cd54d8065bc9a1a09eb8011d4d326596/redis-7.1.0-py3-none-any.whl", hash = "sha256:23c52b208f92b56103e17c5d06bdc1a6c2c0b3106583985a76a18f83b265de2b", size = 354159, upload-time = "2025-11-19T15:54:38.064Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "regex" +version = "2025.11.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/a9/546676f25e573a4cf00fe8e119b78a37b6a8fe2dc95cda877b30889c9c45/regex-2025.11.3.tar.gz", hash = "sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01", size = 414669, upload-time = "2025-11-03T21:34:22.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/74/18f04cb53e58e3fb107439699bd8375cf5a835eec81084e0bddbd122e4c2/regex-2025.11.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41", size = 489312, upload-time = "2025-11-03T21:31:34.343Z" }, + { url = "https://files.pythonhosted.org/packages/78/3f/37fcdd0d2b1e78909108a876580485ea37c91e1acf66d3bb8e736348f441/regex-2025.11.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36", size = 291256, upload-time = "2025-11-03T21:31:35.675Z" }, + { url = "https://files.pythonhosted.org/packages/bf/26/0a575f58eb23b7ebd67a45fccbc02ac030b737b896b7e7a909ffe43ffd6a/regex-2025.11.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1", size = 288921, upload-time = "2025-11-03T21:31:37.07Z" }, + { url = "https://files.pythonhosted.org/packages/ea/98/6a8dff667d1af907150432cf5abc05a17ccd32c72a3615410d5365ac167a/regex-2025.11.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7", size = 798568, upload-time = "2025-11-03T21:31:38.784Z" }, + { url = "https://files.pythonhosted.org/packages/64/15/92c1db4fa4e12733dd5a526c2dd2b6edcbfe13257e135fc0f6c57f34c173/regex-2025.11.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69", size = 864165, upload-time = "2025-11-03T21:31:40.559Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e7/3ad7da8cdee1ce66c7cd37ab5ab05c463a86ffeb52b1a25fe7bd9293b36c/regex-2025.11.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48", size = 912182, upload-time = "2025-11-03T21:31:42.002Z" }, + { url = "https://files.pythonhosted.org/packages/84/bd/9ce9f629fcb714ffc2c3faf62b6766ecb7a585e1e885eb699bcf130a5209/regex-2025.11.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c", size = 803501, upload-time = "2025-11-03T21:31:43.815Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0f/8dc2e4349d8e877283e6edd6c12bdcebc20f03744e86f197ab6e4492bf08/regex-2025.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695", size = 787842, upload-time = "2025-11-03T21:31:45.353Z" }, + { url = "https://files.pythonhosted.org/packages/f9/73/cff02702960bc185164d5619c0c62a2f598a6abff6695d391b096237d4ab/regex-2025.11.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98", size = 858519, upload-time = "2025-11-03T21:31:46.814Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/0e8d1ae71e15bc1dc36231c90b46ee35f9d52fab2e226b0e039e7ea9c10a/regex-2025.11.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74", size = 850611, upload-time = "2025-11-03T21:31:48.289Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f5/70a5cdd781dcfaa12556f2955bf170cd603cb1c96a1827479f8faea2df97/regex-2025.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0", size = 789759, upload-time = "2025-11-03T21:31:49.759Z" }, + { url = "https://files.pythonhosted.org/packages/59/9b/7c29be7903c318488983e7d97abcf8ebd3830e4c956c4c540005fcfb0462/regex-2025.11.3-cp312-cp312-win32.whl", hash = "sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204", size = 266194, upload-time = "2025-11-03T21:31:51.53Z" }, + { url = "https://files.pythonhosted.org/packages/1a/67/3b92df89f179d7c367be654ab5626ae311cb28f7d5c237b6bb976cd5fbbb/regex-2025.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9", size = 277069, upload-time = "2025-11-03T21:31:53.151Z" }, + { url = "https://files.pythonhosted.org/packages/d7/55/85ba4c066fe5094d35b249c3ce8df0ba623cfd35afb22d6764f23a52a1c5/regex-2025.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26", size = 270330, upload-time = "2025-11-03T21:31:54.514Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "rich" +version = "13.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/01/c954e134dc440ab5f96952fe52b4fdc64225530320a910473c1fe270d9aa/rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432", size = 221248, upload-time = "2024-02-28T14:51:19.472Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/67/a37f6214d0e9fe57f6ae54b2956d550ca8365857f42a1ce0392bb21d9410/rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222", size = 240681, upload-time = "2024-02-28T14:51:14.353Z" }, +] + +[[package]] +name = "rich-toolkit" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/97/09/3f9b8d9daaf235195c626f21e03604c05b987404ee3bcacee0c1f67f2a8e/rich_toolkit-0.17.1.tar.gz", hash = "sha256:5af54df8d1dd9c8530e462e1bdcaed625c9b49f5a55b035aa0ba1c17bdb87c9a", size = 187925, upload-time = "2025-12-17T10:49:22.583Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/7b/15e55fa8a76d0d41bf34d965af78acdaf80a315907adb30de8b63c272694/rich_toolkit-0.17.1-py3-none-any.whl", hash = "sha256:96d24bb921ecd225ffce7c526a9149e74006410c05e6d405bd74ffd54d5631ed", size = 31412, upload-time = "2025-12-17T10:49:21.793Z" }, +] + +[[package]] +name = "rignore" +version = "0.7.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/f5/8bed2310abe4ae04b67a38374a4d311dd85220f5d8da56f47ae9361be0b0/rignore-0.7.6.tar.gz", hash = "sha256:00d3546cd793c30cb17921ce674d2c8f3a4b00501cb0e3dd0e82217dbeba2671", size = 57140, upload-time = "2025-11-05T21:41:21.968Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/0e/012556ef3047a2628842b44e753bb15f4dc46806780ff090f1e8fe4bf1eb/rignore-0.7.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:03e82348cb7234f8d9b2834f854400ddbbd04c0f8f35495119e66adbd37827a8", size = 883488, upload-time = "2025-11-05T20:42:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/d4f1f3fe9eb3f8e382d45ce5b0547ea01c4b7e0b4b4eb87bcd66a1d2b888/rignore-0.7.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b9e624f6be6116ea682e76c5feb71ea91255c67c86cb75befe774365b2931961", size = 820411, upload-time = "2025-11-05T20:42:24.782Z" }, + { url = "https://files.pythonhosted.org/packages/4a/c8/dea564b36dedac8de21c18e1851789545bc52a0c22ece9843444d5608a6a/rignore-0.7.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bda49950d405aa8d0ebe26af807c4e662dd281d926530f03f29690a2e07d649a", size = 897821, upload-time = "2025-11-05T20:40:52.613Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/ee96db17ac1835e024c5d0742eefb7e46de60020385ac883dd3d1cde2c1f/rignore-0.7.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5fd5ab3840b8c16851d327ed06e9b8be6459702a53e5ab1fc4073b684b3789e", size = 873963, upload-time = "2025-11-05T20:41:07.49Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8c/ad5a57bbb9d14d5c7e5960f712a8a0b902472ea3f4a2138cbf70d1777b75/rignore-0.7.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ced2a248352636a5c77504cb755dc02c2eef9a820a44d3f33061ce1bb8a7f2d2", size = 1169216, upload-time = "2025-11-05T20:41:23.73Z" }, + { url = "https://files.pythonhosted.org/packages/80/e6/5b00bc2a6bc1701e6878fca798cf5d9125eb3113193e33078b6fc0d99123/rignore-0.7.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a04a3b73b75ddc12c9c9b21efcdaab33ca3832941d6f1d67bffd860941cd448a", size = 942942, upload-time = "2025-11-05T20:41:39.393Z" }, + { url = "https://files.pythonhosted.org/packages/85/e5/7f99bd0cc9818a91d0e8b9acc65b792e35750e3bdccd15a7ee75e64efca4/rignore-0.7.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d24321efac92140b7ec910ac7c53ab0f0c86a41133d2bb4b0e6a7c94967f44dd", size = 959787, upload-time = "2025-11-05T20:42:09.765Z" }, + { url = "https://files.pythonhosted.org/packages/55/54/2ffea79a7c1eabcede1926347ebc2a81bc6b81f447d05b52af9af14948b9/rignore-0.7.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c7aa109d41e593785c55fdaa89ad80b10330affa9f9d3e3a51fa695f739b20", size = 984245, upload-time = "2025-11-05T20:41:54.062Z" }, + { url = "https://files.pythonhosted.org/packages/41/f7/e80f55dfe0f35787fa482aa18689b9c8251e045076c35477deb0007b3277/rignore-0.7.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1734dc49d1e9501b07852ef44421f84d9f378da9fbeda729e77db71f49cac28b", size = 1078647, upload-time = "2025-11-05T21:40:13.463Z" }, + { url = "https://files.pythonhosted.org/packages/d4/cf/2c64f0b6725149f7c6e7e5a909d14354889b4beaadddaa5fff023ec71084/rignore-0.7.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5719ea14ea2b652c0c0894be5dfde954e1853a80dea27dd2fbaa749618d837f5", size = 1139186, upload-time = "2025-11-05T21:40:31.27Z" }, + { url = "https://files.pythonhosted.org/packages/75/95/a86c84909ccc24af0d094b50d54697951e576c252a4d9f21b47b52af9598/rignore-0.7.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8e23424fc7ce35726854f639cb7968151a792c0c3d9d082f7f67e0c362cfecca", size = 1117604, upload-time = "2025-11-05T21:40:48.07Z" }, + { url = "https://files.pythonhosted.org/packages/7f/5e/13b249613fd5d18d58662490ab910a9f0be758981d1797789913adb4e918/rignore-0.7.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3efdcf1dd84d45f3e2bd2f93303d9be103888f56dfa7c3349b5bf4f0657ec696", size = 1127725, upload-time = "2025-11-05T21:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/c7/28/fa5dcd1e2e16982c359128664e3785f202d3eca9b22dd0b2f91c4b3d242f/rignore-0.7.6-cp312-cp312-win32.whl", hash = "sha256:ccca9d1a8b5234c76b71546fc3c134533b013f40495f394a65614a81f7387046", size = 646145, upload-time = "2025-11-05T21:41:51.096Z" }, + { url = "https://files.pythonhosted.org/packages/26/87/69387fb5dd81a0f771936381431780b8cf66fcd2cfe9495e1aaf41548931/rignore-0.7.6-cp312-cp312-win_amd64.whl", hash = "sha256:c96a285e4a8bfec0652e0bfcf42b1aabcdda1e7625f5006d188e3b1c87fdb543", size = 726090, upload-time = "2025-11-05T21:41:36.485Z" }, + { url = "https://files.pythonhosted.org/packages/24/5f/e8418108dcda8087fb198a6f81caadbcda9fd115d61154bf0df4d6d3619b/rignore-0.7.6-cp312-cp312-win_arm64.whl", hash = "sha256:a64a750e7a8277a323f01ca50b7784a764845f6cce2fe38831cb93f0508d0051", size = 656317, upload-time = "2025-11-05T21:41:25.305Z" }, +] + +[[package]] +name = "roman-numerals" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, +] + +[[package]] +name = "roman-numerals-py" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "roman-numerals" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/2c/daca29684cbe9fd4bc711f8246da3c10adca1ccc4d24436b17572eb2590e/roman_numerals_py-4.1.0-py3-none-any.whl", hash = "sha256:553114c1167141c1283a51743759723ecd05604a1b6b507225e91dc1a6df0780", size = 4547, upload-time = "2025-12-17T18:25:40.136Z" }, +] + +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, +] + +[[package]] +name = "rq" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "croniter" }, + { name = "redis" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/6f/a2848f5ba0ca7f1f879c7ad44a2e7b06b98197a7da39be39eda775807f33/rq-2.6.1.tar.gz", hash = "sha256:db5c0d125ac9dbd4438f9a5225ea3e64050542b416fd791d424e2ab5b2853289", size = 675386, upload-time = "2025-11-22T06:45:16.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/cc/919ccbf0c9b4f8b0f68c3f53e6d8e1e94af4d74cee4e6d3cb2e81f7d0da9/rq-2.6.1-py3-none-any.whl", hash = "sha256:5cc88d3bb5263a407fb2ba2dc6fe8dc710dae94b6f74396cdfe1b32beded9408", size = 112578, upload-time = "2025-11-22T06:45:13.529Z" }, +] + +[[package]] +name = "ruff" +version = "0.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/e7/e55dda1c92cdcf34b677ebef17486669800de01e887b7831a1b8fdf5cb08/ruff-0.11.9.tar.gz", hash = "sha256:ebd58d4f67a00afb3a30bf7d383e52d0e036e6195143c6db7019604a05335517", size = 4132134, upload-time = "2025-05-09T16:19:41.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/71/75dfb7194fe6502708e547941d41162574d1f579c4676a8eb645bf1a6842/ruff-0.11.9-py3-none-linux_armv6l.whl", hash = "sha256:a31a1d143a5e6f499d1fb480f8e1e780b4dfdd580f86e05e87b835d22c5c6f8c", size = 10335453, upload-time = "2025-05-09T16:18:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/74/fc/ad80c869b1732f53c4232bbf341f33c5075b2c0fb3e488983eb55964076a/ruff-0.11.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:66bc18ca783b97186a1f3100e91e492615767ae0a3be584e1266aa9051990722", size = 11072566, upload-time = "2025-05-09T16:19:01.432Z" }, + { url = "https://files.pythonhosted.org/packages/87/0d/0ccececef8a0671dae155cbf7a1f90ea2dd1dba61405da60228bbe731d35/ruff-0.11.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bd576cd06962825de8aece49f28707662ada6a1ff2db848d1348e12c580acbf1", size = 10435020, upload-time = "2025-05-09T16:19:03.897Z" }, + { url = "https://files.pythonhosted.org/packages/52/01/e249e1da6ad722278094e183cbf22379a9bbe5f21a3e46cef24ccab76e22/ruff-0.11.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b1d18b4be8182cc6fddf859ce432cc9631556e9f371ada52f3eaefc10d878de", size = 10593935, upload-time = "2025-05-09T16:19:06.455Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9a/40cf91f61e3003fe7bd43f1761882740e954506c5a0f9097b1cff861f04c/ruff-0.11.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0f3f46f759ac623e94824b1e5a687a0df5cd7f5b00718ff9c24f0a894a683be7", size = 10172971, upload-time = "2025-05-09T16:19:10.261Z" }, + { url = "https://files.pythonhosted.org/packages/61/12/d395203de1e8717d7a2071b5a340422726d4736f44daf2290aad1085075f/ruff-0.11.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f34847eea11932d97b521450cf3e1d17863cfa5a94f21a056b93fb86f3f3dba2", size = 11748631, upload-time = "2025-05-09T16:19:12.307Z" }, + { url = "https://files.pythonhosted.org/packages/66/d6/ef4d5eba77677eab511644c37c55a3bb8dcac1cdeb331123fe342c9a16c9/ruff-0.11.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f33b15e00435773df97cddcd263578aa83af996b913721d86f47f4e0ee0ff271", size = 12409236, upload-time = "2025-05-09T16:19:15.006Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8f/5a2c5fc6124dd925a5faf90e1089ee9036462118b619068e5b65f8ea03df/ruff-0.11.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7b27613a683b086f2aca8996f63cb3dd7bc49e6eccf590563221f7b43ded3f65", size = 11881436, upload-time = "2025-05-09T16:19:17.063Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/9683f469ae0b99b95ef99a56cfe8c8373c14eba26bd5c622150959ce9f64/ruff-0.11.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e0d88756e63e8302e630cee3ce2ffb77859797cc84a830a24473939e6da3ca6", size = 13982759, upload-time = "2025-05-09T16:19:19.693Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0b/c53a664f06e0faab596397867c6320c3816df479e888fe3af63bc3f89699/ruff-0.11.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:537c82c9829d7811e3aa680205f94c81a2958a122ac391c0eb60336ace741a70", size = 11541985, upload-time = "2025-05-09T16:19:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/23/a0/156c4d7e685f6526a636a60986ee4a3c09c8c4e2a49b9a08c9913f46c139/ruff-0.11.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:440ac6a7029f3dee7d46ab7de6f54b19e34c2b090bb4f2480d0a2d635228f381", size = 10465775, upload-time = "2025-05-09T16:19:24.401Z" }, + { url = "https://files.pythonhosted.org/packages/43/d5/88b9a6534d9d4952c355e38eabc343df812f168a2c811dbce7d681aeb404/ruff-0.11.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:71c539bac63d0788a30227ed4d43b81353c89437d355fdc52e0cda4ce5651787", size = 10170957, upload-time = "2025-05-09T16:19:27.08Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b8/2bd533bdaf469dc84b45815ab806784d561fab104d993a54e1852596d581/ruff-0.11.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c67117bc82457e4501473c5f5217d49d9222a360794bfb63968e09e70f340abd", size = 11143307, upload-time = "2025-05-09T16:19:29.462Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d9/43cfba291788459b9bfd4e09a0479aa94d05ab5021d381a502d61a807ec1/ruff-0.11.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e4b78454f97aa454586e8a5557facb40d683e74246c97372af3c2d76901d697b", size = 11603026, upload-time = "2025-05-09T16:19:31.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e6/7ed70048e89b01d728ccc950557a17ecf8df4127b08a56944b9d0bae61bc/ruff-0.11.9-py3-none-win32.whl", hash = "sha256:7fe1bc950e7d7b42caaee2a8a3bc27410547cc032c9558ee2e0f6d3b209e845a", size = 10548627, upload-time = "2025-05-09T16:19:33.657Z" }, + { url = "https://files.pythonhosted.org/packages/90/36/1da5d566271682ed10f436f732e5f75f926c17255c9c75cefb77d4bf8f10/ruff-0.11.9-py3-none-win_amd64.whl", hash = "sha256:52edaa4a6d70f8180343a5b7f030c7edd36ad180c9f4d224959c2d689962d964", size = 11634340, upload-time = "2025-05-09T16:19:35.815Z" }, + { url = "https://files.pythonhosted.org/packages/40/f7/70aad26e5877c8f7ee5b161c4c9fa0100e63fc4c944dc6d97b9c7e871417/ruff-0.11.9-py3-none-win_arm64.whl", hash = "sha256:bcf42689c22f2e240f496d0c183ef2c6f7b35e809f12c1db58f75d9aa8d630ca", size = 10741080, upload-time = "2025-05-09T16:19:39.605Z" }, +] + +[[package]] +name = "s3fs" +version = "2025.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiobotocore" }, + { name = "aiohttp" }, + { name = "fsspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/cd/5dde2fed1699ff48120336249d9857a574e39feb8afaff694568ab1499b3/s3fs-2025.3.0.tar.gz", hash = "sha256:446dd539eb0d0678209723cb7ad1bedbb172185b0d34675b09be1ad81843a644", size = 77153, upload-time = "2025-03-07T21:58:32.114Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/3f/35f4041a82a68df89fe4af97c8bb44aa492dad924799cbb02078e9e303e6/s3fs-2025.3.0-py3-none-any.whl", hash = "sha256:88d803615baa04945156ca0e1498009b7acd3132c07198bd81b3e874846e0aa2", size = 30454, upload-time = "2025-03-07T21:58:30.998Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.11.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/24/1390172471d569e281fcfd29b92f2f73774e95972c965d14b6c802ff2352/s3transfer-0.11.3.tar.gz", hash = "sha256:edae4977e3a122445660c7c114bba949f9d191bae3b34a096f18a1c8c354527a", size = 148042, upload-time = "2025-02-26T20:44:57.459Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/81/48c41b554a54d75d4407740abb60e3a102ae416284df04d1dbdcbe3dbf24/s3transfer-0.11.3-py3-none-any.whl", hash = "sha256:ca855bdeb885174b5ffa95b9913622459d4ad8e331fc98eb01e6d5eb6a30655d", size = 84246, upload-time = "2025-02-26T20:44:55.509Z" }, +] + +[[package]] +name = "safetensors" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/9c/6e74567782559a63bd040a236edca26fd71bc7ba88de2ef35d75df3bca5e/safetensors-0.7.0.tar.gz", hash = "sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0", size = 200878, upload-time = "2025-11-19T15:18:43.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/47/aef6c06649039accf914afef490268e1067ed82be62bcfa5b7e886ad15e8/safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517", size = 467781, upload-time = "2025-11-19T15:18:35.84Z" }, + { url = "https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57", size = 447058, upload-time = "2025-11-19T15:18:34.416Z" }, + { url = "https://files.pythonhosted.org/packages/f1/06/578ffed52c2296f93d7fd2d844cabfa92be51a587c38c8afbb8ae449ca89/safetensors-0.7.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542", size = 491748, upload-time = "2025-11-19T15:18:09.79Z" }, + { url = "https://files.pythonhosted.org/packages/ae/33/1debbbb70e4791dde185edb9413d1fe01619255abb64b300157d7f15dddd/safetensors-0.7.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104", size = 503881, upload-time = "2025-11-19T15:18:16.145Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/40c2ca924d60792c3be509833df711b553c60effbd91da6f5284a83f7122/safetensors-0.7.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d", size = 623463, upload-time = "2025-11-19T15:18:21.11Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3a/13784a9364bd43b0d61eef4bea2845039bc2030458b16594a1bd787ae26e/safetensors-0.7.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a", size = 532855, upload-time = "2025-11-19T15:18:25.719Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48", size = 507152, upload-time = "2025-11-19T15:18:33.023Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a8/4b45e4e059270d17af60359713ffd83f97900d45a6afa73aaa0d737d48b6/safetensors-0.7.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981", size = 541856, upload-time = "2025-11-19T15:18:31.075Z" }, + { url = "https://files.pythonhosted.org/packages/06/87/d26d8407c44175d8ae164a95b5a62707fcc445f3c0c56108e37d98070a3d/safetensors-0.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b", size = 674060, upload-time = "2025-11-19T15:18:37.211Z" }, + { url = "https://files.pythonhosted.org/packages/11/f5/57644a2ff08dc6325816ba7217e5095f17269dada2554b658442c66aed51/safetensors-0.7.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85", size = 771715, upload-time = "2025-11-19T15:18:38.689Z" }, + { url = "https://files.pythonhosted.org/packages/86/31/17883e13a814bd278ae6e266b13282a01049b0c81341da7fd0e3e71a80a3/safetensors-0.7.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0", size = 714377, upload-time = "2025-11-19T15:18:40.162Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d8/0c8a7dc9b41dcac53c4cbf9df2b9c83e0e0097203de8b37a712b345c0be5/safetensors-0.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4", size = 677368, upload-time = "2025-11-19T15:18:41.627Z" }, + { url = "https://files.pythonhosted.org/packages/05/e5/cb4b713c8a93469e3c5be7c3f8d77d307e65fe89673e731f5c2bfd0a9237/safetensors-0.7.0-cp38-abi3-win32.whl", hash = "sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba", size = 326423, upload-time = "2025-11-19T15:18:45.74Z" }, + { url = "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" }, +] + +[[package]] +name = "scipy" +version = "1.16.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/ca/d8ace4f98322d01abcd52d381134344bf7b431eba7ed8b42bdea5a3c2ac9/scipy-1.16.3.tar.gz", hash = "sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb", size = 30597883, upload-time = "2025-10-28T17:38:54.068Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/41/5bf55c3f386b1643812f3a5674edf74b26184378ef0f3e7c7a09a7e2ca7f/scipy-1.16.3-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:81fc5827606858cf71446a5e98715ba0e11f0dbc83d71c7409d05486592a45d6", size = 36659043, upload-time = "2025-10-28T17:32:40.285Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0f/65582071948cfc45d43e9870bf7ca5f0e0684e165d7c9ef4e50d783073eb/scipy-1.16.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:c97176013d404c7346bf57874eaac5187d969293bf40497140b0a2b2b7482e07", size = 28898986, upload-time = "2025-10-28T17:32:45.325Z" }, + { url = "https://files.pythonhosted.org/packages/96/5e/36bf3f0ac298187d1ceadde9051177d6a4fe4d507e8f59067dc9dd39e650/scipy-1.16.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2b71d93c8a9936046866acebc915e2af2e292b883ed6e2cbe5c34beb094b82d9", size = 20889814, upload-time = "2025-10-28T17:32:49.277Z" }, + { url = "https://files.pythonhosted.org/packages/80/35/178d9d0c35394d5d5211bbff7ac4f2986c5488b59506fef9e1de13ea28d3/scipy-1.16.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3d4a07a8e785d80289dfe66b7c27d8634a773020742ec7187b85ccc4b0e7b686", size = 23565795, upload-time = "2025-10-28T17:32:53.337Z" }, + { url = "https://files.pythonhosted.org/packages/fa/46/d1146ff536d034d02f83c8afc3c4bab2eddb634624d6529a8512f3afc9da/scipy-1.16.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0553371015692a898e1aa858fed67a3576c34edefa6b7ebdb4e9dde49ce5c203", size = 33349476, upload-time = "2025-10-28T17:32:58.353Z" }, + { url = "https://files.pythonhosted.org/packages/79/2e/415119c9ab3e62249e18c2b082c07aff907a273741b3f8160414b0e9193c/scipy-1.16.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:72d1717fd3b5e6ec747327ce9bda32d5463f472c9dce9f54499e81fbd50245a1", size = 35676692, upload-time = "2025-10-28T17:33:03.88Z" }, + { url = "https://files.pythonhosted.org/packages/27/82/df26e44da78bf8d2aeaf7566082260cfa15955a5a6e96e6a29935b64132f/scipy-1.16.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fb2472e72e24d1530debe6ae078db70fb1605350c88a3d14bc401d6306dbffe", size = 36019345, upload-time = "2025-10-28T17:33:09.773Z" }, + { url = "https://files.pythonhosted.org/packages/82/31/006cbb4b648ba379a95c87262c2855cd0d09453e500937f78b30f02fa1cd/scipy-1.16.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5192722cffe15f9329a3948c4b1db789fbb1f05c97899187dcf009b283aea70", size = 38678975, upload-time = "2025-10-28T17:33:15.809Z" }, + { url = "https://files.pythonhosted.org/packages/c2/7f/acbd28c97e990b421af7d6d6cd416358c9c293fc958b8529e0bd5d2a2a19/scipy-1.16.3-cp312-cp312-win_amd64.whl", hash = "sha256:56edc65510d1331dae01ef9b658d428e33ed48b4f77b1d51caf479a0253f96dc", size = 38555926, upload-time = "2025-10-28T17:33:21.388Z" }, + { url = "https://files.pythonhosted.org/packages/ce/69/c5c7807fd007dad4f48e0a5f2153038dc96e8725d3345b9ee31b2b7bed46/scipy-1.16.3-cp312-cp312-win_arm64.whl", hash = "sha256:a8a26c78ef223d3e30920ef759e25625a0ecdd0d60e5a8818b7513c3e5384cf2", size = 25463014, upload-time = "2025-10-28T17:33:25.975Z" }, +] + +[[package]] +name = "sentencepiece" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/15/2e7a025fc62d764b151ae6d0f2a92f8081755ebe8d4a64099accc6f77ba6/sentencepiece-0.2.1.tar.gz", hash = "sha256:8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad", size = 3228515, upload-time = "2025-08-12T07:00:51.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/be/32ce495aa1d0e0c323dcb1ba87096037358edee539cac5baf8755a6bd396/sentencepiece-0.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:57cae326c8727de58c85977b175af132a7138d84c764635d7e71bbee7e774133", size = 1943152, upload-time = "2025-08-12T06:59:40.048Z" }, + { url = "https://files.pythonhosted.org/packages/88/7e/ff23008899a58678e98c6ff592bf4d368eee5a71af96d0df6b38a039dd4f/sentencepiece-0.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:56dd39a3c4d6493db3cdca7e8cc68c6b633f0d4195495cbadfcf5af8a22d05a6", size = 1325651, upload-time = "2025-08-12T06:59:41.536Z" }, + { url = "https://files.pythonhosted.org/packages/19/84/42eb3ce4796777a1b5d3699dfd4dca85113e68b637f194a6c8d786f16a04/sentencepiece-0.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9381351182ff9888cc80e41c632e7e274b106f450de33d67a9e8f6043da6f76", size = 1253645, upload-time = "2025-08-12T06:59:42.903Z" }, + { url = "https://files.pythonhosted.org/packages/89/fa/d3d5ebcba3cb9e6d3775a096251860c41a6bc53a1b9461151df83fe93255/sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99f955df238021bf11f0fc37cdb54fd5e5b5f7fd30ecc3d93fb48b6815437167", size = 1316273, upload-time = "2025-08-12T06:59:44.476Z" }, + { url = "https://files.pythonhosted.org/packages/04/88/14f2f4a2b922d8b39be45bf63d79e6cd3a9b2f248b2fcb98a69b12af12f5/sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0cdfecef430d985f1c2bcbfff3defd1d95dae876fbd0173376012d2d7d24044b", size = 1387881, upload-time = "2025-08-12T06:59:46.09Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b8/903e5ccb77b4ef140605d5d71b4f9e0ad95d456d6184688073ed11712809/sentencepiece-0.2.1-cp312-cp312-win32.whl", hash = "sha256:a483fd29a34c3e34c39ac5556b0a90942bec253d260235729e50976f5dba1068", size = 999540, upload-time = "2025-08-12T06:59:48.023Z" }, + { url = "https://files.pythonhosted.org/packages/2d/81/92df5673c067148c2545b1bfe49adfd775bcc3a169a047f5a0e6575ddaca/sentencepiece-0.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:4cdc7c36234fda305e85c32949c5211faaf8dd886096c7cea289ddc12a2d02de", size = 1054671, upload-time = "2025-08-12T06:59:49.895Z" }, + { url = "https://files.pythonhosted.org/packages/fe/02/c5e3bc518655d714622bec87d83db9cdba1cd0619a4a04e2109751c4f47f/sentencepiece-0.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:daeb5e9e9fcad012324807856113708614d534f596d5008638eb9b40112cd9e4", size = 1033923, upload-time = "2025-08-12T06:59:51.952Z" }, +] + +[[package]] +name = "sentry-sdk" +version = "2.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f0/0e9dc590513d5e742d7799e2038df3a05167cba084c6ca4f3cdd75b55164/sentry_sdk-2.48.0.tar.gz", hash = "sha256:5213190977ff7fdff8a58b722fb807f8d5524a80488626ebeda1b5676c0c1473", size = 384828, upload-time = "2025-12-16T14:55:41.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/19/8d77f9992e5cbfcaa9133c3bf63b4fbbb051248802e1e803fed5c552fbb2/sentry_sdk-2.48.0-py2.py3-none-any.whl", hash = "sha256:6b12ac256769d41825d9b7518444e57fa35b5642df4c7c5e322af4d2c8721172", size = 414555, upload-time = "2025-12-16T14:55:40.152Z" }, +] + +[[package]] +name = "setproctitle" +version = "1.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/f0/2dc88e842077719d7384d86cc47403e5102810492b33680e7dadcee64cd8/setproctitle-1.3.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2dc99aec591ab6126e636b11035a70991bc1ab7a261da428491a40b84376654e", size = 18049, upload-time = "2025-09-05T12:49:36.241Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b4/50940504466689cda65680c9e9a1e518e5750c10490639fa687489ac7013/setproctitle-1.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdd8aa571b7aa39840fdbea620e308a19691ff595c3a10231e9ee830339dd798", size = 13079, upload-time = "2025-09-05T12:49:38.088Z" }, + { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/cee06af4ffcfb0e8aba047bd44f5262e644199ae7527ae2c1f672b86495c/setproctitle-1.3.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6915964a6dda07920a1159321dcd6d94fc7fc526f815ca08a8063aeca3c204f1", size = 33736, upload-time = "2025-09-05T12:49:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3a/50caca532a9343828e3bf5778c7a84d6c737a249b1796d50dd680290594d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7cb05bd446687ff816a3aaaf831047fc4c364feff7ada94a66024f1367b448c", size = 33143, upload-time = "2025-09-05T12:49:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, + { url = "https://files.pythonhosted.org/packages/ef/dc/ef76a81fac9bf27b84ed23df19c1f67391a753eed6e3c2254ebcb5133f56/setproctitle-1.3.7-cp312-cp312-win32.whl", hash = "sha256:b0304f905efc845829ac2bc791ddebb976db2885f6171f4a3de678d7ee3f7c9f", size = 12552, upload-time = "2025-09-05T12:49:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, +] + +[[package]] +name = "setuptools" +version = "79.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/71/b6365e6325b3290e14957b2c3a804a529968c77a049b2ed40c095f749707/setuptools-79.0.1.tar.gz", hash = "sha256:128ce7b8f33c3079fd1b067ecbb4051a66e8526e7b65f6cec075dfc650ddfa88", size = 1367909, upload-time = "2025-04-23T22:20:59.241Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/6d/b4752b044bf94cb802d88a888dc7d288baaf77d7910b7dedda74b5ceea0c/setuptools-79.0.1-py3-none-any.whl", hash = "sha256:e147c0549f27767ba362f9da434eab9c5dc0045d5304feb602a0af001089fc51", size = 1256281, upload-time = "2025-04-23T22:20:56.768Z" }, +] + +[[package]] +name = "sgl-kernel" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/ac/6537b3fb3dd251ed496ccd48dbc5c539daaa87a8b806e18da2b9b51a3c91/sgl_kernel-0.1.9-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:f762e98cfa890313d1f53e00a380b6c81659f9e4c259ddf50919e4858a5554a2", size = 249857872, upload-time = "2025-06-16T12:03:31.815Z" }, +] + +[[package]] +name = "sglang" +version = "0.4.8.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "ipython" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "requests" }, + { name = "setproctitle" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/58/c5dc95ba8cc08c6db289c7d0103906ec3187bd9859255029c6e845dc134e/sglang-0.4.8.post1.tar.gz", hash = "sha256:50ac9aefa41cc17d1f19c99b1891bb730b124eed889c182c3a669b12ce7a3821", size = 1127761, upload-time = "2025-06-26T09:21:38.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/c9/e1ebdee90f0ae52f6d3f2016142ef8bbd4f32f78772c9d17f35fb6c21a1d/sglang-0.4.8.post1-py3-none-any.whl", hash = "sha256:0c68bd3b461a910f1b135d64df7eb967ab8be73a7f17c9859c957d11c140d0e1", size = 1562438, upload-time = "2025-06-26T09:21:36.429Z" }, +] + +[package.optional-dependencies] +openai = [ + { name = "openai" }, + { name = "tiktoken" }, +] +srt = [ + { name = "blobfile" }, + { name = "compressed-tensors" }, + { name = "cuda-python" }, + { name = "datasets" }, + { name = "einops" }, + { name = "fastapi" }, + { name = "flashinfer-python", version = "0.2.6.post1", source = { registry = "https://pypi.org/simple" } }, + { name = "hf-transfer" }, + { name = "huggingface-hub" }, + { name = "interegular" }, + { name = "llguidance", version = "0.7.30", source = { registry = "https://pypi.org/simple" } }, + { name = "modelscope" }, + { name = "msgspec" }, + { name = "ninja" }, + { name = "orjson" }, + { name = "outlines" }, + { name = "packaging" }, + { name = "partial-json-parser" }, + { name = "pillow" }, + { name = "prometheus-client" }, + { name = "psutil" }, + { name = "pydantic" }, + { name = "pynvml" }, + { name = "python-multipart" }, + { name = "pyzmq" }, + { name = "scipy" }, + { name = "sgl-kernel" }, + { name = "soundfile", version = "0.13.1", source = { registry = "https://pypi.org/simple" } }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchao" }, + { name = "torchaudio", version = "2.7.1", source = { registry = "https://pypi.org/simple" } }, + { name = "torchvision", version = "0.22.1", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.52.3", source = { registry = "https://pypi.org/simple" } }, + { name = "uvicorn", version = "0.29.0", source = { registry = "https://pypi.org/simple" } }, + { name = "uvloop" }, + { name = "xgrammar", version = "0.1.19", source = { registry = "https://pypi.org/simple" } }, +] +torch-memory-saver = [ + { name = "torch-memory-saver" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "skyrl-gym" +version = "0.1.1" +source = { editable = "skyrl-gym" } +dependencies = [ + { name = "func-timeout" }, + { name = "omegaconf" }, + { name = "pandas" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "func-timeout" }, + { name = "omegaconf" }, + { name = "pandas" }, + { name = "pytest", marker = "extra == 'dev'" }, + { name = "requests" }, +] +provides-extras = ["dev"] + +[[package]] +name = "skyrl-train" +version = "0.3.1" +source = { editable = "." } +dependencies = [ + { name = "accelerate" }, + { name = "datasets" }, + { name = "debugpy" }, + { name = "fastapi" }, + { name = "flash-attn", version = "2.8.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "flash-attn", version = "2.8.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "func-timeout" }, + { name = "hf-transfer" }, + { name = "hydra-core" }, + { name = "jaxtyping" }, + { name = "loguru" }, + { name = "ninja" }, + { name = "omegaconf" }, + { name = "peft" }, + { name = "polars" }, + { name = "pybind11" }, + { name = "ray" }, + { name = "s3fs" }, + { name = "setuptools" }, + { name = "skyrl-gym" }, + { name = "tensorboard" }, + { name = "tensordict" }, + { name = "torchdata" }, + { name = "tqdm" }, + { name = "transformers", version = "4.52.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "uvicorn", version = "0.29.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "wandb" }, +] + +[package.optional-dependencies] +dev = [ + { name = "black" }, + { name = "litellm", version = "1.79.2", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm", version = "1.80.10", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pre-commit" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "ruff" }, +] +docs = [ + { name = "myst-parser" }, + { name = "sphinx" }, + { name = "sphinx-autobuild" }, + { name = "sphinx-autodoc-typehints" }, + { name = "sphinx-copybutton" }, + { name = "sphinx-rtd-theme" }, +] +flashrl = [ + { name = "flash-attn", version = "2.8.3", source = { registry = "https://pypi.org/simple" } }, + { name = "flashinfer-python", version = "0.5.3", source = { registry = "https://pypi.org/simple" } }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.22.0", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.22.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-flashrl') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-flashrl') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +mcore = [ + { name = "flash-attn", version = "2.8.1", source = { registry = "https://pypi.org/simple" } }, + { name = "flashinfer-jit-cache" }, + { name = "flashinfer-python", version = "0.5.3", source = { registry = "https://pypi.org/simple" } }, + { name = "megatron-bridge" }, + { name = "megatron-core" }, + { name = "nvidia-modelopt" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.24.0", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.24.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformer-engine", extra = ["pytorch"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "vllm" }, +] +miniswe = [ + { name = "litellm", version = "1.79.2", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm", version = "1.80.10", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "mini-swe-agent" }, +] +sandboxes = [ + { name = "litellm", version = "1.79.2", source = { registry = "https://pypi.org/simple" }, extra = ["proxy"], marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "litellm", version = "1.80.10", source = { registry = "https://pypi.org/simple" }, extra = ["proxy"], marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sglang = [ + { name = "flash-attn", version = "2.8.3", source = { registry = "https://pypi.org/simple" } }, + { name = "sglang", extra = ["openai", "srt", "torch-memory-saver"], marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.22.1", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +vllm = [ + { name = "flash-attn", version = "2.8.3", source = { registry = "https://pypi.org/simple" } }, + { name = "flashinfer-jit-cache" }, + { name = "flashinfer-python", version = "0.5.3", source = { registry = "https://pypi.org/simple" } }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.24.0", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.24.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "vllm" }, +] + +[package.metadata] +requires-dist = [ + { name = "accelerate" }, + { name = "black", marker = "extra == 'dev'", specifier = "==24.10.0" }, + { name = "datasets", specifier = "==4.0.0" }, + { name = "debugpy", specifier = "==1.8.0" }, + { name = "fastapi" }, + { name = "flash-attn" }, + { name = "flash-attn", marker = "extra == 'flashrl'", specifier = "==2.8.3" }, + { name = "flash-attn", marker = "extra == 'mcore'", specifier = "==2.8.1" }, + { name = "flash-attn", marker = "extra == 'sglang'", specifier = "==2.8.3" }, + { name = "flash-attn", marker = "extra == 'vllm'", specifier = "==2.8.3" }, + { name = "flashinfer-jit-cache", marker = "extra == 'mcore'", specifier = "==0.5.3", index = "https://flashinfer.ai/whl/cu128" }, + { name = "flashinfer-jit-cache", marker = "extra == 'vllm'", index = "https://flashinfer.ai/whl/cu128" }, + { name = "flashinfer-python", marker = "(extra == 'flashrl' and extra == 'mcore') or (extra == 'flashrl' and extra != 'sglang') or (extra == 'flashrl' and extra == 'sglang' and extra == 'vllm')" }, + { name = "flashinfer-python", marker = "extra == 'flashrl' and extra != 'mcore' and extra == 'sglang' and extra != 'vllm'", url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" }, + { name = "flashinfer-python", marker = "extra == 'mcore'", specifier = "==0.5.3" }, + { name = "flashinfer-python", marker = "extra != 'mcore' and extra == 'sglang' and extra != 'vllm'", url = "https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl" }, + { name = "flashinfer-python", marker = "(extra == 'mcore' and extra == 'sglang') or (extra == 'sglang' and extra == 'vllm')" }, + { name = "flashinfer-python", marker = "extra == 'vllm'" }, + { name = "func-timeout" }, + { name = "hf-transfer" }, + { name = "hydra-core", specifier = "==1.3.2" }, + { name = "jaxtyping" }, + { name = "litellm", marker = "extra == 'dev'" }, + { name = "litellm", marker = "extra == 'miniswe'" }, + { name = "litellm", extras = ["proxy"], marker = "extra == 'sandboxes'", specifier = ">=1.67.5" }, + { name = "loguru" }, + { name = "megatron-bridge", marker = "extra == 'mcore'", git = "https://github.com/NVIDIA-NeMo/Megatron-Bridge?rev=04e370eedf8cc44a812189a19f2171d90555c07a" }, + { name = "megatron-core", marker = "extra == 'mcore'", specifier = "==0.15.0" }, + { name = "mini-swe-agent", marker = "extra == 'miniswe'", specifier = ">=1.12.0" }, + { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=2.0.0" }, + { name = "ninja" }, + { name = "nvidia-modelopt", marker = "extra == 'mcore'" }, + { name = "omegaconf" }, + { name = "peft" }, + { name = "polars" }, + { name = "pre-commit", marker = "extra == 'dev'" }, + { name = "pybind11" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=6.2.5" }, + { name = "pytest-asyncio", marker = "extra == 'dev'" }, + { name = "ray", specifier = "==2.51.1" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.11.9" }, + { name = "s3fs" }, + { name = "setuptools" }, + { name = "sglang", extras = ["srt", "openai", "torch-memory-saver"], marker = "extra == 'sglang'", specifier = "==0.4.8.post1" }, + { name = "skyrl-gym", editable = "skyrl-gym" }, + { name = "sphinx", marker = "extra == 'docs'", specifier = ">=7.0.0" }, + { name = "sphinx-autobuild", marker = "extra == 'docs'", specifier = ">=2021.3.14" }, + { name = "sphinx-autodoc-typehints", marker = "extra == 'docs'", specifier = ">=1.25.0" }, + { name = "sphinx-copybutton", marker = "extra == 'docs'", specifier = ">=0.5.0" }, + { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = ">=2.0.0" }, + { name = "tensorboard" }, + { name = "tensordict" }, + { name = "torch", marker = "extra == 'flashrl'", specifier = "==2.7.0", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torch", marker = "extra == 'mcore'", specifier = "==2.9.0", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torch", marker = "extra == 'sglang'", specifier = "==2.7.1", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torch", marker = "extra == 'vllm'", specifier = "==2.9.0", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchdata" }, + { name = "torchvision", marker = "extra == 'flashrl'", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "extra == 'mcore'", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "extra == 'sglang'", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "extra == 'vllm'", index = "https://download.pytorch.org/whl/cu128" }, + { name = "tqdm" }, + { name = "transformer-engine", extras = ["pytorch"], marker = "extra == 'mcore'", specifier = "==2.10.0" }, + { name = "transformers", specifier = ">=4.51.0" }, + { name = "uvicorn" }, + { name = "vllm", marker = "extra == 'mcore'", specifier = "==0.13.0" }, + { name = "vllm", marker = "extra == 'vllm'", specifier = "==0.13.0" }, + { name = "wandb" }, +] +provides-extras = ["dev", "docs", "sandboxes", "vllm", "sglang", "mcore", "flashrl", "miniswe"] + +[[package]] +name = "smmap" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "snowballstemmer" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" }, +] + +[[package]] +name = "soundfile" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "cffi", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/96/5ff33900998bad58d5381fd1acfcdac11cbea4f08fc72ac1dc25ffb13f6a/soundfile-0.12.1.tar.gz", hash = "sha256:e8e1017b2cf1dda767aef19d2fd9ee5ebe07e050d430f77a0a7c66ba08b8cdae", size = 43184, upload-time = "2023-02-15T15:37:32.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/bc/cd845c2dbb4d257c744cd58a5bcdd9f6d235ca317e7e22e49564ec88dcd9/soundfile-0.12.1-py2.py3-none-any.whl", hash = "sha256:828a79c2e75abab5359f780c81dccd4953c45a2c4cd4f05ba3e233ddf984b882", size = 24030, upload-time = "2023-02-15T15:37:16.077Z" }, + { url = "https://files.pythonhosted.org/packages/c8/73/059c84343be6509b480013bf1eeb11b96c5f9eb48deff8f83638011f6b2c/soundfile-0.12.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:d922be1563ce17a69582a352a86f28ed8c9f6a8bc951df63476ffc310c064bfa", size = 1213305, upload-time = "2023-02-15T15:37:18.875Z" }, + { url = "https://files.pythonhosted.org/packages/71/87/31d2b9ed58975cec081858c01afaa3c43718eb0f62b5698a876d94739ad0/soundfile-0.12.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:bceaab5c4febb11ea0554566784bcf4bc2e3977b53946dda2b12804b4fe524a8", size = 1075977, upload-time = "2023-02-15T15:37:21.938Z" }, + { url = "https://files.pythonhosted.org/packages/ad/bd/0602167a213d9184fc688b1086dc6d374b7ae8c33eccf169f9b50ce6568c/soundfile-0.12.1-py2.py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:2dc3685bed7187c072a46ab4ffddd38cef7de9ae5eb05c03df2ad569cf4dacbc", size = 1257765, upload-time = "2023-03-24T08:21:58.716Z" }, + { url = "https://files.pythonhosted.org/packages/c1/07/7591f4efd29e65071c3a61b53725036ea8f73366a4920a481ebddaf8d0ca/soundfile-0.12.1-py2.py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:074247b771a181859d2bc1f98b5ebf6d5153d2c397b86ee9e29ba602a8dfe2a6", size = 1174746, upload-time = "2023-02-15T15:37:24.771Z" }, + { url = "https://files.pythonhosted.org/packages/03/0f/49941ed8a2d94e5b36ea94346fb1d2b22e847fede902e05be4c96f26be7d/soundfile-0.12.1-py2.py3-none-win32.whl", hash = "sha256:59dfd88c79b48f441bbf6994142a19ab1de3b9bb7c12863402c2bc621e49091a", size = 888234, upload-time = "2023-02-15T15:37:27.078Z" }, + { url = "https://files.pythonhosted.org/packages/50/ff/26a4ee48d0b66625a4e4028a055b9f25bc9d7c7b2d17d21a45137621a50d/soundfile-0.12.1-py2.py3-none-win_amd64.whl", hash = "sha256:0d86924c00b62552b650ddd28af426e3ff2d4dc2e9047dae5b3d8452e0a49a77", size = 1009109, upload-time = "2023-02-15T15:37:29.41Z" }, +] + +[[package]] +name = "soundfile" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "cffi" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/41/9b873a8c055582859b239be17902a85339bec6a30ad162f98c9b0288a2cc/soundfile-0.13.1.tar.gz", hash = "sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b", size = 46156, upload-time = "2025-01-25T09:17:04.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/28/e2a36573ccbcf3d57c00626a21fe51989380636e821b341d36ccca0c1c3a/soundfile-0.13.1-py2.py3-none-any.whl", hash = "sha256:a23c717560da2cf4c7b5ae1142514e0fd82d6bbd9dfc93a50423447142f2c445", size = 25751, upload-time = "2025-01-25T09:16:44.235Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ab/73e97a5b3cc46bba7ff8650a1504348fa1863a6f9d57d7001c6b67c5f20e/soundfile-0.13.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:82dc664d19831933fe59adad199bf3945ad06d84bc111a5b4c0d3089a5b9ec33", size = 1142250, upload-time = "2025-01-25T09:16:47.583Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e5/58fd1a8d7b26fc113af244f966ee3aecf03cb9293cb935daaddc1e455e18/soundfile-0.13.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:743f12c12c4054921e15736c6be09ac26b3b3d603aef6fd69f9dde68748f2593", size = 1101406, upload-time = "2025-01-25T09:16:49.662Z" }, + { url = "https://files.pythonhosted.org/packages/58/ae/c0e4a53d77cf6e9a04179535766b3321b0b9ced5f70522e4caf9329f0046/soundfile-0.13.1-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9c9e855f5a4d06ce4213f31918653ab7de0c5a8d8107cd2427e44b42df547deb", size = 1235729, upload-time = "2025-01-25T09:16:53.018Z" }, + { url = "https://files.pythonhosted.org/packages/57/5e/70bdd9579b35003a489fc850b5047beeda26328053ebadc1fb60f320f7db/soundfile-0.13.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:03267c4e493315294834a0870f31dbb3b28a95561b80b134f0bd3cf2d5f0e618", size = 1313646, upload-time = "2025-01-25T09:16:54.872Z" }, + { url = "https://files.pythonhosted.org/packages/fe/df/8c11dc4dfceda14e3003bb81a0d0edcaaf0796dd7b4f826ea3e532146bba/soundfile-0.13.1-py2.py3-none-win32.whl", hash = "sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5", size = 899881, upload-time = "2025-01-25T09:16:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/6b761de83277f2f02ded7e7ea6f07828ec78e4b229b80e4ca55dd205b9dc/soundfile-0.13.1-py2.py3-none-win_amd64.whl", hash = "sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9", size = 1019162, upload-time = "2025-01-25T09:16:59.573Z" }, +] + +[[package]] +name = "sphinx" +version = "8.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals-py" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348", size = 8321876, upload-time = "2025-03-02T22:31:59.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/53/136e9eca6e0b9dc0e1962e2c908fbea2e5ac000c2a2fbd9a35797958c48b/sphinx-8.2.3-py3-none-any.whl", hash = "sha256:4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3", size = 3589741, upload-time = "2025-03-02T22:31:56.836Z" }, +] + +[[package]] +name = "sphinx-autobuild" +version = "2025.8.25" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "sphinx" }, + { name = "starlette" }, + { name = "uvicorn", version = "0.29.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "uvicorn", version = "0.31.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "watchfiles" }, + { name = "websockets", version = "13.1", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/20/56411b52f917696995f5ad27d2ea7e9492c84a043c5b49a3a3173573cd93/sphinx_autobuild-2025.8.25-py3-none-any.whl", hash = "sha256:b750ac7d5a18603e4665294323fd20f6dcc0a984117026d1986704fa68f0379a", size = 12535, upload-time = "2025-08-25T18:44:54.164Z" }, +] + +[[package]] +name = "sphinx-autodoc-typehints" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/4f/4fd5583678bb7dc8afa69e9b309e6a99ee8d79ad3a4728f4e52fd7cb37c7/sphinx_autodoc_typehints-3.5.2.tar.gz", hash = "sha256:5fcd4a3eb7aa89424c1e2e32bedca66edc38367569c9169a80f4b3e934171fdb", size = 37839, upload-time = "2025-10-16T00:50:15.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/f2/9657c98a66973b7c35bfd48ba65d1922860de9598fbb535cd96e3f58a908/sphinx_autodoc_typehints-3.5.2-py3-none-any.whl", hash = "sha256:0accd043619f53c86705958e323b419e41667917045ac9215d7be1b493648d8c", size = 21184, upload-time = "2025-10-16T00:50:13.973Z" }, +] + +[[package]] +name = "sphinx-copybutton" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/2b/a964715e7f5295f77509e59309959f4125122d648f86b4fe7d70ca1d882c/sphinx-copybutton-0.5.2.tar.gz", hash = "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd", size = 23039, upload-time = "2023-04-14T08:10:22.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, +] + +[[package]] +name = "sphinx-rtd-theme" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "sphinx" }, + { name = "sphinxcontrib-jquery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/44/c97faec644d29a5ceddd3020ae2edffa69e7d00054a8c7a6021e82f20335/sphinx_rtd_theme-3.0.2.tar.gz", hash = "sha256:b7457bc25dda723b20b086a670b9953c859eab60a2a03ee8eb2bb23e176e5f85", size = 7620463, upload-time = "2024-11-13T11:06:04.545Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/77/46e3bac77b82b4df5bb5b61f2de98637724f246b4966cfc34bc5895d852a/sphinx_rtd_theme-3.0.2-py2.py3-none-any.whl", hash = "sha256:422ccc750c3a3a311de4ae327e82affdaf59eb695ba4936538552f3b00f4ee13", size = 7655561, upload-time = "2024-11-13T11:06:02.094Z" }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, +] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, +] + +[[package]] +name = "sphinxcontrib-jquery" +version = "4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/f3/aa67467e051df70a6330fe7770894b3e4f09436dea6881ae0b4f3d87cad8/sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a", size = 122331, upload-time = "2023-03-14T15:01:01.944Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/85/749bd22d1a68db7291c89e2ebca53f4306c3f205853cf31e9de279034c3c/sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae", size = 121104, upload-time = "2023-03-14T15:01:00.356Z" }, +] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, +] + +[[package]] +name = "sse-starlette" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/8b/54651ad49bce99a50fd61a7f19c2b6a79fbb072e693101fbb1194c362054/sse_starlette-3.0.4.tar.gz", hash = "sha256:5e34286862e96ead0eb70f5ddd0bd21ab1f6473a8f44419dd267f431611383dd", size = 22576, upload-time = "2025-12-14T16:22:52.493Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/22/8ab1066358601163e1ac732837adba3672f703818f693e179b24e0d3b65c/sse_starlette-3.0.4-py3-none-any.whl", hash = "sha256:32c80ef0d04506ced4b0b6ab8fe300925edc37d26f666afb1874c754895f5dc3", size = 11764, upload-time = "2025-12-14T16:22:51.453Z" }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, +] + +[[package]] +name = "starlette" +version = "0.50.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/b8/73a0e6a6e079a9d9cfa64113d771e421640b6f679a52eeb9b32f72d871a1/starlette-0.50.0.tar.gz", hash = "sha256:a2a17b22203254bcbc2e1f926d2d55f3f9497f769416b3190768befe598fa3ca", size = 2646985, upload-time = "2025-11-01T15:25:27.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/52/1064f510b141bd54025f9b55105e26d1fa970b9be67ad766380a3c9b74b0/starlette-0.50.0-py3-none-any.whl", hash = "sha256:9e5391843ec9b6e472eed1365a78c8098cfceb7a74bfd4d6b1c0c0095efb3bca", size = 74033, upload-time = "2025-11-01T15:25:25.461Z" }, +] + +[[package]] +name = "strenum" +version = "0.4.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/ad/430fb60d90e1d112a62ff57bdd1f286ec73a2a0331272febfddd21f330e1/StrEnum-0.4.15.tar.gz", hash = "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff", size = 23384, upload-time = "2023-06-29T22:02:58.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, +] + +[[package]] +name = "supervisor" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/b5/37e7a3706de436a8a2d75334711dad1afb4ddffab09f25e31d89e467542f/supervisor-4.3.0.tar.gz", hash = "sha256:4a2bf149adf42997e1bb44b70c43b613275ec9852c3edacca86a9166b27e945e", size = 468912, upload-time = "2025-08-23T18:25:02.418Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/65/5e726c372da8a5e35022a94388b12252710aad0c2351699c3d76ae8dba78/supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db", size = 320736, upload-time = "2025-08-23T18:25:00.767Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tabulate" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090, upload-time = "2022-10-06T17:21:48.54Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, +] + +[[package]] +name = "tensorboard" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "grpcio" }, + { name = "markdown" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "setuptools" }, + { name = "tensorboard-data-server" }, + { name = "werkzeug" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" }, +] + +[[package]] +name = "tensorboard-data-server" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/dabeaf902892922777492e1d253bb7e1264cadce3cea932f7ff599e53fea/tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60", size = 4823598, upload-time = "2023-10-23T21:23:33.714Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" }, +] + +[[package]] +name = "tensordict" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpickle" }, + { name = "importlib-metadata" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or extra != 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "orjson" }, + { name = "packaging" }, + { name = "pyvers" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/89/2914b6d2796bdbe64ba8d42b568bf02c25673f187079e8795fc668c609fa/tensordict-0.10.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f52321ddec5da5acb3b90785c68b4fd4cce805aab6eb700ec59352e9f9e6214f", size = 801519, upload-time = "2025-09-08T10:07:18.954Z" }, + { url = "https://files.pythonhosted.org/packages/9e/88/2c1bf6c1abdc4d0bfcbdda2d1a5b19c7a9540f67ff6d20fe08d328d78305/tensordict-0.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6a6ba462cc4c04299eb3746e8994df4a625706e60e2dfb88cc5f9513b6cbad2f", size = 445427, upload-time = "2025-09-08T10:07:20.148Z" }, + { url = "https://files.pythonhosted.org/packages/5e/05/6e7d130c5e9af947fad25fb7d40a3aa2fd9ef9d37c9c7ddc94ba11853d23/tensordict-0.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6f0a52524c7c46778bf250444f1cd508f055735667b8d596a1a7e2fb38824e8c", size = 449961, upload-time = "2025-09-08T10:07:21.977Z" }, + { url = "https://files.pythonhosted.org/packages/f4/40/877fd0453c9c79a14063ecd21102a23460d033e3760e83eae7fd6c09b3ef/tensordict-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:76e7c1d6604addd08e026141c3943fa15fbe36db537f9ff311af5d2caee25daf", size = 494502, upload-time = "2025-09-08T10:07:23.2Z" }, +] + +[[package]] +name = "textual" +version = "6.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", extra = ["linkify", "plugins"], marker = "extra == 'extra-11-skyrl-train-miniswe' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "platformdirs" }, + { name = "pygments" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/30/38b615f7d4b16f6fdd73e4dcd8913e2d880bbb655e68a076e3d91181a7ee/textual-6.2.1.tar.gz", hash = "sha256:4699d8dfae43503b9c417bd2a6fb0da1c89e323fe91c4baa012f9298acaa83e1", size = 1570645, upload-time = "2025-10-01T16:11:24.467Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/93/02c7adec57a594af28388d85da9972703a4af94ae1399542555cd9581952/textual-6.2.1-py3-none-any.whl", hash = "sha256:3c7190633cd4d8bfe6049ae66808b98da91ded2edb85cef54e82bf77b03d2a54", size = 710702, upload-time = "2025-10-01T16:11:22.161Z" }, +] + +[[package]] +name = "tiktoken" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/85/be65d39d6b647c79800fd9d29241d081d4eeb06271f383bb87200d74cf76/tiktoken-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8", size = 1050728, upload-time = "2025-10-06T20:21:52.756Z" }, + { url = "https://files.pythonhosted.org/packages/4a/42/6573e9129bc55c9bf7300b3a35bef2c6b9117018acca0dc760ac2d93dffe/tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b", size = 994049, upload-time = "2025-10-06T20:21:53.782Z" }, + { url = "https://files.pythonhosted.org/packages/66/c5/ed88504d2f4a5fd6856990b230b56d85a777feab84e6129af0822f5d0f70/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37", size = 1129008, upload-time = "2025-10-06T20:21:54.832Z" }, + { url = "https://files.pythonhosted.org/packages/f4/90/3dae6cc5436137ebd38944d396b5849e167896fc2073da643a49f372dc4f/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad", size = 1152665, upload-time = "2025-10-06T20:21:56.129Z" }, + { url = "https://files.pythonhosted.org/packages/a3/fe/26df24ce53ffde419a42f5f53d755b995c9318908288c17ec3f3448313a3/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5", size = 1194230, upload-time = "2025-10-06T20:21:57.546Z" }, + { url = "https://files.pythonhosted.org/packages/20/cc/b064cae1a0e9fac84b0d2c46b89f4e57051a5f41324e385d10225a984c24/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3", size = 1254688, upload-time = "2025-10-06T20:21:58.619Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/b8523105c590c5b8349f2587e2fdfe51a69544bd5a76295fc20f2374f470/tiktoken-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd", size = 878694, upload-time = "2025-10-06T20:21:59.876Z" }, +] + +[[package]] +name = "timm" +version = "1.0.22" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.24.0", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.24.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/9d/e4670765d1c033f97096c760b3b907eeb659cf80f3678640e5f060b04c6c/timm-1.0.22.tar.gz", hash = "sha256:14fd74bcc17db3856b1a47d26fb305576c98579ab9d02b36714a5e6b25cde422", size = 2382998, upload-time = "2025-11-05T04:06:09.377Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/14/fc04d491527b774ec7479897f5861959209de1480e4c4cd32ed098ff8bea/timm-1.0.22-py3-none-any.whl", hash = "sha256:888981753e65cbaacfc07494370138b1700a27b1f0af587f4f9b47bc024161d0", size = 2530238, upload-time = "2025-11-05T04:06:06.823Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.21.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "huggingface-hub", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/2f/402986d0823f8d7ca139d969af2917fefaa9b947d1fb32f6168c509f2492/tokenizers-0.21.4.tar.gz", hash = "sha256:fa23f85fbc9a02ec5c6978da172cdcbac23498c3ca9f3645c5c68740ac007880", size = 351253, upload-time = "2025-07-28T15:48:54.325Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/c6/fdb6f72bf6454f52eb4a2510be7fb0f614e541a2554d6210e370d85efff4/tokenizers-0.21.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ccc10a7c3bcefe0f242867dc914fc1226ee44321eb618cfe3019b5df3400133", size = 2863987, upload-time = "2025-07-28T15:48:44.877Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a6/28975479e35ddc751dc1ddc97b9b69bf7fcf074db31548aab37f8116674c/tokenizers-0.21.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e2f601a8e0cd5be5cc7506b20a79112370b9b3e9cb5f13f68ab11acd6ca7d60", size = 2732457, upload-time = "2025-07-28T15:48:43.265Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8f/24f39d7b5c726b7b0be95dca04f344df278a3fe3a4deb15a975d194cbb32/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39b376f5a1aee67b4d29032ee85511bbd1b99007ec735f7f35c8a2eb104eade5", size = 3012624, upload-time = "2025-07-28T13:22:43.895Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/26358925717687a58cb74d7a508de96649544fad5778f0cd9827398dc499/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2107ad649e2cda4488d41dfd031469e9da3fcbfd6183e74e4958fa729ffbf9c6", size = 2939681, upload-time = "2025-07-28T13:22:47.499Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/cc300fea5db2ab5ddc2c8aea5757a27b89c84469899710c3aeddc1d39801/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c73012da95afafdf235ba80047699df4384fdc481527448a078ffd00e45a7d9", size = 3247445, upload-time = "2025-07-28T15:48:39.711Z" }, + { url = "https://files.pythonhosted.org/packages/be/bf/98cb4b9c3c4afd8be89cfa6423704337dc20b73eb4180397a6e0d456c334/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f23186c40395fc390d27f519679a58023f368a0aad234af145e0f39ad1212732", size = 3428014, upload-time = "2025-07-28T13:22:49.569Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/96c1cc780e6ca7f01a57c13235dd05b7bc1c0f3588512ebe9d1331b5f5ae/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc88bb34e23a54cc42713d6d98af5f1bf79c07653d24fe984d2d695ba2c922a2", size = 3193197, upload-time = "2025-07-28T13:22:51.471Z" }, + { url = "https://files.pythonhosted.org/packages/f2/90/273b6c7ec78af547694eddeea9e05de771278bd20476525ab930cecaf7d8/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51b7eabb104f46c1c50b486520555715457ae833d5aee9ff6ae853d1130506ff", size = 3115426, upload-time = "2025-07-28T15:48:41.439Z" }, + { url = "https://files.pythonhosted.org/packages/91/43/c640d5a07e95f1cf9d2c92501f20a25f179ac53a4f71e1489a3dcfcc67ee/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:714b05b2e1af1288bd1bc56ce496c4cebb64a20d158ee802887757791191e6e2", size = 9089127, upload-time = "2025-07-28T15:48:46.472Z" }, + { url = "https://files.pythonhosted.org/packages/44/a1/dd23edd6271d4dca788e5200a807b49ec3e6987815cd9d0a07ad9c96c7c2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1340ff877ceedfa937544b7d79f5b7becf33a4cfb58f89b3b49927004ef66f78", size = 9055243, upload-time = "2025-07-28T15:48:48.539Z" }, + { url = "https://files.pythonhosted.org/packages/21/2b/b410d6e9021c4b7ddb57248304dc817c4d4970b73b6ee343674914701197/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3c1f4317576e465ac9ef0d165b247825a2a4078bcd01cba6b54b867bdf9fdd8b", size = 9298237, upload-time = "2025-07-28T15:48:50.443Z" }, + { url = "https://files.pythonhosted.org/packages/b7/0a/42348c995c67e2e6e5c89ffb9cfd68507cbaeb84ff39c49ee6e0a6dd0fd2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c212aa4e45ec0bb5274b16b6f31dd3f1c41944025c2358faaa5782c754e84c24", size = 9461980, upload-time = "2025-07-28T15:48:52.325Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d3/dacccd834404cd71b5c334882f3ba40331ad2120e69ded32cf5fda9a7436/tokenizers-0.21.4-cp39-abi3-win32.whl", hash = "sha256:6c42a930bc5f4c47f4ea775c91de47d27910881902b0f20e4990ebe045a415d0", size = 2329871, upload-time = "2025-07-28T15:48:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/41/f2/fd673d979185f5dcbac4be7d09461cbb99751554ffb6718d0013af8604cb/tokenizers-0.21.4-cp39-abi3-win_amd64.whl", hash = "sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597", size = 2507568, upload-time = "2025-07-28T15:48:55.456Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "huggingface-hub", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/46/fb6854cec3278fbfa4a75b50232c77622bc517ac886156e6afbfa4d8fc6e/tokenizers-0.22.1.tar.gz", hash = "sha256:61de6522785310a309b3407bac22d99c4db5dba349935e99e4d15ea2226af2d9", size = 363123, upload-time = "2025-09-19T09:49:23.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/33/f4b2d94ada7ab297328fc671fed209368ddb82f965ec2224eb1892674c3a/tokenizers-0.22.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59fdb013df17455e5f950b4b834a7b3ee2e0271e6378ccb33aa74d178b513c73", size = 3069318, upload-time = "2025-09-19T09:49:11.848Z" }, + { url = "https://files.pythonhosted.org/packages/1c/58/2aa8c874d02b974990e89ff95826a4852a8b2a273c7d1b4411cdd45a4565/tokenizers-0.22.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d4e484f7b0827021ac5f9f71d4794aaef62b979ab7608593da22b1d2e3c4edc", size = 2926478, upload-time = "2025-09-19T09:49:09.759Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3b/55e64befa1e7bfea963cf4b787b2cea1011362c4193f5477047532ce127e/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d2962dd28bc67c1f205ab180578a78eef89ac60ca7ef7cbe9635a46a56422a", size = 3256994, upload-time = "2025-09-19T09:48:56.701Z" }, + { url = "https://files.pythonhosted.org/packages/71/0b/fbfecf42f67d9b7b80fde4aabb2b3110a97fac6585c9470b5bff103a80cb/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38201f15cdb1f8a6843e6563e6e79f4abd053394992b9bbdf5213ea3469b4ae7", size = 3153141, upload-time = "2025-09-19T09:48:59.749Z" }, + { url = "https://files.pythonhosted.org/packages/17/a9/b38f4e74e0817af8f8ef925507c63c6ae8171e3c4cb2d5d4624bf58fca69/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1cbe5454c9a15df1b3443c726063d930c16f047a3cc724b9e6e1a91140e5a21", size = 3508049, upload-time = "2025-09-19T09:49:05.868Z" }, + { url = "https://files.pythonhosted.org/packages/d2/48/dd2b3dac46bb9134a88e35d72e1aa4869579eacc1a27238f1577270773ff/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7d094ae6312d69cc2a872b54b91b309f4f6fbce871ef28eb27b52a98e4d0214", size = 3710730, upload-time = "2025-09-19T09:49:01.832Z" }, + { url = "https://files.pythonhosted.org/packages/93/0e/ccabc8d16ae4ba84a55d41345207c1e2ea88784651a5a487547d80851398/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afd7594a56656ace95cdd6df4cca2e4059d294c5cfb1679c57824b605556cb2f", size = 3412560, upload-time = "2025-09-19T09:49:03.867Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c6/dc3a0db5a6766416c32c034286d7c2d406da1f498e4de04ab1b8959edd00/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ef6063d7a84994129732b47e7915e8710f27f99f3a3260b8a38fc7ccd083f4", size = 3250221, upload-time = "2025-09-19T09:49:07.664Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a6/2c8486eef79671601ff57b093889a345dd3d576713ef047776015dc66de7/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ba0a64f450b9ef412c98f6bcd2a50c6df6e2443b560024a09fa6a03189726879", size = 9345569, upload-time = "2025-09-19T09:49:14.214Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/32ce667f14c35537f5f605fe9bea3e415ea1b0a646389d2295ec348d5657/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:331d6d149fa9c7d632cde4490fb8bbb12337fa3a0232e77892be656464f4b446", size = 9271599, upload-time = "2025-09-19T09:49:16.639Z" }, + { url = "https://files.pythonhosted.org/packages/51/7c/a5f7898a3f6baa3fc2685c705e04c98c1094c523051c805cdd9306b8f87e/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:607989f2ea68a46cb1dfbaf3e3aabdf3f21d8748312dbeb6263d1b3b66c5010a", size = 9533862, upload-time = "2025-09-19T09:49:19.146Z" }, + { url = "https://files.pythonhosted.org/packages/36/65/7e75caea90bc73c1dd8d40438adf1a7bc26af3b8d0a6705ea190462506e1/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a0f307d490295717726598ef6fa4f24af9d484809223bbc253b201c740a06390", size = 9681250, upload-time = "2025-09-19T09:49:21.501Z" }, + { url = "https://files.pythonhosted.org/packages/30/2c/959dddef581b46e6209da82df3b78471e96260e2bc463f89d23b1bf0e52a/tokenizers-0.22.1-cp39-abi3-win32.whl", hash = "sha256:b5120eed1442765cd90b903bb6cfef781fd8fe64e34ccaecbae4c619b7b12a82", size = 2472003, upload-time = "2025-09-19T09:49:27.089Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/e33a8c93907b631a99377ef4c5f817ab453d0b34f93529421f42ff559671/tokenizers-0.22.1-cp39-abi3-win_amd64.whl", hash = "sha256:65fd6e3fb11ca1e78a6a93602490f134d1fdeb13bcef99389d5102ea318ed138", size = 2674684, upload-time = "2025-09-19T09:49:24.953Z" }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + +[[package]] +name = "torch" +version = "2.7.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "filelock", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fsspec", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jinja2", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "networkx", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cublas-cu12", version = "12.8.3.14", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-cupti-cu12", version = "12.8.57", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-nvrtc-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-runtime-cu12", version = "12.8.57", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cudnn-cu12", version = "9.7.1.26", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cufft-cu12", version = "11.3.3.41", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cufile-cu12", version = "1.13.0.11", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-curand-cu12", version = "10.3.9.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusolver-cu12", version = "11.7.2.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.7.53", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparselt-cu12", version = "0.6.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nccl-cu12", version = "2.26.2", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvtx-cu12", version = "12.8.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "setuptools", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "sympy", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "triton", version = "3.3.0", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "typing-extensions", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.0%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6bba7dca5d9a729f1e8e9befb98055498e551efaf5ed034824c168b560afc1ac" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7c0f08d1c44a02abad389373dddfce75904b969a410be2f4e5109483dd3dc0ce" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:1704e5dd66c9221e4e8b6ae2d80cbf54e129571e643f5fa9ca78cc6d2096403a" }, +] + +[[package]] +name = "torch" +version = "2.7.1+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "filelock", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fsspec", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jinja2", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "networkx", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cublas-cu12", version = "12.8.3.14", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-cupti-cu12", version = "12.8.57", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-nvrtc-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-runtime-cu12", version = "12.8.57", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cudnn-cu12", version = "9.7.1.26", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cufft-cu12", version = "11.3.3.41", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cufile-cu12", version = "1.13.0.11", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-curand-cu12", version = "10.3.9.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusolver-cu12", version = "11.7.2.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.7.53", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparselt-cu12", version = "0.6.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nccl-cu12", version = "2.26.2", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.61", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvtx-cu12", version = "12.8.55", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'x86_64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "setuptools", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "sympy", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "triton", version = "3.3.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "typing-extensions", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:268e54db9f0bc2b7b9eb089852d3e592c2dea2facc3db494100c3d3b796549fa" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0b64f7d0a6f2a739ed052ba959f7b67c677028c9566ce51997f9f90fe573ddaa" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:2bb8c05d48ba815b316879a18195d53a6472a03e297d971e916753f8e1053d30" }, +] + +[[package]] +name = "torch" +version = "2.9.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "filelock", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "fsspec", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "jinja2", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "networkx", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-cupti-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-nvrtc-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cuda-runtime-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cudnn-cu12", version = "9.10.2.21", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cufft-cu12", version = "11.3.3.83", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cufile-cu12", version = "1.13.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-curand-cu12", version = "10.3.9.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusolver-cu12", version = "11.7.3.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-cusparselt-cu12", version = "0.7.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nccl-cu12", version = "2.27.5", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvshmem-cu12", marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "nvidia-nvtx-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "setuptools", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "sympy", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "triton", version = "3.5.0", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "typing-extensions", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torch-2.9.0%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e1765625084e320f1eb2f4eb5fd9d14d39d08d7a1880c10a307ce5de20831d27" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.9.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:87c62d3b95f1a2270bd116dbd47dc515c0b2035076fbb4a03b4365ea289e89c4" }, + { url = "https://download.pytorch.org/whl/cu128/torch-2.9.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:c97dc47a1f64745d439dd9471a96d216b728d528011029b4f9ae780e985529e0" }, +] + +[[package]] +name = "torch-memory-saver" +version = "0.0.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/28/6c/21dfda5d31afb71f52cedff52370acbb8290485b3f0fee6816a15a3d08f1/torch_memory_saver-0.0.9.tar.gz", hash = "sha256:3bbf76391fb16870b1b0df279fc281c8a05ef8f8809400b309b0a8240e8ee5ba", size = 14220, upload-time = "2025-10-18T02:10:18.163Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/35/b22df9e730d8444d62445a594421992781c7fad271325d41656d8a32d103/torch_memory_saver-0.0.9-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:0cf26332993649f8ea1b95d7307dfba3a95ee6cee53de84a3e561fb21752b584", size = 488722, upload-time = "2025-10-18T02:10:16.825Z" }, +] + +[[package]] +name = "torchao" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/fe/a24225d30775192a4c5d9cea3ecb95e6adc69d0a8b5ed98eb8e58d362344/torchao-0.9.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc708910301a9f98344d43f3fe2aa6d5e1fab706d772b6df47ff05087d664145", size = 5652091, upload-time = "2025-02-28T13:54:15.239Z" }, + { url = "https://files.pythonhosted.org/packages/db/72/01f755514fb61eadc80b974eb4bd4f22f3009b35457773523e3bd497c511/torchao-0.9.0-py3-none-any.whl", hash = "sha256:ea5603c32762f1a9ade1a4dc7b00f5246623b24a28e49e666f614c79a408712a", size = 712541, upload-time = "2025-02-28T13:54:13.671Z" }, +] + +[[package]] +name = "torchaudio" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d1/eb8bc3b3502dddb1b789567b7b19668b1d32817266887b9f381494cfe463/torchaudio-2.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9306dcfc4586cebd7647a93fe9a448e791c4f83934da616b9433b75597a1f978", size = 1846897, upload-time = "2025-06-04T17:44:07.79Z" }, + { url = "https://files.pythonhosted.org/packages/62/7d/6c15f15d3edc5271abc808f70713644b50f0f7bfb85a09dba8b5735fbad3/torchaudio-2.7.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d66bd76b226fdd4135c97650e1b7eb63fb7659b4ed0e3a778898e41dbba21b61", size = 1686680, upload-time = "2025-06-04T17:43:58.986Z" }, + { url = "https://files.pythonhosted.org/packages/48/65/0f46ba74cdc67ea9a8c37c8acfb5194d81639e481e85903c076bcd97188c/torchaudio-2.7.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9cbcdaab77ad9a73711acffee58f4eebc8a0685289a938a3fa6f660af9489aee", size = 3506966, upload-time = "2025-06-04T17:44:06.537Z" }, + { url = "https://files.pythonhosted.org/packages/52/29/06f887baf22cbba85ae331b71b110b115bf11b3968f5914a50c17dde5ab7/torchaudio-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:9cfb8f6ace8e01e2b89de74eb893ba5ce936b88b415383605b0a4d974009dec7", size = 2484265, upload-time = "2025-06-04T17:44:00.277Z" }, +] + +[[package]] +name = "torchaudio" +version = "2.9.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/63/3c0ede3aa3d19a8a6698ddd107fa88660549360b51bf8ce2717cd498d800/torchaudio-2.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab4cbcccfd873b0fb41fcb39c9869e59ef84bb95b093f6f58e2d05172a7500d2", size = 809116, upload-time = "2025-10-15T15:52:00.911Z" }, + { url = "https://files.pythonhosted.org/packages/be/d5/25e58745defe9d05893d3cba5c0e1a76aeaac503ac5ec4d9f83c871df71c/torchaudio-2.9.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7f93388b6e536c14d6015b6f75277a8b45efc532f61b35adc1ed06c98a86003e", size = 476020, upload-time = "2025-10-15T15:51:59.967Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9c/58b8b49dfba2ae85e41ca86b0c52de45bbbea01987490de219c99c523a58/torchaudio-2.9.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:508318a2130b40ad51378f90caf8727a4bd3ac2b296f2b90c900b44e6068a940", size = 2059901, upload-time = "2025-10-15T15:51:54.634Z" }, + { url = "https://files.pythonhosted.org/packages/d7/eb/58b05f75d12f69ccc460893a20c999da082e063082120ed06e05cca3a053/torchaudio-2.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:82117e3a605f2959dc09b4cd8a11178d6e92727d5f85e5d4f9fe47502f84ee96", size = 665350, upload-time = "2025-10-15T15:52:08.384Z" }, +] + +[[package]] +name = "torchdata" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "urllib3" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/d4/af694ef718aedbe95a72760ab9ff7a6a7a44ace2d7f70c27bfeb67c5c503/torchdata-0.11.0-py3-none-any.whl", hash = "sha256:52b940fbbe0e00fb21cabddf528449d1bec5bfb0d0823b7487b15f951658ee33", size = 61968, upload-time = "2025-02-20T22:26:30.666Z" }, +] + +[[package]] +name = "torchprofile" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchvision", version = "0.24.0", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.24.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/36/574c0c46e818533b78b3c09505211162918188325ab4165ef11a3f295755/torchprofile-0.0.4.tar.gz", hash = "sha256:96b6da17d752a06b02977e078aea95614893b31d4117dd5dcd081f30ce65611b", size = 4557, upload-time = "2021-06-22T04:58:03.592Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/15/71ad4ed163b03cba1315f1d96e0bc8e39d5a97f92974ffa610a729b273ab/torchprofile-0.0.4-py3-none-any.whl", hash = "sha256:7151fe88dc770f0eeec241244a4c7feaec2c5e8c7852386bc2d6a8d7dde7384d", size = 7694, upload-time = "2021-06-22T04:58:02.485Z" }, +] + +[[package]] +name = "torchvision" +version = "0.22.0" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, + { name = "pillow", marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6e9752b48c1cdd7f6428bcd30c3d198b30ecea348d16afb651f95035e5252506" }, +] + +[[package]] +name = "torchvision" +version = "0.22.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" }, + { name = "pillow", marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:06c101f40e1ff94869be14487c91fd5352e376f202fdeafb8f53c58cee2fbeb5" }, + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:a87393c86649b7e56b4bf859fe95922ee6ec1c1f3b430246fb1a5b51f8aee37a" }, +] + +[[package]] +name = "torchvision" +version = "0.22.1" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, + { name = "pillow", marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2ad7fe412b821333fc05b4046263d77c14ba86f3965366adbada8dc397ea45b4" }, +] + +[[package]] +name = "torchvision" +version = "0.22.1+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" }, + { name = "pillow", marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f64ef9bb91d71ab35d8384912a19f7419e35928685bc67544d58f45148334373" }, + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:650561ba326d21021243f5e064133dc62dc64d52f79623db5cd76637a9665f96" }, +] + +[[package]] +name = "torchvision" +version = "0.24.0" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pillow", marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.24.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5d6bcd7393774c82531f7ec411f352129fe8c302bba7765c372398a71d5df4a8" }, +] + +[[package]] +name = "torchvision" +version = "0.24.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pillow", marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.24.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e505bd83ee10edb94523d0b805a08f50b8862b58d2cc6f02d14cd4e7ef9302bc" }, + { url = "https://download.pytorch.org/whl/cu128/torchvision-0.24.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:1aa36ac00106e1381c38348611a1ec0eebe942570ebaf0490f026b061dfc212c" }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + +[[package]] +name = "traitlets" +version = "5.14.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/79/72064e6a701c2183016abbbfedaba506d81e30e232a68c9f0d6f6fcd1574/traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", size = 161621, upload-time = "2024-04-19T11:11:49.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, +] + +[[package]] +name = "transformer-engine" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/45/b3402a4931c0850ac662b532888d7cb89d5d8f22324309ae8d24557340ee/transformer_engine-2.10.0-py3-none-any.whl", hash = "sha256:a14ccf4e887409be062c0bd8c4a341df55a77baad6aea6aabfe39c24e38252e5", size = 696221, upload-time = "2025-12-02T20:53:17.688Z" }, +] + +[package.optional-dependencies] +pytorch = [ + { name = "transformer-engine-torch" }, +] + +[[package]] +name = "transformer-engine-cu12" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "packaging" }, + { name = "pydantic" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/3c/9f480a555c4707cd7b091c5341cc96db1af80b5bfb1a2eae834fb704283b/transformer_engine_cu12-2.10.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:ddd6f4f1f2a8f2c450ea0210d04a08a7b8ceff49a4d900f27b3858980502f21b", size = 286567840, upload-time = "2025-12-02T20:50:26.438Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/b63b6989262fcf37402a910112aaee9f3273338d9d1d854478e022f5deb7/transformer_engine_cu12-2.10.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6766d4ea1643a2606d498aa396d4e7da1046fe01580fdef2047c2c8aa37936b0", size = 287067223, upload-time = "2025-12-02T20:52:11.248Z" }, +] + +[[package]] +name = "transformer-engine-torch" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "einops" }, + { name = "onnx" }, + { name = "onnxscript" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "transformer-engine-cu12" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/94/609a7772569d3acdba34261be7fd30b75f5ff4e5f704117c9e0da517b079/transformer_engine_torch-2.10.0.tar.gz", hash = "sha256:71faff8e3def742553ad74b4e32d2d12e91be9acfb13d1699c89e1e18dd4ecd6", size = 220302, upload-time = "2025-12-02T20:53:57.876Z" } + +[[package]] +name = "transformers" +version = "4.52.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "filelock", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "huggingface-hub", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "packaging", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyyaml", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "regex", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "requests", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "safetensors", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tokenizers", version = "0.21.4", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tqdm", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/42/271bcf364788337ac24e7f200005ac7142aaf022206bd6119d2daca22c04/transformers-4.52.3.tar.gz", hash = "sha256:2e1de29374f27920aaf6d589d4e6339f33def2fb08809e1a1d792e040e9fbce7", size = 8951324, upload-time = "2025-05-22T14:40:52.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f8/1f086942bc6a044e4e68dacf6de761a45367795efd5f57ad356765691c79/transformers-4.52.3-py3-none-any.whl", hash = "sha256:cd04059da50e7cf2a617ce3143ba8beffbf119f8c25a0717c3454fd9d0f19609", size = 10460322, upload-time = "2025-05-22T14:40:49.583Z" }, +] + +[[package]] +name = "transformers" +version = "4.57.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", + "platform_machine != 'aarch64' and sys_platform == 'linux'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "filelock", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "huggingface-hub", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "packaging", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pyyaml", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "regex", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "requests", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "safetensors", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tokenizers", version = "0.22.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tqdm", marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/68/a39307bcc4116a30b2106f2e689130a48de8bd8a1e635b5e1030e46fcd9e/transformers-4.57.1.tar.gz", hash = "sha256:f06c837959196c75039809636cd964b959f6604b75b8eeec6fdfc0440b89cc55", size = 10142511, upload-time = "2025-10-14T15:39:26.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/d3/c16c3b3cf7655a67db1144da94b021c200ac1303f82428f2beef6c2e72bb/transformers-4.57.1-py3-none-any.whl", hash = "sha256:b10d05da8fa67dc41644dbbf9bc45a44cb86ae33da6f9295f5fbf5b7890bd267", size = 11990925, upload-time = "2025-10-14T15:39:23.085Z" }, +] + +[[package]] +name = "transformers" +version = "4.57.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "filelock", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "huggingface-hub", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "packaging", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "pyyaml", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "regex", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "requests", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "safetensors", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "tokenizers", version = "0.22.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, + { name = "tqdm", marker = "extra == 'extra-11-skyrl-train-flashrl' or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-sglang')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/70/d42a739e8dfde3d92bb2fff5819cbf331fe9657323221e79415cd5eb65ee/transformers-4.57.3.tar.gz", hash = "sha256:df4945029aaddd7c09eec5cad851f30662f8bd1746721b34cc031d70c65afebc", size = 10139680, upload-time = "2025-11-25T15:51:30.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/6b/2f416568b3c4c91c96e5a365d164f8a4a4a88030aa8ab4644181fdadce97/transformers-4.57.3-py3-none-any.whl", hash = "sha256:c77d353a4851b1880191603d36acb313411d3577f6e2897814f333841f7003f4", size = 11993463, upload-time = "2025-11-25T15:51:26.493Z" }, +] + +[[package]] +name = "triton" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "setuptools", marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/53/ce18470914ab6cfbec9384ee565d23c4d1c55f0548160b1c7b33000b11fd/triton-3.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b68c778f6c4218403a6bd01be7484f6dc9e20fe2083d22dd8aef33e3b87a10a3", size = 156504509, upload-time = "2025-04-09T20:27:40.413Z" }, +] + +[[package]] +name = "triton" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "setuptools", marker = "(sys_platform == 'linux' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/5f/950fb373bf9c01ad4eb5a8cd5eaf32cdf9e238c02f9293557a2129b9c4ac/triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43", size = 155669138, upload-time = "2025-05-29T23:39:51.771Z" }, +] + +[[package]] +name = "triton" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "setuptools", marker = "sys_platform != 'darwin' and sys_platform != 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/66/b1eb52839f563623d185f0927eb3530ee4d5ffe9d377cdaf5346b306689e/triton-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c1d84a5c0ec2c0f8e8a072d7fd150cab84a9c239eaddc6706c081bfae4eb04", size = 155560068, upload-time = "2025-07-30T19:58:37.081Z" }, +] + +[[package]] +name = "triton" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/9b/30988039e1e84df7554fba24e6a734d2d0e847af33cabdf9b532b3c51456/triton-3.5.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7da21fccceafc163e3a5e857abe34351ef76345af06cabf9637a914742671f0b", size = 159946647, upload-time = "2025-10-15T19:15:56.325Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3a/e991574f3102147b642e49637e0281e9bb7c4ba254edb2bab78247c85e01/triton-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9e71db82261c4ffa3921cd050cd5faa18322d2d405c30eb56084afaff3b0833", size = 170476535, upload-time = "2025-10-13T16:38:05.18Z" }, +] + +[[package]] +name = "typer" +version = "0.20.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", marker = "extra == 'extra-11-skyrl-train-mcore' or extra == 'extra-11-skyrl-train-miniswe' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "rich", marker = "extra == 'extra-11-skyrl-train-mcore' or extra == 'extra-11-skyrl-train-miniswe' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "shellingham", marker = "extra == 'extra-11-skyrl-train-mcore' or extra == 'extra-11-skyrl-train-miniswe' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "typing-extensions", marker = "extra == 'extra-11-skyrl-train-mcore' or extra == 'extra-11-skyrl-train-miniswe' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/c1/933d30fd7a123ed981e2a1eedafceab63cb379db0402e438a13bc51bbb15/typer-0.20.1.tar.gz", hash = "sha256:68585eb1b01203689c4199bc440d6be616f0851e9f0eb41e4a778845c5a0fd5b", size = 105968, upload-time = "2025-12-19T16:48:56.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/52/1f2df7e7d1be3d65ddc2936d820d4a3d9777a54f4204f5ca46b8513eff77/typer-0.20.1-py3-none-any.whl", hash = "sha256:4b3bde918a67c8e03d861aa02deca90a95bbac572e71b1b9be56ff49affdb5a8", size = 47381, upload-time = "2025-12-19T16:48:53.679Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/c202b344c5ca7daf398f3b8a477eeb205cf3b6f32e7ec3a6bac0629ca975/tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7", size = 196772, upload-time = "2025-12-13T17:45:35.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + +[[package]] +name = "uc-micro-py" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/7a/146a99696aee0609e3712f2b44c6274566bc368dfe8375191278045186b8/uc-micro-py-1.0.3.tar.gz", hash = "sha256:d321b92cff673ec58027c04015fcaa8bb1e005478643ff4a500882eaab88c48a", size = 6043, upload-time = "2024-02-09T16:52:01.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/87/1f677586e8ac487e29672e4b17455758fce261de06a0d086167bb760361a/uc_micro_py-1.0.3-py3-none-any.whl", hash = "sha256:db1dffff340817673d7b466ec86114a9dc0e9d4d9b5ba229d9d60e5c12600cd5", size = 6229, upload-time = "2024-02-09T16:52:00.371Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/24/a2a2ed9addd907787d7aa0355ba36a6cadf1768b934c652ea78acbd59dcd/urllib3-2.6.2.tar.gz", hash = "sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797", size = 432930, upload-time = "2025-12-11T15:56:40.252Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/b9/4095b668ea3678bf6a0af005527f39de12fb026516fb3df17495a733b7f8/urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd", size = 131182, upload-time = "2025-12-11T15:56:38.584Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.29.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "click", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "h11", marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/8d/5005d39cd79c9ae87baf7d7aafdcdfe0b13aa69d9a1e3b7f1c984a2ac6d2/uvicorn-0.29.0.tar.gz", hash = "sha256:6a69214c0b6a087462412670b3ef21224fa48cae0e452b5883e8e8bdfdd11dd0", size = 40894, upload-time = "2024-03-20T06:43:25.747Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/f5/cbb16fcbe277c1e0b8b3ddd188f2df0e0947f545c49119b589643632d156/uvicorn-0.29.0-py3-none-any.whl", hash = "sha256:2c2aac7ff4f4365c206fd773a39bf4ebd1047c238f8b8268ad996829323473de", size = 60813, upload-time = "2024-03-20T06:43:21.841Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.31.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "click", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "h11", marker = "extra == 'extra-11-skyrl-train-flashrl' or extra == 'extra-11-skyrl-train-mcore' or extra != 'extra-11-skyrl-train-sglang' or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/87/a886eda9ed495a3a4506d5a125cd07c54524280718c4969bde88f075fe98/uvicorn-0.31.1.tar.gz", hash = "sha256:f5167919867b161b7bcaf32646c6a94cdbd4c3aa2eb5c17d36bb9aa5cfd8c493", size = 77368, upload-time = "2024-10-09T19:44:20.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/55/37407280931038a3f21fa0245d60edeaa76f18419581aa3f4397761c78df/uvicorn-0.31.1-py3-none-any.whl", hash = "sha256:adc42d9cac80cf3e51af97c1851648066841e7cfb6993a4ca8de29ac1548ed41", size = 63666, upload-time = "2024-10-09T19:44:18.734Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" } }, +] + +[[package]] +name = "uvloop" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/c0/854216d09d33c543f12a44b393c402e89a920b1a0a7dc634c42de91b9cf6/uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3", size = 2492741, upload-time = "2024-10-14T23:38:35.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/4c/03f93178830dc7ce8b4cdee1d36770d2f5ebb6f3d37d354e061eefc73545/uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c", size = 1471284, upload-time = "2024-10-14T23:37:47.833Z" }, + { url = "https://files.pythonhosted.org/packages/43/3e/92c03f4d05e50f09251bd8b2b2b584a2a7f8fe600008bcc4523337abe676/uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2", size = 821349, upload-time = "2024-10-14T23:37:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ef/a02ec5da49909dbbfb1fd205a9a1ac4e88ea92dcae885e7c961847cd51e2/uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d", size = 4580089, upload-time = "2024-10-14T23:37:51.703Z" }, + { url = "https://files.pythonhosted.org/packages/06/a7/b4e6a19925c900be9f98bec0a75e6e8f79bb53bdeb891916609ab3958967/uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc", size = 4693770, upload-time = "2024-10-14T23:37:54.122Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0c/f07435a18a4b94ce6bd0677d8319cd3de61f3a9eeb1e5f8ab4e8b5edfcb3/uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb", size = 4451321, upload-time = "2024-10-14T23:37:55.766Z" }, + { url = "https://files.pythonhosted.org/packages/8f/eb/f7032be105877bcf924709c97b1bf3b90255b4ec251f9340cef912559f28/uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f", size = 4659022, upload-time = "2024-10-14T23:37:58.195Z" }, +] + +[[package]] +name = "virtualenv" +version = "20.35.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/28/e6f1a6f655d620846bd9df527390ecc26b3805a0c5989048c210e22c5ca9/virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c", size = 6028799, upload-time = "2025-10-29T06:57:40.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b", size = 6005095, upload-time = "2025-10-29T06:57:37.598Z" }, +] + +[[package]] +name = "vllm" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "anthropic" }, + { name = "blake3" }, + { name = "cachetools" }, + { name = "cbor2" }, + { name = "cloudpickle" }, + { name = "compressed-tensors" }, + { name = "depyf" }, + { name = "diskcache" }, + { name = "einops" }, + { name = "fastapi", extra = ["standard"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "filelock" }, + { name = "flashinfer-python", version = "0.5.3", source = { registry = "https://pypi.org/simple" } }, + { name = "gguf" }, + { name = "ijson" }, + { name = "lark" }, + { name = "llguidance", version = "1.3.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, + { name = "lm-format-enforcer" }, + { name = "mcp", version = "1.25.0", source = { registry = "https://pypi.org/simple" } }, + { name = "mistral-common", extra = ["image"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "model-hosting-container-standards" }, + { name = "msgspec" }, + { name = "ninja" }, + { name = "numba" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "openai" }, + { name = "openai-harmony" }, + { name = "opencv-python-headless", version = "4.11.0.86", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "opencv-python-headless", version = "4.12.0.88", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "outlines-core", version = "0.2.11", source = { registry = "https://pypi.org/simple" } }, + { name = "partial-json-parser" }, + { name = "pillow" }, + { name = "prometheus-client" }, + { name = "prometheus-fastapi-instrumentator" }, + { name = "protobuf" }, + { name = "psutil" }, + { name = "py-cpuinfo" }, + { name = "pybase64" }, + { name = "pydantic" }, + { name = "python-json-logger" }, + { name = "pyyaml" }, + { name = "pyzmq" }, + { name = "ray", extra = ["cgraph"], marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "regex" }, + { name = "requests" }, + { name = "scipy" }, + { name = "sentencepiece" }, + { name = "setproctitle" }, + { name = "setuptools" }, + { name = "six" }, + { name = "tiktoken" }, + { name = "tokenizers", version = "0.22.1", source = { registry = "https://pypi.org/simple" } }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "torchaudio", version = "2.9.0", source = { registry = "https://pypi.org/simple" } }, + { name = "torchvision", version = "0.24.0", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "torchvision", version = "0.24.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-mcore') or (platform_machine != 'aarch64' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra == 'extra-11-skyrl-train-mcore') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (sys_platform != 'linux' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "tqdm" }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "typing-extensions" }, + { name = "watchfiles" }, + { name = "xgrammar", version = "0.1.27", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/12/b922f96778d07df1c28dfa9a81fbc9706c13c5d0a4e8d154060818a79705/vllm-0.13.0.tar.gz", hash = "sha256:4ad43db45fef37114b550d03a4f423fb3fa3a31d8bc09ee810ef8b9cdcd4b5fe", size = 17828199, upload-time = "2025-12-19T03:30:32.741Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/82/e6194ac86862c50e9ff3f58ab3eb63d71604f96723bead2fcc610821197f/vllm-0.13.0-cp38-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:464b722c5c5d67a39593ada4a228f7558e860a732cb74a3bfa61c1b442b57581", size = 442031402, upload-time = "2025-12-19T03:31:07.026Z" }, + { url = "https://files.pythonhosted.org/packages/46/ae/36f87f514811c1389ff1a16e4e5b0b55f25ce782eb0eff2d7eaa92ff7deb/vllm-0.13.0-cp38-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:12b3d0a3b91c32a0091349de64b464f1c3d499a5b3a5d0ec387fef94ed5df6ee", size = 474942618, upload-time = "2025-12-19T03:31:35.593Z" }, +] + +[[package]] +name = "wadler-lindig" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/67/cbae4bf7683a64755c2c1778c418fea96d00e34395bb91743f08bd951571/wadler_lindig-0.1.7.tar.gz", hash = "sha256:81d14d3fe77d441acf3ebd7f4aefac20c74128bf460e84b512806dccf7b2cd55", size = 15842, upload-time = "2025-06-18T07:00:42.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/96/04e7b441807b26b794da5b11e59ed7f83b2cf8af202bd7eba8ad2fa6046e/wadler_lindig-0.1.7-py3-none-any.whl", hash = "sha256:e3ec83835570fd0a9509f969162aeb9c65618f998b1f42918cfc8d45122fe953", size = 20516, upload-time = "2025-06-18T07:00:41.684Z" }, +] + +[[package]] +name = "wandb" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/cc/770ae3aa7ae44f6792f7ecb81c14c0e38b672deb35235719bb1006519487/wandb-0.23.1.tar.gz", hash = "sha256:f6fb1e3717949b29675a69359de0eeb01e67d3360d581947d5b3f98c273567d6", size = 44298053, upload-time = "2025-12-03T02:25:10.79Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/0b/c3d7053dfd93fd259a63c7818d9c4ac2ba0642ff8dc8db98662ea0cf9cc0/wandb-0.23.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:358e15471d19b7d73fc464e37371c19d44d39e433252ac24df107aff993a286b", size = 21527293, upload-time = "2025-12-03T02:24:48.011Z" }, + { url = "https://files.pythonhosted.org/packages/ee/9f/059420fa0cb6c511dc5c5a50184122b6aca7b178cb2aa210139e354020da/wandb-0.23.1-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:110304407f4b38f163bdd50ed5c5225365e4df3092f13089c30171a75257b575", size = 22745926, upload-time = "2025-12-03T02:24:50.519Z" }, + { url = "https://files.pythonhosted.org/packages/96/b6/fd465827c14c64d056d30b4c9fcf4dac889a6969dba64489a88fc4ffa333/wandb-0.23.1-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:6cc984cf85feb2f8ee0451d76bc9fb7f39da94956bb8183e30d26284cf203b65", size = 21212973, upload-time = "2025-12-03T02:24:52.828Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ee/9a8bb9a39cc1f09c3060456cc79565110226dc4099a719af5c63432da21d/wandb-0.23.1-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:67431cd3168d79fdb803e503bd669c577872ffd5dadfa86de733b3274b93088e", size = 22887885, upload-time = "2025-12-03T02:24:55.281Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4d/8d9e75add529142e037b05819cb3ab1005679272950128d69d218b7e5b2e/wandb-0.23.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:07be70c0baa97ea25fadc4a9d0097f7371eef6dcacc5ceb525c82491a31e9244", size = 21250967, upload-time = "2025-12-03T02:24:57.603Z" }, + { url = "https://files.pythonhosted.org/packages/97/72/0b35cddc4e4168f03c759b96d9f671ad18aec8bdfdd84adfea7ecb3f5701/wandb-0.23.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:216c95b08e0a2ec6a6008373b056d597573d565e30b43a7a93c35a171485ee26", size = 22988382, upload-time = "2025-12-03T02:25:00.518Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6d/e78093d49d68afb26f5261a70fc7877c34c114af5c2ee0ab3b1af85f5e76/wandb-0.23.1-py3-none-win32.whl", hash = "sha256:fb5cf0f85692f758a5c36ab65fea96a1284126de64e836610f92ddbb26df5ded", size = 22150756, upload-time = "2025-12-03T02:25:02.734Z" }, + { url = "https://files.pythonhosted.org/packages/05/27/4f13454b44c9eceaac3d6e4e4efa2230b6712d613ff9bf7df010eef4fd18/wandb-0.23.1-py3-none-win_amd64.whl", hash = "sha256:21c8c56e436eb707b7d54f705652e030d48e5cfcba24cf953823eb652e30e714", size = 22150760, upload-time = "2025-12-03T02:25:05.106Z" }, + { url = "https://files.pythonhosted.org/packages/30/20/6c091d451e2a07689bfbfaeb7592d488011420e721de170884fedd68c644/wandb-0.23.1-py3-none-win_arm64.whl", hash = "sha256:8aee7f3bb573f2c0acf860f497ca9c684f9b35f2ca51011ba65af3d4592b77c1", size = 20137463, upload-time = "2025-12-03T02:25:08.317Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/d5/f039e7e3c639d9b1d09b07ea412a6806d38123f0508e5f9b48a87b0a76cc/watchfiles-1.1.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:8c89f9f2f740a6b7dcc753140dd5e1ab9215966f7a3530d0c0705c83b401bd7d", size = 404745, upload-time = "2025-10-14T15:04:46.731Z" }, + { url = "https://files.pythonhosted.org/packages/a5/96/a881a13aa1349827490dab2d363c8039527060cfcc2c92cc6d13d1b1049e/watchfiles-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bd404be08018c37350f0d6e34676bd1e2889990117a2b90070b3007f172d0610", size = 391769, upload-time = "2025-10-14T15:04:48.003Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5b/d3b460364aeb8da471c1989238ea0e56bec24b6042a68046adf3d9ddb01c/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8526e8f916bb5b9a0a777c8317c23ce65de259422bba5b31325a6fa6029d33af", size = 449374, upload-time = "2025-10-14T15:04:49.179Z" }, + { url = "https://files.pythonhosted.org/packages/b9/44/5769cb62d4ed055cb17417c0a109a92f007114a4e07f30812a73a4efdb11/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2edc3553362b1c38d9f06242416a5d8e9fe235c204a4072e988ce2e5bb1f69f6", size = 459485, upload-time = "2025-10-14T15:04:50.155Z" }, + { url = "https://files.pythonhosted.org/packages/19/0c/286b6301ded2eccd4ffd0041a1b726afda999926cf720aab63adb68a1e36/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30f7da3fb3f2844259cba4720c3fc7138eb0f7b659c38f3bfa65084c7fc7abce", size = 488813, upload-time = "2025-10-14T15:04:51.059Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2b/8530ed41112dd4a22f4dcfdb5ccf6a1baad1ff6eed8dc5a5f09e7e8c41c7/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8979280bdafff686ba5e4d8f97840f929a87ed9cdf133cbbd42f7766774d2aa", size = 594816, upload-time = "2025-10-14T15:04:52.031Z" }, + { url = "https://files.pythonhosted.org/packages/ce/d2/f5f9fb49489f184f18470d4f99f4e862a4b3e9ac2865688eb2099e3d837a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcc5c24523771db3a294c77d94771abcfcb82a0e0ee8efd910c37c59ec1b31bb", size = 475186, upload-time = "2025-10-14T15:04:53.064Z" }, + { url = "https://files.pythonhosted.org/packages/cf/68/5707da262a119fb06fbe214d82dd1fe4a6f4af32d2d14de368d0349eb52a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db5d7ae38ff20153d542460752ff397fcf5c96090c1230803713cf3147a6803", size = 456812, upload-time = "2025-10-14T15:04:55.174Z" }, + { url = "https://files.pythonhosted.org/packages/66/ab/3cbb8756323e8f9b6f9acb9ef4ec26d42b2109bce830cc1f3468df20511d/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:28475ddbde92df1874b6c5c8aaeb24ad5be47a11f87cde5a28ef3835932e3e94", size = 630196, upload-time = "2025-10-14T15:04:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/78/46/7152ec29b8335f80167928944a94955015a345440f524d2dfe63fc2f437b/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:36193ed342f5b9842edd3532729a2ad55c4160ffcfa3700e0d54be496b70dd43", size = 622657, upload-time = "2025-10-14T15:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/0a/bf/95895e78dd75efe9a7f31733607f384b42eb5feb54bd2eb6ed57cc2e94f4/watchfiles-1.1.1-cp312-cp312-win32.whl", hash = "sha256:859e43a1951717cc8de7f4c77674a6d389b106361585951d9e69572823f311d9", size = 272042, upload-time = "2025-10-14T15:04:59.046Z" }, + { url = "https://files.pythonhosted.org/packages/87/0a/90eb755f568de2688cb220171c4191df932232c20946966c27a59c400850/watchfiles-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:91d4c9a823a8c987cce8fa2690923b069966dabb196dd8d137ea2cede885fde9", size = 288410, upload-time = "2025-10-14T15:05:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/36/76/f322701530586922fbd6723c4f91ace21364924822a8772c549483abed13/watchfiles-1.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:a625815d4a2bdca61953dbba5a39d60164451ef34c88d751f6c368c3ea73d404", size = 278209, upload-time = "2025-10-14T15:05:01.168Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.2.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/30/6b0809f4510673dc723187aeaf24c7f5459922d01e2f794277a3dfb90345/wcwidth-0.2.14.tar.gz", hash = "sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605", size = 102293, upload-time = "2025-09-22T16:29:53.023Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1", size = 37286, upload-time = "2025-09-22T16:29:51.641Z" }, +] + +[[package]] +name = "websockets" +version = "13.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/73/9223dbc7be3dcaf2a7bbf756c351ec8da04b1fa573edaf545b95f6b0c7fd/websockets-13.1.tar.gz", hash = "sha256:a3b3366087c1bc0a2795111edcadddb8b3b59509d5db5d7ea3fdd69f954a8878", size = 158549, upload-time = "2024-09-21T17:34:21.54Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/46/c426282f543b3c0296cf964aa5a7bb17e984f58dde23460c3d39b3148fcf/websockets-13.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d75baf00138f80b48f1eac72ad1535aac0b6461265a0bcad391fc5aba875cfc", size = 157821, upload-time = "2024-09-21T17:32:56.442Z" }, + { url = "https://files.pythonhosted.org/packages/aa/85/22529867010baac258da7c45848f9415e6cf37fef00a43856627806ffd04/websockets-13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9b6f347deb3dcfbfde1c20baa21c2ac0751afaa73e64e5b693bb2b848efeaa49", size = 155480, upload-time = "2024-09-21T17:32:57.698Z" }, + { url = "https://files.pythonhosted.org/packages/29/2c/bdb339bfbde0119a6e84af43ebf6275278698a2241c2719afc0d8b0bdbf2/websockets-13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de58647e3f9c42f13f90ac7e5f58900c80a39019848c5547bc691693098ae1bd", size = 155715, upload-time = "2024-09-21T17:32:59.429Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/8612029ea04c5c22bf7af2fd3d63876c4eaeef9b97e86c11972a43aa0e6c/websockets-13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1b54689e38d1279a51d11e3467dd2f3a50f5f2e879012ce8f2d6943f00e83f0", size = 165647, upload-time = "2024-09-21T17:33:00.495Z" }, + { url = "https://files.pythonhosted.org/packages/56/04/1681ed516fa19ca9083f26d3f3a302257e0911ba75009533ed60fbb7b8d1/websockets-13.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf1781ef73c073e6b0f90af841aaf98501f975d306bbf6221683dd594ccc52b6", size = 164592, upload-time = "2024-09-21T17:33:02.223Z" }, + { url = "https://files.pythonhosted.org/packages/38/6f/a96417a49c0ed132bb6087e8e39a37db851c70974f5c724a4b2a70066996/websockets-13.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d23b88b9388ed85c6faf0e74d8dec4f4d3baf3ecf20a65a47b836d56260d4b9", size = 165012, upload-time = "2024-09-21T17:33:03.288Z" }, + { url = "https://files.pythonhosted.org/packages/40/8b/fccf294919a1b37d190e86042e1a907b8f66cff2b61e9befdbce03783e25/websockets-13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3c78383585f47ccb0fcf186dcb8a43f5438bd7d8f47d69e0b56f71bf431a0a68", size = 165311, upload-time = "2024-09-21T17:33:04.728Z" }, + { url = "https://files.pythonhosted.org/packages/c1/61/f8615cf7ce5fe538476ab6b4defff52beb7262ff8a73d5ef386322d9761d/websockets-13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d6d300f8ec35c24025ceb9b9019ae9040c1ab2f01cddc2bcc0b518af31c75c14", size = 164692, upload-time = "2024-09-21T17:33:05.829Z" }, + { url = "https://files.pythonhosted.org/packages/5c/f1/a29dd6046d3a722d26f182b783a7997d25298873a14028c4760347974ea3/websockets-13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9dcaf8b0cc72a392760bb8755922c03e17a5a54e08cca58e8b74f6902b433cf", size = 164686, upload-time = "2024-09-21T17:33:06.823Z" }, + { url = "https://files.pythonhosted.org/packages/0f/99/ab1cdb282f7e595391226f03f9b498f52109d25a2ba03832e21614967dfa/websockets-13.1-cp312-cp312-win32.whl", hash = "sha256:2f85cf4f2a1ba8f602298a853cec8526c2ca42a9a4b947ec236eaedb8f2dc80c", size = 158712, upload-time = "2024-09-21T17:33:07.877Z" }, + { url = "https://files.pythonhosted.org/packages/46/93/e19160db48b5581feac8468330aa11b7292880a94a37d7030478596cc14e/websockets-13.1-cp312-cp312-win_amd64.whl", hash = "sha256:38377f8b0cdeee97c552d20cf1865695fcd56aba155ad1b4ca8779a5b6ef4ac3", size = 159145, upload-time = "2024-09-21T17:33:09.202Z" }, + { url = "https://files.pythonhosted.org/packages/56/27/96a5cd2626d11c8280656c6c71d8ab50fe006490ef9971ccd154e0c42cd2/websockets-13.1-py3-none-any.whl", hash = "sha256:a9a396a6ad26130cdae92ae10c36af09d9bfe6cafe69670fd3b6da9b07b4044f", size = 152134, upload-time = "2024-09-21T17:34:19.904Z" }, +] + +[[package]] +name = "websockets" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra == 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, + { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, + { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, + { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, + { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/45/ea/b0f8eeb287f8df9066e56e831c7824ac6bab645dd6c7a8f4b2d767944f9b/werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e", size = 864687, upload-time = "2025-11-29T02:15:22.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/f9/9e082990c2585c744734f85bec79b5dae5df9c974ffee58fe421652c8e91/werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905", size = 224960, upload-time = "2025-11-29T02:15:21.13Z" }, +] + +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] + +[[package]] +name = "wrapt" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +] + +[[package]] +name = "xgrammar" +version = "0.1.19" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')", + "sys_platform != 'linux'", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", +] +dependencies = [ + { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, + { name = "ninja" }, + { name = "pydantic" }, + { name = "sentencepiece" }, + { name = "tiktoken" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "transformers", version = "4.52.3", source = { registry = "https://pypi.org/simple" } }, + { name = "triton", version = "3.3.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/55/73e1e4f918ade656c4fa7f3a5fcfb3d521a429fe305d2cb8ca58bfb201d4/xgrammar-0.1.19.tar.gz", hash = "sha256:75bf3e814283b1cbaee9252234c5d4081f0058d29b26d8984f1cdf031c99b775", size = 1714056, upload-time = "2025-05-08T07:13:46.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/a8/886975ef77106ba8fad8f7c253d8aead02e1d285a831857f4a67365a7c6e/xgrammar-0.1.19-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:70f1bb54e9bdb92830424713629e37ffcd4f8e4ebbbf03a72860503e25d349bf", size = 504554, upload-time = "2025-05-08T07:13:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9d/e27686ad71be897cda26289d7f899250f41a3fd8a12b472f1ba3ea8fc5ae/xgrammar-0.1.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:70ee7d359386e816eb85f9f763d68a0f2dfedb3da8601ed38e6e8e8391aa9b98", size = 457619, upload-time = "2025-05-08T07:13:25.283Z" }, + { url = "https://files.pythonhosted.org/packages/bc/64/e64c7a06fbbe8d610dd520cb00045c109ad4f56457198220d63830efd426/xgrammar-0.1.19-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16439a86378f7e07d2db91f8a9645d1ff9959b018f1fae6768a057b4b3926dc7", size = 5717888, upload-time = "2025-05-08T07:13:26.85Z" }, + { url = "https://files.pythonhosted.org/packages/c6/68/df91740b23287d06c9d67fadd5d0dc096bb1beaf6079ab083f143545f520/xgrammar-0.1.19-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9beb2cb2b55c9524f24b3cbf8181c47e435586976aa0c37e220661f786c601f", size = 5834560, upload-time = "2025-05-08T07:13:29.006Z" }, + { url = "https://files.pythonhosted.org/packages/df/42/d0248e8af1c69a92c409ee06e6f07fb047567c366e4d08676b6a3bc356f6/xgrammar-0.1.19-cp312-cp312-win_amd64.whl", hash = "sha256:4a430dbf229c04539f0929069df245f5f652298e37dc3f04ce0a6aa8639546ef", size = 527418, upload-time = "2025-05-08T07:13:31.229Z" }, +] + +[[package]] +name = "xgrammar" +version = "0.1.27" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm') or (platform_python_implementation != 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')", + "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine == 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "platform_machine != 'aarch64' and sys_platform == 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform == 'darwin' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", + "sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra != 'extra-11-skyrl-train-vllm'", +] +dependencies = [ + { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, + { name = "ninja" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "pydantic" }, + { name = "torch", version = "2.9.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, + { name = "transformers", version = "4.57.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-11-skyrl-train-mcore' or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra == 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "transformers", version = "4.57.3", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-mcore') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-miniswe') or (extra == 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-sglang') or (extra == 'extra-11-skyrl-train-mcore' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-flashrl' and extra == 'extra-11-skyrl-train-vllm') or (extra != 'extra-11-skyrl-train-mcore' and extra != 'extra-11-skyrl-train-miniswe' and extra != 'extra-11-skyrl-train-sglang' and extra == 'extra-11-skyrl-train-vllm')" }, + { name = "triton", version = "3.5.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/e1/b522b1e50fddd773d368c2945ef5ed628aa90c0c972027f9aa5a51d6d4f9/xgrammar-0.1.27.tar.gz", hash = "sha256:40af7bb2891f1633ec7f660723c74a92a963307d283aca9e3b4e53a0feaf1d46", size = 2303435, upload-time = "2025-11-04T03:11:53.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/b6/09b43e2adff45d30ebcf9110d0ff753f4c96b368adaa2d166df3dee88d5f/xgrammar-0.1.27-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:6404a7714440eb86ab0379d749f33591274eeef04787dc00d61f22069f3ed51d", size = 663319, upload-time = "2025-11-04T03:11:28.682Z" }, + { url = "https://files.pythonhosted.org/packages/88/8b/53eb5c6d0df8df9f6350f182516a5b8c7b8b11d62650300d2c04af2bc4ea/xgrammar-0.1.27-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d01fa9894bc44a7f6a70b0301b59f3e310c0e0e7b7ea4cf5ce190b12d8220dd8", size = 636168, upload-time = "2025-11-04T03:11:30.373Z" }, + { url = "https://files.pythonhosted.org/packages/08/1b/53d30395bb973f13255d3e3a72961f95fdfb4083877c3f93bb626e3d1522/xgrammar-0.1.27-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:906c0601bac9170e1bab77ca985259035ff9c386c347efcb191555eab86e984e", size = 8676340, upload-time = "2025-11-04T03:11:32.203Z" }, + { url = "https://files.pythonhosted.org/packages/48/74/70cfac0171d9f309cfe18c5384330e3edc9466c436b258495fd30ecf29a3/xgrammar-0.1.27-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb68988a122f544301c496f2cac8ee82960ca7f5b3a42a952b2a00c0a55e6ca5", size = 8870650, upload-time = "2025-11-04T03:11:34.322Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a1/0392aa9c7669c56f7f88e4423b246476a74a72c3bb9db944e1bfc029985e/xgrammar-0.1.27-cp312-cp312-win_amd64.whl", hash = "sha256:3aac335ea052afc8f8dc34b9f2afcb9462a68189423aed9f60b0941db6cfc310", size = 708811, upload-time = "2025-11-04T03:11:36.214Z" }, +] + +[[package]] +name = "xxhash" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c", size = 32744, upload-time = "2025-10-02T14:34:34.622Z" }, + { url = "https://files.pythonhosted.org/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204", size = 30816, upload-time = "2025-10-02T14:34:36.043Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490", size = 194035, upload-time = "2025-10-02T14:34:37.354Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2", size = 212914, upload-time = "2025-10-02T14:34:38.6Z" }, + { url = "https://files.pythonhosted.org/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa", size = 212163, upload-time = "2025-10-02T14:34:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0", size = 445411, upload-time = "2025-10-02T14:34:41.569Z" }, + { url = "https://files.pythonhosted.org/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2", size = 193883, upload-time = "2025-10-02T14:34:43.249Z" }, + { url = "https://files.pythonhosted.org/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9", size = 210392, upload-time = "2025-10-02T14:34:45.042Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e", size = 197898, upload-time = "2025-10-02T14:34:46.302Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374", size = 210655, upload-time = "2025-10-02T14:34:47.571Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d", size = 414001, upload-time = "2025-10-02T14:34:49.273Z" }, + { url = "https://files.pythonhosted.org/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae", size = 191431, upload-time = "2025-10-02T14:34:50.798Z" }, + { url = "https://files.pythonhosted.org/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb", size = 30617, upload-time = "2025-10-02T14:34:51.954Z" }, + { url = "https://files.pythonhosted.org/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c", size = 31534, upload-time = "2025-10-02T14:34:53.276Z" }, + { url = "https://files.pythonhosted.org/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829", size = 27876, upload-time = "2025-10-02T14:34:54.371Z" }, +] + +[[package]] +name = "yarl" +version = "1.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/63/0c6ebca57330cd313f6102b16dd57ffaf3ec4c83403dcb45dbd15c6f3ea1/yarl-1.22.0.tar.gz", hash = "sha256:bebf8557577d4401ba8bd9ff33906f1376c877aa78d1fe216ad01b4d6745af71", size = 187169, upload-time = "2025-10-06T14:12:55.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/ff/46736024fee3429b80a165a732e38e5d5a238721e634ab41b040d49f8738/yarl-1.22.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e340382d1afa5d32b892b3ff062436d592ec3d692aeea3bef3a5cfe11bbf8c6f", size = 142000, upload-time = "2025-10-06T14:09:44.631Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9a/b312ed670df903145598914770eb12de1bac44599549b3360acc96878df8/yarl-1.22.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f1e09112a2c31ffe8d80be1b0988fa6a18c5d5cad92a9ffbb1c04c91bfe52ad2", size = 94338, upload-time = "2025-10-06T14:09:46.372Z" }, + { url = "https://files.pythonhosted.org/packages/ba/f5/0601483296f09c3c65e303d60c070a5c19fcdbc72daa061e96170785bc7d/yarl-1.22.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:939fe60db294c786f6b7c2d2e121576628468f65453d86b0fe36cb52f987bd74", size = 94909, upload-time = "2025-10-06T14:09:48.648Z" }, + { url = "https://files.pythonhosted.org/packages/60/41/9a1fe0b73dbcefce72e46cf149b0e0a67612d60bfc90fb59c2b2efdfbd86/yarl-1.22.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e1651bf8e0398574646744c1885a41198eba53dc8a9312b954073f845c90a8df", size = 372940, upload-time = "2025-10-06T14:09:50.089Z" }, + { url = "https://files.pythonhosted.org/packages/17/7a/795cb6dfee561961c30b800f0ed616b923a2ec6258b5def2a00bf8231334/yarl-1.22.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b8a0588521a26bf92a57a1705b77b8b59044cdceccac7151bd8d229e66b8dedb", size = 345825, upload-time = "2025-10-06T14:09:52.142Z" }, + { url = "https://files.pythonhosted.org/packages/d7/93/a58f4d596d2be2ae7bab1a5846c4d270b894958845753b2c606d666744d3/yarl-1.22.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:42188e6a615c1a75bcaa6e150c3fe8f3e8680471a6b10150c5f7e83f47cc34d2", size = 386705, upload-time = "2025-10-06T14:09:54.128Z" }, + { url = "https://files.pythonhosted.org/packages/61/92/682279d0e099d0e14d7fd2e176bd04f48de1484f56546a3e1313cd6c8e7c/yarl-1.22.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6d2cb59377d99718913ad9a151030d6f83ef420a2b8f521d94609ecc106ee82", size = 396518, upload-time = "2025-10-06T14:09:55.762Z" }, + { url = "https://files.pythonhosted.org/packages/db/0f/0d52c98b8a885aeda831224b78f3be7ec2e1aa4a62091f9f9188c3c65b56/yarl-1.22.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50678a3b71c751d58d7908edc96d332af328839eea883bb554a43f539101277a", size = 377267, upload-time = "2025-10-06T14:09:57.958Z" }, + { url = "https://files.pythonhosted.org/packages/22/42/d2685e35908cbeaa6532c1fc73e89e7f2efb5d8a7df3959ea8e37177c5a3/yarl-1.22.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e8fbaa7cec507aa24ea27a01456e8dd4b6fab829059b69844bd348f2d467124", size = 365797, upload-time = "2025-10-06T14:09:59.527Z" }, + { url = "https://files.pythonhosted.org/packages/a2/83/cf8c7bcc6355631762f7d8bdab920ad09b82efa6b722999dfb05afa6cfac/yarl-1.22.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:433885ab5431bc3d3d4f2f9bd15bfa1614c522b0f1405d62c4f926ccd69d04fa", size = 365535, upload-time = "2025-10-06T14:10:01.139Z" }, + { url = "https://files.pythonhosted.org/packages/25/e1/5302ff9b28f0c59cac913b91fe3f16c59a033887e57ce9ca5d41a3a94737/yarl-1.22.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b790b39c7e9a4192dc2e201a282109ed2985a1ddbd5ac08dc56d0e121400a8f7", size = 382324, upload-time = "2025-10-06T14:10:02.756Z" }, + { url = "https://files.pythonhosted.org/packages/bf/cd/4617eb60f032f19ae3a688dc990d8f0d89ee0ea378b61cac81ede3e52fae/yarl-1.22.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31f0b53913220599446872d757257be5898019c85e7971599065bc55065dc99d", size = 383803, upload-time = "2025-10-06T14:10:04.552Z" }, + { url = "https://files.pythonhosted.org/packages/59/65/afc6e62bb506a319ea67b694551dab4a7e6fb7bf604e9bd9f3e11d575fec/yarl-1.22.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a49370e8f711daec68d09b821a34e1167792ee2d24d405cbc2387be4f158b520", size = 374220, upload-time = "2025-10-06T14:10:06.489Z" }, + { url = "https://files.pythonhosted.org/packages/e7/3d/68bf18d50dc674b942daec86a9ba922d3113d8399b0e52b9897530442da2/yarl-1.22.0-cp312-cp312-win32.whl", hash = "sha256:70dfd4f241c04bd9239d53b17f11e6ab672b9f1420364af63e8531198e3f5fe8", size = 81589, upload-time = "2025-10-06T14:10:09.254Z" }, + { url = "https://files.pythonhosted.org/packages/c8/9a/6ad1a9b37c2f72874f93e691b2e7ecb6137fb2b899983125db4204e47575/yarl-1.22.0-cp312-cp312-win_amd64.whl", hash = "sha256:8884d8b332a5e9b88e23f60bb166890009429391864c685e17bd73a9eda9105c", size = 87213, upload-time = "2025-10-06T14:10:11.369Z" }, + { url = "https://files.pythonhosted.org/packages/44/c5/c21b562d1680a77634d748e30c653c3ca918beb35555cff24986fff54598/yarl-1.22.0-cp312-cp312-win_arm64.whl", hash = "sha256:ea70f61a47f3cc93bdf8b2f368ed359ef02a01ca6393916bc8ff877427181e74", size = 81330, upload-time = "2025-10-06T14:10:13.112Z" }, + { url = "https://files.pythonhosted.org/packages/73/ae/b48f95715333080afb75a4504487cbe142cae1268afc482d06692d605ae6/yarl-1.22.0-py3-none-any.whl", hash = "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", size = 46814, upload-time = "2025-10-06T14:12:53.872Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] diff --git a/skyrl-tx/.gitignore b/skyrl-tx/.gitignore new file mode 100644 index 0000000000..5aa7c78151 --- /dev/null +++ b/skyrl-tx/.gitignore @@ -0,0 +1,87 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +uv.lock + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Jupyter Notebook +.ipynb_checkpoints + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# MkDocs build output +site/ + +# IDEs and editors +.idea/ +.vscode/ + +# OS generated files +.DS_Store +Thumbs.db + +# Local artifacts +tinker.db +uv.lock + +# Alembic - don't track pycache +tx/tinker/alembic/__pycache__/ + +# SQLite databases (tracked in git by default, but ignore if created locally) +*.db +*.db-journal +*.db-wal +*.db-shm diff --git a/skyrl-tx/LICENSE b/skyrl-tx/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/skyrl-tx/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/skyrl-tx/README.md b/skyrl-tx/README.md new file mode 100644 index 0000000000..781fdf2311 --- /dev/null +++ b/skyrl-tx/README.md @@ -0,0 +1,291 @@ +
+ +# 🧸 SkyRL tx: Unifying LLM training and inference + +

+ GitHub • + Tinker Docs • + Tinker Cookbook • + Slack +

+ +
+ +SkyRL tx is an open-source library that implements a backend for the [Tinker API](https://thinkingmachines.ai/tinker/), allowing you to set up your own Tinker-like service running on your own hardware. It provides a unified interface for both training and inference, enabling seamless online learning, cost-effective multi-tenancy through LoRA, and simplified ML infrastructure. + +## ✨ Key Features + +- **Unified Training & Inference** — Single engine for forward passes, backward passes, and sampling +- **Multi-User LoRA Support** — Efficient GPU sharing across users with individual adapters +- **SFT & RL Support** — Supervised fine-tuning and reinforcement learning with PPO and custom loss functions +- **Multi-Node Training** — FSDP and tensor parallelism for distributed training +- **Multiple Model Architectures** — Support for Qwen3 (dense & MoE) and Llama 3 +- **External Inference Engine** — Optional vLLM integration for optimized inference +- **Production Ready** — PostgreSQL support, cloud storage checkpoints, and database migrations + +## 🏗️ Architecture + +SkyRL tx consists of four main components: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ REST API Server │ +│ (FastAPI - handles requests) │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Database │ +│ (SQLite/PostgreSQL - metadata, job queue) │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Engine │ +│ (Scheduling & batching across users/adapters) │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Worker │ +│ (Model execution, forward/backward, optimizer) │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## 🚀 Quick Start + +### Installation + +```bash +git clone https://github.com/NovaSky-AI/SkyRL +cd SkyRL/skyrl-tx + +# For GPU +uv run --extra gpu --extra tinker -m tx.tinker.api --base-model + +# For TPU +uv run --extra tpu --extra tinker -m tx.tinker.api --base-model +``` + +### Basic Training Example (Pig Latin) + +Start the server: + +```bash +uv run --extra gpu --extra tinker -m tx.tinker.api --base-model "Qwen/Qwen3-0.6B" +``` + +Run a simple training loop: + +```python +import tinker +import numpy as np +from tinker import types + +# Connect to the local server +service_client = tinker.ServiceClient(base_url="http://localhost:8000", api_key="dummy") +training_client = service_client.create_lora_training_client(base_model="Qwen/Qwen3-0.6B") +tokenizer = training_client.get_tokenizer() + +# Training examples +examples = [ + {"input": "banana split", "output": "anana-bay plit-say"}, + {"input": "quantum physics", "output": "uantum-qay ysics-phay"}, + {"input": "coding wizard", "output": "oding-cay izard-way"}, +] + +def process_example(example, tokenizer): + prompt = f"English: {example['input']}\nPig Latin:" + prompt_tokens = tokenizer.encode(prompt, add_special_tokens=True) + completion_tokens = tokenizer.encode(f" {example['output']}\n\n", add_special_tokens=False) + + tokens = prompt_tokens + completion_tokens + weights = [0] * len(prompt_tokens) + [1] * len(completion_tokens) + + return types.Datum( + model_input=types.ModelInput.from_ints(tokens=tokens[:-1]), + loss_fn_inputs=dict(weights=weights[1:], target_tokens=tokens[1:]) + ) + +processed = [process_example(ex, tokenizer) for ex in examples] + +# Training loop +for _ in range(6): + fwdbwd = training_client.forward_backward(processed, "cross_entropy").result() + training_client.optim_step(types.AdamParams(learning_rate=1e-4)).result() + + logprobs = np.concatenate([o['logprobs'].tolist() for o in fwdbwd.loss_fn_outputs]) + weights = np.concatenate([e.loss_fn_inputs['weights'].tolist() for e in processed]) + print(f"Loss: {-np.dot(logprobs, weights) / weights.sum():.4f}") +``` + +### Sampling + +```python +# After training, create a sampling client +sampling_client = training_client.save_weights_and_get_sampling_client(name='my-model') + +# Sample from the model +prompt = types.ModelInput.from_ints(tokenizer.encode("English: coffee break\nPig Latin:")) +params = types.SamplingParams(max_tokens=20, temperature=0.0) +result = sampling_client.sample(prompt=prompt, sampling_params=params, num_samples=8).result() + +for i, seq in enumerate(result.sequences): + print(f"{i}: {tokenizer.decode(seq.tokens)}") +``` + +## 📖 Usage Examples + +### Dense Model Training (Qwen3-8B on 8×H100) + +```bash +# Start the server +uv run --extra gpu --extra tinker -m tx.tinker.api \ + --base-model Qwen/Qwen3-8B \ + --backend-config '{"max_lora_adapters": 2, "max_lora_rank": 1, "tensor_parallel_size": 8, "train_micro_batch_size": 1}' + +# Run training (using tinker-cookbook) +export TINKER_API_KEY="dummy" +uv run --with wandb --with tinker sl_loop.py \ + base_url=http://localhost:8000 \ + model_name=Qwen/Qwen3-8B lora_rank=1 +``` + +### MoE Model Training (Qwen/Qwen3-30B-A3B) + +```bash +# Start the server +uv run --extra gpu --extra tinker -m tx.tinker.api \ + --base-model Qwen/Qwen3-30B-A3B \ + --backend-config '{"max_lora_adapters": 2, "max_lora_rank": 1, "tensor_parallel_size": 8, "train_micro_batch_size": 1, "shard_attention_heads": false}' + +# Run training (using tinker-cookbook) +export TINKER_API_KEY="dummy" +uv run --with wandb --with tinker sl_loop.py \ + base_url=http://localhost:8000 \ + model_name=Qwen/Qwen3-30B-A3B lora_rank=1 max_length=512 +``` + +### Reinforcement Learning (Qwen/Qwen3-8B) + +```bash +# Start server +uv run --extra gpu --extra tinker -m tx.tinker.api \ + --base-model Qwen/Qwen3-8B \ + --backend-config '{"max_lora_adapters": 3, "max_lora_rank": 1, "tensor_parallel_size": 8, "train_micro_batch_size": 8, "sample_max_num_sequences": 256}' > out.log + +# Run RL loop +uv run --with wandb --with tinker rl_loop.py \ + base_url=http://localhost:8000 \ + model_name="Qwen/Qwen3-8B" \ + lora_rank=1 max_length=1024 +``` + +### Multi-Node Training + +```bash +# Node 0 (coordinator + API server) +CUDA_VISIBLE_DEVICES=0,1,2,3 uv run --extra gpu --extra tinker -m tx.tinker.api \ + --base-model Qwen/Qwen3-8B \ + --backend-config '{ + "max_lora_adapters": 3, + "max_lora_rank": 1, + "tensor_parallel_size": 4, + "fully_sharded_data_parallel_size": 2, + "train_micro_batch_size": 8, + "sample_max_num_sequences": 256, + "coordinator_address": "node0:7777", + "num_processes": 2 + }' > out.log + +# Node 1 (worker) +CUDA_VISIBLE_DEVICES=4,5,6,7 uv run --extra gpu --extra tinker -m tx.tinker.backends.jax \ + --coordinator-address "node0:7777" \ + --num-processes 2 \ + --process-id 1 +``` + +### With External vLLM Inference + +```bash +# Start vLLM +VLLM_ALLOW_RUNTIME_LORA_UPDATING=True \ +VLLM_PLUGINS=lora_filesystem_resolver \ +VLLM_LORA_RESOLVER_CACHE_DIR=/tmp/lora_models/ \ +CUDA_VISIBLE_DEVICES=4,5,6,7 uv run --with vllm vllm serve Qwen/Qwen3-4B \ + --tensor-parallel-size 4 --port 7999 --enable-lora + +# Start SkyRL tx with external inference +CUDA_VISIBLE_DEVICES=0,1,2,3 uv run --extra gpu --extra tinker -m tx.tinker.api \ + --base-model Qwen/Qwen3-4B \ + --external-inference-url "http://0.0.0.0:7999" \ + --backend-config '{"max_lora_adapters": 3, "max_lora_rank": 1, "tensor_parallel_size": 4, "train_micro_batch_size": 8}' > out.log +``` + +## 🎯 Supported Features + +| Feature | Status | +|---------|--------| +| Qwen3 Dense Models | ✅ | +| Qwen3 MoE Models | ✅ | +| Llama 3 Models | ✅ | +| Multi-User LoRA | ✅ | +| LoRA (all layers) | ✅ | +| Forward/Backward | ✅ | +| Sampling | ✅ | +| Gradient Accumulation | ✅ | +| Gradient Checkpointing | ✅ | +| JIT Compilation | ✅ | +| Tensor Parallelism | ✅ | +| FSDP | ✅ | +| Multi-Node | ✅ | +| PostgreSQL | ✅ | +| Cloud Storage Checkpoints | ✅ | +| Custom Loss Functions | ✅ | +| External Inference (vLLM) | ✅ | +| Local Model Loading | ✅ | + +## 🗺️ Roadmap + +- **Performance** — Expert parallelism, context parallelism, optimized kernels +- **Models** — More architectures, PyTorch model definitions via torchax +- **API Coverage** — Full Tinker API compatibility +- **Operations** — Dashboard/frontend, improved logging and metrics +- **Integration** — SkyRL-train Tinkerification + +## 🤝 Contributing + +We welcome contributions! The project is early and hackable — now is a great time to get involved. + +**Ways to contribute:** +- Try examples from the [Tinker documentation](https://tinker-docs.thinkingmachines.ai/) or [cookbook](https://github.com/thinking-machines-lab/tinker-cookbook) +- Fix issues or implement features from our [issue tracker](https://github.com/NovaSky-AI/SkyRL/issues?q=is%3Aissue%20state%3Aopen%20label%3Atx) +- Improve documentation +- Add support for more models +- Performance optimizations + +## 📚 Resources + +- **[Ray Summit Talk](https://www.youtube.com/watch?v=_JLnESEu2gw)** — SkyRL tx: A unified training and inference engine +- **[Slides](https://docs.google.com/presentation/d/1g-u8zxz7FsnlQXXShBVoqjUJhS48c6rxkJJJn0sj78A/)** — Presentation slides +- **[Tinker Documentation](https://tinker-docs.thinkingmachines.ai/)** — Official Tinker API docs +- **[Tinker Cookbook](https://github.com/thinking-machines-lab/tinker-cookbook)** — Example recipes + +## 📝 Blog Posts + +- **[Introducing SkyRL tx](https://novasky-ai.notion.site/skyrl-tx)** +- **[SkyRL tx v0.0.2](https://novasky-ai.notion.site/skyrl-tx-v002)** +- **[SkyRL tx v0.0.3](https://novasky-ai.notion.site/skyrl-tx-003)** +- **[SkyRL tx v0.1.0](https://novasky-ai.notion.site/skyrl-tx-v010)** +- **[SkyRL tx v0.2.0](https://novasky-ai.notion.site/skyrl-tx-v02)** +- **[SkyRL tx v0.2.1](https://novasky-ai.notion.site/skyrl-tx-v021)** + +## 📬 Contact + +- **Slack**: [#skyrl-tx](https://skyrl.slack.com/archives/C09K1JGNPJS) +- **GitHub**: [NovaSky-AI/SkyRL/skyrl-tx](https://github.com/NovaSky-AI/SkyRL/tree/main/skyrl-tx) +- **Twitter/X**: [@NovaSkyAI](https://x.com/NovaSkyAI) + +## 📄 License + +See [LICENSE](LICENSE) for details. diff --git a/skyrl-tx/benchmarks/benchmark_engine.py b/skyrl-tx/benchmarks/benchmark_engine.py new file mode 100644 index 0000000000..da2cb47226 --- /dev/null +++ b/skyrl-tx/benchmarks/benchmark_engine.py @@ -0,0 +1,148 @@ +"""Benchmark forward/backward and sampling for the TinkerEngine.""" + +import argparse +import time + +import jax + +from tx.tinker.engine import TinkerEngine +from tx.tinker.config import EngineConfig, add_model +from tx.tinker import types + + +def make_fwd_bwd_input(token_lists: list[list[int]]) -> types.ForwardBackwardInput: + samples = [] + for tokens in token_lists: + targets = tokens[1:] + [0] + weights = [1] * len(tokens) + samples.append( + types.Datum( + model_input=types.ModelInput(chunks=[types.ModelInputChunk(tokens=tokens)]), + loss_fn_inputs=types.LossFnInputs( + target_tokens=types.TensorData(data=targets), + weights=types.TensorData(data=weights), + advantages=types.TensorData(data=[]), + logprobs=types.TensorData(data=[]), + ), + ) + ) + return types.ForwardBackwardInput(data=samples, loss_fn="cross_entropy") + + +def make_sample_input(prompt_tokens: list[int], max_tokens: int, checkpoint_id: str) -> types.SampleInput: + return types.SampleInput( + base_model=None, + prompt=types.ModelInput(chunks=[types.ModelInputChunk(tokens=prompt_tokens)]), + sampling_params=types.SamplingParams(temperature=1.0, max_tokens=max_tokens, seed=42), + num_samples=1, + checkpoint_id=checkpoint_id, + prompt_logprobs=False, + ) + + +def build_engine(config: EngineConfig, num_adapters: int) -> TinkerEngine: + engine = TinkerEngine(config) + for i in range(num_adapters): + model_id = f"adapter_{i}" + engine.process_single_request( + types.RequestType.CREATE_MODEL, + model_id, + {"lora_config": {"rank": config.backend_config["max_lora_rank"], "alpha": 32, "seed": i}}, + ) + # Mark as loaded so sampling uses in-memory weights + engine.backend.models[model_id].loaded_checkpoint_id = model_id + return engine + + +def run_fwd_bwd_bench(engine: TinkerEngine, args: argparse.Namespace): + print("\n=== Forward/Backward Benchmark ===") + + token_lists = [ + [int(x) for x in jax.random.randint(jax.random.PRNGKey(i), (args.seq_len,), 1, 1000)] + for i in range(args.samples_per_request) + ] + fb_input = make_fwd_bwd_input(token_lists) + model_ids = list(engine.backend.models.keys()) + reqs = {str(i): (model_ids[i % len(model_ids)], fb_input) for i in range(args.num_requests)} + + print(f"Warming up ({args.num_warmup_steps} steps)...") + for _ in range(args.num_warmup_steps): + engine.process_forward_backward(reqs) + + print(f"Running benchmark ({args.num_steps} steps)...") + start = time.perf_counter() + for _ in range(args.num_steps): + engine.process_forward_backward(reqs) + elapsed = time.perf_counter() - start + + total_tokens = args.num_steps * args.num_requests * args.samples_per_request * args.seq_len + print("\nResults:") + print(f" steps: {args.num_steps}") + print(f" elapsed: {elapsed:.3f} s") + print(f" tokens/sec: {total_tokens / elapsed:.0f}") + print(f" sec/step: {elapsed / args.num_steps:.2f}") + + +def run_sample_bench(engine: TinkerEngine, args: argparse.Namespace): + print("\n=== Sampling Benchmark ===") + + model_ids = list(engine.backend.models.keys()) + reqs = {} + for i in range(args.num_requests): + prompt_tokens = [int(x) for x in jax.random.randint(jax.random.PRNGKey(i), (args.seq_len,), 1, 1000)] + model_id = model_ids[i % len(model_ids)] + reqs[str(i)] = (model_id, make_sample_input(prompt_tokens, args.sample_max_tokens, checkpoint_id=model_id)) + + print(f"Warming up ({args.num_warmup_steps} steps)...") + for _ in range(args.num_warmup_steps): + engine.process_sample(reqs) + + print(f"Running benchmark ({args.num_steps} steps)...") + start = time.perf_counter() + for _ in range(args.num_steps): + engine.process_sample(reqs) + elapsed = time.perf_counter() - start + + total_tokens = args.num_steps * args.num_requests * args.sample_max_tokens + print("\nResults:") + print(f" steps: {args.num_steps}") + print(f" elapsed: {elapsed:.3f} s") + print(f" tokens generated/sec: {total_tokens / elapsed:.0f}") + print(f" sec/step: {elapsed / args.num_steps:.2f}") + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark forward/backward and sampling for TinkerEngine") + add_model(parser, EngineConfig) + + parser.add_argument("--benchmark", choices=["fwd_bwd", "sample", "all"], default="all", help="Benchmark to run") + parser.add_argument("--num-steps", type=int, default=5, help="Number of benchmark steps") + parser.add_argument("--num-warmup-steps", type=int, default=2, help="Number of warmup steps") + parser.add_argument("--num-requests", type=int, default=256, help="Number of requests per batch") + parser.add_argument("--seq-len", type=int, default=128, help="Sequence length for inputs") + parser.add_argument("--samples-per-request", type=int, default=1, help="Samples per request (fwd_bwd only)") + parser.add_argument("--num-adapters", type=int, default=2, help="Number of LoRA adapters to create") + parser.add_argument("--sample-max-tokens", type=int, default=128, help="Max tokens to generate (sampling only)") + + args = parser.parse_args() + + # Build EngineConfig from parsed args + config_fields = {name: getattr(args, name) for name in EngineConfig.model_fields.keys() if hasattr(args, name)} + config = EngineConfig(**config_fields) + + bench_config = {k: v for k, v in vars(args).items() if k not in EngineConfig.model_fields} + print(f"EngineConfig: {config}") + print(f"BenchmarkConfig: {bench_config}") + print("Building engine...") + + engine = build_engine(config, args.num_adapters) + + if args.benchmark in ("fwd_bwd", "all"): + run_fwd_bwd_bench(engine, args) + + if args.benchmark in ("sample", "all"): + run_sample_bench(engine, args) + + +if __name__ == "__main__": + main() diff --git a/skyrl-tx/docs/contributing.md b/skyrl-tx/docs/contributing.md new file mode 100644 index 0000000000..f5229a13f4 --- /dev/null +++ b/skyrl-tx/docs/contributing.md @@ -0,0 +1,20 @@ +# Contributing + +We happily accept contributions! + +## Development setup + +You can run the unit tests with + +```bash +uv run --extra dev pytest -s tests +``` + +You can build and view the documentation by running the following in the project directory: + +```bash +uv run --extra dev mkdocs serve +``` + +The commands in [the CI workflow](https://github.com/tx-project/tx/tree/main/.github/workflows) +are the best source of useful commands to run during development. \ No newline at end of file diff --git a/skyrl-tx/docs/index.md b/skyrl-tx/docs/index.md new file mode 100644 index 0000000000..d5a07e64b7 --- /dev/null +++ b/skyrl-tx/docs/index.md @@ -0,0 +1,20 @@ +# tx: Transformers x-platform + +tx is a simple but powerful cross-platform training library. + +## Quickstart + +Here is an example of how to fine-tune a model on a HuggingFace dataset: + +```bash +# Download Qwen3-8B checkpoint +uv run --with huggingface_hub hf download Qwen/Qwen3-4B --local-dir /tmp/qwen3 + +# Fine-tune the model on a chat dataset (use --extra tpu if you are running on TPUs) +# This needs to be run in the repository https://github.com/tx-project/tx directory +uv run --extra gpu --with jinja2 tx train --model Qwen/Qwen3-4B --dataset HuggingFaceH4/ultrachat_200k --loader tx.loaders.chat --split train_sft --output-dir /tmp/ultrachat --batch-size 8 --load-checkpoint-path /tmp/qwen3 + +# Try out the model +uv run --with vllm vllm serve /tmp/ultrachat +uv run --with vllm vllm chat +``` diff --git a/skyrl-tx/docs/reference.md b/skyrl-tx/docs/reference.md new file mode 100644 index 0000000000..fd5f0293bd --- /dev/null +++ b/skyrl-tx/docs/reference.md @@ -0,0 +1,58 @@ +# `tx` + +**Usage**: + +```console +$ tx [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--install-completion`: Install completion for the current shell. +* `--show-completion`: Show completion for the current shell, to copy it or customize the installation. +* `--help`: Show this message and exit. + +**Commands**: + +* `train`: Train a model +* `version` + +## `tx train` + +Train a model + +**Usage**: + +```console +$ tx train [OPTIONS] +``` + +**Options**: + +* `--model TEXT`: HuggingFace model ID or local model path [required] +* `--dataset TEXT`: HuggingFace dataset to use for training [required] +* `--loader TEXT`: Loader used for loading the dataset [default: tx.loaders.text] +* `--split TEXT`: The dataset split to use [default: train] +* `--output-dir PATH`: The output directory where the model predictions and checkpoints will be written [required] +* `--load-checkpoint-path PATH`: If specified, resume training from this checkpoint +* `--save-steps INTEGER`: Number of steps between checkpoints [default: 500] +* `--max-steps INTEGER`: The maximum number of training steps +* `--batch-size INTEGER`: Batch size of each training batch [required] +* `--optimizer [adamw]`: Which optax optimizer to use [default: adamw] +* `--optimizer-args LOADS`: Arguments for the optax optimizer (in JSON format) [default: {"learning_rate": 1e-5, "weight_decay": 0.1}] +* `--tp-size INTEGER`: Tensor parallelism degree to use for the model [default: 1] +* `--tracker [wandb]`: Experiment tracker to report results to +* `--tracker-args LOADS`: Arguments that will be passed to the experiment tracker (in JSON format) [default: {}] +* `--help`: Show this message and exit. + +## `tx version` + +**Usage**: + +```console +$ tx version [OPTIONS] +``` + +**Options**: + +* `--help`: Show this message and exit. diff --git a/skyrl-tx/docs/running/gpus.md b/skyrl-tx/docs/running/gpus.md new file mode 100644 index 0000000000..e811b12c7d --- /dev/null +++ b/skyrl-tx/docs/running/gpus.md @@ -0,0 +1,32 @@ +# Running on GPUs + +We assume your are logged into a single GPU node with one or more GPUs. +Multi-node instructions will be added later. + +## Setting up tx + +Install `uv` and clone the `tx` repository with + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +git clone https://github.com/tx-project/tx +cd tx +``` + +## Starting the training + +Next, download the dataset with + +```bash +uv run --with huggingface_hub hf download Qwen/Qwen3-4B --local-dir /tmp/qwen3 +``` + +You can then start the training with + +```bash +uv run --extra gpu --with jinja2 tx train --model Qwen/Qwen3-4B --dataset HuggingFaceH4/ultrachat_200k --loader tx.loaders.chat --split train_sft --output-dir /tmp/ultrachat --batch-size 8 --load-checkpoint-path /tmp/qwen3 --tp-size 8 +``` + +In this example we assume you have 8 GPUs in the node. If you have a different number of GPUs, you can modify the `--tp-size` parameter appropriately. + +See the full set of options of `tx` in [the CLI reference](../reference.md). diff --git a/skyrl-tx/docs/running/tpus.md b/skyrl-tx/docs/running/tpus.md new file mode 100644 index 0000000000..6d06c1fd4d --- /dev/null +++ b/skyrl-tx/docs/running/tpus.md @@ -0,0 +1,51 @@ +# Running on TPUs + +Currently we only have instructions how to run `tx` on a single TPU VM. Multi-node instructions will be added later. + +## Setting up the TPU VM + +First start the TPU VM: + +```bash +gcloud compute tpus tpu-vm create --project= --zone= --accelerator-type=v6e-8 --version=v2-alpha-tpuv6e --scopes=https://www.googleapis.com/auth/cloud-platform.read-only,https://www.googleapis.com/auth/devstorage.read_write --network= --subnetwork= --spot +``` + +After the VM is started, you can ssh into it via + +```bash +gcloud compute tpus tpu-vm ssh +``` + +## Setting up tx + +Once you are logged into the VM, install `uv` and clone the `tx` repository with + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +git clone https://github.com/tx-project/tx +cd tx +``` + +## Starting the training + +Next, download the dataset with + +```bash +uv run --with huggingface_hub hf download Qwen/Qwen3-4B --local-dir /tmp/qwen3 +``` + +You can then start the training with + +```bash +uv run --extra tpu --with jinja2 tx train --model Qwen/Qwen3-4B --dataset HuggingFaceH4/ultrachat_200k --loader tx.loaders.chat --split train_sft --output-dir /tmp/ultrachat --batch-size 8 --load-checkpoint-path /tmp/qwen3 --tp-size 8 +``` + +Note that at the beginning the training is a little slow since the JIT compiler needs to compile kernels for the various shapes. + +See the full set of options of `tx` in [the CLI reference](../reference.md). + +You can visualize TPU usage with + +```bash +uv run --with libtpu --with git+https://github.com/google/cloud-accelerator-diagnostics/#subdirectory=tpu_info tpu-info +``` diff --git a/skyrl-tx/mkdocs.yml b/skyrl-tx/mkdocs.yml new file mode 100644 index 0000000000..88fdd303eb --- /dev/null +++ b/skyrl-tx/mkdocs.yml @@ -0,0 +1,15 @@ +site_name: tx documentation +theme: + name: material + palette: + primary: yellow + +repo_url: https://github.com/tx-project/tx + +nav: + - "Overview": index.md + - "Running": + - running/gpus.md + - running/tpus.md + - "CLI Reference": reference.md + - "Contributing": contributing.md diff --git a/skyrl-tx/pyproject.toml b/skyrl-tx/pyproject.toml new file mode 100644 index 0000000000..8ec58d2437 --- /dev/null +++ b/skyrl-tx/pyproject.toml @@ -0,0 +1,77 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "skyrl-tx" +dynamic = ["version"] +description = "Unified API for training and inference" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "datasets>=4.0.0", + "flax>=0.12.2", + "optax>=0.2.5", + "pillow>=11.3.0", + "rich>=14.1.0", + "safetensors>=0.6.2", + "tokenizers>=0.21.2", + "transformers>=4.56.1", + "typer>=0.17.4", + # "wandb>=0.22.0", + "peft", + "hf_transfer", + "cloudpathlib>=0.23.0", + "jax>=0.8", +] + +[project.optional-dependencies] +gpu = [ + "jax[cuda12]>=0.7.2", +] + +tpu = [ + "jax[tpu]>=0.7.2", +] + +tinker = [ + "tinker>=0.3.0", + "fastapi[standard]", + "sqlmodel", + "sqlalchemy[asyncio]", + "aiosqlite", + "asyncpg", + "psycopg2-binary", +] + +aws = [ + "cloudpathlib[s3]", +] + +gcp = [ + "cloudpathlib[gs]", +] + +azure = [ + "cloudpathlib[azure]", +] + +dev = [ + "mkdocs", + "mkdocs-material", + "pytest", + "pytest-forked", + "torch", + "ty", + "cloudpathlib[s3]", + "alembic", +] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.dynamic] +version = {attr = "tx.__version__"} + +[project.scripts] +tx = "tx.run.main:app" diff --git a/skyrl-tx/tests/README.md b/skyrl-tx/tests/README.md new file mode 100644 index 0000000000..747e748b0e --- /dev/null +++ b/skyrl-tx/tests/README.md @@ -0,0 +1,7 @@ +# SkyRL tx: Tests + +## Run all tests +uv run --extra dev --extra tinker pytest -v + +## Run Specific tests +uv run --extra dev --extra tinker pytest -v -s tests/models/test_qwen3_generate.py::test_qwen3_generate_speed diff --git a/skyrl-tx/tests/__init__.py b/skyrl-tx/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-tx/tests/conftest.py b/skyrl-tx/tests/conftest.py new file mode 100644 index 0000000000..0e6763a158 --- /dev/null +++ b/skyrl-tx/tests/conftest.py @@ -0,0 +1,9 @@ +import jax +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def configure_jax_cpu_devices(): + """Configure JAX to use 2 CPU devices for testing parallelism.""" + if not jax._src.xla_bridge.backends_are_initialized(): + jax.config.update("jax_num_cpu_devices", 2) diff --git a/skyrl-tx/tests/layers/test_ragged_dot.py b/skyrl-tx/tests/layers/test_ragged_dot.py new file mode 100644 index 0000000000..23f346028b --- /dev/null +++ b/skyrl-tx/tests/layers/test_ragged_dot.py @@ -0,0 +1,32 @@ +import jax +import pytest +import jax.numpy as jnp +from tx.layers.util import ragged_dot + + +@pytest.mark.parametrize( + "group_sizes,group_offset,g_local,expected_scale", + [ + ([2, 2, 2], 1, 2, [0, 0, 1, 1, 2, 2]), # middle shard + ([2, 2, 2], 0, 2, [1, 1, 2, 2, 0, 0]), # first shard + ([2, 2, 2], 2, 1, [0, 0, 0, 0, 1, 1]), # last shard + ([6], 0, 1, [1, 1, 1, 1, 1, 1]), # single group + ([2, 0, 0, 4], 1, 2, [0, 0, 0, 0, 0, 0]), # empty groups in shard + ([1, 3, 2], 1, 2, [0, 1, 1, 1, 2, 2]), # uneven sizes + ], +) +def test_ragged_dot_with_group_offset(group_sizes, group_offset, g_local, expected_scale): + """Test ragged_dot with group_offset for various edge cases.""" + group_sizes = jnp.array(group_sizes) + m, d = 6, 2 + + lhs = jnp.arange(m * d, dtype=jnp.float32).reshape(m, d) + rhs = jnp.stack([(i + 1) * jnp.eye(d) for i in range(g_local)]) # 1*I, 2*I, ... + + result = jax.jit(ragged_dot)(lhs, rhs, group_sizes, group_offset=jnp.array([group_offset])) + + # expected_scale: 0 for masked tokens, else local_group_idx + 1 + scale = jnp.array(expected_scale, dtype=jnp.float32)[:, None] + expected = lhs * scale + + assert jnp.allclose(result, expected), f"Got:\n{result}\nExpected:\n{expected}" diff --git a/skyrl-tx/tests/models/test_llama3.py b/skyrl-tx/tests/models/test_llama3.py new file mode 100644 index 0000000000..7666d54287 --- /dev/null +++ b/skyrl-tx/tests/models/test_llama3.py @@ -0,0 +1,53 @@ +import os +import tempfile + +from flax import nnx +import jax +import jax.numpy as jnp +import numpy as np +import pytest +import torch +from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer + +from tx.models.llama3 import Llama3ForCausalLM +from tx.models.configs import Llama3Config +from tx.utils.models import load_safetensors + + +@pytest.mark.parametrize("tp", [1, 2]) +def test_llama3(tp: int): + """Test LLama3 model against HuggingFace reference implementation.""" + if os.getenv("CI"): + pytest.skip("Test currently runs out of memory in the CI") + + # Use a small LLama model for testing + model_name = "unsloth/Llama-3.2-1B" + tokenizer = AutoTokenizer.from_pretrained(model_name) + hf_model = AutoModelForCausalLM.from_pretrained(model_name, attn_implementation="eager", use_safetensors=True) + + inputs = ["The capital of France is", "The most popular programming language is"] + batch = tokenizer(inputs, return_tensors="pt", padding=True) + + with torch.no_grad(): + hf_outputs = hf_model( + batch.input_ids, attention_mask=batch.attention_mask, output_hidden_states=True, return_dict=True + ) + + # Save the HF model checkpoint so we can load our model from it + with tempfile.TemporaryDirectory() as tmp: + hf_model.save_pretrained(tmp, safe_serialization=True) + + base_config = AutoConfig.from_pretrained(model_name) + config = Llama3Config(base_config, max_lora_adapters=1, max_lora_rank=1, shard_attention_heads=True) + mesh = jax.make_mesh((1, tp), ("dp", "tp")) + with jax.set_mesh(mesh): + model = Llama3ForCausalLM(config, dtype=jnp.float32, rngs=nnx.Rngs(0)) + load_safetensors(tmp, config, model) + + outputs = model(batch.input_ids.numpy(), attention_mask=batch.attention_mask.numpy(), output_hidden_states=True) + + assert outputs.hidden_states is not None + assert np.allclose(hf_outputs.hidden_states[0], outputs.hidden_states[0], rtol=1e-6) + assert np.allclose(hf_outputs.hidden_states[1], outputs.hidden_states[1], rtol=1e-3, atol=1e-3) + # Higher tolerance for final layer due to accumulated numerical differences between PyTorch and JAX + assert np.allclose(hf_outputs.hidden_states[-1], outputs.hidden_states[-1], rtol=5e-2, atol=5e-2) diff --git a/skyrl-tx/tests/models/test_llama3_lora_training.py b/skyrl-tx/tests/models/test_llama3_lora_training.py new file mode 100644 index 0000000000..012878af2e --- /dev/null +++ b/skyrl-tx/tests/models/test_llama3_lora_training.py @@ -0,0 +1,100 @@ +from flax import nnx +import jax +import jax.numpy as jnp +import optax +from huggingface_hub import snapshot_download +from transformers import PretrainedConfig + +from tx.models.configs import Llama3Config +from tx.models.llama3 import Llama3ForCausalLM +from tx.utils.models import get_dtype, load_safetensors +from tx.layers.lora import init_lora_adapter +from tx.tinker.types import LoraConfig + + +def test_lora_training(): + base_model = "unsloth/Llama-3.2-1B" + base_config = PretrainedConfig.from_pretrained(base_model) + config = Llama3Config(base_config, max_lora_adapters=5, max_lora_rank=32, shard_attention_heads=True) + + checkpoint_path = snapshot_download(base_model, allow_patterns=["*.safetensors"]) + mesh = jax.make_mesh((1, 1), ("dp", "tp")) + with jax.set_mesh(mesh): + model = Llama3ForCausalLM(config, dtype=get_dtype(config.dtype), rngs=nnx.Rngs(0)) + load_safetensors(checkpoint_path, config, model) + + # Set different ranks for each adapter (0: rank 16, 1: rank 8) + init_lora_adapter(model, adapter_index=0, lora_config=LoraConfig(rank=16, alpha=16, seed=0)) + init_lora_adapter(model, adapter_index=1, lora_config=LoraConfig(rank=8, alpha=8, seed=1)) + + # Create optimizer that only targets LoRA A and B parameters + optimizer = nnx.Optimizer(model, optax.adamw(1e-4), wrt=model.is_lora_param) + + batch = jnp.array([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [11, 12, 13, 14, 15, 16, 17, 18, 19, 20]], dtype=jnp.int32) + target_ids = batch[:, 1:] + input_ids = batch[:, :-1] + adapter_indices = jnp.array([0, 1], dtype=jnp.int32) + attention_mask = jnp.ones_like(input_ids) + + def loss_fn(model, input_ids, target_ids, attention_mask): + outputs = model(input_ids, attention_mask=attention_mask, adapter_indices=adapter_indices) + logits = outputs.logits + return optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=target_ids).mean() + + # Compute gradients - we need to use nnx.split to separate parameters + # that we want to compute gradients for + graphdef, lora_params, non_lora_params = nnx.split(model, model.is_lora_param, ...) + + # Helper to extract adapter params at specific index + def get_adapter_params(params, adapter_idx): + return jax.tree.map(lambda p: p[adapter_idx].copy(), params) + + # Helper to extract out-of-rank params for an adapter + def get_out_of_rank_params(params, adapter_idx, rank): + def slice_param(path, p): + if "lora_A" in str(path): + return p[adapter_idx, :, rank:].copy() + elif "lora_B" in str(path): + return p[adapter_idx, rank:, :].copy() + return p + + return jax.tree.map_with_path(slice_param, params) + + # Save initial states + initial_adapter_2_params = get_adapter_params(lora_params, 2) + initial_adapter_0_out_of_rank = get_out_of_rank_params(lora_params, 0, 16) + initial_adapter_1_out_of_rank = get_out_of_rank_params(lora_params, 1, 8) + + # Training loop + for step in range(10): + + def loss_for_lora(lora_params): + merged_model = nnx.merge(graphdef, lora_params, non_lora_params) + return loss_fn(merged_model, input_ids, target_ids, attention_mask) + + loss_and_grad_fn = nnx.value_and_grad(loss_for_lora) + loss, lora_grads = loss_and_grad_fn(lora_params) + + optimizer.update(lora_params, lora_grads) + + print(f"Step {step}: loss = {float(loss):.4f}") + + def verify_params_unchanged(initial_params, final_params, error_msg_prefix): + for (path, initial), (_, final) in zip( + jax.tree.leaves_with_path(initial_params), jax.tree.leaves_with_path(final_params) + ): + assert jnp.allclose(initial, final), f"{error_msg_prefix} for {path}" + + # Verify adapter 2 (unused) was not modified + final_adapter_2_params = get_adapter_params(lora_params, 2) + verify_params_unchanged(initial_adapter_2_params, final_adapter_2_params, "Adapter 2 was modified") + + # Verify out-of-rank params were not modified + final_adapter_0_out_of_rank = get_out_of_rank_params(lora_params, 0, 16) + verify_params_unchanged( + initial_adapter_0_out_of_rank, final_adapter_0_out_of_rank, "Adapter 0 out-of-rank params modified" + ) + final_adapter_1_out_of_rank = get_out_of_rank_params(lora_params, 1, 8) + verify_params_unchanged( + initial_adapter_1_out_of_rank, final_adapter_1_out_of_rank, "Adapter 1 out-of-rank params modified" + ) diff --git a/skyrl-tx/tests/models/test_qwen3.py b/skyrl-tx/tests/models/test_qwen3.py new file mode 100644 index 0000000000..c450efbf8c --- /dev/null +++ b/skyrl-tx/tests/models/test_qwen3.py @@ -0,0 +1,277 @@ +import os +import tempfile + +from flax import nnx +import jax +import jax.numpy as jnp +import numpy as np +from peft import LoraConfig, get_peft_model +import pytest +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, PretrainedConfig +from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock as HFQwen3MoeSparseMoeBlock + +from tx.layers.lora import LoRAMixin +from tx.models.configs import Qwen3Config +from tx.models.qwen3 import Qwen3ForCausalLM, Qwen3MoeSparseMoeBlock +from tx.utils.models import load_safetensors + + +@pytest.mark.parametrize("tp", [1, 2]) +def test_qwen3(tp: int): + if tp > 1 and os.getenv("CI"): + pytest.skip("TP > 1 currently runs out of memory in the CI") + + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + hf_model = AutoModelForCausalLM.from_pretrained( + "Qwen/Qwen3-0.6B", attn_implementation="eager", use_safetensors=True + ) + + inputs = ["The capital of France is", "The most popular programming language is"] + batch = tokenizer(inputs, return_tensors="pt", padding=True) + with torch.no_grad(): + hf_outputs = hf_model( + batch.input_ids, attention_mask=batch.attention_mask, output_hidden_states=True, return_dict=True + ) + + # Save the HF model checkpoint so we can load our model from it + with tempfile.TemporaryDirectory() as tmp: + hf_model.save_pretrained(tmp, safe_serialization=True) + + base_config = PretrainedConfig.from_pretrained("Qwen/Qwen3-0.6B") + config = Qwen3Config(base_config, max_lora_adapters=32, max_lora_rank=32, shard_attention_heads=True) + mesh = jax.make_mesh((1, tp), ("fsdp", "tp")) + with jax.set_mesh(mesh): + model = Qwen3ForCausalLM(config, dtype=jnp.float32, rngs=nnx.Rngs(0)) + load_safetensors(tmp, config, model) + + outputs = model(batch.input_ids.numpy(), attention_mask=batch.attention_mask.numpy(), output_hidden_states=True) + assert outputs.hidden_states is not None + assert np.allclose(hf_outputs.hidden_states[0], outputs.hidden_states[0], rtol=1e-6) + assert np.allclose(hf_outputs.hidden_states[1], outputs.hidden_states[1], rtol=1e-3, atol=1e-3) + assert np.allclose(hf_outputs.hidden_states[-1], outputs.hidden_states[-1], rtol=1e-3, atol=1e-3) + + +def load_moe_base_weights(jax_moe_layer: Qwen3MoeSparseMoeBlock, hf_moe_layer: HFQwen3MoeSparseMoeBlock) -> None: + """Load base weights from HF MoE layer to JAX MoE layer.""" + jax_moe_layer.gate.kernel[:] = hf_moe_layer.gate.weight.detach().numpy().T + for i, expert in enumerate(hf_moe_layer.experts): + jax_moe_layer.experts.gate_proj.weight[i, :, :] = expert.gate_proj.weight.detach().numpy().T + jax_moe_layer.experts.up_proj.weight[i, :, :] = expert.up_proj.weight.detach().numpy().T + jax_moe_layer.experts.down_proj.weight[i, :, :] = expert.down_proj.weight.detach().numpy().T + + +@pytest.mark.parametrize("ep,tp", [(1, 1), (1, 2), (2, 1)]) +def test_qwen3_moe_layer(ep: int, tp: int): + model_name = "trl-internal-testing/tiny-Qwen3MoeForCausalLM" + hf_model = AutoModelForCausalLM.from_pretrained(model_name, attn_implementation="eager", use_safetensors=True) + base_config = PretrainedConfig.from_pretrained(model_name) + config = Qwen3Config(base_config, max_lora_adapters=0, max_lora_rank=0, shard_attention_heads=True) + + hf_moe_layer = hf_model.model.layers[0].mlp + x = torch.randn(4, 2, config.hidden_size) + with torch.no_grad(): + hf_final_hidden_states, hf_router_logits = hf_moe_layer.forward(x) + + mesh = jax.make_mesh((1, ep, tp), ("fsdp", "ep", "tp")) + with jax.set_mesh(mesh): + moe_layer = Qwen3MoeSparseMoeBlock(config, dtype=jnp.float32, rngs=nnx.Rngs(0)) + load_moe_base_weights(moe_layer, hf_moe_layer) + + final_hidden_states, router_logits = moe_layer(x.numpy(), return_router_logits=True) + + assert np.allclose(hf_router_logits, router_logits, rtol=1e-4) + assert np.allclose(hf_final_hidden_states, final_hidden_states, rtol=1e-2, atol=1e-2) + + +def load_lora_weights( + jax_module: LoRAMixin, + adapter_idx: int, + lora_A_weights: np.ndarray, + lora_B_weights: np.ndarray, + scaling: float, + rank: int, +) -> None: + """Load LoRA weights from numpy arrays to JAX module.""" + assert ( + jax_module.lora_A is not None + and jax_module.lora_B is not None + and jax_module.lora_scaling is not None + and jax_module.lora_ranks is not None + ) + jax_module.lora_A.value = jax_module.lora_A.value.at[adapter_idx].set(jnp.array(lora_A_weights)) + jax_module.lora_B.value = jax_module.lora_B.value.at[adapter_idx].set(jnp.array(lora_B_weights)) + jax_module.lora_scaling.value = jax_module.lora_scaling.value.at[adapter_idx].set(scaling) + jax_module.lora_ranks.value = jax_module.lora_ranks.value.at[adapter_idx].set(rank) + + +@pytest.mark.parametrize("ep,tp", [(1, 1), (1, 2), (2, 1)]) +def test_qwen3_moe_layer_lora(ep: int, tp: int): + """Test MoE LoRA by merging adapter into base weights and comparing outputs.""" + model_name = "trl-internal-testing/tiny-Qwen3MoeForCausalLM" + hf_model = AutoModelForCausalLM.from_pretrained(model_name, attn_implementation="eager", use_safetensors=True) + base_config = PretrainedConfig.from_pretrained(model_name) + config = Qwen3Config(base_config, max_lora_adapters=3, max_lora_rank=4, shard_attention_heads=True) + + hf_moe_layer = hf_model.model.layers[0].mlp + x = torch.randn(3, 4, config.hidden_size) + + mesh = jax.make_mesh((1, ep, tp), ("fsdp", "ep", "tp")) + with jax.set_mesh(mesh): + moe_layer = Qwen3MoeSparseMoeBlock(config, dtype=jnp.float32, rngs=nnx.Rngs(0)) + load_moe_base_weights(moe_layer, hf_moe_layer) + + # Set LoRA weights for all adapters + rng = np.random.default_rng(42) + scaling = 2.0 + rank = config.max_lora_rank + for adapter_idx in range(config.max_lora_adapters): + for proj in [moe_layer.experts.gate_proj, moe_layer.experts.up_proj, moe_layer.experts.down_proj]: + assert proj.lora_A is not None and proj.lora_B is not None + lora_A = rng.normal(0, 1.0, proj.lora_A.value.shape[1:]) + lora_B = rng.normal(0, 1.0, proj.lora_B.value.shape[1:]) + load_lora_weights(proj, adapter_idx, lora_A, lora_B, scaling, rank) + + # Test with different adapters per sample + adapter_indices = jnp.array([0, 2, 1]) + output_with_lora, _ = moe_layer(x.numpy(), adapter_indices=adapter_indices, return_router_logits=True) + + # Test each sample by comparing with merged weights for its adapter + for sample_idx in range(len(adapter_indices)): + adapter_idx = int(adapter_indices[sample_idx]) + + # Create merged model by adding LoRA weights to base weights + moe_layer_merged = Qwen3MoeSparseMoeBlock(config, dtype=jnp.float32, rngs=nnx.Rngs(1 + adapter_idx)) + moe_layer_merged.gate.kernel[:] = moe_layer.gate.kernel[:] + + for proj_name in ["gate_proj", "up_proj", "down_proj"]: + proj = getattr(moe_layer.experts, proj_name) + proj_merged = getattr(moe_layer_merged.experts, proj_name) + + # For each expert, merge: base + scaling * (lora_A @ lora_B) + for expert_idx in range(config.num_experts): + lora_A = proj.lora_A.value[adapter_idx, expert_idx, :, :] + lora_B = proj.lora_B.value[adapter_idx, expert_idx, :, :] + lora_delta = scaling * (lora_A @ lora_B) + + merged_weight = proj.weight[expert_idx, :, :] + lora_delta + proj_merged.weight.value = proj_merged.weight.value.at[expert_idx, :, :].set(merged_weight) + + # Run merged model on this sample + x_sample = x[sample_idx : sample_idx + 1].numpy() + output_merged, _ = moe_layer_merged(x_sample, return_router_logits=True) + + assert np.allclose(output_with_lora[sample_idx : sample_idx + 1], output_merged, rtol=1e-3, atol=1e-3) + + +def test_qwen3_lora(): + """Test multi-LoRA implementation by comparing with HuggingFace PEFT model using two different adapters.""" + base_model_name = "Qwen/Qwen3-0.6B" + lora_adapters = ["pcmoritz/qwen3-0.6b-lora-random", "pcmoritz/qwen3-0.6b-lora-random2"] + + tokenizer = AutoTokenizer.from_pretrained(base_model_name) + # Use two different inputs to test with different adapters + inputs = ["The capital of France is", "My name is"] + batch = tokenizer(inputs, return_tensors="pt", padding=True) + + with tempfile.TemporaryDirectory() as base_tmp: + base_hf_model = AutoModelForCausalLM.from_pretrained( + base_model_name, attn_implementation="eager", use_safetensors=True + ) + base_hf_model.save_pretrained(base_tmp, safe_serialization=True) + + # Create HF models with different adapters + hf_lora_models = [] + lora_configs = [] + for adapter_name in lora_adapters: + lora_config = LoraConfig.from_pretrained(adapter_name) + lora_config.target_modules = [ + "embed_tokens", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", + ] + lora_configs.append(lora_config) + + hf_model = get_peft_model( + AutoModelForCausalLM.from_pretrained( + base_model_name, attn_implementation="eager", use_safetensors=True + ), + lora_config, + ) + hf_model.eval() + hf_model.load_adapter(adapter_name, adapter_name="default") + hf_lora_models.append(hf_model) + + base_config = PretrainedConfig.from_pretrained(base_model_name) + config = Qwen3Config( + base_config, + max_lora_adapters=len(lora_adapters), + max_lora_rank=max(cfg.r for cfg in lora_configs), + shard_attention_heads=True, + ) + + mesh = jax.make_mesh((1, 1), ("fsdp", "tp")) + with jax.set_mesh(mesh): + model = Qwen3ForCausalLM(config, dtype=jnp.float32, rngs=nnx.Rngs(0)) + load_safetensors(base_tmp, config, model) + + # Get outputs from all HF models + hf_outputs_list = [] + with torch.no_grad(): + for idx in range(len(lora_adapters)): + hf_output = hf_lora_models[idx]( + batch.input_ids[idx : idx + 1], + attention_mask=batch.attention_mask[idx : idx + 1], + output_hidden_states=True, + return_dict=True, + ) + hf_outputs_list.append(hf_output) + + # Load LoRA adapter weights from all adapters + for adapter_idx, (hf_model, lora_config) in enumerate(zip(hf_lora_models, lora_configs)): + # Load embed_tokens LoRA weights + hf_embed_tokens = hf_model.base_model.model.model.embed_tokens + load_lora_weights( + model.model.embed_tokens, + adapter_idx=adapter_idx, + lora_A_weights=hf_embed_tokens.lora_embedding_A["default"].detach().numpy().T, + lora_B_weights=hf_embed_tokens.lora_embedding_B["default"].detach().numpy().T, + scaling=lora_config.lora_alpha / lora_config.r, + rank=lora_config.r, + ) + + # Load layer LoRA weights + for i, layer in enumerate(model.model.layers): + hf_layer = hf_model.base_model.model.model.layers[i] + for module, projections in [ + ("mlp", ["gate_proj", "up_proj", "down_proj"]), + ("self_attn", ["q_proj", "k_proj", "v_proj", "o_proj"]), + ]: + for proj_name in projections: + hf_proj = getattr(getattr(hf_layer, module), proj_name) + load_lora_weights( + getattr(getattr(layer, module), proj_name), + adapter_idx=adapter_idx, + lora_A_weights=hf_proj.lora_A["default"].weight.detach().numpy().T, + lora_B_weights=hf_proj.lora_B["default"].weight.detach().numpy().T, + scaling=lora_config.lora_alpha / lora_config.r, + rank=lora_config.r, + ) + + # Use different adapter indices for each input + adapter_indices = jnp.arange(len(lora_adapters), dtype=jnp.int32) + outputs = model( + batch.input_ids.numpy(), + attention_mask=batch.attention_mask.numpy(), + output_hidden_states=True, + adapter_indices=adapter_indices, + ) + + # Compare outputs with corresponding adapters + for idx in range(len(lora_adapters)): + assert np.allclose(hf_outputs_list[idx].logits[0], outputs.logits[idx], rtol=1e-3, atol=1e-3) diff --git a/skyrl-tx/tests/models/test_qwen3_config.py b/skyrl-tx/tests/models/test_qwen3_config.py new file mode 100644 index 0000000000..fc2783b994 --- /dev/null +++ b/skyrl-tx/tests/models/test_qwen3_config.py @@ -0,0 +1,29 @@ +"""Tests for Qwen3Config.""" + +from tx.models.configs import Qwen3Config +from transformers import PretrainedConfig + + +def test_config_wraps_pretrained_config(): + """Test that Qwen3Config wraps a PretrainedConfig and adds LoRA params.""" + hf_config = PretrainedConfig.from_pretrained("Qwen/Qwen3-0.6B") + config = Qwen3Config(hf_config, max_lora_adapters=8, max_lora_rank=16, shard_attention_heads=False) + + # Check LoRA params were set + assert config.max_lora_adapters == 8 + assert config.max_lora_rank == 16 + assert config.shard_attention_heads is False + + # Check base config attributes were copied + assert config.vocab_size > 0 + assert config.hidden_size > 0 + assert config.num_hidden_layers > 0 + + +def test_config_preserves_moe_config(): + """Test that MoE-specific configs are preserved.""" + hf_config = PretrainedConfig.from_pretrained("trl-internal-testing/tiny-Qwen3MoeForCausalLM") + config = Qwen3Config(hf_config, max_lora_adapters=3, max_lora_rank=4, shard_attention_heads=True) + + # Check that MoE-specific attributes are preserved + assert config.num_experts > 0 diff --git a/skyrl-tx/tests/models/test_qwen3_generate.py b/skyrl-tx/tests/models/test_qwen3_generate.py new file mode 100644 index 0000000000..378a37162e --- /dev/null +++ b/skyrl-tx/tests/models/test_qwen3_generate.py @@ -0,0 +1,161 @@ +import os +import tempfile +import time + +from flax import nnx +import jax +import jax.numpy as jnp +import numpy as np +import pytest +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, PretrainedConfig + +from tx.models.configs import Qwen3Config +from tx.models.qwen3 import Qwen3ForCausalLM +from tx.tinker import types +from tx.utils.models import load_safetensors + + +def test_qwen3_generate(): + """Test batched text generation with KV caching matches HuggingFace.""" + model_name = "Qwen/Qwen3-0.6B" + tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + hf_model = AutoModelForCausalLM.from_pretrained(model_name, attn_implementation="eager", use_safetensors=True) + + inputs = ["My name is", "The capital of France is", "Test stopping", "Test stopping"] + batch = tokenizer(inputs, return_tensors="pt", padding=True) + + # Generate with HuggingFace (reference) + with torch.no_grad(): + hf_output = hf_model.generate( + batch.input_ids, + attention_mask=batch.attention_mask, + max_new_tokens=20, + do_sample=False, + return_dict_in_generate=True, + output_scores=True, + ) + + # Generate with our implementation + with tempfile.TemporaryDirectory() as tmp: + hf_model.save_pretrained(tmp, safe_serialization=True) + base_config = PretrainedConfig.from_pretrained(model_name) + config = Qwen3Config(base_config, max_lora_adapters=32, max_lora_rank=32, shard_attention_heads=True) + + mesh = jax.make_mesh((1, 1), ("fsdp", "tp")) + with jax.set_mesh(mesh): + model = Qwen3ForCausalLM(config, dtype=jnp.float32, rngs=nnx.Rngs(0)) + load_safetensors(tmp, config, model) + + sampling_params = [ + types.SamplingParams(max_tokens=10, temperature=0.0, seed=42), + types.SamplingParams(max_tokens=20, temperature=0.0, seed=42), + types.SamplingParams(max_tokens=50, temperature=0.0, seed=42, stop_tokens=[6149]), + # Stop token at position 3, but max_tokens=2 should cap output first + types.SamplingParams(max_tokens=2, temperature=0.0, seed=42, stop_tokens=[6149]), + ] + result = model.generate( + batch.input_ids.numpy(), + batch.attention_mask.numpy(), + sampling_params=sampling_params, + ) + + # Compare generated tokens + for i, (our_tokens, hf_tokens, sampling_param) in enumerate( + zip(result.generated_ids, hf_output.sequences, sampling_params) + ): + prompt_length = batch.input_ids.shape[1] + hf_tokens_truncated = hf_tokens[prompt_length : prompt_length + sampling_param.max_tokens].tolist() + + if sampling_param.stop_tokens and result.stop_reasons[i] == "stop": + assert our_tokens[-1] in sampling_param.stop_tokens + # We need to truncate it manually here since if we use the `eos_token_id` + # in huggingface generate, it will pad the sequence with padding tokens + hf_tokens_truncated = hf_tokens_truncated[: len(our_tokens)] + + assert our_tokens == hf_tokens_truncated, ( + f"Generated tokens for request {i} don't match HuggingFace. " + f"Ours: {our_tokens}, HF: {hf_tokens_truncated}" + ) + + # Verify request 2: stop token should be hit + assert result.stop_reasons[2] == "stop" + # Verify request 3: max_tokens=2 should cap output before stop token at position 3 + assert len(result.generated_ids[3]) == 2 + assert result.stop_reasons[3] == "length" + + # Compare logprobs for sampled tokens + for i, (our_tokens, our_logprobs) in enumerate(zip(result.generated_ids, result.logprobs)): + # Compute expected logprobs from HF scores + for step_idx, (token_id, our_logprob) in enumerate(zip(our_tokens, our_logprobs)): + hf_logits = hf_output.scores[step_idx][i] + hf_logprobs = torch.nn.functional.log_softmax(hf_logits, dim=-1) + expected_logprob = float(hf_logprobs[token_id]) + + assert np.isclose(our_logprob, expected_logprob, rtol=1e-3, atol=1e-3), ( + f"Request {i}, step {step_idx}: Logprob mismatch. " + f"Ours: {our_logprob}, HF: {expected_logprob}, diff: {abs(our_logprob - expected_logprob)}" + ) + + +@pytest.mark.skipif(os.environ.get("CI") is not None, reason="Skip speed test in CI due to memory limits") +def test_qwen3_generate_speed(): + """Profile batched text generation with KV caching.""" + model_name = "Qwen/Qwen3-0.6B" + tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + hf_model = AutoModelForCausalLM.from_pretrained(model_name, attn_implementation="eager", use_safetensors=True) + base_config = PretrainedConfig.from_pretrained(model_name) + config = Qwen3Config(base_config, max_lora_adapters=32, max_lora_rank=32, shard_attention_heads=True) + + inputs = [ + "Why do humans need sleep and what happens when we dream", + "Explain the meaning of life and consciousness", + "Describe the process of photosynthesis in plants", + "How do airplanes fly through the air efficiently", + "What are black holes and how are they formed", + "Tell me about the solar system and its planets", + "Explain the difference between AI and machine learning", + "How does the human brain process language", + "What is quantum computing and how does it work", + ] + + batch = tokenizer(inputs, return_tensors="pt", padding=True) + + with tempfile.TemporaryDirectory() as tmp: + hf_model.save_pretrained(tmp, safe_serialization=True) + mesh = jax.make_mesh((1, 1), ("fsdp", "tp")) + with jax.set_mesh(mesh): + model = Qwen3ForCausalLM(config, dtype=jnp.bfloat16, rngs=nnx.Rngs(0)) + load_safetensors(tmp, config, model) + sampling_params = [types.SamplingParams(max_tokens=50, temperature=0.0, seed=42) for i in range(len(inputs))] + + # Warmup + model.generate( + batch.input_ids.numpy(), + batch.attention_mask.numpy(), + sampling_params=sampling_params, + ) + + runs = 1 + times = [] + + for i in range(runs): + start = time.perf_counter() + result = model.generate( + batch.input_ids.numpy(), + batch.attention_mask.numpy(), + sampling_params=sampling_params, + ) + elapsed = time.perf_counter() - start + times.append(elapsed) + + times = np.array(times) + mean_time = times.mean() + std_time = times.std() + + total_new_tokens = len(result.generated_ids) * 50 + + print(f"Generation stats (50 tokens, {runs} runs):") + print(f"Mean time: {mean_time*1000:.2f} ± {std_time*1000:.2f} ms") + print(f"Min/Max: {times.min()*1000:.2f} / {times.max()*1000:.2f} ms") + print(f"New tokens/sec: {total_new_tokens / mean_time:.2f}") diff --git a/skyrl-tx/tests/models/test_qwen3_lora_training.py b/skyrl-tx/tests/models/test_qwen3_lora_training.py new file mode 100644 index 0000000000..88d41f433d --- /dev/null +++ b/skyrl-tx/tests/models/test_qwen3_lora_training.py @@ -0,0 +1,100 @@ +from flax import nnx +import jax +import jax.numpy as jnp +import optax +from huggingface_hub import snapshot_download +from transformers import PretrainedConfig + +from tx.models.configs import Qwen3Config +from tx.models.qwen3 import Qwen3ForCausalLM +from tx.utils.models import get_dtype, load_safetensors +from tx.layers.lora import init_lora_adapter +from tx.tinker.types import LoraConfig + + +def test_lora_training(): + base_model = "Qwen/Qwen3-0.6B" + base_config = PretrainedConfig.from_pretrained(base_model) + config = Qwen3Config(base_config, max_lora_adapters=5, max_lora_rank=32, shard_attention_heads=True) + + checkpoint_path = snapshot_download(base_model, allow_patterns=["*.safetensors"]) + mesh = jax.make_mesh((1, 1), ("fsdp", "tp")) + with jax.set_mesh(mesh): + model = Qwen3ForCausalLM(config, dtype=get_dtype(config.dtype), rngs=nnx.Rngs(0)) + load_safetensors(checkpoint_path, config, model) + + # Set different ranks for each adapter (0: rank 16, 1: rank 8) + init_lora_adapter(model, adapter_index=0, lora_config=LoraConfig(rank=16, alpha=16, seed=0)) + init_lora_adapter(model, adapter_index=1, lora_config=LoraConfig(rank=8, alpha=8, seed=1)) + + # Create optimizer that only targets LoRA A and B parameters + optimizer = nnx.Optimizer(model, optax.adamw(1e-4), wrt=model.is_lora_param) + + batch = jnp.array([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [11, 12, 13, 14, 15, 16, 17, 18, 19, 20]], dtype=jnp.int32) + target_ids = batch[:, 1:] + input_ids = batch[:, :-1] + adapter_indices = jnp.array([0, 1], dtype=jnp.int32) + attention_mask = jnp.ones_like(input_ids) + + def loss_fn(model, input_ids, target_ids, attention_mask): + outputs = model(input_ids, attention_mask=attention_mask, adapter_indices=adapter_indices) + logits = outputs.logits + return optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=target_ids).mean() + + # Compute gradients - we need to use nnx.split to separate parameters + # that we want to compute gradients for + graphdef, lora_params, non_lora_params = nnx.split(model, model.is_lora_param, ...) + + # Helper to extract adapter params at specific index + def get_adapter_params(params, adapter_idx): + return jax.tree.map(lambda p: p[adapter_idx].copy(), params) + + # Helper to extract out-of-rank params for an adapter + def get_out_of_rank_params(params, adapter_idx, rank): + def slice_param(path, p): + if "lora_A" in str(path): + return p[adapter_idx, :, rank:].copy() + elif "lora_B" in str(path): + return p[adapter_idx, rank:, :].copy() + return p + + return jax.tree.map_with_path(slice_param, params) + + # Save initial states + initial_adapter_2_params = get_adapter_params(lora_params, 2) + initial_adapter_0_out_of_rank = get_out_of_rank_params(lora_params, 0, 16) + initial_adapter_1_out_of_rank = get_out_of_rank_params(lora_params, 1, 8) + + # Training loop + for step in range(10): + + def loss_for_lora(lora_params): + merged_model = nnx.merge(graphdef, lora_params, non_lora_params) + return loss_fn(merged_model, input_ids, target_ids, attention_mask) + + loss_and_grad_fn = nnx.value_and_grad(loss_for_lora) + loss, lora_grads = loss_and_grad_fn(lora_params) + + optimizer.update(lora_params, lora_grads) + + print(f"Step {step}: loss = {float(loss):.4f}") + + def verify_params_unchanged(initial_params, final_params, error_msg_prefix): + for (path, initial), (_, final) in zip( + jax.tree.leaves_with_path(initial_params), jax.tree.leaves_with_path(final_params) + ): + assert jnp.allclose(initial, final), f"{error_msg_prefix} for {path}" + + # Verify adapter 2 (unused) was not modified + final_adapter_2_params = get_adapter_params(lora_params, 2) + verify_params_unchanged(initial_adapter_2_params, final_adapter_2_params, "Adapter 2 was modified") + + # Verify out-of-rank params were not modified + final_adapter_0_out_of_rank = get_out_of_rank_params(lora_params, 0, 16) + verify_params_unchanged( + initial_adapter_0_out_of_rank, final_adapter_0_out_of_rank, "Adapter 0 out-of-rank params modified" + ) + final_adapter_1_out_of_rank = get_out_of_rank_params(lora_params, 1, 8) + verify_params_unchanged( + initial_adapter_1_out_of_rank, final_adapter_1_out_of_rank, "Adapter 1 out-of-rank params modified" + ) diff --git a/skyrl-tx/tests/tinker/__init__.py b/skyrl-tx/tests/tinker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-tx/tests/tinker/conftest.py b/skyrl-tx/tests/tinker/conftest.py new file mode 100644 index 0000000000..398c6deb56 --- /dev/null +++ b/skyrl-tx/tests/tinker/conftest.py @@ -0,0 +1,18 @@ +"""Shared test utilities and fixtures for tinker tests.""" + +import time +from typing import Callable + + +def wait_for_condition( + condition_fn: Callable[[], bool], + timeout_sec: float = 10, + poll_interval_sec: float = 0.1, +) -> bool: + """Poll until condition_fn returns True or timeout is reached. Returns True if condition was met.""" + start_time = time.time() + while time.time() - start_time < timeout_sec: + if condition_fn(): + return True + time.sleep(poll_interval_sec) + return False diff --git a/skyrl-tx/tests/tinker/test_api.py b/skyrl-tx/tests/tinker/test_api.py new file mode 100644 index 0000000000..b619206323 --- /dev/null +++ b/skyrl-tx/tests/tinker/test_api.py @@ -0,0 +1,403 @@ +"""Tests for the Tinker API mock server using the real tinker client.""" + +import asyncio +import os +import subprocess +import tempfile +import urllib.request +from contextlib import contextmanager +from urllib.parse import urlparse + +import pytest +import tinker +from tinker import types +from transformers import AutoTokenizer + +from tests.tinker.conftest import wait_for_condition + + +BASE_MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM" + + +TEST_SERVER_PORT = 8000 + +# Configs for the fast cleanup test +TEST_SERVER_PORT_FAST_CLEANUP = 8001 +FAST_CLEANUP_INTERVAL_SEC = 1 # How often to check for stale sessions +FAST_CLEANUP_TIMEOUT_SEC = 3 # Seconds without heartbeat before session is stale + + +def verify_training_client(training_client: tinker.TrainingClient): + """Verify a training client works with a forward pass.""" + tokenizer = training_client.get_tokenizer() + data = [make_datum(tokenizer, "Hello", " world")] + result = training_client.forward(data, "cross_entropy").result() + assert result is not None + + +def create_service_and_training_client(base_url: str, skip_verify: bool = False): + """Create a service client and a training client, verifying it works.""" + service_client = tinker.ServiceClient(base_url=base_url, api_key="dummy") + training_client = service_client.create_lora_training_client(base_model=BASE_MODEL) + if not skip_verify: + verify_training_client(training_client) + return service_client, training_client + + +@contextmanager +def start_api_server(overrides: dict[str, str] | None = None): + """Start the FastAPI server with optional config overrides. Prints log on failure.""" + with tempfile.TemporaryDirectory() as tmp_dir: + log_path = os.path.join(tmp_dir, "server.log") + db_path = os.path.join(tmp_dir, "server.db") + + with open(log_path, "w") as log_file: + defaults = { + "host": "0.0.0.0", + "port": str(TEST_SERVER_PORT), + "base-model": BASE_MODEL, + "backend-config": '{"max_lora_adapters": 4}', + "database-url": f"sqlite:///{db_path}", + } + if overrides: + defaults.update(overrides) + cmd = ["uv", "run", "--extra", "tinker", "-m", "tx.tinker.api"] + for key, value in defaults.items(): + cmd.extend([f"--{key}", value]) + process = subprocess.Popen(cmd, stdout=log_file, stderr=log_file) + print(f"Starting API server: {' '.join(cmd)}") + try: + yield process, log_path + except Exception: + with open(log_path) as f: + print(f"=== Test failed. Server log ({log_path}) ===\n{f.read()}") + raise + finally: + process.terminate() + process.wait(timeout=5) + + +@pytest.fixture(scope="module") +def api_server(): + """Start the FastAPI server for testing.""" + with start_api_server() as server: + yield server + + +@pytest.fixture +def service_client(api_server): + """Create a service client connected to the test server.""" + return tinker.ServiceClient(base_url=f"http://0.0.0.0:{TEST_SERVER_PORT}/", api_key="dummy") + + +def make_datum(tokenizer, prompt: str, completion: str, weight: tuple[float, float] | None = (0.0, 1.0)): + """Helper to create a Datum from prompt and completion with configurable weights.""" + prompt_tokens = tokenizer.encode(prompt, add_special_tokens=True) + completion_tokens = tokenizer.encode(f"{completion}\n\n", add_special_tokens=False) + all_tokens = prompt_tokens + completion_tokens + target_tokens = all_tokens[1:] + [tokenizer.eos_token_id] + + loss_fn_inputs = {"target_tokens": target_tokens} + if weight is not None: + prompt_weight, completion_weight = weight + all_weights = [prompt_weight] * len(prompt_tokens) + [completion_weight] * len(completion_tokens) + loss_fn_inputs["weights"] = all_weights[1:] + [completion_weight] + + return types.Datum( + model_input=types.ModelInput.from_ints(all_tokens), + loss_fn_inputs=loss_fn_inputs, + ) + + +def test_capabilities(service_client): + """Test the get_server_capabilities endpoint.""" + capabilities = service_client.get_server_capabilities() + model_names = [item.model_name for item in capabilities.supported_models] + assert BASE_MODEL in model_names + + +def custom_cross_entropy_loss(data, model_logprobs): + total_loss = 0.0 + for datum, log_p in zip(data, model_logprobs): + weights = datum.loss_fn_inputs.get("weights") + weights = weights.to_torch() if weights else 1.0 + total_loss += -(log_p * weights).sum() + return total_loss, {} + + +def test_training_workflow(service_client): + """Test a complete training workflow.""" + training_client = service_client.create_lora_training_client(base_model=BASE_MODEL) + + tokenizer = training_client.get_tokenizer() + + # Create training examples + processed_examples = [ + make_datum(tokenizer, "Question: What is 2+2?\nAnswer:", " 4", weight=(0.0, 0.0)), + make_datum(tokenizer, "Question: What color is the sky?\nAnswer:", " Blue"), + make_datum(tokenizer, "Question: What is 3+3?\nAnswer:", " 6", weight=None), + ] + + # Save the optimizer state + resume_path = training_client.save_state(name="0000").result().path + # Make sure if we save the sampler weights it will not override training weights + training_client.save_weights_for_sampler(name="0000").result() + # Get the training run ID from the first save + parsed_resume = urlparse(resume_path) + original_training_run_id = parsed_resume.netloc + + # Run training step + fwdbwd_future = training_client.forward_backward(processed_examples, "cross_entropy") + optim_future = training_client.optim_step(types.AdamParams(learning_rate=1e-4)) + + # Get results + fwdbwd_result = fwdbwd_future.result() + optim_result = optim_future.result() + + assert fwdbwd_result is not None + assert optim_result is not None + assert fwdbwd_result.loss_fn_output_type == "scalar" + assert len(fwdbwd_result.loss_fn_outputs) == 3 + + # The first example has all 0 weights, so all losses should be 0 + assert all(v == 0.0 for v in fwdbwd_result.loss_fn_outputs[0]["elementwise_loss"].data) + + # The second example has default weights (0 for prompt, 1 for completion), so should have non-zero losses + assert any(v != 0.0 for v in fwdbwd_result.loss_fn_outputs[1]["elementwise_loss"].data) + + # The third example omits weights (auto-filled with 1s), so all losses should be non-zero + assert all(v != 0.0 for v in fwdbwd_result.loss_fn_outputs[2]["elementwise_loss"].data) + + # Load the optimizer state and verify another forward_backward pass has the same loss + training_client.load_state(resume_path) + fwdbwd_result2 = training_client.forward_backward(processed_examples, "cross_entropy").result() + assert fwdbwd_result2.loss_fn_outputs == fwdbwd_result.loss_fn_outputs + # Also check that custom loss function produces the same loss + fwdbwd_result_custom = training_client.forward_backward_custom( + processed_examples, loss_fn=custom_cross_entropy_loss + ).result() + assert fwdbwd_result_custom.loss_fn_outputs == fwdbwd_result.loss_fn_outputs + + # Test that we can restore the training run + training_client = service_client.create_training_client_from_state(resume_path) + # Verify the restored client has the same state by running forward_backward again + fwdbwd_result3 = training_client.forward_backward(processed_examples, "cross_entropy").result() + assert fwdbwd_result3.loss_fn_outputs == fwdbwd_result.loss_fn_outputs + + sampling_path = training_client.save_weights_for_sampler(name="final").result().path + parsed = urlparse(sampling_path) + training_run_id = parsed.netloc + checkpoint_id = parsed.path.lstrip("/") + rest_client = service_client.create_rest_client() + # Download the checkpoint + checkpoint_response = rest_client.get_checkpoint_archive_url(training_run_id, checkpoint_id).result() + with tempfile.NamedTemporaryFile() as tmp_archive: + urllib.request.urlretrieve(checkpoint_response.url, tmp_archive.name) + assert os.path.getsize(tmp_archive.name) > 0 + + # List all checkpoints for the original training run + checkpoints_response = rest_client.list_checkpoints(original_training_run_id).result() + assert checkpoints_response is not None + assert len(checkpoints_response.checkpoints) > 0 + # Verify that the checkpoint we created is in the list + checkpoint_ids = [ckpt.checkpoint_id for ckpt in checkpoints_response.checkpoints] + assert "0000" in checkpoint_ids + + # Verify the training run appears in list_training_runs with correct fields + training_runs = rest_client.list_training_runs().result() + assert training_runs.cursor.total_count == len(training_runs.training_runs) + training_run = next(tr for tr in training_runs.training_runs if tr.training_run_id == original_training_run_id) + assert training_run.base_model == BASE_MODEL + assert training_run.is_lora is True + assert training_run.lora_rank == 32 + assert training_run.corrupted is False + + +@pytest.mark.parametrize("use_lora", [False, True], ids=["base_model", "lora_model"]) +def test_sample(service_client, use_lora): + """Test the sample endpoint with base model or LoRA adapter.""" + tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL) + + if use_lora: + training_client = service_client.create_lora_training_client(base_model=BASE_MODEL) + sampling_client = training_client.save_weights_and_get_sampling_client(name="test_sample") + else: + sampling_client = service_client.create_sampling_client(base_model=BASE_MODEL) + + # Sample from the model (base or LoRA) + prompt = types.ModelInput.from_ints(tokenizer.encode("Hello, how are you doing today? ", add_special_tokens=True)) + num_samples_per_request = [1, 2] + max_tokens_per_request = [20, 10] + requests = [] + for num_samples, max_tokens in zip(num_samples_per_request, max_tokens_per_request): + request = sampling_client.sample( + prompt=prompt, + sampling_params=types.SamplingParams(temperature=0.0, max_tokens=max_tokens, seed=42), + num_samples=num_samples, + ) + requests.append(request) + + # Verify we got the right number of sequences and tokens back + for request, num_samples, max_tokens in zip(requests, num_samples_per_request, max_tokens_per_request): + sample_result = request.result() + assert sample_result is not None + assert len(sample_result.sequences) == num_samples + assert len(sample_result.sequences[0].tokens) == max_tokens + + # Test stop tokens: generate once, then use the 5th token as a stop token + initial_result = sampling_client.sample( + prompt=prompt, + sampling_params=types.SamplingParams(temperature=0.0, max_tokens=10, seed=42), + num_samples=1, + ).result() + + stop_token = initial_result.sequences[0].tokens[4] + stopped_result = sampling_client.sample( + prompt=prompt, + sampling_params=types.SamplingParams(temperature=0.0, max_tokens=50, seed=42, stop=[stop_token]), + num_samples=1, + ).result() + + assert len(stopped_result.sequences[0].tokens) == 5 + assert stopped_result.sequences[0].stop_reason == "stop" + assert stopped_result.sequences[0].tokens[-1] == stop_token + + +def test_sample_top_k(service_client): + """Test that top_k sampling restricts token selection.""" + tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL) + sampling_client = service_client.create_sampling_client(base_model=BASE_MODEL) + prompt = types.ModelInput.from_ints(tokenizer.encode("Hello, how are you doing today? ", add_special_tokens=True)) + + def sample_with_top_k(top_k, num_runs=3): + return [ + sampling_client.sample( + prompt=prompt, + sampling_params=types.SamplingParams(temperature=1.0, max_tokens=5, seed=42 + i, top_k=top_k), + num_samples=1, + ) + .result() + .sequences[0] + .tokens + for i in range(num_runs) + ] + + # top_k=1 should produce identical outputs regardless of seed + results_top_1 = sample_with_top_k(top_k=1) + assert all(r == results_top_1[0] for r in results_top_1), "top_k=1 should produce identical outputs" + + # top_k=-1 (disabled) should vary with different seeds + results_no_top_k = sample_with_top_k(top_k=-1) + assert not all(seq == results_no_top_k[0] for seq in results_no_top_k), "Without top_k, outputs should vary" + + +def test_sample_with_stop_strings(service_client): + """Test the sample endpoint with string stop sequences.""" + tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL) + sampling_client = service_client.create_sampling_client(base_model=BASE_MODEL) + + prompt = types.ModelInput.from_ints(tokenizer.encode("Hello, how are you doing today? ", add_special_tokens=True)) + + # Generate a baseline sample without stop strings + baseline_result = sampling_client.sample( + prompt=prompt, + sampling_params=types.SamplingParams(temperature=0.0, max_tokens=50, seed=42), + num_samples=1, + ).result() + + baseline_tokens = baseline_result.sequences[0].tokens + baseline_text = tokenizer.decode(baseline_tokens) + + # Find a substring that appears in the generated text to use as stop string + # Use a short substring from the middle of the text + mid_point = len(baseline_text) // 2 + stop_string = baseline_text[mid_point : mid_point + 5] + + # Skip test if we couldn't find a good stop string + if not stop_string or stop_string not in baseline_text: + pytest.skip("Could not find a suitable stop string in generated text") + + # Generate with the stop string + stopped_result = sampling_client.sample( + prompt=prompt, + sampling_params=types.SamplingParams(temperature=0.0, max_tokens=50, seed=42, stop=[stop_string]), + num_samples=1, + ).result() + + stopped_tokens = stopped_result.sequences[0].tokens + stopped_text = tokenizer.decode(stopped_tokens) + + # Verify the stop string handling + assert stopped_result.sequences[0].stop_reason == "stop" + # The output should be shorter than or equal to baseline + assert len(stopped_tokens) <= len(baseline_tokens) + # The stop string should appear at or near the end of the decoded text + assert stop_string in stopped_text + + +@pytest.fixture(scope="function") +def api_server_fast_cleanup(): + """Start the FastAPI server with fast cleanup settings for testing.""" + with start_api_server( + overrides={ + "port": str(TEST_SERVER_PORT_FAST_CLEANUP), + "session-cleanup-interval-sec": str(FAST_CLEANUP_INTERVAL_SEC), + "session-timeout-sec": str(FAST_CLEANUP_TIMEOUT_SEC), + }, + ) as server: + yield server + + +def test_unload_model(api_server): + """Test that unload_model properly unloads a model.""" + # Create a training client (which creates a model) and verify it works + _, training_client = create_service_and_training_client(base_url=f"http://0.0.0.0:{TEST_SERVER_PORT}/") + + async def unload_model(): + async with tinker._client.AsyncTinker( + api_key="dummy", base_url=f"http://0.0.0.0:{TEST_SERVER_PORT}/" + ) as client: + future = await client.models.unload(request=types.UnloadModelRequest(model_id=training_client.model_id)) + while True: + result = await client.futures.retrieve( + request=types.FutureRetrieveRequest(request_id=future.request_id) + ) + if isinstance(result, types.UnloadModelResponse): + return result + await asyncio.sleep(0.1) + + assert isinstance(asyncio.run(unload_model()), types.UnloadModelResponse) + + # Verify model no longer works after unload + with pytest.raises(Exception): + verify_training_client(training_client) + + +def test_stale_session_cleanup(api_server_fast_cleanup): + """Test that stale sessions are automatically cleaned up and models are unloaded. + + This test only checks server logs for cleanup messages rather than verifying + adapter slot reuse, since that behavior is already covered by unit tests in + test_jax_backend.py and test_engine.py. + """ + _, log_path = api_server_fast_cleanup + base_url = f"http://0.0.0.0:{TEST_SERVER_PORT_FAST_CLEANUP}/" + # Skip verification because we've reduced cleanup timeout in this test. + # When running on a slow machine, the model may have already be cleaned + # up at verification time. + service_client, training_client = create_service_and_training_client( + base_url=base_url, + skip_verify=True, + ) + + # Stop heartbeating by deleting the clients + del training_client + del service_client + + # Poll for cleanup log messages + def cleanup_logs_found(): + with open(log_path) as f: + log_output = f.read() + return "Auto-unloaded stale model" in log_output and "Deleted model" in log_output + + assert wait_for_condition(cleanup_logs_found, timeout_sec=10, poll_interval_sec=1), "Cleanup logs not found" diff --git a/skyrl-tx/tests/tinker/test_db.py b/skyrl-tx/tests/tinker/test_db.py new file mode 100644 index 0000000000..e515f7c701 --- /dev/null +++ b/skyrl-tx/tests/tinker/test_db.py @@ -0,0 +1,55 @@ +import os +from pathlib import Path +import subprocess +import tempfile + + +ALEMBIC_CMD_PREFIX = ["uv", "run", "--extra", "dev"] + + +def test_alembic_migration_generation(): + """Test that Alembic can generate migrations from SQLModel definitions.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_db_path = Path(tmpdir) / "test_alembic.db" + test_db_url = f"sqlite:///{test_db_path}" + + tinker_dir = Path(__file__).parent.parent.parent / "tx" / "tinker" + + # Test: alembic upgrade head creates tables + result = subprocess.run( + ALEMBIC_CMD_PREFIX + ["alembic", "upgrade", "head"], + cwd=tinker_dir, + capture_output=True, + text=True, + env={**os.environ, "TX_DATABASE_URL": test_db_url}, + ) + + # Should succeed (even if no migrations exist, it shouldn't error) + assert result.returncode == 0, f"Alembic upgrade failed: {result.stderr}" + + # Test: alembic current shows version + result = subprocess.run( + ALEMBIC_CMD_PREFIX + ["alembic", "current"], + cwd=tinker_dir, + capture_output=True, + text=True, + env={**os.environ, "TX_DATABASE_URL": test_db_url}, + ) + + assert result.returncode == 0, f"Alembic current failed: {result.stderr}" + + +def test_alembic_history(): + """Test that Alembic history command works.""" + tinker_dir = Path(__file__).parent.parent.parent / "tx" / "tinker" + + # Test: alembic history + result = subprocess.run( + ["uv", "run", "alembic", "history"], + cwd=tinker_dir, + capture_output=True, + text=True, + ) + + # Should work even with no migrations + assert result.returncode == 0, f"Alembic history failed: {result.stderr}" diff --git a/skyrl-tx/tests/tinker/test_engine.py b/skyrl-tx/tests/tinker/test_engine.py new file mode 100644 index 0000000000..3319a8c3e2 --- /dev/null +++ b/skyrl-tx/tests/tinker/test_engine.py @@ -0,0 +1,82 @@ +from cloudpathlib import AnyPath +from datetime import datetime, timedelta, timezone + +from sqlmodel import Session, SQLModel + +from tx.tinker.engine import TinkerEngine +from tx.tinker.config import EngineConfig +from tx.tinker import types +from tx.tinker.db_models import SessionDB, ModelDB + + +BASE_MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM" + + +def test_process_unload_model(): + """Test that process_unload_model removes model from backend.""" + config = EngineConfig( + base_model=BASE_MODEL, + checkpoints_base=AnyPath(""), + backend_config={"max_lora_adapters": 4, "max_lora_rank": 32}, + ) + engine = TinkerEngine(config) + SQLModel.metadata.create_all(engine.db_engine) + + model_id = "test_model" + _ = engine.process_single_request( + types.RequestType.CREATE_MODEL, model_id, {"lora_config": {"rank": 8, "alpha": 16, "seed": 0}} + ) + assert engine.backend.has_model(model_id) + + result = engine.process_unload_model(model_id, types.UnloadModelInput()) + assert result.status == "unloaded" + assert not engine.backend.has_model(model_id) + + +def test_cleanup_stale_sessions(): + """Test that cleanup_stale_sessions unloads models from expired sessions.""" + config = EngineConfig( + base_model=BASE_MODEL, + checkpoints_base=AnyPath(""), + backend_config={"max_lora_adapters": 4, "max_lora_rank": 32}, + session_timeout_sec=60, + database_url="sqlite:///:memory:", # Use in-memory DB for test isolation + ) + engine = TinkerEngine(config) + SQLModel.metadata.create_all(engine.db_engine) + + model_id = "stale_model" + session_id = "stale_session" + + # Create model in backend + _ = engine.process_single_request( + types.RequestType.CREATE_MODEL, model_id, {"lora_config": {"rank": 8, "alpha": 16, "seed": 0}} + ) + assert engine.backend.has_model(model_id) + + # Insert stale session and model into DB + stale_heartbeat = datetime.now(timezone.utc) - timedelta(seconds=120) + with Session(engine.db_engine) as session: + session.add( + SessionDB( + session_id=session_id, + sdk_version="test", + status="active", + last_heartbeat_at=stale_heartbeat, + ) + ) + session.add( + ModelDB( + model_id=model_id, + base_model=BASE_MODEL, + lora_config=types.LoraConfig(rank=8, alpha=16, seed=0).model_dump(), + status="ready", + request_id=1, + session_id=session_id, + ) + ) + session.commit() + + # Run cleanup and assert one model was unloaded + assert engine.cleanup_stale_sessions() == 1 + assert not engine.backend.has_model(model_id) diff --git a/skyrl-tx/tests/tinker/test_jax_backend.py b/skyrl-tx/tests/tinker/test_jax_backend.py new file mode 100644 index 0000000000..2edd9d82b7 --- /dev/null +++ b/skyrl-tx/tests/tinker/test_jax_backend.py @@ -0,0 +1,558 @@ +"""Unit tests for JaxBackend.""" + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +import optax + +from tx.tinker.backends.jax import JaxBackend, JaxBackendConfig +from tx.tinker.engine import prepare_model_pass_batch, prepare_sample_batch +from tx.tinker.types import LoraConfig, OptimStepInput +from tx.tinker import api +from tx.tinker import types +from tx.layers.lora import LoRALinear + + +BASE_MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM" +MAX_LORA_ADAPTERS = 4 +LORA_RANK = 8 + + +def create_backend(max_lora_adapters: int = MAX_LORA_ADAPTERS): + """Create a JaxBackend.""" + config = JaxBackendConfig(max_lora_adapters=max_lora_adapters, max_lora_rank=32) + return JaxBackend(BASE_MODEL, config) + + +def create_model(backend: JaxBackend, model_id: str) -> int: + """Create a model and return its adapter index.""" + lora_config = LoraConfig(rank=LORA_RANK, alpha=16, seed=0) + backend.create_model(model_id, lora_config) + return backend.models[model_id].adapter_index + + +def test_delete_model_basic(): + """Test basic model deletion.""" + backend = create_backend() + model_id = "test_model" + + # Create model + _ = create_model(backend, model_id) + assert backend.has_model(model_id) + + # Delete model + backend.delete_model(model_id) + assert not backend.has_model(model_id) + + +def test_delete_non_existent_model(): + """Test deleting a non-existent model raises ValueError.""" + backend = create_backend() + with pytest.raises(ValueError, match="not found"): + backend.delete_model("nonexistent_model") + + +def test_adapter_slot_reuse(): + """Test that deleted adapter slots are reused.""" + backend = create_backend() + + # Create 3 models and check adapter indices + assert create_model(backend, "model_1") == 1 + assert create_model(backend, "model_2") == 2 + assert create_model(backend, "model_3") == 3 + + # Delete first model, new model should reuse index 1 + backend.delete_model("model_1") + assert create_model(backend, "model_4") == 1 + + # Delete middle model, new model should fill gap at index 1 + backend.delete_model("model_2") + assert create_model(backend, "model_5") == 2 + + +def test_max_adapters_limit(): + """Test that creating more than available adapters raises ValueError.""" + backend = create_backend() + + # Index 0 is reserved for base model, so we have max_lora_adapters - 1 slots + num_available = MAX_LORA_ADAPTERS - 1 + for i in range(num_available): + _ = create_model(backend, f"model_{i}") + + # Try to create one more - should fail + with pytest.raises(ValueError, match="Maximum number of LoRA adapters"): + _ = create_model(backend, "model_overflow") + + +def test_max_adapters_after_delete(): + """Test that deleting a model frees a slot for new models.""" + backend = create_backend() + # Index 0 is reserved for base model, so we have max_lora_adapters - 1 slots + num_available = MAX_LORA_ADAPTERS - 1 + for i in range(num_available): + _ = create_model(backend, f"model_{i}") + + # Delete one model + backend.delete_model("model_0") + + # Now we should be able to create a new model which should reuse the freed slot + assert create_model(backend, "model_new") == 1 + + +def test_clear_lora_adapter(): + """Test that clear_lora_adapter zeros out adapter state.""" + backend = create_backend() + model_id = "test_model" + adapter_idx = create_model(backend, model_id) + + # Verify adapter has non-zero rank after creation + model = backend.model + lora_layer: LoRALinear = model.model.layers[0].self_attn.q_proj + assert lora_layer.lora_ranks[adapter_idx] > 0 + + # Delete the model (calls clear_lora_adapter internally) + backend.delete_model(model_id) + + # Verify adapter state is zeroed + assert lora_layer.lora_ranks[adapter_idx] == 0 + assert lora_layer.lora_scaling[adapter_idx] == 0.0 + assert (lora_layer.lora_A[adapter_idx] == 0.0).all() + assert (lora_layer.lora_B[adapter_idx] == 0.0).all() + + +def make_fwd_bwd_input(token_lists: list[list[int]]) -> types.ForwardBackwardInput: + """Build a ForwardBackwardInput for testing.""" + samples = [] + for tokens in token_lists: + targets = tokens[1:] + [0] + weights = [1] * len(tokens) + samples.append( + types.Datum( + model_input=types.ModelInput(chunks=[types.ModelInputChunk(tokens=tokens)]), + loss_fn_inputs=types.LossFnInputs( + target_tokens=types.TensorData(data=targets), + weights=types.TensorData(data=weights), + advantages=types.TensorData(data=[]), + logprobs=types.TensorData(data=[]), + ), + ) + ) + return types.ForwardBackwardInput(data=samples, loss_fn="cross_entropy") + + +def _assert_tree_allclose(t1, t2, rtol=1e-3, atol=1e-3, min_match_pct=99.0): + """Assert that at least min_match_pct% of elements in two trees are close.""" + leaves1 = jax.tree.leaves(t1) + leaves2 = jax.tree.leaves(t2) + assert len(leaves1) == len(leaves2), "Gradient trees differ in structure/leaf count" + for a, b in zip(leaves1, leaves2): + a_arr = np.asarray(a) + b_arr = np.asarray(b) + + # Check how many elements are close + matches = np.isclose(a_arr, b_arr, rtol=rtol, atol=atol) + match_pct = 100.0 * np.sum(matches) / a_arr.size + if match_pct < min_match_pct: + # Show statistics about mismatches + diff = np.abs(a_arr - b_arr) + rel_diff = np.abs((a_arr - b_arr) / (np.abs(b_arr) + 1e-10)) + failing = ~matches + raise AssertionError( + f"Only {match_pct:.2f}% of elements match (required: {min_match_pct}%)\n" + f" Max absolute diff: {np.max(diff[failing])}\n" + f" Max relative diff: {np.max(rel_diff[failing])}\n" + f" Mean of mismatches: {np.mean(diff[failing])}" + ) + + +def test_adapter_gradient_calculation(): + """Test that gradients for one adapter are not affected by another adapter's batch size.""" + config = JaxBackendConfig(max_lora_adapters=8, max_lora_rank=32) + backend = JaxBackend(BASE_MODEL, config) + + adapter1_id = "adapter1" + adapter2_id = "adapter2" + backend.create_model(adapter1_id, LoraConfig(rank=32, alpha=32, seed=0)) + backend.create_model(adapter2_id, LoraConfig(rank=32, alpha=32, seed=0)) + + # Adapter1 samples (fixed across both rounds) + a1_input = make_fwd_bwd_input([[1, 2, 3, 4], [5, 6, 7, 8]]) + # Adapter2 samples (round 1: 2 samples; round 2: 4 samples) + a2_input1 = make_fwd_bwd_input( + [ + [9, 10, 11, 12], + [13, 14, 15, 16], + ] + ) + reqs_round1 = { + "101": (adapter1_id, a1_input), + "102": (adapter2_id, a2_input1), + } + + # Process round 1 batch + backend.forward_backward(prepare_model_pass_batch(reqs_round1)) + + adapter1_idx = backend.models[adapter1_id].adapter_index + adapter2_idx = backend.models[adapter2_id].adapter_index + + # Extract gradients for adapter 1 + grads_A1_round1 = jax.tree.map(lambda x: x[adapter1_idx], backend.accumulated_grads.grad_sum) + + # Clear stored grads so we can run another fwd/bwd without optimizer update. + backend.accumulated_grads = backend.accumulated_grads.reset_adapter(adapter1_idx) + backend.accumulated_grads = backend.accumulated_grads.reset_adapter(adapter2_idx) + + a1_input = make_fwd_bwd_input([[1, 2, 3, 4], [5, 6, 7, 8]]) + a2_input2 = make_fwd_bwd_input([[9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]]) + reqs_round2 = { + "201": (adapter1_id, a1_input), + "202": (adapter2_id, a2_input2), + } + + # Process round 2 batch + backend.forward_backward(prepare_model_pass_batch(reqs_round2)) + + grads_A1_round2 = jax.tree.map(lambda x: x[adapter1_idx], backend.accumulated_grads.grad_sum) + + # Compare gradients using 99% match threshold + _assert_tree_allclose(grads_A1_round1, grads_A1_round2, rtol=1e-3, atol=1e-2, min_match_pct=99.0) + + +def test_micro_batch_grad_accumulation(): + """ + Verifies that fwd-bwd with micro-batching produces the same + per-adapter mean gradients as without micro-batching. + """ + adapter1_id = "adapter1" + adapter2_id = "adapter2" + + # Build backend with micro-batching enabled (batch size 4) + config_micro = JaxBackendConfig(max_lora_adapters=8, max_lora_rank=32, train_micro_batch_size=4) + backend_micro = JaxBackend(BASE_MODEL, config_micro) + backend_micro.create_model(adapter1_id, LoraConfig(rank=32, alpha=32, seed=0)) + backend_micro.create_model(adapter2_id, LoraConfig(rank=32, alpha=32, seed=0)) + + # Fused batch with 6 total examples: 2 for adapter1, 4 for adapter2. + a1_input = make_fwd_bwd_input([[1, 2, 3, 4], [5, 6, 7, 8]]) # 2 samples + a2_input = make_fwd_bwd_input( + [ + [9, 10, 11, 12], + [13, 14, 15, 16], + [17, 18, 19, 20], + [21, 22, 23, 24], + ] + ) + + reqs = { + "1001": (adapter1_id, a1_input), + "1002": (adapter2_id, a2_input), + } + + # Run 1: micro-batching enabled + backend_micro.forward_backward(prepare_model_pass_batch(reqs)) + + adapter1_idx = backend_micro.models[adapter1_id].adapter_index + adapter2_idx = backend_micro.models[adapter2_id].adapter_index + + mean_micro_a1 = backend_micro.accumulated_grads.get_mean(adapter1_idx) + mean_micro_a2 = backend_micro.accumulated_grads.get_mean(adapter2_idx) + + # Sanity check gradient sum denominators with micro-batching + assert backend_micro.accumulated_grads.counts[adapter1_idx] == 2 + assert backend_micro.accumulated_grads.counts[adapter2_idx] == 4 + + # Build a second backend without micro-batching + config_full = JaxBackendConfig(max_lora_adapters=8, max_lora_rank=32, train_micro_batch_size=0) + backend_full = JaxBackend(BASE_MODEL, config_full) + backend_full.create_model(adapter1_id, LoraConfig(rank=32, alpha=32, seed=0)) + backend_full.create_model(adapter2_id, LoraConfig(rank=32, alpha=32, seed=0)) + + # Run 2: micro-batching disabled + backend_full.forward_backward(prepare_model_pass_batch(reqs)) + + # Note: adapter indices might be different in new backend instance if logic changed, + # but here we create them in same order so it should be fine. + # Better to fetch them again to be safe. + adapter1_idx_full = backend_full.models[adapter1_id].adapter_index + adapter2_idx_full = backend_full.models[adapter2_id].adapter_index + + mean_full_a1 = backend_full.accumulated_grads.get_mean(adapter1_idx_full) + mean_full_a2 = backend_full.accumulated_grads.get_mean(adapter2_idx_full) + + # Sanity check gradient sum denominators without micro-batching + assert backend_full.accumulated_grads.counts[adapter1_idx_full] == 2 + assert backend_full.accumulated_grads.counts[adapter2_idx_full] == 4 + + # Compare MEAN gradients with and without micro-batching + _assert_tree_allclose(mean_micro_a1, mean_full_a1, rtol=1e-3, atol=5e-3) + _assert_tree_allclose(mean_micro_a2, mean_full_a2, rtol=1e-3, atol=5e-3) + + +def test_process_optim_step_hyperparams_behavior(): + """Request-scoped overrides apply for the step, base hyperparameters stay unchanged, and update size shifts.""" + config = JaxBackendConfig(max_lora_adapters=8, max_lora_rank=32) + backend = JaxBackend(BASE_MODEL, config) + + low_adapter = "adapter_low" + default_adapter = "adapter_default" + for model_id in (low_adapter, default_adapter): + backend.create_model(model_id, LoraConfig(rank=32, alpha=32, seed=0)) + + tokens = [[1, 2, 3, 4], [5, 6, 7, 8]] + + def apply_step(request_id: int, model_id: str, optim_input: OptimStepInput) -> float: + reqs = {str(request_id): (model_id, make_fwd_bwd_input(tokens))} + backend.forward_backward(prepare_model_pass_batch(reqs)) + params_before = jax.tree.map(jnp.copy, backend.lora_params) + backend.optim_step(model_id, optim_input) + delta = jax.tree.map( + lambda old, new: (new - old).astype(jnp.float32), + params_before, + backend.lora_params, + ) + return float(optax.global_norm(delta)) + + tiny_input = OptimStepInput( + adam_params=types.AdamParams(learning_rate=1e-8, beta1=1e-8, beta2=1e-8, eps=1e-9, weight_decay=0.0) + ) + default_input = OptimStepInput(adam_params=api.AdamParams().to_types()) + # Apply override step on the first adapter. + tiny_norm = apply_step(1, low_adapter, tiny_input) + + # Apply fallback/default step on the second adapter (same engine). + default_norm = apply_step(2, default_adapter, default_input) + + # Expect a large gap in update magnitude between the two adapters. + assert tiny_norm > 0 + assert default_norm / tiny_norm == pytest.approx(1e4, rel=5e-3) + + +def test_gradient_checkpointing(): + """ + Verify gradient checkpointing doesn't affect loss values. + """ + losses = [] + for use_gradient_checkpointing in (False, True): + config = JaxBackendConfig( + max_lora_adapters=1, + max_lora_rank=4, + train_micro_batch_size=1, + gradient_checkpointing=use_gradient_checkpointing, + ) + backend = JaxBackend(BASE_MODEL, config) + + # Create batch + B, T = 2, 8 + vocab = backend.model.config.vocab_size + input_ids = jnp.arange(B * T, dtype=jnp.int32).reshape(B, T) % vocab + attention_mask = jnp.ones((B, T), dtype=jnp.int32) + adapter_indices = jnp.zeros((B,), dtype=jnp.int32) + target_ids = input_ids + loss_mask = jnp.ones((B, T), dtype=jnp.float32) + loss_fn_types = jnp.zeros((B,), dtype=jnp.int32) + sampling_logprobs = jnp.zeros((B, T), dtype=jnp.float32) + advantages = jnp.zeros((B, T), dtype=jnp.float32) + + # Compute loss, using gradient checkpointing if enabled + _, per_token_losses, _ = backend._forward_backward_and_accumulate( + backend.accumulated_grads, + backend.lora_params, + backend.non_lora_params, + input_ids, + attention_mask, + adapter_indices, + target_ids, + loss_mask, + loss_fn_types, + sampling_logprobs, + advantages, + ) + losses.append(float(per_token_losses.mean())) + + # Check relative difference between losses is small + assert abs(losses[0] - losses[1]) / abs(losses[0]) < 5e-3 + + +def make_sample_input(tokens: list[int], prompt_logprobs: bool = False, max_tokens: int = 16) -> types.SampleInput: + """Build a SampleInput for testing.""" + return types.SampleInput( + base_model=BASE_MODEL, # Sample from base model (no LoRA) + prompt=types.ModelInput(chunks=[types.ModelInputChunk(tokens=tokens)]), + sampling_params=api.SamplingParams(temperature=0.0, max_tokens=max_tokens, seed=42).to_types(), + num_samples=1, + checkpoint_id="", # Empty for base model sampling + prompt_logprobs=prompt_logprobs, + ) + + +def test_sample_max_num_sequences(): + """ + Verify sampling with sample_max_num_sequences constraint. + """ + config = JaxBackendConfig( + max_lora_adapters=2, + max_lora_rank=32, + sample_max_num_sequences=2, # Set max sample batch size to 2 + ) + backend = JaxBackend(BASE_MODEL, config) + + # Five prompts, resulting in 3 batches (2 of size 2, 1 of size 1) + prompts = [ + [1, 2, 3], + [4, 5, 6, 7], + [8, 9], + [10, 11, 12, 13, 14], + [15, 16, 17], + ] + + # Build a batch of 5 sample requests + reqs = {str(request_id): ("", make_sample_input(tokens)) for request_id, tokens in enumerate(prompts)} + + # Process sample requests. + results = backend.sample(prepare_sample_batch(reqs)) + + # Verify results + assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}" + for request_id in reqs: + result = results[request_id] + + assert len(result.sequences) == 1, f"Request {request_id}: expected 1 sequence, got {len(result.sequences)}" + seq = result.sequences[0] + tokens = seq.tokens + + # Should have generated some tokens (max_tokens=16) + assert len(tokens) > 0, f"Request {request_id}: no tokens generated" + assert len(tokens) <= 16, f"Request {request_id}: generated {len(tokens)} tokens, max was 16" + + # Stop reason should be valid + assert seq.stop_reason in ["length", "stop"], f"Request {request_id}: invalid stop_reason '{seq.stop_reason}'" + + # If we have logprobs, they should match the number of tokens + if seq.logprobs: + assert len(seq.logprobs) == len( + tokens + ), f"Request {request_id}: {len(tokens)} tokens but {len(seq.logprobs)} logprobs" + + +def test_sample_with_prompt_logprobs(): + """Test correct handling of prompt_logprobs in sampling requests.""" + config = JaxBackendConfig(max_lora_adapters=2, max_lora_rank=32) + backend = JaxBackend(BASE_MODEL, config) + + prompts = [ + [1, 2, 3, 4], + [5, 6, 7, 8, 9], + [10, 11, 12], + ] + + # Test with prompt_logprobs enabled + reqs_with_logprobs = { + f"req_{i}": ("", make_sample_input(tokens, prompt_logprobs=True, max_tokens=8)) + for i, tokens in enumerate(prompts) + } + + results_with = backend.sample(prepare_sample_batch(reqs_with_logprobs)) + + for i, tokens in enumerate(prompts): + request_id = f"req_{i}" + result = results_with[request_id] + + # Verify prompt_logprobs are returned + assert result.prompt_logprobs is not None, f"Request {request_id}: prompt_logprobs should not be None" + # Prompt logprobs should have length = prompt_length - 1 + expected_length = len(tokens) - 1 + assert ( + len(result.prompt_logprobs) == expected_length + ), f"Request {request_id}: expected {expected_length} prompt_logprobs, got {len(result.prompt_logprobs)}" + + # Test mixed batch: one request with prompt_logprobs=True and one with =False + reqs_mixed = { + "req_with_0": ("", make_sample_input(prompts[0], prompt_logprobs=True, max_tokens=8)), + "req_without_1": ("", make_sample_input(prompts[1], prompt_logprobs=False, max_tokens=8)), + } + + results_mixed = backend.sample(prepare_sample_batch(reqs_mixed)) + + # Verify request with prompt_logprobs=True has logprobs + assert results_mixed["req_with_0"].prompt_logprobs is not None + assert len(results_mixed["req_with_0"].prompt_logprobs) == len(prompts[0]) - 1 + + # Verify request with prompt_logprobs=False has None + assert results_mixed["req_without_1"].prompt_logprobs is None + + +def test_sample_prompt_logprobs_with_microbatching(): + """Test that prompt_logprobs work correctly with micro-batching.""" + config = JaxBackendConfig( + max_lora_adapters=2, + max_lora_rank=32, + sample_max_num_sequences=2, # Force micro-batching with batch size of 2 + ) + backend = JaxBackend(BASE_MODEL, config) + + # Create 5 prompts, which will be split into 3 micro-batches (2, 2, 1) + prompts = [ + [1, 2, 3], + [4, 5, 6, 7], + [8, 9, 10], + [11, 12, 13, 14], + [15, 16], + ] + + # All requests ask for prompt_logprobs + reqs = { + f"req_{i}": ("", make_sample_input(tokens, prompt_logprobs=True, max_tokens=8)) + for i, tokens in enumerate(prompts) + } + + results = backend.sample(prepare_sample_batch(reqs)) + + # Verify that each request got its correct prompt_logprobs + for i, tokens in enumerate(prompts): + request_id = f"req_{i}" + result = results[request_id] + + # Verify prompt_logprobs are returned + assert result.prompt_logprobs is not None, f"Request {request_id}: prompt_logprobs should not be None" + + # Verify correct length + expected_length = len(tokens) - 1 + assert ( + len(result.prompt_logprobs) == expected_length + ), f"Request {request_id}: expected {expected_length} prompt_logprobs, got {len(result.prompt_logprobs)}" + + +def test_adapter_reuse_initializes_lora_adapter(): + """Test that reusing an adapter slot initializes the lora adapter properly.""" + # Use max_lora_adapters=2 so only slot 1 is available + # (slot 0 is reserved for base model) + backend = create_backend(max_lora_adapters=2) + model = backend.model + lora_layer: LoRALinear = model.model.layers[0].self_attn.q_proj + + # Create first model + model_id_1 = "model_1" + adapter_idx = create_model(backend, model_id_1) + + # Verify lora_A is non-zero after creation + assert not ( + lora_layer.lora_A[adapter_idx, ..., :LORA_RANK] == 0.0 + ).all(), "lora_A should be initialized with he_uniform (non-zero)" + + # Delete the model (clears both lora_A and lora_B to zeros) + backend.delete_model(model_id_1) + assert (lora_layer.lora_A[adapter_idx] == 0.0).all(), "lora_A should be zeroed after clear_lora_adapter" + + # Create a new model that reuses the same adapter slot + model_id_2 = "model_2" + new_adapter_idx = create_model(backend, model_id_2) + assert new_adapter_idx == adapter_idx, "Should reuse the same adapter slot" + + # Verify lora_A is initialized (non-zero) + assert not ( + lora_layer.lora_A[adapter_idx, ..., :LORA_RANK] == 0.0 + ).all(), "lora_A should be initialized with he_uniform after adapter reuse" + + # Verify lora_B is zeros + assert (lora_layer.lora_B[adapter_idx] == 0.0).all(), "lora_B should be zeros" diff --git a/skyrl-tx/tests/utils/test_generator.py b/skyrl-tx/tests/utils/test_generator.py new file mode 100644 index 0000000000..b1cbc552c1 --- /dev/null +++ b/skyrl-tx/tests/utils/test_generator.py @@ -0,0 +1,185 @@ +from flax import nnx +import jax.numpy as jnp +from tx.models.types import CausalLMOutput +from tx.tinker.types import SamplingParams +from tx.utils.generator import GenerateOutput, GeneratorMixin, KVCache, apply_top_k_batch, apply_top_p_batch + + +class DummyModel(GeneratorMixin, nnx.Module): + def __init__(self, vocab_size: int = 16): + self.vocab_size = vocab_size + + def __call__(self, input_ids, attention_mask=None, positions=None, kv_cache=None, adapter_indices=None): + """Simple dummy model for testing generator behavior.""" + batch_size, seq_len = input_ids.shape + base = jnp.arange(self.vocab_size, dtype=jnp.float32) + + if kv_cache is None: + # Prefill: deterministic logits + logits = jnp.tile(base[None, None, :], (batch_size, seq_len, 1)) + keys = [jnp.zeros((batch_size, seq_len, 1, 1), dtype=jnp.float32)] + values = [jnp.zeros((batch_size, seq_len, 1, 1), dtype=jnp.float32)] + kv_cache = KVCache(keys=keys, values=values, cache_position=seq_len) + else: + # Step: logits vary with cache_position + logits = jnp.tile(base[None, None, :] + kv_cache.cache_position, (batch_size, 1, 1)) + kv_cache = KVCache(keys=kv_cache.keys, values=kv_cache.values, cache_position=kv_cache.cache_position + 1) + + return CausalLMOutput(logits=logits, last_hidden_state=logits, kv_cache=kv_cache) + + +def make_inputs(batch_size: int, prompt_length: int): + input_ids = jnp.tile(jnp.arange(prompt_length, dtype=jnp.int32)[None, :], (batch_size, 1)) + attention_mask = jnp.ones((batch_size, prompt_length), dtype=jnp.int32) + return input_ids, attention_mask + + +def generator_outputs_equal(output1: GenerateOutput, index1: int, output2: GenerateOutput, index2: int) -> bool: + """Check if two GenerateOutput objects are equal at the given indices.""" + return ( + output1.generated_ids[index1] == output2.generated_ids[index2] + and jnp.allclose(jnp.array(output1.logprobs[index1]), jnp.array(output2.logprobs[index2])) + and output1.stop_reasons[index1] == output2.stop_reasons[index2] + ) + + +def test_deterministic_generation(): + """Repeated generation with same seed should be deterministic.""" + model = DummyModel(vocab_size=8) + input_ids, attention_mask = make_inputs(batch_size=1, prompt_length=3) + sampling = SamplingParams(max_tokens=4, temperature=1.0, seed=12345) + + res1 = model.generate(input_ids, attention_mask, sampling_params=[sampling]) + res2 = model.generate(input_ids, attention_mask, sampling_params=[sampling]) + + assert generator_outputs_equal(res1, 0, res2, 0) + + +def test_batch_independence(): + """Batch generation should be equivalent to individual generation with same seeds.""" + model = DummyModel(vocab_size=12) + input_ids, attention_mask = make_inputs(batch_size=2, prompt_length=4) + + sp1 = SamplingParams(max_tokens=5, temperature=1.0, seed=111) + sp2 = SamplingParams(max_tokens=5, temperature=1.0, seed=222) + + batch_result = model.generate(input_ids, attention_mask, sampling_params=[sp1, sp2]) + + res_a = model.generate(input_ids[:1], attention_mask[:1], sampling_params=[sp1]) + res_b = model.generate(input_ids[1:], attention_mask[1:], sampling_params=[sp2]) + + assert generator_outputs_equal(batch_result, 0, res_a, 0) + assert generator_outputs_equal(batch_result, 1, res_b, 0) + + +def test_greedy_vs_sampled(): + """Greedy and sampled generation should be independent in batch.""" + model = DummyModel(vocab_size=10) + input_ids, attention_mask = make_inputs(batch_size=2, prompt_length=2) + + sp_greedy = SamplingParams(max_tokens=3, temperature=0.0, seed=999) + sp_sample = SamplingParams(max_tokens=3, temperature=1.0, seed=2020) + + batch_result = model.generate(input_ids, attention_mask, sampling_params=[sp_greedy, sp_sample]) + + single_greedy = model.generate(input_ids[:1], attention_mask[:1], sampling_params=[sp_greedy]) + single_sample = model.generate(input_ids[1:], attention_mask[1:], sampling_params=[sp_sample]) + + assert generator_outputs_equal(batch_result, 0, single_greedy, 0) + assert generator_outputs_equal(batch_result, 1, single_sample, 0) + + +def test_prompt_logprobs(): + """Test prompt logprobs computation.""" + model = DummyModel(vocab_size=16) + prompt_length = 5 + expected_length = prompt_length - 1 # We skip the first token + + # Test with single sequence (batch_size=1) + input_ids, attention_mask = make_inputs(batch_size=1, prompt_length=prompt_length) + sampling = SamplingParams(max_tokens=4, temperature=0.0, seed=42) + + # Test with prompt_logprobs=True + result_with = model.generate(input_ids, attention_mask, sampling_params=[sampling], prompt_logprobs=True) + assert result_with.prompt_logprobs is not None, "prompt_logprobs should not be None when enabled" + assert len(result_with.prompt_logprobs) == 1, "Should have prompt_logprobs for 1 sequence in batch" + assert ( + len(result_with.prompt_logprobs[0]) == expected_length + ), f"prompt_logprobs should have length {expected_length} (prompt_length - 1)" + + # Test with prompt_logprobs=False + result_without = model.generate(input_ids, attention_mask, sampling_params=[sampling], prompt_logprobs=False) + assert result_without.prompt_logprobs is None, "prompt_logprobs should be None when disabled" + + # Test with batched generation + batch_size = 3 + input_ids_batch, attention_mask_batch = make_inputs(batch_size=batch_size, prompt_length=prompt_length) + result_batch = model.generate( + input_ids_batch, attention_mask_batch, sampling_params=[sampling] * batch_size, prompt_logprobs=True + ) + + assert result_batch.prompt_logprobs is not None + assert len(result_batch.prompt_logprobs) == batch_size, f"Should have prompt_logprobs for {batch_size} sequences" + for i in range(batch_size): + assert ( + len(result_batch.prompt_logprobs[i]) == expected_length + ), f"Sequence {i}: expected prompt_logprobs length {expected_length}" + + +def test_top_k_filtering(): + """Test apply_top_k_batch function directly.""" + # Create test logits [batch_size, vocab_size] + logits = jnp.array([[1.0, 2.0, 3.0, 4.0, 5.0]]) + + # Test k=2: should keep only top 2 values (4.0 and 5.0) + filtered = apply_top_k_batch(logits, k_values=jnp.array([2]), max_k=2) + # Values below threshold should be -inf, and top 2 values should be unchanged + expected = jnp.array([[-jnp.inf, -jnp.inf, -jnp.inf, 4.0, 5.0]]) + assert jnp.array_equal(filtered, expected) + + # Test k<=0 with max_k>0: should not filter that example + filtered = apply_top_k_batch(logits, k_values=jnp.array([-1]), max_k=5) + assert jnp.array_equal(filtered, logits) + + # Test per-example k values in batch (second row has ties: two 3.0s) + logits_batch = jnp.array([[1.0, 2.0, 3.0, 4.0, 5.0], [5.0, 4.0, 3.0, 3.0, 1.0]]) + filtered = apply_top_k_batch(logits_batch, k_values=jnp.array([2, 3]), max_k=3) + # Second row keeps exactly 3 values despite ties (5.0, 4.0, and first 3.0) + expected = jnp.array( + [ + [-jnp.inf, -jnp.inf, -jnp.inf, 4.0, 5.0], + [5.0, 4.0, 3.0, -jnp.inf, -jnp.inf], + ] + ) + assert jnp.array_equal(filtered, expected) + + +def test_top_p_filtering(): + """Test apply_top_p_batch function directly.""" + # probs = [0.002, 0.006, 0.015, 0.041, 0.112, 0.825] + logits = jnp.array([[0.0, 1.0, 2.0, 3.0, 4.0, 6.0]]) + + # Test p=1.0: should not filter anything + filtered = apply_top_p_batch(logits, p_values=jnp.array([1.0])) + assert jnp.array_equal(filtered, logits) + + # Test p=0.0: should only keep the top token (index 5) + filtered = apply_top_p_batch(logits, p_values=jnp.array([0.0])) + expected = jnp.array([[-jnp.inf, -jnp.inf, -jnp.inf, -jnp.inf, -jnp.inf, 6.0]]) + assert jnp.array_equal(filtered, expected) + + # Test p=0.9: cumsum_exclusive < 0.9 keeps top 2 tokens + filtered = apply_top_p_batch(logits, p_values=jnp.array([0.9])) + expected = jnp.array([[-jnp.inf, -jnp.inf, -jnp.inf, -jnp.inf, 4.0, 6.0]]) + assert jnp.array_equal(filtered, expected) + + # Test per-example p values in batch + logits_batch = jnp.array([[0.0, 1.0, 2.0, 3.0, 4.0, 6.0], [0.0, 1.0, 2.0, 3.0, 4.0, 6.0]]) + filtered = apply_top_p_batch(logits_batch, p_values=jnp.array([1.0, 0.0])) + expected = jnp.array( + [ + [0.0, 1.0, 2.0, 3.0, 4.0, 6.0], + [-jnp.inf, -jnp.inf, -jnp.inf, -jnp.inf, -jnp.inf, 6.0], + ] + ) + assert jnp.array_equal(filtered, expected) diff --git a/skyrl-tx/tests/utils/test_models.py b/skyrl-tx/tests/utils/test_models.py new file mode 100644 index 0000000000..1fbe95df13 --- /dev/null +++ b/skyrl-tx/tests/utils/test_models.py @@ -0,0 +1,84 @@ +from pathlib import Path + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +import torch +from cloudpathlib import CloudPath, implementation_registry +from cloudpathlib.local import local_s3_implementation +from flax import nnx +from peft import PeftModel +from transformers import AutoConfig, AutoModelForCausalLM + +from tx.layers.lora import init_lora_adapter +from tx.models.configs import Qwen3Config +from tx.models.qwen3 import Qwen3ForCausalLM +from tx.tinker.types import LoraConfig +from tx.utils import models +from tx.utils.storage import download_and_unpack + + +def create_test_model(base_model_name: str, rank: int, alpha: int, adapter_index: int): + """Create a small Qwen3 model for testing with LoRA enabled.""" + base_config = AutoConfig.from_pretrained(base_model_name) + # Make it smaller for testing + base_config.num_hidden_layers = 1 + base_config.hidden_size = 64 + base_config.intermediate_size = 128 + base_config.num_attention_heads = 2 + base_config.num_key_value_heads = 2 + + config = Qwen3Config(base_config, max_lora_adapters=5, max_lora_rank=32, shard_attention_heads=True) + + mesh = jax.make_mesh((1, 1), ("fsdp", "tp")) + with jax.set_mesh(mesh): + model = Qwen3ForCausalLM(config, dtype=jnp.float32, rngs=nnx.Rngs(0)) + init_lora_adapter(model, adapter_index=adapter_index, lora_config=LoraConfig(rank=rank, alpha=alpha, seed=0)) + + return config, base_config, model + + +@pytest.mark.parametrize("storage_type", ["local", "cloud"]) +def test_save_load_lora_checkpoint(storage_type: str, monkeypatch, tmp_path: Path): + base_model_name = "Qwen/Qwen3-0.6B" + # Setup output path for tar.gz file based on storage type + if storage_type == "cloud": + monkeypatch.setitem(implementation_registry, "s3", local_s3_implementation) + client = local_s3_implementation.client_class(local_storage_dir=tmp_path) + output_path = CloudPath("s3://bucket/checkpoint.tar.gz", client=client) + else: + output_path = tmp_path / "checkpoint.tar.gz" + + rank, alpha, adapter_index = 8, 16, 2 + config, base_config, model = create_test_model(base_model_name, rank, alpha, adapter_index) + adapter_config = LoraConfig(rank=rank, alpha=alpha, seed=0) + + # Set LoRA weights to random values for testing (to catch transpose bugs) + q_proj = model.model.layers[0].self_attn.q_proj + rng1, rng2 = jax.random.split(jax.random.PRNGKey(42)) + q_proj.lora_A.value = jax.random.normal(rng1, q_proj.lora_A.value.shape) + q_proj.lora_B.value = jax.random.normal(rng2, q_proj.lora_B.value.shape) + + # Store expected values (trimmed to rank and transposed) + expected_lora_A = np.array(q_proj.lora_A.value[adapter_index, :, :rank].T) + expected_lora_B = np.array(q_proj.lora_B.value[adapter_index, :rank, :].T) + + # Save and verify checkpoint exists + models.save_lora_checkpoint(model, base_model_name, adapter_config, adapter_index, output_path) + assert output_path.exists() + + # Load with peft and verify + with download_and_unpack(output_path) as extracted_dir: + base_model = AutoModelForCausalLM.from_config(base_config) + peft_model = PeftModel.from_pretrained(base_model, extracted_dir) + + assert peft_model.peft_config["default"].r == rank + assert peft_model.peft_config["default"].lora_alpha == alpha + + q_proj_adapter = peft_model.base_model.model.model.layers[0].self_attn.q_proj + lora_A = q_proj_adapter.lora_A["default"].weight + lora_B = q_proj_adapter.lora_B["default"].weight + + assert torch.allclose(lora_A, torch.from_numpy(expected_lora_A), atol=1e-6) + assert torch.allclose(lora_B, torch.from_numpy(expected_lora_B), atol=1e-6) diff --git a/skyrl-tx/tx/__init__.py b/skyrl-tx/tx/__init__.py new file mode 100644 index 0000000000..3ced3581bb --- /dev/null +++ b/skyrl-tx/tx/__init__.py @@ -0,0 +1 @@ +__version__ = "0.2.1" diff --git a/skyrl-tx/tx/layers/layernorm.py b/skyrl-tx/tx/layers/layernorm.py new file mode 100644 index 0000000000..e061d6d09c --- /dev/null +++ b/skyrl-tx/tx/layers/layernorm.py @@ -0,0 +1,24 @@ +"""Layer normalization implementations.""" + +from flax import nnx +import jax +from jax import numpy as jnp + +from tx.layers.util import Param + + +class RMSNorm(nnx.Module): + """Root Mean Square Layer Normalization. + + Reference: https://arxiv.org/abs/1910.07467 + """ + + def __init__(self, size: int, *, eps: float = 1e-6, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.eps = eps + self.weight = Param( + size, dtype=dtype, kernel_init=nnx.with_partitioning(nnx.initializers.normal(), jax.P(None)), rngs=rngs + ) + + def __call__(self, x: jax.Array) -> jax.Array: + rms = jnp.sqrt(jnp.mean(x**2, axis=-1, keepdims=True) + self.eps) + return self.weight * x / rms diff --git a/skyrl-tx/tx/layers/lora.py b/skyrl-tx/tx/layers/lora.py new file mode 100644 index 0000000000..9bb1ac8081 --- /dev/null +++ b/skyrl-tx/tx/layers/lora.py @@ -0,0 +1,349 @@ +from flax import nnx +import jax +from jax import numpy as jnp + +from tx.utils.models import filter_lora +from tx.layers.util import Param, prepare_routing, ragged_dot +from tx.models.types import ModelForCausalLM +from tx.tinker.types import LoraConfig + + +class LoRAMixin: + """A mixin for flax NNX modules to add multi-adapter LoRA support. + This mixin adds LoRA parameters (lora_A, lora_B) and methods to apply + the low-rank adaptation to a base module's output. It is designed to + be used with layers like nnx.Linear. + """ + + lora_scaling: nnx.Variable | None + lora_ranks: nnx.Variable | None + lora_A: nnx.Param | None + lora_B: nnx.Param | None + + def init_lora( + self, + *, + max_lora_adapters: int, + max_lora_rank: int, + shape_A: tuple[int, ...], + shape_B: tuple[int, ...], + sharding_A: tuple[str | None, ...], + sharding_B: tuple[str | None, ...], + dtype: jnp.dtype, + rngs: nnx.Rngs, + ) -> None: + """Initialize LoRA parameter tensors.""" + self.max_lora_adapters = max_lora_adapters + self.max_lora_rank = max_lora_rank + + if max_lora_adapters == 0: + self.lora_scaling = None + self.lora_ranks = None + self.lora_A = None + self.lora_B = None + else: + self.lora_scaling = nnx.Variable(jnp.full((max_lora_adapters,), 1.0, dtype=dtype)) + self.lora_ranks = nnx.Variable(jnp.full((max_lora_adapters,), max_lora_rank, dtype=jnp.int32)) + # lora_A and lora_B are initialized to zeros here. The actual weight + # initialization (he_uniform for lora_A, zeros for lora_B) happens in + # init_lora_adapter(), which must be called before training. + self.lora_A = Param( + *shape_A, + dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.zeros_init(), sharding_A), + rngs=rngs, + ) + self.lora_B = Param( + *shape_B, + dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.zeros_init(), sharding_B), + rngs=rngs, + ) + + def apply_lora( + self, + x: jax.Array, + base_output: jax.Array, + adapter_indices: jax.Array | None, + ) -> jax.Array: + if self.max_lora_adapters == 0 or adapter_indices is None: + return base_output + + if self.lora_A is None or self.lora_B is None or self.lora_scaling is None: + raise RuntimeError("LoRA parameters are not initialized. `init_lora` must be called.") + + (batch_size, seq_len, *dims) = x.shape + assert len(self.lora_A.shape) == 3 + assert len(dims) == 0 if isinstance(self, nnx.Embed) else tuple(dims) == self.lora_A[...].shape[1:-1] + assert adapter_indices.shape[0] == batch_size + + x_flat = x.reshape(-1, *dims) + adapter_indices_expanded = jnp.repeat(adapter_indices, seq_len) + + # Sort tokens to prepare for ragged_dot + x_sorted, group_sizes, unsort_indices, adapter_indices_sorted = prepare_routing( + x_flat, adapter_indices_expanded, self.max_lora_adapters, adapter_indices=adapter_indices_expanded + ) + + # Apply LoRA using ragged_dot: x @ A @ B + if isinstance(self, nnx.Embed): + # Embedding path: A[x] + intermediate = self.lora_A[...][adapter_indices_sorted, x_sorted, :] + else: + # Linear path: x @ A + intermediate = jax.lax.ragged_dot(x_sorted, self.lora_A[...], group_sizes) + lora_output_sorted = jax.lax.ragged_dot(intermediate, self.lora_B[...], group_sizes) + + # Unsort, reshape, scale + lora_output = lora_output_sorted[unsort_indices].reshape(batch_size, seq_len, -1) + lora_output = lora_output * self.lora_scaling[...][adapter_indices, None, None] + return base_output + lora_output.reshape(base_output.shape) + + +class LoRAEmbed(LoRAMixin, nnx.Embed): + """An nnx.Embed layer with multi-adapter LoRA support""" + + def __init__( + self, + num_embeddings: int, + features: int, + *, + max_lora_adapters: int = 0, + max_lora_rank: int = 8, + dtype: jnp.dtype = jnp.float32, + param_dtype: jnp.dtype | None = None, + embedding_init: nnx.Initializer, + rngs: nnx.Rngs, + ) -> None: + param_dtype = param_dtype or dtype + + super().__init__( + num_embeddings=num_embeddings, + features=features, + dtype=dtype, + param_dtype=param_dtype, + embedding_init=embedding_init, + rngs=rngs, + ) + assert ( + self.embedding[...].sharding is not None + ), "LoRAEmbed layer needs sharding, you can specify it by using nnx.with_partitioning on the embedding_init" + sharding = self.embedding[...].sharding.spec + + self.init_lora( + max_lora_adapters=max_lora_adapters, + max_lora_rank=max_lora_rank, + shape_A=(max_lora_adapters, num_embeddings, max_lora_rank), + shape_B=(max_lora_adapters, max_lora_rank, features), + sharding_A=(None, sharding[0], None), + sharding_B=(None, None, sharding[1]), + dtype=param_dtype, + rngs=rngs, + ) + + def __call__(self, x: jax.Array, adapter_indices: jax.Array | None = None) -> jax.Array: + base_out = super().__call__(x) + return self.apply_lora(x, base_out, adapter_indices) + + +class LoRALinear(LoRAMixin, nnx.Linear): + """An nnx.Linear layer with multi-adapter LoRA support.""" + + def __init__( + self, + in_features: int, + out_features: int, + *, + max_lora_adapters: int = 0, + max_lora_rank: int = 8, + dtype: jnp.dtype = jnp.float32, + param_dtype: jnp.dtype | None = None, + use_bias: bool, + kernel_init: nnx.Initializer, + bias_init: nnx.Initializer | None = None, + rngs: nnx.Rngs, + ) -> None: + param_dtype = param_dtype or dtype + if bias_init is None: + bias_init = nnx.initializers.zeros_init() + + super().__init__( + in_features, + out_features, + use_bias=use_bias, + dtype=dtype, + param_dtype=param_dtype, + kernel_init=kernel_init, + bias_init=bias_init, + rngs=rngs, + ) + assert ( + self.kernel[...].sharding is not None + ), "LoRALinear layer needs sharding, you can specify it by using nnx.with_partitioning on the kernel_init" + sharding = self.kernel[...].sharding.spec + self.init_lora( + max_lora_adapters=max_lora_adapters, + max_lora_rank=max_lora_rank, + shape_A=(max_lora_adapters, in_features, max_lora_rank), + shape_B=(max_lora_adapters, max_lora_rank, out_features), + sharding_A=(None, sharding[0], None), + sharding_B=(None, None, sharding[1]), + dtype=param_dtype, + rngs=rngs, + ) + + def __call__(self, x: jax.Array, adapter_indices: jax.Array | None = None) -> jax.Array: + base_out = super().__call__(x) + return self.apply_lora(x, base_out, adapter_indices) + + +class LoRAExpert(LoRAMixin, nnx.Module): + """Expert layer with multi-adapter LoRA support.""" + + def __init__( + self, + num_experts: int, + in_features: int, + out_features: int, + *, + max_lora_adapters: int = 0, + max_lora_rank: int = 8, + dtype: jnp.dtype = jnp.float32, + kernel_init: nnx.Initializer, + rngs: nnx.Rngs, + ) -> None: + self.num_experts = num_experts + self.in_features = in_features + self.out_features = out_features + + self.weight = Param(num_experts, in_features, out_features, dtype=dtype, kernel_init=kernel_init, rngs=rngs) + + assert self.weight.value.sharding is not None, "LoRAExpert layer needs sharding" + sharding = self.weight.value.sharding.spec + self.init_lora( + max_lora_adapters=max_lora_adapters, + max_lora_rank=max_lora_rank, + shape_A=(max_lora_adapters, num_experts, in_features, max_lora_rank), + shape_B=(max_lora_adapters, num_experts, max_lora_rank, out_features), + sharding_A=(None, sharding[0], sharding[1], None), + sharding_B=(None, sharding[0], None, sharding[2]), + dtype=dtype, + rngs=rngs, + ) + + def __call__( + self, + x: jax.Array, + group_sizes: jax.Array, + adapter_indices_sorted: jax.Array | None = None, + *, + group_offset: jax.Array | None = None, + ) -> jax.Array: + base_out = ragged_dot(x, self.weight.value, group_sizes, group_offset=group_offset) + + if self.max_lora_adapters == 0 or adapter_indices_sorted is None: + return base_out + + if self.lora_A is None or self.lora_B is None or self.lora_scaling is None: + raise RuntimeError("LoRA parameters are not initialized. `init_lora` must be called.") + + # Reconstruct expert indices from group_sizes (global indices) + expert_indices = jnp.repeat(jnp.arange(self.num_experts), group_sizes, total_repeat_length=x.shape[0]) + + # Expert-first flattening so local expert groups are contiguous + flattened_indices = expert_indices * self.max_lora_adapters + adapter_indices_sorted + num_flattened_groups = self.num_experts * self.max_lora_adapters + num_local_experts = self.lora_A.value.shape[1] + + # Reshape LoRA weights in expert-first order + lora_A = self.lora_A.value.transpose((1, 0, 2, 3)).reshape( + self.max_lora_adapters * num_local_experts, self.in_features, self.max_lora_rank + ) + lora_B = self.lora_B.value.transpose((1, 0, 2, 3)).reshape( + self.max_lora_adapters * num_local_experts, self.max_lora_rank, self.out_features + ) + + # Sort tokens by combined index + x_sorted, combined_group_sizes, unsort_indices, _ = prepare_routing(x, flattened_indices, num_flattened_groups) + + # Compute group_offset for LoRA (scaled by max_lora_adapters) + lora_group_offset = group_offset * self.max_lora_adapters if group_offset is not None else None + + # Apply LoRA using ragged_dot: x @ A @ B + intermediate = ragged_dot(x_sorted, lora_A, combined_group_sizes, group_offset=lora_group_offset) + lora_output_sorted = ragged_dot(intermediate, lora_B, combined_group_sizes, group_offset=lora_group_offset) + + # Unsort and apply scaling + lora_output = lora_output_sorted[unsort_indices] + lora_output = lora_output * self.lora_scaling.value[adapter_indices_sorted, None] + + return base_out + lora_output + + +def init_lora_adapter(model: ModelForCausalLM, adapter_index: int, lora_config: LoraConfig): + """Initialize a LoRA adapter for training. + + Initializes the adapter: lora_A with he_uniform, lora_B with zeros, + and sets the appropriate rank and scaling. This must be called BEFORE + training begins on an adapter slot. + + Args: + model: The model containing LoRA layers + adapter_index: Index of the adapter to initialize + lora_config: LoraConfig object containing rank, alpha, seed, and training flags + """ + rngs = nnx.Rngs(lora_config.seed) + state = nnx.state(model) + + def init_adapter(path, value): + effective_rank = lora_config.rank + normalized_path = tuple(p.key if hasattr(p, "key") else p.name for p in path) + + # Apply rank normalization for MoE expert layers + # Following Thinking Machines' approach: divide rank by num_experts + # to keep total LoRA parameters similar to non-MoE models + if "experts" in normalized_path: + effective_rank = max(1, lora_config.rank // model.config.num_experts) + + if not filter_lora(lora_config, normalized_path): + effective_rank = 0 + + key_name = path[-2].key + if key_name == "lora_ranks": + return value.at[adapter_index].set(effective_rank) + if key_name == "lora_scaling": + # Set scaling to 0.0 if rank is 0 + return value.at[adapter_index].set(lora_config.alpha / effective_rank if effective_rank > 0 else 0.0) + if key_name == "lora_A": + # Reinitialize with he_uniform, then zero columns beyond rank + shape = value[adapter_index].shape + new_A = nnx.initializers.he_uniform()(rngs.params(), shape, value.dtype) + new_A = new_A.at[..., effective_rank:].set(0.0) + return value.at[adapter_index].set(new_A) + if key_name == "lora_B": + # Explicitly zero lora_B + return value.at[adapter_index].set(0.0) + return value + + updated_state = jax.tree.map_with_path(init_adapter, state) + nnx.update(model, updated_state) + + +def clear_lora_adapter(model: ModelForCausalLM, adapter_index: int): + """Clear/reset a LoRA adapter, freeing it for reuse. + + Sets rank=0, scaling=0, and zeros out lora_A and lora_B for the adapter. + Before reusing this adapter for training again, `init_lora_adapter` must be called. + """ + state = nnx.state(model) + + def clear_adapter(path, value): + key = path[-2].key + if key == "lora_ranks": + return value.at[adapter_index].set(0) + if key in ("lora_scaling", "lora_A", "lora_B"): + return value.at[adapter_index].set(0.0) + return value + + updated_state = jax.tree.map_with_path(clear_adapter, state) + nnx.update(model, updated_state) diff --git a/skyrl-tx/tx/layers/rotary_embedding.py b/skyrl-tx/tx/layers/rotary_embedding.py new file mode 100644 index 0000000000..3ff67655fb --- /dev/null +++ b/skyrl-tx/tx/layers/rotary_embedding.py @@ -0,0 +1,24 @@ +"""Rotary Position Embeddings (RoPE) implementation.""" + +import jax +from jax import numpy as jnp + + +def apply_rope(inputs: jax.Array, position_ids: jax.Array, head_dim: int, theta: float) -> jax.Array: + """Apply Rotary Position Embeddings (RoPE). + + Args: + inputs: Input tensor of shape [B, T, num_heads, head_dim] + position_ids: Position indices of shape [B, T] + head_dim: Dimension of each attention head + theta: Base for the geometric progression (rope_theta) + + Returns: + Tensor with RoPE applied, same shape as inputs + """ + fraction = 2 * jnp.arange(0, head_dim // 2, dtype=jnp.float32) / head_dim + timescale = jnp.pow(theta, fraction) + x = (position_ids[..., None] / timescale[None, None, :])[..., None, :] + sin, cos = jnp.sin(x), jnp.cos(x) + a, b = jnp.split(inputs, 2, axis=-1) + return jnp.concatenate([a * cos - b * sin, b * cos + a * sin], axis=-1).astype(inputs.dtype) diff --git a/skyrl-tx/tx/layers/util.py b/skyrl-tx/tx/layers/util.py new file mode 100644 index 0000000000..0030c604de --- /dev/null +++ b/skyrl-tx/tx/layers/util.py @@ -0,0 +1,112 @@ +from flax import nnx +import jax +from jax import lax +from jax import numpy as jnp +from jax.sharding import get_abstract_mesh, PartitionSpec + + +def ragged_dot( + lhs: jax.Array, + rhs: jax.Array, + group_sizes: jax.Array, + precision=None, + preferred_element_type=None, + group_offset: jax.Array | None = None, +) -> jax.Array: + """Ragged dot product with group_offset support. + + When group_offset is specified, rhs contains groups [offset, offset + g_local). + Tokens outside this range are routed to boundary groups and masked to zero. + """ + if group_offset is None: + return lax.ragged_dot( + lhs, + rhs, + group_sizes, + precision=precision, + preferred_element_type=preferred_element_type, + ) + + assert group_offset.shape == (1,), "group_offset must have shape (1,)" + offset = group_offset[0] + m = lhs.shape[0] + g_local = rhs.shape[0] + + assert g_local > 0, "rhs must have at least one group" + + # Compute token boundaries for local groups + cumsum = jnp.cumulative_sum(group_sizes, include_initial=True) + shard_start = cumsum[offset] + shard_end = cumsum[offset + g_local] + + # Valid mask for tokens in local groups + token_idx = jnp.arange(m) + valid_mask = (token_idx >= shard_start) & (token_idx < shard_end) + + # Adjust group sizes: absorb extra tokens at boundaries + local_group_sizes = lax.dynamic_slice_in_dim(group_sizes, offset, g_local, axis=0) + adjusted_group_sizes = local_group_sizes.at[0].add(shard_start).at[-1].add(m - shard_end) + + # Call ragged_dot - extra tokens use boundary groups but get masked out + result = lax.ragged_dot( + lhs, + rhs, + adjusted_group_sizes, + precision=precision, + preferred_element_type=preferred_element_type, + ) + + return jnp.where(valid_mask[:, None], result, 0) + + +def Param(*shape: int, dtype: jnp.dtype, kernel_init: nnx.Initializer, rngs: nnx.Rngs): + return nnx.Param(kernel_init(rngs.param(), shape, dtype)) + + +def prepare_routing( + tokens: jax.Array, indices: jax.Array, num_groups: int, adapter_indices: jax.Array | None = None +) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array | None]: + """Prepare inputs for ragged_dot operations by sorting tokens by group. + + Args: + tokens: Array of shape (num_tokens, ...) to be sorted by group + indices: Array of shape (num_tokens,) indicating group assignment for each token + num_groups: Total number of groups + adapter_indices: Optional array of shape (num_tokens,) to be sorted together with tokens + + Returns: + sorted_tokens: Tokens sorted by group index + group_sizes: Number of tokens in each group + unsort_indices: Indices to restore original order after ragged operations + """ + sort_indices = jnp.argsort(indices) + sorted_tokens = tokens[sort_indices] + sorted_adapter_indices = None if adapter_indices is None else adapter_indices[sort_indices] + group_sizes = jnp.bincount(indices, length=num_groups) + unsort_indices = jnp.argsort(sort_indices) + return sorted_tokens, group_sizes, unsort_indices, sorted_adapter_indices + + +def shard_map_ep(module: nnx.Module, func, *args): + """Apply shard_map over the 'ep' axis for a stateful nnx.Module. + + Args: + module: The NNX module (will be split into graph/state). + func: Function to run inside shard_map. Signature: (module, *args). + *args: Arguments to pass to func (replicated across shards). + """ + graphdef, state = nnx.split(module) + # Extract only 'ep' dims from PartitionSpecs, replacing others with None + state_specs = jax.tree.map( + lambda s: PartitionSpec(*(p if p == "ep" else None for p in s)) if isinstance(s, PartitionSpec) else s, + nnx.get_partition_spec(state), + is_leaf=lambda x: isinstance(x, PartitionSpec), + ) + in_specs = (state_specs,) + (PartitionSpec(),) * len(args) + + @jax.shard_map(mesh=get_abstract_mesh(), in_specs=in_specs, out_specs=PartitionSpec(), axis_names={"ep"}) + def _body(state, *fn_args): + module_shard = nnx.merge(graphdef, state) + return func(module_shard, *fn_args) + + return _body(state, *args) diff --git a/skyrl-tx/tx/loaders/__init__.py b/skyrl-tx/tx/loaders/__init__.py new file mode 100644 index 0000000000..cb405dc300 --- /dev/null +++ b/skyrl-tx/tx/loaders/__init__.py @@ -0,0 +1,10 @@ +from tx.loaders.chat import chat +from tx.loaders.common import get_loader +from tx.loaders.text import text + + +__all__ = [ + "get_loader", + "chat", + "text", +] diff --git a/skyrl-tx/tx/loaders/chat.py b/skyrl-tx/tx/loaders/chat.py new file mode 100644 index 0000000000..e2ada93fb6 --- /dev/null +++ b/skyrl-tx/tx/loaders/chat.py @@ -0,0 +1,20 @@ +import jax.numpy as jnp +from datasets import Dataset +from transformers import PreTrainedTokenizer + +from tx.loaders.common import LoaderIterator + + +def chat(tokenizer: PreTrainedTokenizer, dataset: Dataset, batch_size: int) -> LoaderIterator: + "Data loader that applies the chat template. It returns an iterator over (batch, metrics) elements." + + for data in dataset.shuffle().iter(batch_size=batch_size): + batch = tokenizer.apply_chat_template(data["messages"], tokenize=False) + # We pad to multiples of 512 here so jax needs to compile less different shapes + batch = tokenizer(batch, return_tensors="np", padding=True, pad_to_multiple_of=512) + batch = {k: jnp.asarray(v) for k, v in batch.items()} + yield { + "text": batch["input_ids"][:, :-1], + "attention_mask": batch["attention_mask"][:, :-1], + "target": batch["input_ids"][:, 1:], + }, {"shape": batch["input_ids"].shape, "tokens": batch["attention_mask"].sum()} diff --git a/skyrl-tx/tx/loaders/common.py b/skyrl-tx/tx/loaders/common.py new file mode 100644 index 0000000000..3f18855a83 --- /dev/null +++ b/skyrl-tx/tx/loaders/common.py @@ -0,0 +1,18 @@ +import importlib +from operator import attrgetter +from typing import Callable, Iterator + +import jax +from datasets import Dataset +from transformers import PreTrainedTokenizer + +LoaderIterator = Iterator[tuple[dict[str, jax.Array], dict[str, str]]] + + +def get_loader(loader_name: str) -> Callable[[PreTrainedTokenizer, Dataset, int], LoaderIterator]: + module_name, function_name = loader_name.split(".", 1) + try: + module = importlib.import_module(module_name) + return attrgetter(function_name)(module) + except (ImportError, AttributeError) as e: + raise ImportError(f"Could not load {function_name} from {module_name}: {e}") diff --git a/skyrl-tx/tx/loaders/text.py b/skyrl-tx/tx/loaders/text.py new file mode 100644 index 0000000000..24888de3d4 --- /dev/null +++ b/skyrl-tx/tx/loaders/text.py @@ -0,0 +1,19 @@ +import jax.numpy as jnp +from datasets import Dataset +from transformers import PreTrainedTokenizer + +from tx.loaders.common import LoaderIterator + + +def text(tokenizer: PreTrainedTokenizer, dataset: Dataset, batch_size: int) -> LoaderIterator: + "Data loader for text data. It returns an iterator over (batch, metrics) elements." + + for data in dataset.iter(batch_size=batch_size): + # We pad to multiples of 128 here so jax needs to compile less different shapes + batch = tokenizer(data["text"], return_tensors="np", padding=True, pad_to_multiple_of=128) + batch = {k: jnp.asarray(v) for k, v in batch.items()} + yield { + "text": batch["input_ids"][:, :-1], + "attention_mask": batch["attention_mask"][:, :-1], + "target": batch["input_ids"][:, 1:], + }, {"shape": batch["input_ids"].shape, "tokens": batch["attention_mask"].sum()} diff --git a/skyrl-tx/tx/models/__init__.py b/skyrl-tx/tx/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skyrl-tx/tx/models/configs.py b/skyrl-tx/tx/models/configs.py new file mode 100644 index 0000000000..adc2b57ab4 --- /dev/null +++ b/skyrl-tx/tx/models/configs.py @@ -0,0 +1,43 @@ +"""Configuration classes for models with LoRA support.""" + +from transformers import PretrainedConfig + + +class ModelConfig(PretrainedConfig): + """Configuration for tx models with LoRA support. + + Wraps a HuggingFace PretrainedConfig with additional parameters + for Multi-LoRA training and tensor parallelism. + + Args: + config: A HuggingFace PretrainedConfig object (e.g., from AutoConfig.from_pretrained()) + max_lora_adapters: Maximum number of concurrent LoRA adapters + max_lora_rank: Maximum rank for LoRA adapters + shard_attention_heads: Whether to shard attention across tensor parallel devices + """ + + # Type hints for LoRA attributes + max_lora_adapters: int + max_lora_rank: int + shard_attention_heads: bool + + def __init__( + self, + config: PretrainedConfig, + *, + max_lora_adapters: int, + max_lora_rank: int, + shard_attention_heads: bool, + ): + # Copy all attributes from the base config + super().__init__(**config.to_dict()) + + # Add LoRA-specific parameters + self.max_lora_adapters = max_lora_adapters + self.max_lora_rank = max_lora_rank + self.shard_attention_heads = shard_attention_heads + + +# Model-specific aliases for clarity and backwards compatibility +Llama3Config = ModelConfig +Qwen3Config = ModelConfig diff --git a/skyrl-tx/tx/models/llama3.py b/skyrl-tx/tx/models/llama3.py new file mode 100644 index 0000000000..2fb1652903 --- /dev/null +++ b/skyrl-tx/tx/models/llama3.py @@ -0,0 +1,323 @@ +from flax import nnx +import jax +from jax import numpy as jnp +from jax.sharding import get_abstract_mesh +from transformers import LlamaConfig + +from tx.layers.lora import LoRAEmbed, LoRALinear +from tx.layers.rotary_embedding import apply_rope +from tx.layers.layernorm import RMSNorm +from tx.models.types import CausalLMOutput, ModelOutput +from tx.utils.generator import GeneratorMixin, KVCache, compute_positions + + +class Llama3Attention(nnx.Module): + """Multi-head attention with Grouped Query Attention (GQA) support.""" + + def __init__(self, config: LlamaConfig, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + self.num_heads = config.num_attention_heads + self.num_kv_heads = config.num_key_value_heads + tp = get_abstract_mesh().shape.get("tp", 1) + shard_attention_heads = config.shard_attention_heads + if shard_attention_heads: + assert self.num_heads % tp == 0, f"num_heads={self.num_heads} must be divisible by tp={tp}" + assert self.num_kv_heads % tp == 0, f"num_kv_heads={self.num_kv_heads} must be divisible by tp={tp}" + tp_shard = "tp" if shard_attention_heads else None + self.head_dim = getattr(config, "head_dim", None) or config.hidden_size // self.num_heads + + self.q_proj = LoRALinear( + in_features=config.hidden_size, + out_features=self.num_heads * self.head_dim, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, tp_shard)), + rngs=rngs, + ) + + self.k_proj = LoRALinear( + in_features=config.hidden_size, + out_features=self.num_kv_heads * self.head_dim, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, tp_shard)), + rngs=rngs, + ) + + self.v_proj = LoRALinear( + in_features=config.hidden_size, + out_features=self.num_kv_heads * self.head_dim, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, tp_shard)), + rngs=rngs, + ) + + self.o_proj = LoRALinear( + in_features=self.num_heads * self.head_dim, + out_features=config.hidden_size, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (tp_shard, None)), + rngs=rngs, + ) + + def __call__( + self, + x: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array, + adapter_indices: jax.Array | None = None, + kv_cache: tuple[jax.Array, jax.Array, int] | None = None, + ) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + B, T, _ = x.shape + + # Project and reshape to [B, T, num_heads, head_dim] + q = self.q_proj(x, adapter_indices=adapter_indices).reshape(B, T, self.num_heads, self.head_dim) + k = self.k_proj(x, adapter_indices=adapter_indices).reshape(B, T, self.num_kv_heads, self.head_dim) + v = self.v_proj(x, adapter_indices=adapter_indices).reshape(B, T, self.num_kv_heads, self.head_dim) + + # Apply RoPE + q = apply_rope(q, positions, self.head_dim, self.config.rope_theta) + k = apply_rope(k, positions, self.head_dim, self.config.rope_theta) + + # Handle KV cache + if kv_cache is not None: + k_cache, v_cache, cache_position = kv_cache + k = jax.lax.dynamic_update_slice(k_cache, k, (0, cache_position, 0, 0)) + v = jax.lax.dynamic_update_slice(v_cache, v, (0, cache_position, 0, 0)) + + updated_cache = (k, v) + + # Attention (causal only during prefill, GQA handled natively by dot_product_attention) + attn_output = jax.nn.dot_product_attention( + q, + k, + v, + scale=1.0 / self.head_dim**0.5, + mask=attention_mask[:, None, None, :].astype(bool), + is_causal=kv_cache is None, + ) + + output = attn_output.reshape(B, T, self.num_heads * self.head_dim) + return self.o_proj(output, adapter_indices=adapter_indices), updated_cache + + +class Llama3MLP(nnx.Module): + + def __init__(self, config: LlamaConfig, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.gate_proj = LoRALinear( + config.hidden_size, + config.intermediate_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, "tp")), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + self.up_proj = LoRALinear( + config.hidden_size, + config.intermediate_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, "tp")), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + self.down_proj = LoRALinear( + config.intermediate_size, + config.hidden_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("tp", None)), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + + def __call__(self, x: jax.Array, adapter_indices: jax.Array | None = None) -> jax.Array: + gate_out = self.gate_proj(x, adapter_indices) + up_out = self.up_proj(x, adapter_indices) + return self.down_proj(nnx.silu(gate_out) * up_out, adapter_indices) + + +class Llama3DecoderLayer(nnx.Module): + + def __init__(self, config: LlamaConfig, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + self.self_attn = Llama3Attention(config, dtype=dtype, rngs=rngs) + self.mlp = Llama3MLP(config, dtype=dtype, rngs=rngs) + + def __call__( + self, + hidden_states: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array, + adapter_indices: jax.Array | None = None, + kv_cache: tuple[jax.Array, jax.Array, int] | None = None, + ) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states, updated_cache = self.self_attn( + hidden_states, + attention_mask=attention_mask, + positions=positions, + adapter_indices=adapter_indices, + kv_cache=kv_cache, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + mlp_output = self.mlp(hidden_states, adapter_indices=adapter_indices) + hidden_states = residual + mlp_output + + return hidden_states, updated_cache + + +class Llama3Model(nnx.Module): + + def __init__(self, config: LlamaConfig, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + + self.embed_tokens = LoRAEmbed( + num_embeddings=config.vocab_size, + features=config.hidden_size, + dtype=dtype, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + param_dtype=dtype, + embedding_init=nnx.with_partitioning(nnx.initializers.normal(), ("tp", None)), + rngs=rngs, + ) + self.layers = nnx.List( + [Llama3DecoderLayer(config, dtype=dtype, rngs=rngs) for _ in range(config.num_hidden_layers)] + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + + def __call__( + self, + input_ids: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array, + output_hidden_states: bool | None = None, + adapter_indices: jax.Array | None = None, + kv_cache: KVCache | None = None, + ) -> ModelOutput: + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + hidden_states = self.embed_tokens(input_ids, adapter_indices=adapter_indices) + all_hidden_states: list[jax.Array] = [] + updated_keys, updated_values = [], [] + + for layer_idx, layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states.append(hidden_states) + + hidden_states, (k, v) = layer( + hidden_states, + attention_mask=attention_mask, + positions=positions, + adapter_indices=adapter_indices, + kv_cache=kv_cache and (kv_cache.keys[layer_idx], kv_cache.values[layer_idx], kv_cache.cache_position), + ) + updated_keys.append(k) + updated_values.append(v) + + hidden_states = self.norm(hidden_states) + if output_hidden_states: + all_hidden_states.append(hidden_states) + + # Increment cache_position if cache exists, or use sequence length for new cache + new_cache_position = kv_cache.cache_position + 1 if kv_cache is not None else input_ids.shape[1] + + return ModelOutput( + last_hidden_state=hidden_states, + kv_cache=KVCache(keys=updated_keys, values=updated_values, cache_position=new_cache_position), + hidden_states=all_hidden_states if output_hidden_states else None, + ) + + +class Llama3ForCausalLM(nnx.Module, GeneratorMixin): + + def __init__(self, config: LlamaConfig, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + self.model = Llama3Model(config, dtype=dtype, rngs=rngs) + + if not self.config.tie_word_embeddings: + self.lm_head = LoRALinear( + config.hidden_size, + config.vocab_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, "tp")), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + + @staticmethod + def is_lora_param(path: tuple, _value) -> bool: + """Return True if a parameter path corresponds to LoRA weights.""" + return any(name in path for name in ("lora_A", "lora_B")) + + def __call__( + self, + input_ids: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array | None = None, + output_hidden_states: bool | None = None, + adapter_indices: jax.Array | None = None, + kv_cache: KVCache | None = None, + ) -> CausalLMOutput: + if positions is None: + positions = compute_positions(attention_mask) + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + positions=positions, + output_hidden_states=output_hidden_states, + adapter_indices=adapter_indices, + kv_cache=kv_cache, + ) + hidden_states = outputs.last_hidden_state + if self.config.tie_word_embeddings: + logits = hidden_states @ self.model.embed_tokens.embedding.value.T + else: + logits = self.lm_head(hidden_states, adapter_indices=adapter_indices) + + return CausalLMOutput( + logits=logits, + last_hidden_state=outputs.last_hidden_state, + kv_cache=outputs.kv_cache, + hidden_states=outputs.hidden_states, + ) + + +LlamaForCausalLM = Llama3ForCausalLM diff --git a/skyrl-tx/tx/models/mnist.py b/skyrl-tx/tx/models/mnist.py new file mode 100644 index 0000000000..31c87c011d --- /dev/null +++ b/skyrl-tx/tx/models/mnist.py @@ -0,0 +1,24 @@ +from flax import nnx +from functools import partial + + +class Mnist(nnx.Module): + + def __init__(self, *, rngs: nnx.Rngs): + self.conv1 = nnx.Conv(1, 32, kernel_size=(3, 3), rngs=rngs) + self.batch_norm1 = nnx.BatchNorm(32, rngs=rngs) + self.dropout1 = nnx.Dropout(rate=0.025, rngs=rngs) + self.conv2 = nnx.Conv(32, 64, kernel_size=(3, 3), rngs=rngs) + self.batch_norm2 = nnx.BatchNorm(64, rngs=rngs) + self.avg_pool = partial(nnx.avg_pool, window_shape=(2, 2), strides=(2, 2)) + self.linear1 = nnx.Linear(3136, 256, rngs=rngs) + self.dropout2 = nnx.Dropout(rate=0.025, rngs=rngs) + self.linear2 = nnx.Linear(256, 10, rngs=rngs) + + def __call__(self, x): + x = self.avg_pool(nnx.relu(self.batch_norm1(self.dropout1(self.conv1(x))))) + x = self.avg_pool(nnx.relu(self.batch_norm2(self.conv2(x)))) + x = x.reshape(x.shape[0], -1) # flatten + x = nnx.relu(self.dropout2(self.linear1(x))) + x = self.linear2(x) + return x diff --git a/skyrl-tx/tx/models/qwen3.py b/skyrl-tx/tx/models/qwen3.py new file mode 100644 index 0000000000..cdc9c3a760 --- /dev/null +++ b/skyrl-tx/tx/models/qwen3.py @@ -0,0 +1,437 @@ +from flax import nnx +import jax +from jax import numpy as jnp +from jax.sharding import get_abstract_mesh + +from tx.layers.lora import LoRAEmbed, LoRAExpert, LoRALinear +from tx.layers.util import prepare_routing, shard_map_ep +from tx.layers.rotary_embedding import apply_rope +from tx.models.configs import Qwen3Config +from tx.layers.layernorm import RMSNorm +from tx.models.types import CausalLMOutput, ModelOutput +from tx.utils.generator import GeneratorMixin, KVCache, compute_positions + + +class Qwen3Attention(nnx.Module): + """Multi-head attention with Grouped Query Attention (GQA) support.""" + + def __init__(self, config: Qwen3Config, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + self.num_heads = config.num_attention_heads + self.num_kv_heads = config.num_key_value_heads + tp = get_abstract_mesh().shape.get("tp", 1) + shard_attention_heads = config.shard_attention_heads + if shard_attention_heads: + assert self.num_heads % tp == 0, f"num_heads={self.num_heads} must be divisible by tp={tp}" + assert self.num_kv_heads % tp == 0, f"num_kv_heads={self.num_kv_heads} must be divisible by tp={tp}" + tp_shard = "tp" if shard_attention_heads else None + self.head_dim = getattr(config, "head_dim", None) or config.hidden_size // self.num_heads + + self.q_proj = LoRALinear( + in_features=config.hidden_size, + out_features=self.num_heads * self.head_dim, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("fsdp", tp_shard)), + rngs=rngs, + ) + self.k_proj = LoRALinear( + in_features=config.hidden_size, + out_features=self.num_kv_heads * self.head_dim, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("fsdp", tp_shard)), + rngs=rngs, + ) + self.v_proj = LoRALinear( + in_features=config.hidden_size, + out_features=self.num_kv_heads * self.head_dim, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("fsdp", tp_shard)), + rngs=rngs, + ) + self.o_proj = LoRALinear( + in_features=self.num_heads * self.head_dim, + out_features=config.hidden_size, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + param_dtype=dtype, + use_bias=False, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (tp_shard, "fsdp")), + rngs=rngs, + ) + + self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + + def __call__( + self, + x: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array, + adapter_indices: jax.Array | None = None, + kv_cache: tuple[jax.Array, jax.Array, int] | None = None, + ) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + B, T, _ = x.shape + + # Project and reshape to [B, T, num_heads, head_dim] + q = self.q_norm(self.q_proj(x, adapter_indices=adapter_indices).reshape(B, T, self.num_heads, self.head_dim)) + k = self.k_norm(self.k_proj(x, adapter_indices=adapter_indices).reshape(B, T, self.num_kv_heads, self.head_dim)) + v = self.v_proj(x, adapter_indices=adapter_indices).reshape(B, T, self.num_kv_heads, self.head_dim) + + # Apply RoPE + q = apply_rope(q, positions, self.head_dim, self.config.rope_theta) + k = apply_rope(k, positions, self.head_dim, self.config.rope_theta) + + # Handle KV cache + if kv_cache is not None: + k_cache, v_cache, cache_position = kv_cache + k = jax.lax.dynamic_update_slice(k_cache, k, (0, cache_position, 0, 0)) + v = jax.lax.dynamic_update_slice(v_cache, v, (0, cache_position, 0, 0)) + + updated_cache = (k, v) + + # Attention (causal only during prefill, GQA handled natively by dot_product_attention) + attn_output = jax.nn.dot_product_attention( + q, + k, + v, + scale=1.0 / self.head_dim**0.5, + mask=attention_mask[:, None, None, :].astype(bool), + is_causal=kv_cache is None, + ) + + output = attn_output.reshape(B, T, self.num_heads * self.head_dim) + return self.o_proj(output, adapter_indices=adapter_indices), updated_cache + + +class Qwen3MLP(nnx.Module): + + def __init__(self, config: Qwen3Config, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.gate_proj = LoRALinear( + config.hidden_size, + config.intermediate_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("fsdp", "tp")), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + self.up_proj = LoRALinear( + config.hidden_size, + config.intermediate_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("fsdp", "tp")), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + self.down_proj = LoRALinear( + config.intermediate_size, + config.hidden_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("tp", "fsdp")), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + + def __call__(self, x: jax.Array, adapter_indices: jax.Array | None = None) -> jax.Array: + gate_out = self.gate_proj(x, adapter_indices) + up_out = self.up_proj(x, adapter_indices) + return self.down_proj(nnx.silu(gate_out) * up_out, adapter_indices) + + +class Qwen3Experts(nnx.Module): + + def __init__(self, config: Qwen3Config, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + self.gate_proj = LoRAExpert( + config.num_experts, + config.hidden_size, + config.moe_intermediate_size, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("ep", "fsdp", "tp")), + rngs=rngs, + ) + self.up_proj = LoRAExpert( + config.num_experts, + config.hidden_size, + config.moe_intermediate_size, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("ep", "fsdp", "tp")), + rngs=rngs, + ) + self.down_proj = LoRAExpert( + config.num_experts, + config.moe_intermediate_size, + config.hidden_size, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), ("ep", "tp", "fsdp")), + rngs=rngs, + ) + + def __call__( + self, hidden_states: jax.Array, router_logits: jax.Array, adapter_indices: jax.Array | None = None + ) -> jax.Array: + routing_weights, selected_experts = jax.lax.top_k(router_logits, k=self.config.num_experts_per_tok) + routing_weights = nnx.softmax(routing_weights, axis=-1) + + num_experts = self.config.num_experts + num_experts_per_tok = self.config.num_experts_per_tok + hidden_size = self.config.hidden_size + + ep = get_abstract_mesh().shape.get("ep", 1) + assert num_experts % ep == 0, f"num_experts={num_experts} must be divisible by ep={ep}" + + # Prepare routing (inputs are replicated, so every rank generates the same sorted lists) + hidden_expanded = jnp.repeat(hidden_states, num_experts_per_tok, axis=0) + adapter_expanded = jnp.repeat(adapter_indices, num_experts_per_tok) if adapter_indices is not None else None + hidden_sorted, group_sizes, unsort_indices, adapter_sorted = prepare_routing( + hidden_expanded, selected_experts.ravel(), num_experts, adapter_indices=adapter_expanded + ) + + def forward(experts, hidden_sorted, group_sizes, unsort_indices, adapter_sorted, routing_weights): + # Calculate local offset for this shard + ep_rank = jax.lax.axis_index("ep") + experts_per_rank = num_experts // jax.lax.axis_size("ep") + group_offset = jnp.array([ep_rank * experts_per_rank], dtype=jnp.int32) + + # Expert computation + gate = experts.gate_proj(hidden_sorted, group_sizes, adapter_sorted, group_offset=group_offset) + up = experts.up_proj(hidden_sorted, group_sizes, adapter_sorted, group_offset=group_offset) + down = experts.down_proj(nnx.silu(gate) * up, group_sizes, adapter_sorted, group_offset=group_offset) + + # Unsort and combine + out = down[unsort_indices].reshape(-1, num_experts_per_tok, hidden_size) + local_out = jnp.sum(out * routing_weights[..., None], axis=1) + return jax.lax.psum(local_out, axis_name="ep") + + return shard_map_ep(self, forward, hidden_sorted, group_sizes, unsort_indices, adapter_sorted, routing_weights) + + +class Qwen3MoeSparseMoeBlock(nnx.Module): + + def __init__(self, config: Qwen3Config, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + self.gate = nnx.Linear( + config.hidden_size, + config.num_experts, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, None)), + rngs=rngs, + ) + self.experts = Qwen3Experts(config, dtype=dtype, rngs=rngs) + + def __call__( + self, + hidden_states: jax.Array, + *, + adapter_indices: jax.Array | None = None, + return_router_logits: bool = False, + ) -> jax.Array | tuple[jax.Array, jax.Array]: + (batch_size, seq_len, hidden_size) = hidden_states.shape + hidden_states = hidden_states.reshape(-1, hidden_size) + # Expand adapter_indices to match flattened hidden_states + if adapter_indices is not None: + adapter_indices = jnp.repeat(adapter_indices, seq_len) + router_logits = self.gate(hidden_states) + + hidden_states = self.experts(hidden_states, router_logits, adapter_indices) + hidden_states = hidden_states.reshape(batch_size, seq_len, hidden_size) + + if return_router_logits: + return hidden_states, router_logits + return hidden_states + + +class Qwen3DecoderLayer(nnx.Module): + + def __init__(self, config: Qwen3Config, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + self.self_attn = Qwen3Attention(config, dtype=dtype, rngs=rngs) + if getattr(config, "num_experts", None): + self.mlp = Qwen3MoeSparseMoeBlock(config, dtype=dtype, rngs=rngs) + else: + self.mlp = Qwen3MLP(config, dtype=dtype, rngs=rngs) + + def __call__( + self, + hidden_states: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array, + adapter_indices: jax.Array | None = None, + kv_cache: tuple[jax.Array, jax.Array, int] | None = None, + ) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states, updated_cache = self.self_attn( + hidden_states, + attention_mask=attention_mask, + positions=positions, + adapter_indices=adapter_indices, + kv_cache=kv_cache, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + mlp_output = self.mlp(hidden_states, adapter_indices=adapter_indices) + hidden_states = residual + mlp_output + + return hidden_states, updated_cache + + +class Qwen3Model(nnx.Module): + + def __init__(self, config: Qwen3Config, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + + self.embed_tokens = LoRAEmbed( + num_embeddings=config.vocab_size, + features=config.hidden_size, + dtype=dtype, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + param_dtype=dtype, + embedding_init=nnx.with_partitioning(nnx.initializers.normal(), ("tp", None)), + rngs=rngs, + ) + self.layers = nnx.List( + [Qwen3DecoderLayer(config, dtype=dtype, rngs=rngs) for _ in range(config.num_hidden_layers)] + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype, rngs=rngs) + + def __call__( + self, + input_ids: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array, + output_hidden_states: bool | None = None, + adapter_indices: jax.Array | None = None, + kv_cache: KVCache | None = None, + ) -> ModelOutput: + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + hidden_states = self.embed_tokens(input_ids, adapter_indices=adapter_indices) + all_hidden_states: list[jax.Array] = [] + updated_keys, updated_values = [], [] + + for layer_idx, layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states.append(hidden_states) + + hidden_states, (k, v) = layer( + hidden_states, + attention_mask=attention_mask, + positions=positions, + adapter_indices=adapter_indices, + kv_cache=kv_cache and (kv_cache.keys[layer_idx], kv_cache.values[layer_idx], kv_cache.cache_position), + ) + updated_keys.append(k) + updated_values.append(v) + + hidden_states = self.norm(hidden_states) + if output_hidden_states: + all_hidden_states.append(hidden_states) + + # Increment cache_position if cache exists, or use sequence length for new cache + new_cache_position = kv_cache.cache_position + 1 if kv_cache is not None else input_ids.shape[1] + + return ModelOutput( + last_hidden_state=hidden_states, + kv_cache=KVCache(keys=updated_keys, values=updated_values, cache_position=new_cache_position), + hidden_states=all_hidden_states if output_hidden_states else None, + ) + + +class Qwen3ForCausalLM(nnx.Module, GeneratorMixin): + + def __init__(self, config: Qwen3Config, *, dtype: jnp.dtype, rngs: nnx.Rngs) -> None: + self.config = config + self.model = Qwen3Model(config, dtype=dtype, rngs=rngs) + if not self.config.tie_word_embeddings: + self.lm_head = LoRALinear( + config.hidden_size, + config.vocab_size, + use_bias=False, + dtype=dtype, + param_dtype=dtype, + kernel_init=nnx.with_partitioning(nnx.initializers.lecun_normal(), (None, "tp")), + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + rngs=rngs, + ) + + @staticmethod + def is_lora_param(path: tuple, _value) -> bool: + """Return True if a parameter path corresponds to LoRA weights.""" + return any(name in path for name in ("lora_A", "lora_B")) + + def __call__( + self, + input_ids: jax.Array, + *, + attention_mask: jax.Array, + positions: jax.Array | None = None, + output_hidden_states: bool | None = None, + adapter_indices: jax.Array | None = None, + kv_cache: KVCache | None = None, + ) -> CausalLMOutput: + if positions is None: + positions = compute_positions(attention_mask) + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + positions=positions, + output_hidden_states=output_hidden_states, + adapter_indices=adapter_indices, + kv_cache=kv_cache, + ) + hidden_states = outputs.last_hidden_state + if self.config.tie_word_embeddings: + logits = hidden_states @ self.model.embed_tokens.embedding.value.T + else: + logits = self.lm_head(hidden_states, adapter_indices=adapter_indices) + + return CausalLMOutput( + logits=logits, + last_hidden_state=outputs.last_hidden_state, + kv_cache=outputs.kv_cache, + hidden_states=outputs.hidden_states, + ) + + +Qwen3MoeForCausalLM = Qwen3ForCausalLM diff --git a/skyrl-tx/tx/models/types.py b/skyrl-tx/tx/models/types.py new file mode 100644 index 0000000000..0369a3750a --- /dev/null +++ b/skyrl-tx/tx/models/types.py @@ -0,0 +1,48 @@ +"""Model output dataclasses.""" + +from __future__ import annotations +from dataclasses import dataclass +from typing import Protocol + +import jax +from transformers import PretrainedConfig + +from tx.utils.generator import KVCache + + +class ModelForCausalLM(Protocol): + config: PretrainedConfig + + +@jax.tree_util.register_dataclass +@dataclass +class ModelOutput: + """Output type for models like Qwen3Model. + + Attributes: + last_hidden_state: The last hidden state from the model. + kv_cache: The updated key-value cache. + hidden_states: All hidden states if output_hidden_states=True. + """ + + last_hidden_state: jax.Array + kv_cache: KVCache + hidden_states: list[jax.Array] | None = None + + +@jax.tree_util.register_dataclass +@dataclass +class CausalLMOutput: + """Output type for causal language models like Qwen3ForCausalLM. + + Attributes: + logits: The language modeling logits. + last_hidden_state: The last hidden state from the model. + kv_cache: The updated key-value cache. + hidden_states: All hidden states, if output_hidden_states=True. + """ + + logits: jax.Array + last_hidden_state: jax.Array + kv_cache: KVCache + hidden_states: list[jax.Array] | None = None diff --git a/skyrl-tx/tx/run/main.py b/skyrl-tx/tx/run/main.py new file mode 100644 index 0000000000..25f47f0b2b --- /dev/null +++ b/skyrl-tx/tx/run/main.py @@ -0,0 +1,17 @@ +import typer + +import tx +from tx.run.train import train + +app = typer.Typer() + +app.command(help="Train a model")(train) + + +@app.command() +def version(): + typer.echo(f"tx v{tx.__version__}") + + +if __name__ == "__main__": + app() diff --git a/skyrl-tx/tx/run/train.py b/skyrl-tx/tx/run/train.py new file mode 100644 index 0000000000..1d6ff22d0d --- /dev/null +++ b/skyrl-tx/tx/run/train.py @@ -0,0 +1,109 @@ +import json +from pathlib import Path +import sys + +from datasets import Dataset, load_dataset +import jax +from flax import nnx +import optax +from transformers import AutoConfig, AutoTokenizer +import typer + +from tx.loaders import get_loader +from tx.models.configs import Qwen3Config +from tx.utils.models import OptimizerName, get_dtype, get_model_class, get_optimizer, load_safetensors, save_safetensors +from tx.utils.log import ExperimentTracker, add_file_handler, get_tracker, logger + +app = typer.Typer() + + +def loss_fn(model, batch): + output = model(batch["text"], attention_mask=batch["attention_mask"]) + loss = optax.softmax_cross_entropy_with_integer_labels(logits=output.logits, labels=batch["target"]) + return loss.mean(), output.logits + + +@nnx.jit +def train_step(model, optimizer: nnx.Optimizer, batch): + grad_fn = nnx.value_and_grad(loss_fn, has_aux=True) + (loss, logits), grads = grad_fn(model, batch) + gradnorm = optax.global_norm(grads) + optimizer.update(model, grads) + return loss, gradnorm + + +def train( + model_name: str = typer.Option(..., "--model", help="HuggingFace model ID or local model path"), + dataset: str = typer.Option(..., "--dataset", help="HuggingFace dataset to use for training"), + loader_name: str = typer.Option("tx.loaders.text", "--loader", help="Loader used for loading the dataset"), + split: str = typer.Option("train", "--split", help="The dataset split to use"), + output_dir: Path = typer.Option( + ..., "--output-dir", help="The output directory where the model predictions and checkpoints will be written" + ), + load_checkpoint_path: Path | None = typer.Option( + None, "--load-checkpoint-path", help="If specified, resume training from this checkpoint" + ), + save_steps: int = typer.Option(500, "--save-steps", help="Number of steps between checkpoints"), + max_steps: int | None = typer.Option(None, "--max-steps", help="The maximum number of training steps"), + batch_size: int = typer.Option(..., "--batch-size", help="Batch size of each training batch"), + optimizer_name: OptimizerName = typer.Option("adamw", "--optimizer", help="Which optax optimizer to use"), + optimizer_args: dict = typer.Option( + '{"learning_rate": 1e-5, "weight_decay": 0.1}', + "--optimizer-args", + help="Arguments for the optax optimizer (in JSON format)", + parser=json.loads, + ), + tp_size: int = typer.Option(1, "--tp-size", help="Tensor parallelism degree to use for the model"), + tracker_name: ExperimentTracker | None = typer.Option( + None, "--tracker", help="Experiment tracker to report results to" + ), + tracker_args: dict = typer.Option( + "{}", + "--tracker-args", + help="Arguments that will be passed to the experiment tracker (in JSON format)", + parser=json.loads, + ), +) -> None: + if not jax._src.xla_bridge.backends_are_initialized(): # ty: ignore + jax.config.update("jax_num_cpu_devices", tp_size) + # If you want to debug NaNs, add the following: + # jax.config.update("jax_debug_nans", True) + + output_dir.mkdir(parents=True, exist_ok=True) + add_file_handler(output_dir / "tx.log") + logger.info(f"tx was invoked with 'tx {' '.join(sys.argv[1:])}'") + + train_dataset = load_dataset(dataset, split=split) + assert isinstance(train_dataset, Dataset) + base_config = AutoConfig.from_pretrained(model_name) + config = Qwen3Config(base_config, max_lora_adapters=0, max_lora_rank=0, shard_attention_heads=True) + tokenizer = AutoTokenizer.from_pretrained(model_name) + tracker = get_tracker(tracker_name, base_config, **tracker_args) + loader = get_loader(loader_name) + + model_class = get_model_class(base_config) + mesh = jax.make_mesh((1, 1, tp_size), ("fsdp", "ep", "tp")) + with jax.set_mesh(mesh): + model = model_class(config, dtype=get_dtype(config.dtype), rngs=nnx.Rngs(0)) + optimizer = nnx.Optimizer(model, get_optimizer(optimizer_name, optimizer_args), wrt=nnx.Param) + + if load_checkpoint_path: + load_safetensors(load_checkpoint_path, base_config, model) + + num_steps = train_dataset.num_rows / batch_size + for step, (batch, metrics) in enumerate(loader(tokenizer, train_dataset, batch_size)): + if max_steps and step >= max_steps: + break + + model.train() + loss, gradnorm = train_step(model, optimizer, batch) + tracker.log({"epoch": step / num_steps, **metrics, "gradnorm": gradnorm.item(), "loss": loss.item()}, step) + + if step % save_steps == 0: + logger.info(f"Saving checkpoint to {output_dir}") + save_safetensors(base_config, model, output_dir / "model.safetensors") + + logger.info(f"Saving final checkpoint to {output_dir}") + base_config.save_pretrained(output_dir) + tokenizer.save_pretrained(output_dir) + save_safetensors(base_config, model, output_dir / "model.safetensors") diff --git a/skyrl-tx/tx/tinker/alembic.ini b/skyrl-tx/tx/tinker/alembic.ini new file mode 100644 index 0000000000..d3eae3e16f --- /dev/null +++ b/skyrl-tx/tx/tinker/alembic.ini @@ -0,0 +1,145 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# Use absolute path to make it work from any directory +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/skyrl-tx/tx/tinker/alembic/README b/skyrl-tx/tx/tinker/alembic/README new file mode 100644 index 0000000000..98e4f9c44e --- /dev/null +++ b/skyrl-tx/tx/tinker/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/skyrl-tx/tx/tinker/alembic/env.py b/skyrl-tx/tx/tinker/alembic/env.py new file mode 100644 index 0000000000..d4a75c2159 --- /dev/null +++ b/skyrl-tx/tx/tinker/alembic/env.py @@ -0,0 +1,83 @@ +from logging.config import fileConfig +import os +import sys +from pathlib import Path + +from sqlalchemy import pool + +from alembic import context + +# Add parent directory to path so we can import tx modules +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +# Import SQLModel and database models +from sqlmodel import SQLModel + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# Use SQLModel metadata which includes all our table definitions +target_metadata = SQLModel.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + from sqlalchemy import create_engine + + # Get database URL - ignore whatever is in config, use our helper + db_url = os.environ["TX_DATABASE_URL"] + connectable = create_engine(db_url, poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/skyrl-tx/tx/tinker/alembic/script.py.mako b/skyrl-tx/tx/tinker/alembic/script.py.mako new file mode 100644 index 0000000000..11016301e7 --- /dev/null +++ b/skyrl-tx/tx/tinker/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/skyrl-tx/tx/tinker/api.py b/skyrl-tx/tx/tinker/api.py new file mode 100644 index 0000000000..ba425e6a1d --- /dev/null +++ b/skyrl-tx/tx/tinker/api.py @@ -0,0 +1,1135 @@ +import fastapi +from fastapi import FastAPI, HTTPException, Depends, Request +from fastapi.responses import StreamingResponse, RedirectResponse +from pydantic import BaseModel, Field, model_validator +from typing import Literal, Any, AsyncGenerator +from datetime import datetime, timedelta, timezone +from uuid import uuid4 +from contextlib import asynccontextmanager +from sqlmodel import SQLModel, select, func +from sqlmodel.ext.asyncio.session import AsyncSession +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.exc import IntegrityError, TimeoutError as SATimeoutError +import asyncio +import subprocess +import random +import time + +from tx.tinker import types +from tx.tinker.config import EngineConfig, add_model, config_to_argv +from tx.tinker.db_models import ( + CheckpointDB, + ModelDB, + FutureDB, + RequestStatus, + CheckpointStatus, + SessionDB, + SamplingSessionDB, + get_async_database_url, +) +from tx.tinker.extra import ExternalInferenceClient +from tx.utils.storage import download_file +from tx.utils.log import logger + +# Validation patterns for train_run_ids, model_ids and checkpoint_ids +ID_PATTERN = r"^[a-zA-Z0-9_-]+$" +ID_MAX_LENGTH = 255 + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Lifespan event handler for startup and shutdown.""" + + db_url = get_async_database_url(app.state.engine_config.database_url) + app.state.db_engine = create_async_engine(db_url, echo=False) + + async with app.state.db_engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + + # Setup external inference client if configured + if app.state.engine_config.external_inference_url: + app.state.external_inference_client = ExternalInferenceClient(app.state.engine_config, app.state.db_engine) + logger.info(f"External engine configured: {app.state.engine_config.external_inference_url}") + else: + app.state.external_inference_client = None + logger.info("Using internal engine for inference") + + # Build subprocess command with engine config parameters + cmd = ["uv", "run", "--extra", "tinker", "-m", "tx.tinker.engine"] + cmd.extend(config_to_argv(app.state.engine_config)) + + background_engine = subprocess.Popen(cmd) + logger.info(f"Started background engine with PID {background_engine.pid}: {' '.join(cmd)}") + + yield + + logger.info(f"Stopping background engine (PID {background_engine.pid})") + background_engine.terminate() + try: + background_engine.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.warning(f"Background engine (PID {background_engine.pid}) did not terminate gracefully, killing") + background_engine.kill() + background_engine.wait() + logger.info("Background engine stopped") + + +app = FastAPI(title="Tinker API Mock", version="0.0.1", lifespan=lifespan) + + +async def get_session(request: Request) -> AsyncGenerator[AsyncSession, None]: + """Dependency to get a database session.""" + async with AsyncSession(request.app.state.db_engine) as session: + yield session + + +async def get_model(session: AsyncSession, model_id: str) -> ModelDB: + """Fetch a model by ID, raising 404 if not found.""" + statement = select(ModelDB).where(ModelDB.model_id == model_id) + result = await session.exec(statement) + model = result.first() + if not model: + raise HTTPException(status_code=404, detail="Model not found") + return model + + +async def create_future( + session: AsyncSession, + request_type: types.RequestType, + model_id: str | None, + request_data: BaseModel, +) -> int: + """Create a FutureDB entry and return its auto-generated request_id.""" + future_db = FutureDB( + request_type=request_type, + model_id=model_id, + request_data=request_data.model_dump(), + status=RequestStatus.PENDING, + ) + session.add(future_db) + await session.flush() # Flush to generate auto-increment request_id + assert future_db.request_id + return future_db.request_id + + +async def create_checkpoint( + session: AsyncSession, + model_id: str, + checkpoint_id: str, + checkpoint_type: types.CheckpointType, +): + """Create a pending CheckpointDB entry, relying on database constraints for validation.""" + checkpoint_db = CheckpointDB( + model_id=model_id, + checkpoint_id=checkpoint_id, + checkpoint_type=checkpoint_type, + status=CheckpointStatus.PENDING, + ) + session.add(checkpoint_db) + + try: + await session.flush() + except IntegrityError: + # Determine which constraint failed by checking if the model exists + statement = select(ModelDB).where(ModelDB.model_id == model_id) + result = await session.exec(statement) + + if not result.first(): + raise HTTPException(status_code=404, detail=f"Model '{model_id}' not found") + else: + raise HTTPException( + status_code=409, detail=f"Checkpoint '{checkpoint_id}' already exists for model '{model_id}'" + ) + + +class LoRAConfig(BaseModel): + rank: int + seed: int | None = Field( + default=None, description="Seed for LoRA weight initialization. If None, a random seed is used." + ) + + +class CreateModelRequest(BaseModel): + session_id: str + base_model: str + lora_config: LoRAConfig + + +class CreateModelResponse(BaseModel): + model_id: str + base_model: str + lora_config: LoRAConfig | None = None + status: str = "created" + request_id: str + + +class UnloadModelRequest(BaseModel): + model_id: str + type: str | None = None + + +class UnloadModelResponse(BaseModel): + request_id: str + model_id: str + + +class ModelData(BaseModel): + base_model: str + lora_config: LoRAConfig | None = None + model_name: str | None = None + + +class ModelInfoResponse(BaseModel): + model_id: str + status: str + model_data: ModelData + + +class Checkpoint(BaseModel): + checkpoint_id: str + checkpoint_type: Literal["training", "sampler"] + time: datetime + tinker_path: str + + +class TrainingRun(BaseModel): + training_run_id: str + base_model: str + model_owner: str = "default" + is_lora: bool = True + corrupted: bool = False + lora_rank: int | None = None + last_request_time: datetime + last_checkpoint: Checkpoint | None = None + last_sampler_checkpoint: Checkpoint | None = None + user_metadata: dict[str, str] | None = None + + +class ModelInputChunk(BaseModel): + tokens: list[int] + + def to_types(self) -> types.ModelInputChunk: + return types.ModelInputChunk(tokens=self.tokens) + + +class ModelInput(BaseModel): + chunks: list[ModelInputChunk] + + def to_types(self) -> types.ModelInput: + return types.ModelInput(chunks=[chunk.to_types() for chunk in self.chunks]) + + +class TensorData(BaseModel): + data: list[int] | list[float] + + def to_types(self) -> types.TensorData: + return types.TensorData(data=self.data) + + +class Datum(BaseModel): + loss_fn_inputs: dict[str, TensorData] + model_input: ModelInput + + def to_types(self) -> types.Datum: + inp = self.loss_fn_inputs + + if "weights" not in inp: + weights = types.TensorData(data=[1.0] * len(inp["target_tokens"].data)) + else: + weights = inp["weights"].to_types() + + return types.Datum( + loss_fn_inputs=types.LossFnInputs( + target_tokens=inp["target_tokens"].to_types(), + weights=weights, + advantages=inp["advantages"].to_types() if "advantages" in inp else types.TensorData(data=[]), + logprobs=inp["logprobs"].to_types() if "logprobs" in inp else types.TensorData(data=[]), + ), + model_input=self.model_input.to_types(), + ) + + +class ForwardBackwardInput(BaseModel): + data: list[Datum] + loss_fn: Literal["cross_entropy", "importance_sampling", "ppo"] + + def to_types(self) -> types.ForwardBackwardInput: + return types.ForwardBackwardInput(data=[datum.to_types() for datum in self.data], loss_fn=self.loss_fn) + + +class ForwardBackwardRequest(BaseModel): + model_id: str + forward_backward_input: ForwardBackwardInput + + +class ForwardRequest(BaseModel): + model_id: str + forward_input: ForwardBackwardInput + + +class AdamParams(BaseModel): + learning_rate: float = Field(default=1e-4, ge=0.0) + beta1: float = Field(default=0.9, ge=0.0, lt=1.0) + beta2: float = Field(default=0.95, ge=0.0, lt=1.0) + eps: float = Field(default=1e-12, gt=0.0) + weight_decay: float = Field(default=0.0, ge=0.0) + + def to_types(self) -> types.AdamParams: + return types.AdamParams( + learning_rate=self.learning_rate, + beta1=self.beta1, + beta2=self.beta2, + eps=self.eps, + weight_decay=self.weight_decay, + ) + + +class OptimStepRequest(BaseModel): + model_id: str + adam_params: AdamParams + + +class SaveWeightsForSamplerRequest(BaseModel): + model_id: str + path: str | None = Field(default=None, pattern=ID_PATTERN, max_length=ID_MAX_LENGTH) + sampling_session_seq_id: int | None = None + seq_id: int | None = None + type: Literal["save_weights_for_sampler"] = "save_weights_for_sampler" + + @model_validator(mode="after") + def check_path_or_ids(self): + if not self.path and (self.sampling_session_seq_id is None or self.seq_id is None): + raise ValueError("Either 'path' or both 'sampling_session_seq_id' and 'seq_id' must be provided") + return self + + +class SamplingParams(BaseModel): + max_tokens: int | None = None + seed: int | None = None + stop: list[int] | list[str] | None = None + temperature: float = 1 + top_k: int = -1 + top_p: float = 1 + + def to_types(self) -> types.SamplingParams: + if self.max_tokens is None: + raise HTTPException(status_code=400, detail="max_tokens is currently required") + + # Generate a random seed if not provided + seed = self.seed if self.seed is not None else random.randint(0, 2**31 - 1) + + # Determine if stop values are token IDs (int) or strings + stop_tokens = None + stop_strings = None + if self.stop: + if all(isinstance(s, int) for s in self.stop): + stop_tokens = list(self.stop) + elif all(isinstance(s, str) for s in self.stop): + stop_strings = list(self.stop) + else: + raise HTTPException( + status_code=400, + detail="stop must be either all integers (token IDs) or all strings, not mixed", + ) + + return types.SamplingParams( + temperature=self.temperature, + max_tokens=self.max_tokens, + seed=seed, + stop_tokens=stop_tokens, + stop_strings=stop_strings, + top_k=self.top_k, + top_p=self.top_p, + ) + + +class SampleRequest(BaseModel): + num_samples: int = 1 + prompt: ModelInput + sampling_params: SamplingParams + base_model: str | None = None + model_path: str | None = None + sampling_session_id: str | None = None + seq_id: int | None = None + prompt_logprobs: bool | None = None + topk_prompt_logprobs: int = 0 + type: Literal["sample"] = "sample" + + @model_validator(mode="after") + def validate_model_source(self): + """Valid if: + - sampling_session_id is provided AND seq_id is provided + - OR exactly one of base_model or model_path is provided + """ + if self.sampling_session_id is not None: + if self.seq_id is None: + raise ValueError("'seq_id' must be provided when 'sampling_session_id' is used") + return self + if (self.base_model is None) == (self.model_path is None): + raise ValueError( + "When 'sampling_session_id' is not provided, exactly one of 'base_model' or 'model_path' must be provided" + ) + return self + + +class SaveWeightsRequest(BaseModel): + model_id: str + path: str = Field(..., pattern=ID_PATTERN, max_length=ID_MAX_LENGTH) + type: Literal["save_weights"] | None = None + + +class LoadWeightsRequest(BaseModel): + model_id: str + path: str + type: Literal["load_weights"] | None = None + + +class FutureResponse(BaseModel): + future_id: str + status: str = "pending" + request_id: str + + +class TelemetryEvent(BaseModel): + event: str + event_id: str + event_session_index: int + severity: str + timestamp: str + properties: dict[str, Any] | None = None + + +class TelemetryRequest(BaseModel): + events: list[TelemetryEvent] + platform: str + sdk_version: str + session_id: str + + +class TelemetryResponse(BaseModel): + status: Literal["accepted"] = "accepted" + + +class HealthResponse(BaseModel): + status: Literal["ok"] + + +class CreateSessionRequest(BaseModel): + tags: list[str] + user_metadata: dict[str, Any] | None = None + sdk_version: str + type: Literal["create_session"] = "create_session" + + +class CreateSessionResponse(BaseModel): + type: Literal["create_session"] = "create_session" + info_message: str | None = None + warning_message: str | None = None + error_message: str | None = None + session_id: str + + +class SessionHeartbeatRequest(BaseModel): + session_id: str + type: Literal["session_heartbeat"] = "session_heartbeat" + + +class SessionHeartbeatResponse(BaseModel): + type: Literal["session_heartbeat"] = "session_heartbeat" + + +class CreateSamplingSessionRequest(BaseModel): + session_id: str + sampling_session_seq_id: int + base_model: str | None = None + model_path: str | None = None + type: Literal["create_sampling_session"] = "create_sampling_session" + + +class CreateSamplingSessionResponse(BaseModel): + type: Literal["create_sampling_session"] = "create_sampling_session" + sampling_session_id: str + + +class SupportedModel(BaseModel): + model_name: str + + +class GetServerCapabilitiesResponse(BaseModel): + supported_models: list[SupportedModel] + + +class ListCheckpointsResponse(BaseModel): + checkpoints: list[Checkpoint] + + +class Cursor(BaseModel): + offset: int + limit: int + total_count: int + + +class TrainingRunsResponse(BaseModel): + training_runs: list[TrainingRun] + cursor: Cursor + + +class WeightsInfoRequest(BaseModel): + tinker_path: str + + +class WeightsInfoResponse(BaseModel): + """Minimal information for loading public checkpoints.""" + + # from: https://github.com/thinking-machines-lab/tinker/blob/main/src/tinker/types/weights_info_response.py + base_model: str + is_lora: bool + lora_rank: int | None = None + + +@app.get("/api/v1/healthz", response_model=HealthResponse) +async def healthz(): + """Checks if the API server is ready.""" + return HealthResponse(status="ok") + + +@app.post("/api/v1/create_session", response_model=CreateSessionResponse) +async def create_session(request: CreateSessionRequest, session: AsyncSession = Depends(get_session)): + """Create a new session + persist in DB""" + session_id = f"session_{uuid4().hex[:8]}" + session_db = SessionDB( + session_id=session_id, + tags=request.tags, + user_metadata=request.user_metadata or {}, + sdk_version=request.sdk_version, + status="active", + ) + session.add(session_db) + await session.commit() + return CreateSessionResponse(session_id=session_id) + + +@app.post("/api/v1/session_heartbeat", response_model=SessionHeartbeatResponse) +async def session_heartbeat(request: SessionHeartbeatRequest, session: AsyncSession = Depends(get_session)): + """Heartbeat for an active session to keep it alive.""" + session_db = await session.get(SessionDB, request.session_id) + if session_db is None: + raise HTTPException(status_code=404, detail="Session not found") + session_db.last_heartbeat_at = datetime.now(timezone.utc) + session_db.heartbeat_count += 1 + await session.commit() + return SessionHeartbeatResponse() + + +@app.post("/api/v1/create_sampling_session", response_model=CreateSamplingSessionResponse) +async def create_sampling_session(request: CreateSamplingSessionRequest, session: AsyncSession = Depends(get_session)): + """Create a new sampling session within an existing session.""" + session_db = await session.get(SessionDB, request.session_id) + if session_db is None: + raise HTTPException(status_code=404, detail="Session not found") + # Exactly one of base_model or model_path must be provided + if (request.base_model is None) == (request.model_path is None): + raise HTTPException(status_code=400, detail="Exactly one of base_model or model_path must be provided") + sampling_session_id = f"sampling_{uuid4().hex[:8]}" + sampling_db = SamplingSessionDB( + sampling_session_id=sampling_session_id, + session_id=request.session_id, + sampling_session_seq_id=request.sampling_session_seq_id, + base_model=request.base_model, + model_path=request.model_path, + ) + session.add(sampling_db) + await session.commit() + return CreateSamplingSessionResponse(sampling_session_id=sampling_session_id) + + +@app.post("/api/v1/create_model", response_model=CreateModelResponse) +async def create_model(request: CreateModelRequest, session: AsyncSession = Depends(get_session)): + """Create a new model, optionally with a LoRA adapter.""" + # Validate session exists + session_db = await session.get(SessionDB, request.session_id) + if session_db is None: + raise HTTPException(status_code=404, detail="Session not found") + + model_id = f"model_{uuid4().hex[:8]}" + + # alpha = 32 seems to be the tinker default (see https://thinkingmachines.ai/blog/lora/) + # Generate a random seed if not provided + seed = request.lora_config.seed if request.lora_config.seed is not None else random.randint(0, 2**31 - 1) + lora_config = types.LoraConfig(rank=request.lora_config.rank, alpha=32.0, seed=seed) + request_id = await create_future( + session=session, + request_type=types.RequestType.CREATE_MODEL, + model_id=model_id, + request_data=types.CreateModelInput(lora_config=lora_config), + ) + + model_db = ModelDB( + model_id=model_id, + base_model=request.base_model, + lora_config=lora_config.model_dump(), + status="created", + request_id=request_id, + session_id=request.session_id, + ) + session.add(model_db) + + await session.commit() + + return CreateModelResponse( + model_id=model_id, + base_model=request.base_model, + lora_config=request.lora_config, + status="created", + request_id=str(request_id), + ) + + +@app.post("/api/v1/unload_model", response_model=UnloadModelResponse) +async def unload_model(request: UnloadModelRequest, session: AsyncSession = Depends(get_session)): + """Unload a model and free all associated resources.""" + # Validate model exists + model_db = await session.get(ModelDB, request.model_id) + if model_db is None: + raise HTTPException(status_code=404, detail="Model not found") + + # Update model status + model_db.status = "unloading" + + # Create future request + request_id = await create_future( + session=session, + request_type=types.RequestType.UNLOAD_MODEL, + model_id=request.model_id, + request_data=types.UnloadModelInput(), + ) + + await session.commit() + + return UnloadModelResponse(request_id=str(request_id), model_id=request.model_id) + + +class GetInfoRequest(BaseModel): + model_id: str + type: str | None = None + + +@app.post("/api/v1/get_info", response_model=ModelInfoResponse) +async def get_model_info(request: GetInfoRequest, session: AsyncSession = Depends(get_session)): + """Retrieve information about the current model.""" + model = await get_model(session, request.model_id) + + lora_config = types.LoraConfig.model_validate(model.lora_config) + model_data = ModelData( + base_model=model.base_model, lora_config=LoRAConfig(rank=lora_config.rank), model_name=model.base_model + ) + + return ModelInfoResponse(model_id=model.model_id, status=model.status, model_data=model_data) + + +@app.get("/api/v1/training_runs/{model_id}", response_model=TrainingRun) +async def get_training_run(model_id: str, session: AsyncSession = Depends(get_session)): + """Get training run for session resumption.""" + model = await get_model(session, model_id) + + lora_config = types.LoraConfig.model_validate(model.lora_config) + + return TrainingRun( + training_run_id=model.model_id, + base_model=model.base_model, + model_owner="default", + is_lora=True, + corrupted=False, + lora_rank=lora_config.rank, + # TODO: Once we track modified_at timestamps, update this + last_request_time=model.created_at, + last_checkpoint=None, + last_sampler_checkpoint=None, + user_metadata=None, + ) + + +@app.post("/api/v1/forward_backward", response_model=FutureResponse) +async def forward_backward(request: ForwardBackwardRequest, session: AsyncSession = Depends(get_session)): + """Compute and accumulate gradients.""" + await get_model(session, request.model_id) + + request_id = await create_future( + session=session, + request_type=types.RequestType.FORWARD_BACKWARD, + model_id=request.model_id, + request_data=request.forward_backward_input.to_types(), + ) + + await session.commit() + + return FutureResponse(future_id=str(request_id), status="pending", request_id=str(request_id)) + + +@app.post("/api/v1/forward", response_model=FutureResponse) +async def forward(request: ForwardRequest, session: AsyncSession = Depends(get_session)): + """Forward pass to obtain logprobs without accumulating gradients""" + await get_model(session, request.model_id) + + request_id = await create_future( + session=session, + request_type=types.RequestType.FORWARD, + model_id=request.model_id, + request_data=request.forward_input.to_types(), + ) + + await session.commit() + + return FutureResponse(future_id=str(request_id), status="pending", request_id=str(request_id)) + + +@app.post("/api/v1/optim_step", response_model=FutureResponse) +async def optim_step(request: OptimStepRequest, session: AsyncSession = Depends(get_session)): + """Update model using accumulated gradients.""" + await get_model(session, request.model_id) + + request_id = await create_future( + session=session, + request_type=types.RequestType.OPTIM_STEP, + model_id=request.model_id, + request_data=types.OptimStepInput(adam_params=request.adam_params.to_types()), + ) + + await session.commit() + + return FutureResponse(future_id=str(request_id), status="pending", request_id=str(request_id)) + + +@app.post("/api/v1/load_weights", response_model=FutureResponse) +async def load_weights(request: LoadWeightsRequest, req: Request, session: AsyncSession = Depends(get_session)): + """Loads weights and training state.""" + await get_model(session, request.model_id) + + path = types.TinkerPath.parse(request.path) + if ( + not path + or path.kind != "weights" + or not (source_model_id := path.primary_id) + or not (checkpoint_id := path.secondary_id) + ): + raise HTTPException( + status_code=400, detail="request.path must be in format tinker://source_model_id/weights/checkpoint_id" + ) + + await validate_checkpoint(req, source_model_id, checkpoint_id, types.CheckpointType.TRAINING, session) + + request_id = await create_future( + session=session, + request_type=types.RequestType.LOAD_WEIGHTS, + model_id=request.model_id, + request_data=types.LoadWeightsInput(source_model_id=source_model_id, checkpoint_id=checkpoint_id), + ) + + await session.commit() + + return FutureResponse(future_id=str(request_id), status="pending", request_id=str(request_id)) + + +@app.post("/api/v1/save_weights", response_model=FutureResponse) +async def save_weights(request: SaveWeightsRequest, session: AsyncSession = Depends(get_session)): + """Saves weights and training state.""" + # Create pending checkpoint entry (validates model exists) + await create_checkpoint( + session=session, + model_id=request.model_id, + checkpoint_id=request.path, + checkpoint_type=types.CheckpointType.TRAINING, + ) + + request_id = await create_future( + session=session, + request_type=types.RequestType.SAVE_WEIGHTS, + model_id=request.model_id, + request_data=types.SaveWeightsInput(path=request.path), + ) + + await session.commit() + + return FutureResponse(future_id=str(request_id), status="pending", request_id=str(request_id)) + + +@app.post("/api/v1/save_weights_for_sampler", response_model=FutureResponse) +async def save_weights_for_sampler(request: SaveWeightsForSamplerRequest, session: AsyncSession = Depends(get_session)): + """Saves weights in a format compatible with sampling/inference servers.""" + # Get the model (validates it exists and gives us the session_id) + model = await get_model(session, request.model_id) + + checkpoint_id = request.path or f"ss{request.sampling_session_seq_id}_seq{request.seq_id}" + sampling_session_id = None + if request.sampling_session_seq_id is not None and request.seq_id is not None: + # Create the sampling session using the model's session + sampling_session_id = f"sampling_{uuid4().hex[:8]}" + sampling_db = SamplingSessionDB( + sampling_session_id=sampling_session_id, + session_id=model.session_id, + sampling_session_seq_id=request.sampling_session_seq_id, + base_model=None, + model_path=f"tinker://{request.model_id}/sampler_weights/{checkpoint_id}", + ) + session.add(sampling_db) + + # Create pending checkpoint entry + await create_checkpoint( + session=session, + model_id=request.model_id, + checkpoint_id=checkpoint_id, + checkpoint_type=types.CheckpointType.SAMPLER, + ) + + request_id = await create_future( + session=session, + request_type=types.RequestType.SAVE_WEIGHTS_FOR_SAMPLER, + model_id=request.model_id, + request_data=types.SaveWeightsForSamplerInput( + path=checkpoint_id, + sampling_session_seq_id=request.sampling_session_seq_id, + seq_id=request.seq_id, + sampling_session_id=sampling_session_id, + ), + ) + + await session.commit() + + return FutureResponse(future_id=str(request_id), status="pending", request_id=str(request_id)) + + +async def get_sampling_model(request: SampleRequest, session: AsyncSession) -> (str | None, str | None): + """Return (base_model, model_path) for a sampling request.""" + # Resolve model/base from sampling_session_id if provided + if request.sampling_session_id is not None: + sampling_session = await session.get(SamplingSessionDB, request.sampling_session_id) + if sampling_session is None: + raise HTTPException(status_code=404, detail="Sampling session not found") + return (sampling_session.base_model, sampling_session.model_path) + return (request.base_model, request.model_path) + + +@app.post("/api/v1/asample", response_model=FutureResponse) +async def asample(request: SampleRequest, req: Request, session: AsyncSession = Depends(get_session)): + """Generates samples from the model (async version).""" + base_model, model_path = await get_sampling_model(request, session) + + if base_model: + model_id = checkpoint_id = "" + else: + assert model_path is not None + path = types.TinkerPath.parse(model_path) + if ( + not path + # Accept either tinker://model_id/checkpoint_id or tinker://model_id/sampler_weights/checkpoint_id + or path.kind not in ("", "sampler_weights") + or not (model_id := path.primary_id) + or not (checkpoint_id := path.secondary_id) + ): + raise HTTPException( + status_code=400, + detail="model_path must be tinker://model_id/checkpoint_id or tinker://model_id/sampler_weights/checkpoint_id", + ) + await get_model(session, model_id) + # Validate that the checkpoint exists and is ready + await validate_checkpoint(req, model_id, checkpoint_id, types.CheckpointType.SAMPLER, session) + + request_id = await create_future( + session=session, + request_type=( + types.RequestType.EXTERNAL if req.app.state.external_inference_client else types.RequestType.SAMPLE + ), + model_id=model_id, + request_data=types.SampleInput( + base_model=base_model, + prompt=request.prompt.to_types(), + sampling_params=request.sampling_params.to_types(), + num_samples=request.num_samples, + checkpoint_id=checkpoint_id, + prompt_logprobs=request.prompt_logprobs, + ), + ) + + await session.commit() + + if req.app.state.external_inference_client: + asyncio.create_task( + req.app.state.external_inference_client.call_and_store_result(request_id, request, model_id, checkpoint_id) + ) + + return FutureResponse(future_id=str(request_id), status="pending", request_id=str(request_id)) + + +@app.get("/api/v1/get_server_capabilities", response_model=GetServerCapabilitiesResponse) +async def get_server_capabilities(request: Request): + """Retrieve information about supported models and server capabilities.""" + supported_models = [ + SupportedModel(model_name=request.app.state.engine_config.base_model), + ] + return GetServerCapabilitiesResponse(supported_models=supported_models) + + +class RetrieveFutureRequest(BaseModel): + request_id: str + + +@app.post("/api/v1/retrieve_future") +async def retrieve_future(request: RetrieveFutureRequest, req: Request): + """Retrieve the result of an async operation, waiting until it's available.""" + timeout = 300 # 5 minutes + deadline = time.perf_counter() + timeout + + # Start with 100ms, grow to 1s + poll = 0.1 + max_poll = 1.0 + + while time.perf_counter() < deadline: + try: + async with AsyncSession(req.app.state.db_engine) as session: + # First, only query the status to avoid deserializing JSON data + statement = select(FutureDB.status).where(FutureDB.request_id == int(request.request_id)) + result = await session.exec(statement) + status = result.first() + + if not status: + raise HTTPException(status_code=404, detail="Future not found") + + # Only fetch full record if status is terminal (completed or failed) + if status in (RequestStatus.COMPLETED, RequestStatus.FAILED): + statement = select(FutureDB).where(FutureDB.request_id == int(request.request_id)) + result = await session.exec(statement) + future = result.first() + + if future.status == RequestStatus.COMPLETED: + return future.result_data + + if future.status == RequestStatus.FAILED: + # Return 400 for handled errors (validation, etc.), 500 for unexpected failures + if future.result_data and "error" in future.result_data: + raise HTTPException(status_code=400, detail=future.result_data["error"]) + else: + raise HTTPException(status_code=500, detail="Unknown error") + except SATimeoutError: + pass + + # Exponential backoff + await asyncio.sleep(poll) + poll = min(poll * 1.5, max_poll) + + raise HTTPException(status_code=408, detail="Timeout waiting for result") + + +@app.post("/api/v1/telemetry", response_model=TelemetryResponse) +async def send_telemetry(request: TelemetryRequest): + """Accept batches of SDK telemetry events for analytics and diagnostics.""" + # Just acknowledge receipt without doing anything + return TelemetryResponse(status="accepted") + + +async def validate_checkpoint( + request: Request, unique_id: str, checkpoint_id: str, checkpoint_type: types.CheckpointType, session: AsyncSession +): + """Validate that a model and checkpoint exist in the database, returning the checkpoint path.""" + checkpoint_db = await session.get(CheckpointDB, (unique_id, checkpoint_id, checkpoint_type)) + + if not checkpoint_db: + raise HTTPException(status_code=404, detail=f"Checkpoint not found: {unique_id}/{checkpoint_id}") + + if checkpoint_db.status == CheckpointStatus.PENDING: + raise HTTPException(status_code=425, detail="Checkpoint is still being created") + + if checkpoint_db.status == CheckpointStatus.FAILED: + raise HTTPException(status_code=500, detail=f"Checkpoint creation failed: {checkpoint_db.error_message}") + + subdir = "sampler_weights" if checkpoint_type == types.CheckpointType.SAMPLER else "" + return request.app.state.engine_config.checkpoints_base / unique_id / subdir / f"{checkpoint_id}.tar.gz" + + +@app.get("/api/v1/training_runs") +async def list_training_runs( + limit: int = 20, offset: int = 0, session: AsyncSession = Depends(get_session) +) -> TrainingRunsResponse: + """List all training runs""" + + # Use window function to get total count alongside paginated results in a single query + statement = select(ModelDB, func.count().over().label("total_count")).offset(offset).limit(limit) + result = await session.exec(statement) + rows = result.all() + + total_count = rows[0].total_count if rows else 0 + + training_runs = [] + for row in rows: + model = row.ModelDB + lora_config = types.LoraConfig.model_validate(model.lora_config) + + training_runs.append( + TrainingRun( + training_run_id=model.model_id, + base_model=model.base_model, + model_owner="default", + is_lora=True, + corrupted=False, + lora_rank=lora_config.rank, + last_request_time=model.created_at, # TODO: Once we track modified_at timestamps, update this + last_checkpoint=None, + last_sampler_checkpoint=None, + user_metadata=None, + ) + ) + + return TrainingRunsResponse( + training_runs=training_runs, cursor=Cursor(offset=offset, limit=limit, total_count=total_count) + ) + + +@app.get("/api/v1/training_runs/{unique_id}/checkpoints/{checkpoint_id}/archive") +async def get_checkpoint_archive_url( + request: Request, + unique_id: str = fastapi.Path(..., pattern=ID_PATTERN, max_length=ID_MAX_LENGTH), + checkpoint_id: str = fastapi.Path(..., pattern=ID_PATTERN, max_length=ID_MAX_LENGTH), + session: AsyncSession = Depends(get_session), +): + """Return a 302 redirect to the download URL (SDK expects this pattern)""" + await validate_checkpoint(request, unique_id, checkpoint_id, types.CheckpointType.SAMPLER, session) + + # Generate URL to the download endpoint and return 302 redirect + download_url = str(request.url_for("download_checkpoint_archive", unique_id=unique_id, checkpoint_id=checkpoint_id)) + expires = datetime.utcnow() + timedelta(minutes=120) + + response = RedirectResponse(url=download_url, status_code=302) + response.headers["Expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT") + return response + + +@app.get("/api/v1/training_runs/{unique_id}/checkpoints/{checkpoint_id}/download") +async def download_checkpoint_archive( + request: Request, + unique_id: str = fastapi.Path(..., pattern=ID_PATTERN, max_length=ID_MAX_LENGTH), + checkpoint_id: str = fastapi.Path(..., pattern=ID_PATTERN, max_length=ID_MAX_LENGTH), + session: AsyncSession = Depends(get_session), +): + """Actually download the checkpoint archive bytes""" + checkpoint_path = await validate_checkpoint( + request, unique_id, checkpoint_id, types.CheckpointType.SAMPLER, session + ) + + file_buffer = await asyncio.to_thread(download_file, checkpoint_path) + + filename = f"{unique_id}_{checkpoint_id}.tar.gz" + headers = { + "Content-Disposition": f'attachment; filename="{filename}"', + "Content-Length": str(file_buffer.getbuffer().nbytes), + } + + return StreamingResponse(file_buffer, media_type="application/octet-stream", headers=headers) + + +@app.get("/api/v1/training_runs/{unique_id}/checkpoints") +async def list_checkpoints( + unique_id: str = fastapi.Path(..., pattern=ID_PATTERN, max_length=ID_MAX_LENGTH), + session: AsyncSession = Depends(get_session), +): + """List checkpoints for a model.""" + statement = ( + select(CheckpointDB) + .where(CheckpointDB.model_id == unique_id) + .where(CheckpointDB.status == CheckpointStatus.COMPLETED) + ) + result = await session.exec(statement) + + checkpoints = [] + for checkpoint in result.all(): + # Construct tinker_path based on checkpoint type + path_kind = "weights" if checkpoint.checkpoint_type == types.CheckpointType.TRAINING else "sampler_weights" + tinker_path = f"tinker://{unique_id}/{path_kind}/{checkpoint.checkpoint_id}" + + checkpoints.append( + Checkpoint( + checkpoint_id=checkpoint.checkpoint_id, + checkpoint_type=checkpoint.checkpoint_type.value, + time=checkpoint.completed_at, + tinker_path=tinker_path, + ) + ) + + return ListCheckpointsResponse(checkpoints=checkpoints) + + +@app.get("/api/v1/models/{unique_id}/checkpoints") +async def list_checkpoints_models( + unique_id: str = fastapi.Path(..., pattern=ID_PATTERN, max_length=ID_MAX_LENGTH), + session: AsyncSession = Depends(get_session), +): + """Just to be compatible with tinker SDK""" + return await list_checkpoints(unique_id=unique_id, session=session) + + +@app.post("/api/v1/weights_info", response_model=WeightsInfoResponse) +async def get_weights_info(request: WeightsInfoRequest, req: Request, session: AsyncSession = Depends(get_session)): + """Get information about weights/checkpoint from a tinker path.""" + path = types.TinkerPath.parse(request.tinker_path) + + if not path or path.kind != "weights": + raise HTTPException( + status_code=400, detail="Invalid tinker path format. Expected: tinker://model_id/weights/checkpoint_id" + ) + + model_id = path.primary_id + checkpoint_id = path.secondary_id + + # Get model info (this will raise 404 if model doesn't exist) + model = await get_model(session, model_id) + + # Validate checkpoint exists and is completed + await validate_checkpoint(req, model_id, checkpoint_id, types.CheckpointType.TRAINING, session) + + lora_config = types.LoraConfig.model_validate(model.lora_config) + is_lora = lora_config.rank > 0 + + return WeightsInfoResponse( + base_model=model.base_model, + is_lora=is_lora, + lora_rank=lora_config.rank, + ) + + +@app.get("/") +async def root(): + """Root endpoint with API information.""" + return { + "name": "Tinker API Mock", + "version": "0.0.1", + "endpoints": { + "models": ["/api/v1/create_model", "/api/v1/get_info", "/api/v1/training_runs/{model_id}"], + "training": ["/api/v1/forward_backward", "/api/v1/optim_step"], + "futures": ["/api/v1/retrieve_future"], + "service": ["/api/v1/get_server_capabilities"], + "telemetry": ["/api/v1/telemetry"], + "checkpoints": ["/api/v1/training_runs/{unique_id}/checkpoints"], + "download": [ + "/api/v1/training_runs/{unique_id}/checkpoints/{checkpoint_id}/archive", + "/api/v1/training_runs/{unique_id}/checkpoints/{checkpoint_id}/download", + ], + }, + } + + +if __name__ == "__main__": + import argparse + import uvicorn + + # Parse command-line arguments + parser = argparse.ArgumentParser(description="SkyRL tx tinker API server") + add_model(parser, EngineConfig) + parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to bind to") + parser.add_argument("--port", type=int, default=8000, help="Port to bind to") + args = parser.parse_args() + + # Create EngineConfig from parsed arguments (only EngineConfig fields) + engine_config = EngineConfig.model_validate({k: v for k, v in vars(args).items() if k in EngineConfig.model_fields}) + + # Store config in app.state so lifespan can access it + app.state.engine_config = engine_config + + uvicorn.run(app, host=args.host, port=args.port) diff --git a/skyrl-tx/tx/tinker/backends/__init__.py b/skyrl-tx/tx/tinker/backends/__init__.py new file mode 100644 index 0000000000..64cab64084 --- /dev/null +++ b/skyrl-tx/tx/tinker/backends/__init__.py @@ -0,0 +1,5 @@ +"""Tinker engine backends.""" + +from tx.tinker.backends.backend import AbstractBackend + +__all__ = ["AbstractBackend"] diff --git a/skyrl-tx/tx/tinker/backends/backend.py b/skyrl-tx/tx/tinker/backends/backend.py new file mode 100644 index 0000000000..7e12fd8f4e --- /dev/null +++ b/skyrl-tx/tx/tinker/backends/backend.py @@ -0,0 +1,164 @@ +"""Abstract backend interface for TinkerEngine. + +Backends handle all model state, adapter management, and computation. +The engine handles database operations and scheduling. + +Design: + 1. AbstractBackend (backend.py) + Clean interface defining what backends must implement: + - create_model (manages model metadata, adapter allocation, and optimizer lifecycle) + - forward_backward, forward, optim_step, sample + - load_checkpoint, save_checkpoint, save_sampler_checkpoint + + 2. JaxBackend (jax.py) + - Implements all abstract methods in Jax, fully supporting MultiLoRA for training and sampling + - Uses jax.value_and_grad for gradient computation + - Uses 2D mesh (fsdp, tp) + - Multi-adapter AccumulatedGradients with counts array + - Manages model metadata and adapter_index allocation internally + + 3. TinkerEngine (engine.py) + - Instantiates backend based on config + - Delegates computation and model management to self.backend + - Handles all database operations +""" + +from abc import ABC, abstractmethod + +from pydantic import BaseModel + +from tx.tinker import types + + +class AbstractBackend(ABC): + """Abstract base class for TinkerEngine backends. + + Backends handle computation and model state manipulation. + Database operations are handled by TinkerEngine. + """ + + @abstractmethod + def __init__(self, base_model: str, config: BaseModel): + """Initialize the backend.""" + pass + + @abstractmethod + def create_model(self, model_id: str, lora_config: types.LoraConfig) -> None: + """Create a new model in the backend. + + Creates optimizer and configures LoRA adapter. + + Args: + model_id: The model identifier + lora_config: LoRA configuration with rank and alpha + """ + pass + + @abstractmethod + def forward_backward( + self, + prepared_batch: types.PreparedModelPassBatch, + ) -> dict[str, types.ForwardBackwardOutput | types.ErrorResponse]: + """Run forward and backward pass on a batch. + + Args: + prepared_batch: PreparedModelPassBatch with all data extracted from requests + + Returns: + Dict mapping request_id to result or error + """ + pass + + @abstractmethod + def forward( + self, + prepared_batch: types.PreparedModelPassBatch, + ) -> dict[str, types.ForwardBackwardOutput | types.ErrorResponse]: + """Run forward-only pass on a batch (no gradient computation). + + Args: + prepared_batch: PreparedModelPassBatch with all data extracted from requests + + Returns: + Dict mapping request_id to result or error + """ + pass + + @abstractmethod + def optim_step(self, model_id: str, request_data: types.OptimStepInput) -> types.OptimStepOutput: + """Apply an optimizer step using accumulated gradients. + + Args: + model_id: The model identifier + request_data: The optimizer step input parameters + + Returns: + OptimStepOutput result + """ + pass + + @abstractmethod + def sample( + self, + prepared_batch: types.PreparedSampleBatch, + ) -> dict[str, types.SampleOutput | types.ErrorResponse]: + """Generate samples for a batch of requests. + + Args: + prepared_batch: PreparedSampleBatch with all data extracted from requests + + Returns: + Dict mapping request_id to result or error + """ + pass + + @abstractmethod + def save_checkpoint(self, output_path, model_id: str) -> None: + """Save training checkpoint to disk. + + Args: + output_path: Path to save the checkpoint + model_id: The model identifier + """ + pass + + @abstractmethod + def load_checkpoint(self, checkpoint_path, model_id: str) -> None: + """Load training checkpoint from disk. + + Args: + checkpoint_path: Path to the checkpoint file + model_id: The model identifier + """ + pass + + @abstractmethod + def save_sampler_checkpoint(self, output_path, model_id: str) -> None: + """Save sampler checkpoint to disk as tar.gz. + + Args: + output_path: Path to save the checkpoint tar.gz file + model_id: The model identifier + """ + pass + + @abstractmethod + def has_model(self, model_id: str) -> bool: + """Check if a model is registered with the backend. + + Args: + model_id: The model identifier + + Returns: + True if the model is registered, False otherwise + """ + pass + + @abstractmethod + def delete_model(self, model_id: str) -> None: + """Delete a model and free all associated resources. + + Args: + model_id: The model identifier + """ + pass diff --git a/skyrl-tx/tx/tinker/backends/jax.py b/skyrl-tx/tx/tinker/backends/jax.py new file mode 100644 index 0000000000..720b760eb9 --- /dev/null +++ b/skyrl-tx/tx/tinker/backends/jax.py @@ -0,0 +1,1056 @@ +"""JAX LoRA backend for TinkerEngine. + +This backend implements the full training and inference pipeline for models +with LoRA adapters. It uses jax.value_and_grad for gradient computation and supports +multiple LoRA adapters via the AccumulatedGradients dataclass. + +In multi-host mode, process 0 (coordinator) runs the engine with JaxBackend, +which broadcasts commands to workers. Workers run separately using `run_worker()` +or by running this module directly with `python -m tx.tinker.backends.jax`. + +Usage: + # Coordinator (process 0) - runs the full engine: + uv run -m tx.tinker.engine --base-model Qwen/Qwen3-8B --backend-config '{ + "coordinator_address": "localhost:7777", + "num_processes": 2, + ... + }' + + # Workers (process 1+) - run only the worker loop (receives config from coordinator): + uv run -m tx.tinker.backends.jax --coordinator-address localhost:7777 --num-processes 2 --process-id 1 +""" + +import time +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Callable, get_type_hints + +from cloudpathlib import AnyPath +import numpy as np +import jax +import jax.numpy as jnp +from jax.experimental import multihost_utils +import optax +from flax import nnx +from flax.training import checkpoints +from pydantic import BaseModel, Field, TypeAdapter +from transformers import AutoTokenizer, PretrainedConfig + +from tx.models.configs import Qwen3Config +from tx.layers.lora import clear_lora_adapter, init_lora_adapter +from tx.tinker import types +from tx.tinker.backends.backend import AbstractBackend +from tx.tinker.backends.utils import pad, pad_batch, pad_to_fsdp +from tx.tinker.loss_fns import LOSS_FUNCTIONS +from tx.utils.models import ( + get_dtype, + get_model_class, + load_safetensors, + load_lora_checkpoint, + save_lora_checkpoint, + extract_adapter_state, + insert_adapter_state, + round_up_seq_len, + resolve_model_path, +) +from tx.utils.log import logger + + +class JaxBackendConfig(BaseModel, extra="forbid"): + """Configuration specific to the JAX backend.""" + + max_lora_adapters: int = Field(default=32, description="Maximum number of LoRA adapters") + max_lora_rank: int = Field(default=32, description="Maximum LoRA rank") + tensor_parallel_size: int = Field(default=1, description="Tensor parallelism degree to use for the model") + expert_parallel_size: int = Field(default=1, description="Expert parallelism degree for MoE layers") + fully_sharded_data_parallel_size: int = Field( + default=1, description="Fully sharded data parallelism degree for the model" + ) + train_micro_batch_size: int = Field( + default=0, + description="Micro-batch size (measured in number of sequences) for gradient accumulation; 0 means disabled (use full batch)", + ) + sample_max_num_sequences: int = Field( + default=0, + description="Maximum batch size (measured in number of sequences) for sampling/generation; 0 means disabled (use full batch)", + ) + enforce_eager: bool = Field(default=False, description="Disable JAX JIT compilation") + shard_attention_heads: bool = Field( + default=True, + description="Whether to shard attention linear layers (qkvo projections) across tensor parallel devices", + ) + gradient_checkpointing: bool = Field( + default=False, + description="Whether to use gradient checkpointing (full recomputation strategy)", + ) + # Multi-node configuration + coordinator_address: str | None = Field( + default=None, + description="JAX coordinator address (host:port) for multi-node training. If not set, runs in single-node mode.", + ) + num_processes: int | None = Field( + default=None, + description="Total number of processes in the multi-node cluster", + ) + + +@jax.tree_util.register_dataclass +@dataclass +class AccumulatedGradients: + """Stores accumulated gradients for all LoRA adapters.""" + + grad_sum: nnx.State + counts: jax.Array + + @classmethod + def create(cls, lora_params: nnx.State, max_adapters: int) -> "AccumulatedGradients": + """Initialize with zeros.""" + return cls( + grad_sum=jax.tree.map(jnp.zeros_like, lora_params), + counts=jnp.zeros((max_adapters,), dtype=jnp.int32), + ) + + def add(self, lora_grads: nnx.State, adapter_indices: jax.Array) -> "AccumulatedGradients": + """Accumulate gradients and increment counts.""" + # Count occurrences of each adapter index in the batch + batch_counts = jnp.bincount(adapter_indices, length=self.counts.shape[0]) + return AccumulatedGradients( + grad_sum=jax.tree.map(lambda a, b: a + b, self.grad_sum, lora_grads), + counts=self.counts + batch_counts, + ) + + def get_mean(self, adapter_index: jax.Array) -> nnx.State: + """Compute mean gradients for a specific adapter, with zeros for all other adapters.""" + count = self.counts[adapter_index] + return jax.tree.map( + lambda g: jnp.zeros_like(g).at[adapter_index].set(g[adapter_index] / count.astype(g.dtype)), + self.grad_sum, + ) + + def reset_adapter(self, adapter_index: jax.Array) -> "AccumulatedGradients": + """Reset gradients and count for a specific adapter.""" + return AccumulatedGradients( + grad_sum=jax.tree.map(lambda g: g.at[adapter_index].set(0.0), self.grad_sum), + counts=self.counts.at[adapter_index].set(0), + ) + + +class JaxBackendImpl(AbstractBackend): + """JAX backend implementation for models with LoRA adapters. + + This is the core implementation class. Use JaxBackend (the distributed wrapper) + for multi-host coordination. + + This backend: + - Uses jax.value_and_grad for gradient computation + - Uses 2D mesh (fsdp, tp) for fully sharded data parallelism and tensor parallelism + - Supports multiple LoRA adapters via AccumulatedGradients with counts array + - Supports both FORWARD and FORWARD_BACKWARD request types + """ + + def __init__(self, base_model: str, config: JaxBackendConfig): + """Initialize JAX LoRA backend.""" + self.base_model = base_model + self.config = config + self.metrics = types.EngineMetrics() + + # Initialize the shared base model with LoRA config + checkpoint_path = resolve_model_path(base_model) + self.tokenizer = AutoTokenizer.from_pretrained(checkpoint_path) + base_config = PretrainedConfig.from_pretrained(checkpoint_path) + self.model_config = Qwen3Config( + base_config, + max_lora_adapters=config.max_lora_adapters, + max_lora_rank=config.max_lora_rank, + shard_attention_heads=config.shard_attention_heads, + ) + + model_class = get_model_class(self.model_config) + + # Create model and load weights + self.mesh = jax.make_mesh( + ( + config.fully_sharded_data_parallel_size, + config.expert_parallel_size, + config.tensor_parallel_size, + ), + ("fsdp", "ep", "tp"), + ) + with jax.set_mesh(self.mesh), nnx.use_eager_sharding(True): + self.model = model_class(self.model_config, dtype=get_dtype(self.model_config.dtype), rngs=nnx.Rngs(0)) + load_safetensors(checkpoint_path, self.model_config, self.model) + + # Split model into LoRA and non-LoRA parameters + self.graphdef, self.lora_params, self.non_lora_params = nnx.split(self.model, self.model.is_lora_param, ...) + + # Initialize adapter 0 with minimal config (required for base model sampling path) + init_lora_adapter(self.model, adapter_index=0, lora_config=types.LoraConfig(rank=1, alpha=1.0, seed=0)) + + # Initialize global accumulated gradients + self.accumulated_grads = AccumulatedGradients.create(self.lora_params, config.max_lora_adapters) + + # Per-model optimizer storage (managed internally) + self.optimizers: dict[str, nnx.Optimizer] = {} + + # Store LoRA model metadata (model_id -> metadata) + self.models: dict[str, types.ModelMetadata] = {} + + logger.info( + f"Initialized base model {base_model} with " + f"max_lora_adapters={config.max_lora_adapters}, max_lora_rank={config.max_lora_rank}" + ) + + self._create_loss_and_grad_fn() + + def _micro_batch_size(self, total: int) -> int: + """Return effective micro-batch size; 0/absent => disabled (use full fused batch).""" + mb = self.config.train_micro_batch_size + return total if mb <= 0 else max(1, min(mb, total)) + + @contextmanager + def _jit_timing_context(self, seq_len: int, mode: str): + """Context manager to track JIT compilation times for different sequence lengths. + + Args: + seq_len: The sequence length being compiled + mode: Either 'train' or 'sample' to track separately + """ + jit_times = self.metrics.train_seq_len_jit_times if mode == "train" else self.metrics.sample_seq_len_jit_times + if not self.config.enforce_eager and seq_len not in jit_times: + logger.info(f"JIT compiling for {mode} seq_len={seq_len} in progress...") + start_time = time.time() + yield + elapsed = time.time() - start_time + jit_times[seq_len] = elapsed + logger.info(f"JIT compilation for {mode} seq_len={seq_len} took {elapsed:.2f}s") + else: + yield + + def _create_loss_and_grad_fn(self): + """Compile and cache the loss function to avoid re-jitting on every call.""" + + def _model_forward( + graphdef: nnx.GraphDef, + lora_params: nnx.State, + non_lora_params: nnx.State, + input_ids: jax.Array, + attention_mask: jax.Array, + adapter_indices: jax.Array, + ) -> jax.Array: + model = nnx.merge(graphdef, lora_params, non_lora_params) + output = model(input_ids, attention_mask=attention_mask, adapter_indices=adapter_indices) + return output.logits + + if self.config.gradient_checkpointing: + # Wrap the model forward call to use jax.checkpoint for gradient checkpointing + # policy=None corresponds to full activation recomputation + _model_forward = jax.checkpoint(_model_forward, policy=None) + + def loss_for_lora( + lora_params: nnx.State, + non_lora_params: nnx.State, + input_ids: jax.Array, + attention_mask: jax.Array, + adapter_indices: jax.Array, + target_ids: jax.Array, + loss_mask: jax.Array, + loss_fn_types: jax.Array, + sampling_logprobs: jax.Array, + advantages: jax.Array, + ) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + logits = _model_forward( + self.graphdef, lora_params, non_lora_params, input_ids, attention_mask, adapter_indices + ) # [B, T, V] + + log_sum_exp = jax.nn.logsumexp(logits, axis=-1, keepdims=True) + target_logits = jnp.take_along_axis(logits, target_ids[..., None], axis=-1) + target_logprobs = (target_logits - log_sum_exp).squeeze(-1) + + def compute_loss_per_example(loss_fn_type, target_logprobs, loss_mask, sampling_logprobs, advantages): + return jax.lax.switch( + loss_fn_type, + LOSS_FUNCTIONS, + target_logprobs, + loss_mask, + sampling_logprobs, + advantages, + ) + + per_token_losses = jax.vmap(compute_loss_per_example)( + loss_fn_types, + target_logprobs, + loss_mask, + sampling_logprobs, + advantages, + ) + + per_seq_loss = per_token_losses.sum(axis=-1) / jnp.maximum(loss_mask.sum(axis=-1), 1e-9) + # Return sum of losses (we'll divide gradients by per-adapter batch size later) + return per_seq_loss.sum(), (target_logprobs, per_token_losses) + + # Only differentiate with respect to lora_params (argnums=0) + loss_and_grad_fn = jax.value_and_grad(loss_for_lora, argnums=0, has_aux=True) + + def forward_only( + accumulated_grads: AccumulatedGradients, + lora_params: nnx.State, + non_lora_params: nnx.State, + input_ids: jax.Array, + attention_mask: jax.Array, + adapter_indices: jax.Array, + target_ids: jax.Array, + loss_mask: jax.Array, + loss_fn_types: jax.Array, + sampling_logprobs: jax.Array, + advantages: jax.Array, + ) -> tuple[AccumulatedGradients, jax.Array, jax.Array]: + _, (target_logprobs, per_token_losses) = loss_for_lora( + lora_params, + non_lora_params, + input_ids, + attention_mask, + adapter_indices, + target_ids, + loss_mask, + loss_fn_types, + sampling_logprobs, + advantages, + ) + return accumulated_grads, per_token_losses, target_logprobs + + def forward_backward_and_accumulate( + accumulated_grads: AccumulatedGradients, + lora_params: nnx.State, + non_lora_params: nnx.State, + input_ids: jax.Array, + attention_mask: jax.Array, + adapter_indices: jax.Array, + target_ids: jax.Array, + loss_mask: jax.Array, + loss_fn_types: jax.Array, + sampling_logprobs: jax.Array, + advantages: jax.Array, + ) -> tuple[AccumulatedGradients, jax.Array, jax.Array]: + """Fused forward-backward-accumulate operation.""" + # Forward-backward + (_, (target_logprobs, per_token_losses)), lora_grads = loss_and_grad_fn( + lora_params, + non_lora_params, + input_ids, + attention_mask, + adapter_indices, + target_ids, + loss_mask, + loss_fn_types, + sampling_logprobs, + advantages, + ) + # Accumulate gradients + new_accumulated_grads = accumulated_grads.add(lora_grads, adapter_indices) + return new_accumulated_grads, per_token_losses, target_logprobs + + if self.config.enforce_eager: + # Disable JIT compilation for debugging + self._forward_backward_and_accumulate = forward_backward_and_accumulate + self._forward = forward_only + + else: + # Retrieve the sharding of lora and non_lora params and compute the sharding of inputs and outputs + lora_shardings = jax.tree.map( + lambda spec: jax.NamedSharding(self.mesh, spec), nnx.get_partition_spec(self.lora_params) + ) + non_lora_shardings = jax.tree.map( + lambda spec: jax.NamedSharding(self.mesh, spec), nnx.get_partition_spec(self.non_lora_params) + ) + # Get sharding for AccumulatedGradients + accumulated_grads_shardings = jax.tree.map( + lambda spec: jax.NamedSharding(self.mesh, spec), nnx.get_partition_spec(self.accumulated_grads) + ) + + # Shard batch inputs along the FSDP axis (batch, seq_len) + batch_sharded_2d = jax.NamedSharding(self.mesh, jax.P("fsdp", None)) + + # JIT the fused function + # Input order: input_ids, attention_mask, adapter_indices, target_ids, + # loss_mask, loss_fn_types, sampling_logprobs, advantages + # All batch arrays are sharded along batch dimension + batch_sharded_1d = jax.NamedSharding(self.mesh, jax.P("fsdp")) + input_shardings = ( + batch_sharded_2d, # input_ids + batch_sharded_2d, # attention_mask + batch_sharded_1d, # adapter_indices (sharded, bincount runs per-device) + batch_sharded_2d, # target_ids + batch_sharded_2d, # loss_mask + batch_sharded_1d, # loss_fn_types (sharded, used in vmap over batch) + batch_sharded_2d, # sampling_logprobs + batch_sharded_2d, # advantages + ) + self._forward_backward_and_accumulate = jax.jit( + forward_backward_and_accumulate, + in_shardings=(accumulated_grads_shardings, lora_shardings, non_lora_shardings) + input_shardings, + out_shardings=(accumulated_grads_shardings, batch_sharded_2d, batch_sharded_2d), + donate_argnames=("accumulated_grads",), + ) + self._forward = jax.jit( + forward_only, + in_shardings=(accumulated_grads_shardings, lora_shardings, non_lora_shardings) + input_shardings, + out_shardings=(accumulated_grads_shardings, batch_sharded_2d, batch_sharded_2d), + ) + + # JIT-compiled function to compute full gradients and apply optimizer update + def compute_grads_and_update( + accumulated_grads: AccumulatedGradients, + lora_params: nnx.State, + optimizer: nnx.Optimizer, + adapter_index: jax.Array, + ) -> AccumulatedGradients: + """Compute full gradients, apply optimizer update, and reset accumulated grads.""" + optimizer.update(lora_params, accumulated_grads.get_mean(adapter_index)) + return accumulated_grads.reset_adapter(adapter_index) + + if self.config.enforce_eager: + self._compute_grads_and_update = compute_grads_and_update + else: + self._compute_grads_and_update = nnx.jit(compute_grads_and_update) + + def has_model(self, model_id: str) -> bool: + """Check if a model is registered with the backend.""" + return model_id in self.models + + def create_model(self, model_id: str, lora_config: types.LoraConfig) -> None: + """Create a new model in the backend. + + Creates optimizer and configures LoRA adapter. Allocates adapter_index internally. + """ + # Allocate adapter index for this model_id (find first available slot) + # Index 0 is reserved for base model, so user models use indices 1 to max_lora_adapters-1 + used_indices = {m.adapter_index for m in self.models.values()} + available_indices = set(range(1, self.config.max_lora_adapters)) - used_indices + if not available_indices: + raise ValueError(f"Maximum number of LoRA adapters ({self.config.max_lora_adapters}) reached") + adapter_index = min(available_indices) + assert 1 <= adapter_index <= self.config.max_lora_adapters - 1 + + # Validate rank doesn't exceed max + if not (0 < lora_config.rank <= self.config.max_lora_rank): + raise ValueError(f"LoRA rank {lora_config.rank} must be between 1 and {self.config.max_lora_rank}") + + # Store model metadata + self.models[model_id] = types.ModelMetadata( + adapter_index=adapter_index, + lora_config=lora_config, + ) + + # Create optimizer + with jax.set_mesh(self.mesh): + tx = optax.inject_hyperparams(optax.adamw)(learning_rate=0.0) + self.optimizers[model_id] = nnx.Optimizer(self.model, tx, wrt=self.model.is_lora_param) + + # Configure adapter + init_lora_adapter(self.model, adapter_index, lora_config) + logger.info(f"Created model {model_id} with adapter_index={adapter_index}, config={lora_config}") + + def delete_model(self, model_id: str) -> None: + """Delete a model and free all associated resources.""" + if model_id not in self.models: + raise ValueError(f"Model {model_id} not found") + + # Get adapter index before deleting metadata + adapter_index = self.models[model_id].adapter_index + + # Clear LoRA adapter weights + with jax.set_mesh(self.mesh): + clear_lora_adapter(self.model, adapter_index) + + # Delete optimizer + del self.optimizers[model_id] + + # Delete model metadata + del self.models[model_id] + + logger.info(f"Deleted model {model_id} (adapter_index={adapter_index})") + + def _model_pass( + self, + prepared_batch: types.PreparedModelPassBatch, + model_pass_fn: Callable, + ) -> dict[str, types.ForwardBackwardOutput | types.ErrorResponse]: + """Common batch processing logic for forward-only and forward-backward operations. + + Args: + prepared_batch: PreparedModelPassBatch with all data extracted from requests + model_pass_fn: Callable to perform the model pass (forward or forward_backward) + + Returns: + Dict mapping request_id to result_data or error info + """ + if not prepared_batch.all_input_ids: + return {} + + results = {} + + # Extract data from prepared batch + all_input_ids = prepared_batch.all_input_ids + all_targets = prepared_batch.all_targets + all_token_weights = prepared_batch.all_token_weights + all_sampling_logprobs = prepared_batch.all_sampling_logprobs + all_advantages = prepared_batch.all_advantages + all_loss_fn_types = prepared_batch.all_loss_fn_types + request_batch_slices = prepared_batch.request_batch_slices + + # Convert model_ids to adapter_indices + all_adapter_indices = [self.models[model_id].adapter_index for model_id in prepared_batch.all_model_ids] + + # Pad sequences to same length. Also bin it so the JIT has to compile fewer kernels. + max_len = round_up_seq_len(max(len(seq) for seq in all_input_ids)) + + input_ids = pad_batch(all_input_ids, max_len, np.int32) + target_ids = pad_batch(all_targets, max_len, np.int32) + adapter_indices = np.array(all_adapter_indices, dtype=np.int32) + loss_fn_types = np.array(all_loss_fn_types, dtype=np.int32) + + # Create attention mask (1 for real tokens, 0 for padding) + attention_mask = pad_batch([[1] * len(seq) for seq in all_input_ids], max_len, np.int32) + loss_mask = pad_batch(all_token_weights, max_len, np.float32) + sampling_logprobs = pad_batch(all_sampling_logprobs, max_len, np.float32) + advantages = pad_batch(all_advantages, max_len, np.float32) + + total_bs = int(input_ids.shape[0]) + micro_bs = self._micro_batch_size(total_bs) + seq_lens = [len(seq) for seq in all_input_ids] + + # Collect full padded arrays on device, slice after transfer + token_losses_device = [] + logprobs_device = [] + seq_len = input_ids.shape[1] + + # Sharding specs for batch inputs + sharding_2d = jax.NamedSharding(self.mesh, jax.P("fsdp", None)) + sharding_1d = jax.NamedSharding(self.mesh, jax.P("fsdp")) + fsdp_size = self.mesh.shape["fsdp"] + + with jax.set_mesh(self.mesh), self._jit_timing_context(seq_len, mode="train"): + for mb_start in range(0, total_bs, micro_bs): + mb_end = min(mb_start + micro_bs, total_bs) + + # Pad and shard the micro-batch inputs + ( + mb_input_ids, + mb_attention_mask, + mb_target_ids, + mb_loss_mask, + mb_sampling_logprobs, + mb_advantages, + mb_adapter_indices, + mb_loss_fn_types, + ) = jax.device_put( + ( + pad_to_fsdp(input_ids[mb_start:mb_end], fsdp_size), + pad_to_fsdp(attention_mask[mb_start:mb_end], fsdp_size), + pad_to_fsdp(target_ids[mb_start:mb_end], fsdp_size), + pad_to_fsdp(loss_mask[mb_start:mb_end], fsdp_size), + pad_to_fsdp(sampling_logprobs[mb_start:mb_end], fsdp_size), + pad_to_fsdp(advantages[mb_start:mb_end], fsdp_size), + pad_to_fsdp(adapter_indices[mb_start:mb_end], fsdp_size), + pad_to_fsdp(loss_fn_types[mb_start:mb_end], fsdp_size), + ), + (sharding_2d,) * 6 + (sharding_1d,) * 2, + ) + + self.accumulated_grads, per_token_losses, target_logprobs = model_pass_fn( + self.accumulated_grads, + self.lora_params, + self.non_lora_params, + mb_input_ids, + mb_attention_mask, + mb_adapter_indices, + mb_target_ids, + mb_loss_mask, + mb_loss_fn_types, + mb_sampling_logprobs, + mb_advantages, + ) + # Slice back to original size (remove FSDP padding) + token_losses_device.append(per_token_losses[: mb_end - mb_start]) + logprobs_device.append(target_logprobs[: mb_end - mb_start]) + + # Gather results from all hosts before device_get + if jax.process_count() > 1: + token_losses_device = [multihost_utils.process_allgather(x, tiled=True) for x in token_losses_device] + logprobs_device = [multihost_utils.process_allgather(x, tiled=True) for x in logprobs_device] + + # Single batched device-to-host transfer for all arrays + token_losses_host, logprobs_host = jax.device_get((token_losses_device, logprobs_device)) + + # Flatten microbatches and slice to actual sequence lengths + token_losses_out = [] + logprobs_out = [] + idx = 0 + for mb_losses, mb_logprobs in zip(token_losses_host, logprobs_host): + for i in range(mb_losses.shape[0]): + token_losses_out.append(mb_losses[i, : seq_lens[idx]].astype(jnp.float32)) + logprobs_out.append(mb_logprobs[i, : seq_lens[idx]].astype(jnp.float32)) + idx += 1 + + # Compute per-request results + for request_id, _, start_idx, end_idx in request_batch_slices: + loss_fn_outputs = [] + # Compute per-example losses + for i in range(start_idx, end_idx): + # Extract losses for this example's tokens + token_losses = token_losses_out[i] + token_logprobs = logprobs_out[i] + loss_fn_outputs.append( + { + "elementwise_loss": { + "data": token_losses.tolist(), + "dtype": "float32", + "shape": [token_losses.shape[0]], + }, + "logprobs": { + "data": token_logprobs.tolist(), + "dtype": "float32", + "shape": [token_logprobs.shape[0]], + }, + } + ) + + results[request_id] = types.ForwardBackwardOutput( + loss_fn_output_type="scalar", + loss_fn_outputs=loss_fn_outputs, + metrics={}, + ) + + return results + + def forward_backward( + self, + prepared_batch: types.PreparedModelPassBatch, + ) -> dict[str, types.ForwardBackwardOutput | types.ErrorResponse]: + """Run forward and backward pass on a batch.""" + return self._model_pass(prepared_batch, self._forward_backward_and_accumulate) + + def forward( + self, + prepared_batch: types.PreparedModelPassBatch, + ) -> dict[str, types.ForwardBackwardOutput | types.ErrorResponse]: + """Run forward-only pass on a batch (no gradient computation).""" + return self._model_pass(prepared_batch, self._forward) + + def optim_step(self, model_id: str, request_data: types.OptimStepInput) -> types.OptimStepOutput: + """Apply an optimizer step using accumulated gradients.""" + adapter_index = self.models[model_id].adapter_index + optimizer = self.optimizers[model_id] + + # Check if we have any gradients accumulated (count > 0) + if self.accumulated_grads.counts[adapter_index] == 0: + logger.warning(f"No accumulated gradients for model {model_id}, skipping optimizer step") + return types.OptimStepOutput() + + # Update hyperparameters from the request + hp = optimizer.opt_state.hyperparams + hp["learning_rate"][...] = request_data.adam_params.learning_rate + hp["b1"][...] = request_data.adam_params.beta1 + hp["b2"][...] = request_data.adam_params.beta2 + hp["eps"][...] = request_data.adam_params.eps + hp["weight_decay"][...] = request_data.adam_params.weight_decay + + # JIT-compiled: compute full gradients, apply optimizer update, and reset accumulated grads + with jax.set_mesh(self.mesh): + self.accumulated_grads = self._compute_grads_and_update( + self.accumulated_grads, + self.lora_params, + optimizer, + jnp.int32(adapter_index), + ) + + logger.info(f"Applied optimizer step for model {model_id} (adapter {adapter_index})") + return types.OptimStepOutput() + + def sample( + self, + prepared_batch: types.PreparedSampleBatch, + ) -> dict[str, types.SampleOutput | types.ErrorResponse]: + """Generate samples for a batch of requests. + + Args: + prepared_batch: PreparedSampleBatch with all data extracted from requests + + Returns: + Dict mapping request_id to result or error + """ + if not prepared_batch.all_prompts: + return {} + + results = {} + + # Extract data from prepared batch + all_prompts = prepared_batch.all_prompts + all_sampling_params = prepared_batch.all_sampling_params + request_batch_slices = prepared_batch.request_batch_slices + needs_prompt_logprobs = prepared_batch.needs_prompt_logprobs + + # Load sampler weights and get adapter indices + all_adapter_indices = self.load_sampler_weights(prepared_batch) + + total_batch_size = len(all_prompts) + max_batch_size = ( + self.config.sample_max_num_sequences if self.config.sample_max_num_sequences > 0 else total_batch_size + ) + # Collect generated sequences and prompt logprobs across batches + all_sequences: list[types.GeneratedSequence] = [] + all_prompt_logprobs: list[list[float]] = [] + + # Sharding specs for sampling inputs + sharding_2d = jax.NamedSharding(self.mesh, jax.P("fsdp", None)) + sharding_1d = jax.NamedSharding(self.mesh, jax.P("fsdp")) + + with jax.set_mesh(self.mesh): + model = nnx.merge(self.graphdef, self.lora_params, self.non_lora_params) + for batch_start in range(0, total_batch_size, max_batch_size): + batch_end = min(batch_start + max_batch_size, total_batch_size) + batch_prompts = pad(all_prompts[batch_start:batch_end], max_batch_size, fill=[]) + batch_adapter_indices = pad(all_adapter_indices[batch_start:batch_end], max_batch_size, fill=0) + sampling_params = pad( + all_sampling_params[batch_start:batch_end], max_batch_size, fill=all_sampling_params[batch_start] + ) + + # Pad sequences to same length within the batch to minimize memory usage. + # Also bin it so the JIT has to compile fewer kernels. + # Use left-padding for sampling so the last position is always the last real token. + max_len = round_up_seq_len(max((len(seq) for seq in batch_prompts), default=0)) + input_ids = pad_batch(batch_prompts, max_len, np.int32, left=True) + attention_mask = pad_batch([[1] * len(seq) for seq in batch_prompts], max_len, np.int32, left=True) + + # Shard inputs along FSDP axis (already padded to max_batch_size) + input_ids, attention_mask, adapter_indices = jax.device_put( + (input_ids, attention_mask, np.array(batch_adapter_indices, dtype=np.int32)), + (sharding_2d, sharding_2d, sharding_1d), + ) + + with self._jit_timing_context(max_len, mode="sample"): + result = model.generate( + input_ids, + attention_mask, + sampling_params=sampling_params, + adapter_indices=adapter_indices, + prompt_logprobs=needs_prompt_logprobs, + tokenizer=self.tokenizer, + ) + # Only take the actual results, not the padded ones + batch_size = batch_end - batch_start + all_sequences.extend( + types.GeneratedSequence(stop_reason=stop_reason, tokens=tokens, logprobs=logprobs) + for stop_reason, tokens, logprobs in zip( + result.stop_reasons[:batch_size], + result.generated_ids[:batch_size], + result.logprobs[:batch_size], + ) + ) + if needs_prompt_logprobs and result.prompt_logprobs: + all_prompt_logprobs.extend(result.prompt_logprobs[:batch_size]) + + for request_id, _, start_idx, end_idx, prompt_logprobs_requested in request_batch_slices: + sequences = [all_sequences[i] for i in range(start_idx, end_idx)] + # Each of `num_samples` samples in a request share the same prompt; use the first's prompt logprobs + prompt_logprobs = ( + all_prompt_logprobs[start_idx] if prompt_logprobs_requested and all_prompt_logprobs else None + ) + results[request_id] = types.SampleOutput(sequences=sequences, prompt_logprobs=prompt_logprobs) + + return results + + def save_checkpoint(self, output_path: AnyPath, model_id: str) -> None: + """Save training checkpoint using Flax checkpoints.""" + checkpoint_data = self._extract_checkpoint_data(model_id) + checkpoints.save_checkpoint_multiprocess( + target=checkpoint_data, + ckpt_dir=output_path, + step=0, + prefix="checkpoint_", + overwrite=True, + ) + logger.info(f"Saved training checkpoint to {output_path}") + + def _extract_checkpoint_data(self, model_id: str) -> dict: + """Extract adapter state and optimizer state for checkpointing.""" + adapter_index = self.models[model_id].adapter_index + rank = self.models[model_id].lora_config.rank + lora_weights = extract_adapter_state(adapter_index, self.lora_params, rank) + optimizer_state = extract_adapter_state(adapter_index, nnx.state(self.optimizers[model_id]), rank) + return { + "lora_weights": lora_weights, + "optimizer_state": optimizer_state, + "lora_config": self.models[model_id].lora_config.model_dump(), + } + + def _insert_checkpoint_data(self, model_id: str, checkpoint_data: dict) -> None: + """Insert checkpoint data into model state.""" + adapter_index = self.models[model_id].adapter_index + rank = checkpoint_data["lora_config"]["rank"] + + if self.models[model_id].lora_config.rank != rank: + raise ValueError( + f"Rank mismatch: checkpoint has rank {rank}, " + f"model configured with rank {self.models[model_id].lora_config.rank}" + ) + + insert_adapter_state(adapter_index, self.lora_params, checkpoint_data["lora_weights"], rank) + insert_adapter_state( + adapter_index, nnx.state(self.optimizers[model_id]), checkpoint_data["optimizer_state"], rank + ) + + def load_checkpoint(self, checkpoint_path: AnyPath, model_id: str) -> None: + """Load training checkpoint using Flax checkpoints.""" + checkpoint = checkpoints.restore_checkpoint( + ckpt_dir=checkpoint_path, + target=self._extract_checkpoint_data(model_id), + prefix="checkpoint_", + ) + + if checkpoint is None: + raise FileNotFoundError(f"Training checkpoint not found in {checkpoint_path}") + + self._insert_checkpoint_data(model_id, checkpoint) + logger.info(f"Loaded training checkpoint from {checkpoint_path}") + + def save_sampler_checkpoint(self, output_path: AnyPath, model_id: str) -> None: + """Save sampler checkpoint as tar.gz using save_lora_checkpoint.""" + lora_model = self.models[model_id] + save_lora_checkpoint( + self.model, + self.base_model, + lora_model.lora_config, + lora_model.adapter_index, + output_path, + ) + logger.info(f"Saved LoRA sampler checkpoint to {output_path}") + + def load_sampler_checkpoint(self, model_id: str, checkpoint_id: str, checkpoint_path: AnyPath) -> None: + """Insert sampler weights from checkpoint file.""" + adapter_index = self.models[model_id].adapter_index + adapter_config = self.models[model_id].lora_config + load_lora_checkpoint(self.model, adapter_config, adapter_index, checkpoint_path) + self.models[model_id].loaded_checkpoint_id = checkpoint_id + logger.info(f"Loaded LoRA sampler weights for model {model_id} at adapter index {adapter_index}") + + def load_sampler_weights(self, prepared_batch: types.PreparedSampleBatch) -> list[int]: + """Load sampler weights for all requests and return adapter indices array. + + Ensures all required checkpoints are loaded before sampling. + + Args: + prepared_batch: PreparedSampleBatch with model_ids, checkpoint_ids, and other batch data + + Returns: + The adapter_indices array for LoRA sampling [batch_size] + Uses adapter index 0 for base model sampling (no LoRA) + """ + adapter_indices = [] + loaded_adapters = set() # Track adapters already used in this batch + + for model_id, checkpoint_id, checkpoint_path in zip( + prepared_batch.all_model_ids, prepared_batch.all_checkpoint_ids, prepared_batch.all_checkpoint_paths + ): + if model_id: + # This code path is for sampling from a LoRA adapter + assert checkpoint_id != "", "checkpoint_id must be not empty" + + adapter_index = self.models[model_id].adapter_index + if self.models[model_id].loaded_checkpoint_id == checkpoint_id: + # Weights already loaded in RAM + adapter_indices.append(adapter_index) + else: + # Need to load from disk + assert adapter_index not in loaded_adapters, "Cannot override already used adapter" + + logger.info(f"Loading LoRA sampler checkpoint from {checkpoint_path}") + self.load_sampler_checkpoint(model_id, checkpoint_id, AnyPath(checkpoint_path)) + adapter_indices.append(adapter_index) + + loaded_adapters.add(adapter_index) + else: + # This code path is for sampling from the base model + adapter_indices.append(0) + + return adapter_indices + + +# ============================================================================= +# Multi-host coordination +# ============================================================================= + + +class RpcPayload(BaseModel): + """Generic RPC payload container using runtime type introspection. + + Instead of defining separate command classes for each method, this single + generic container holds the method name and raw kwargs. The worker uses + type hints from the target method to automatically re-hydrate the kwargs + into the correct Pydantic models. + """ + + method: str + kwargs: dict[str, Any] # Contains raw dicts/JSON types + + +RpcPayloadAdapter: TypeAdapter[RpcPayload] = TypeAdapter(RpcPayload) + + +def _broadcast_command(cmd: RpcPayload | None) -> RpcPayload: + """Broadcast an RpcPayload from coordinator to all workers using JSON. + + On coordinator (process 0): serializes and broadcasts the payload. + On workers: receives and deserializes the payload (pass None). + """ + if jax.process_index() == 0: + assert cmd is not None, "Coordinator must provide a command to broadcast." + data = RpcPayloadAdapter.dump_json(cmd) + size = np.array([len(data)], dtype=np.int64) + else: + size = np.array([0], dtype=np.int64) + + # Broadcast size first + size = multihost_utils.broadcast_one_to_all(size) + + # Broadcast data + if jax.process_index() == 0: + data_arr = np.frombuffer(data, dtype=np.uint8) + else: + data_arr = np.zeros(size[0], dtype=np.uint8) + + data_arr = multihost_utils.broadcast_one_to_all(data_arr) + + return RpcPayloadAdapter.validate_json(data_arr.tobytes()) + + +class JaxBackend(JaxBackendImpl): + """Distributed wrapper that broadcasts commands before calling JaxBackendImpl methods. + + Workers use runtime type introspection to re-hydrate arguments automatically. + """ + + def __init__(self, base_model: str, config: JaxBackendConfig): + if config.coordinator_address is not None: + jax.distributed.initialize( + coordinator_address=config.coordinator_address, + num_processes=config.num_processes, + process_id=0, + ) + logger.info( + f"JAX distributed initialized: process_id={jax.process_index()} ({jax.process_count()} total), " + f"local devices: {jax.local_device_count()}, total devices: {jax.device_count()}" + ) + + self._broadcast_and_call("__init__", base_model=base_model, config=config) + + def _broadcast_and_call(self, method: str, **kwargs): + """Broadcast method call to workers and execute locally via super().""" + if jax.process_count() > 1: + clean = {k: v.model_dump() if isinstance(v, BaseModel) else v for k, v in kwargs.items()} + _broadcast_command(RpcPayload(method=method, kwargs=clean)) + return getattr(super(), method)(**kwargs) + + def create_model(self, model_id: str, lora_config: types.LoraConfig) -> None: + self._broadcast_and_call("create_model", model_id=model_id, lora_config=lora_config) + + def forward_backward(self, prepared_batch: types.PreparedModelPassBatch): + return self._broadcast_and_call("forward_backward", prepared_batch=prepared_batch) + + def forward(self, prepared_batch: types.PreparedModelPassBatch): + return self._broadcast_and_call("forward", prepared_batch=prepared_batch) + + def optim_step(self, model_id: str, request_data: types.OptimStepInput): + return self._broadcast_and_call("optim_step", model_id=model_id, request_data=request_data) + + def sample(self, prepared_batch: types.PreparedSampleBatch): + return self._broadcast_and_call("sample", prepared_batch=prepared_batch) + + def save_checkpoint(self, output_path: AnyPath, model_id: str) -> None: + self._broadcast_and_call("save_checkpoint", output_path=output_path, model_id=model_id) + + def load_checkpoint(self, checkpoint_path: AnyPath, model_id: str) -> None: + self._broadcast_and_call("load_checkpoint", checkpoint_path=checkpoint_path, model_id=model_id) + + def save_sampler_checkpoint(self, output_path: AnyPath, model_id: str) -> None: + self._broadcast_and_call("save_sampler_checkpoint", output_path=output_path, model_id=model_id) + + +def run_worker(coordinator_address: str, num_processes: int, process_id: int): + """Entry point for worker processes. + + Initializes JAX distributed, receives config from coordinator, then runs + the worker loop using runtime type introspection to re-hydrate arguments. + + Args: + coordinator_address: JAX coordinator address (host:port) + num_processes: Total number of processes in the cluster + process_id: This process's ID (must be > 0 for workers) + """ + if process_id == 0: + raise ValueError("Worker process_id must be > 0 (process 0 is the coordinator)") + + # Initialize JAX distributed first (before any other JAX operations) + jax.distributed.initialize( + coordinator_address=coordinator_address, + num_processes=num_processes, + process_id=process_id, + ) + logger.info( + f"Worker process_id={jax.process_index()} ({jax.process_count()} total) initialized, waiting for config from coordinator..." + ) + + # Receive INIT payload with base_model and config from coordinator + init_payload = _broadcast_command(None) + assert init_payload.method == "__init__", f"Expected __init__, got {init_payload.method}" + config = JaxBackendConfig.model_validate(init_payload.kwargs["config"]) + logger.info(f"Worker received config: base_model={init_payload.kwargs['base_model']}, config={config}") + + backend = JaxBackendImpl(init_payload.kwargs["base_model"], config) + + logger.info(f"Worker process_id={jax.process_index()} entering command loop") + + while True: + payload: RpcPayload = _broadcast_command(None) + + if not hasattr(backend, payload.method): + logger.error(f"Unknown method: {payload.method}") + continue + + method = getattr(backend, payload.method) + + # Re-hydrate raw dicts into Pydantic models using type hints + hints = get_type_hints(method) + kwargs = {k: TypeAdapter(hints[k]).validate_python(v) if k in hints else v for k, v in payload.kwargs.items()} + method(**kwargs) + + +def main(): + """Entry point for running as a worker process.""" + import argparse + + parser = argparse.ArgumentParser(description="SkyRL tx tinker worker process") + parser.add_argument( + "--coordinator-address", + required=True, + help="JAX coordinator address (host:port)", + ) + parser.add_argument( + "--num-processes", + type=int, + required=True, + help="Total number of processes in the cluster", + ) + parser.add_argument( + "--process-id", + type=int, + required=True, + help="This process's ID (must be > 0 for workers)", + ) + + args = parser.parse_args() + run_worker(args.coordinator_address, args.num_processes, args.process_id) + + +if __name__ == "__main__": + main() diff --git a/skyrl-tx/tx/tinker/backends/utils.py b/skyrl-tx/tx/tinker/backends/utils.py new file mode 100644 index 0000000000..b3f5969db1 --- /dev/null +++ b/skyrl-tx/tx/tinker/backends/utils.py @@ -0,0 +1,58 @@ +"""Shared helper utilities for TinkerEngine backends.""" + +import time +from contextlib import contextmanager + +import numpy as np + +from tx.utils.log import logger + + +@contextmanager +def log_timing(request: str): + """Context manager to log execution time for a request.""" + start_time = time.perf_counter() + try: + yield + finally: + elapsed = time.perf_counter() - start_time + logger.info(f"(timing) {request} took {elapsed:.3f}s") + + +def pad(xs, pad_to: int, *, fill): + """Pad a list to a specified length with a fill value.""" + return xs + ([fill] * (pad_to - len(xs))) + + +def pad_batch(sequences: list[list], max_length: int, dtype, left: bool = False) -> np.ndarray: + """Pad a batch of sequences to max_length. + + Args: + sequences: List of sequences to pad. + max_length: Target length for all sequences. + dtype: NumPy dtype for the output array. + left: If True, use left-padding (tokens at end). Required for autoregressive + generation so the last position corresponds to the last real token. + If False (default), use right-padding (tokens at start). + + Returns: + A NumPy array of shape (batch_size, max_length) with the padded sequences. + """ + batch_size = len(sequences) + padded = np.zeros((batch_size, max_length), dtype=dtype) + for i, seq in enumerate(sequences): + assert len(seq) <= max_length, f"Sequence length {len(seq)} exceeds max_length {max_length}" + if left: + padded[i, max_length - len(seq) :] = seq + else: + padded[i, : len(seq)] = seq + return padded + + +def pad_to_fsdp(arr: np.ndarray, fsdp_size: int) -> np.ndarray: + """Pad array's first dimension to be divisible by FSDP size.""" + batch_size = arr.shape[0] + pad_size = (fsdp_size - batch_size % fsdp_size) % fsdp_size + if pad_size == 0: + return arr + return np.pad(arr, [(0, pad_size)] + [(0, 0)] * (arr.ndim - 1)) diff --git a/skyrl-tx/tx/tinker/config.py b/skyrl-tx/tx/tinker/config.py new file mode 100644 index 0000000000..e126e5499e --- /dev/null +++ b/skyrl-tx/tx/tinker/config.py @@ -0,0 +1,131 @@ +"""Configuration for the Tinker engine.""" + +import argparse +import json +import os +from pathlib import Path + +from cloudpathlib import AnyPath +from pydantic import BaseModel, Field + + +class EngineConfig(BaseModel): + """Configuration for the Tinker engine.""" + + base_model: str = Field(..., description="Base model name (e.g., Qwen/Qwen3-0.6B)") + backend: str = Field(default="jax", description="Backend to use for training and inference") + backend_config: dict = Field( + default_factory=dict, + description="Backend-specific configuration as JSON string", + json_schema_extra={"argparse_type": json.loads}, + ) + checkpoints_base: AnyPath = Field( + default=AnyPath("/tmp/tx_checkpoints"), + description="Base path where checkpoints will be stored", + ) + database_url: str = Field( + default=f'sqlite:///{Path(__file__).parent / "tinker.db"}', + description="Database URL (e.g., postgresql://user:password@localhost:5432/tinker). If not set, uses TX_DATABASE_URL env var or defaults to SQLite", + json_schema_extra={"argparse_type": str, "env_var": "TX_DATABASE_URL"}, + ) + external_inference_url: str | None = Field( + default=None, + description="URL of the external inference engine. If set, sample requests will be sent to the external engine instead (currently only VLLM is supported).", + json_schema_extra={"argparse_type": str}, + ) + external_inference_api_key: str = Field( + default="EMPTY", + description="API key for an external inference engine. If not provided will use vLLM 'EMPTY' key convention", + ) + external_inference_lora_base: Path = Field( + default=Path("/tmp/lora_models"), + description="Directory where LoRA models will be extracted for external inference engines", + ) + session_cleanup_interval_sec: int = Field( + default=60, + description="How often to check for stale sessions (seconds). Set to -1 to disable cleanup.", + ) + # The tinker client sends heartbeats every 10 seconds by default. + # https://github.com/thinking-machines-lab/tinker/blob/2d8e9d5e00f746f39148a5d0cb760dff3f2eed43/src/tinker/lib/internal_client_holder.py#L182 + session_timeout_sec: int = Field( + default=300, + description="Seconds without heartbeat before session is considered stale. Set to -1 to disable cleanup.", + ) + + +def convert_env_var(env_name: str, env_value: str, expected_type: type): + """Convert environment variable to expected type.""" + if expected_type is bool: + if env_value not in ("0", "1"): + raise ValueError( + f"Environment variable '{env_name}' for a boolean flag must be '0' or '1', but got '{env_value}'." + ) + return env_value == "1" + else: + return env_value + + +def add_model(parser: argparse.ArgumentParser, model: type[BaseModel]) -> None: + """Add Pydantic model fields to an ArgumentParser. + + The priority order of how options are handled: 1. Explicitly specified command line options, + 2. environment variables and 3. default values. + + Args: + parser: The ArgumentParser to add arguments to + model: The Pydantic model class + """ + for name, field in model.model_fields.items(): + arg_name = name.replace("_", "-") + kwargs = { + "help": field.description, + } + + # Check for default value, with env_var support + default_value = field.default + if field.json_schema_extra and "env_var" in field.json_schema_extra: + env_name = field.json_schema_extra["env_var"] + if env_value := os.environ.get(env_name): + default_value = convert_env_var(env_name, env_value, field.annotation) + + if field.annotation is bool: + # For boolean flags, use BooleanOptionalAction to support both --{arg_name} and --no-{arg_name} + kwargs = {**kwargs, "action": argparse.BooleanOptionalAction, "dest": name, "default": default_value} + else: + # Check if explicit argparse_type is specified in field metadata + argparse_type = field.json_schema_extra.get("argparse_type") if field.json_schema_extra else None + if argparse_type is not None: + kwargs["type"] = argparse_type + elif field.annotation is not None: + kwargs["type"] = field.annotation + + if field.is_required(): + # Mark as required in argparse if no default is provided + kwargs["required"] = True + else: + # For optional fields, provide the default value to argparse + kwargs["default"] = default_value + + parser.add_argument(f"--{arg_name}", **kwargs) + + +def config_to_argv(cfg: BaseModel) -> list[str]: + """This should 'unparse' a config parsed by an ArgumentParser constructed by add_model.""" + argv = [] + for field_name, value in cfg.model_dump().items(): + field = cfg.model_fields[field_name] + arg_name = field_name.replace("_", "-") + + if field.annotation is bool: + argv.append(f"--{arg_name}" if value else f"--no-{arg_name}") + elif field.annotation is dict: + # Serialize dict to JSON string + if value: + argv.append(f"--{arg_name}") + argv.append(json.dumps(value)) + else: + # Skip None values - let them use defaults or environment variables + if value is not None: + argv.append(f"--{arg_name}") + argv.append(str(value)) + return argv diff --git a/skyrl-tx/tx/tinker/db_models.py b/skyrl-tx/tx/tinker/db_models.py new file mode 100644 index 0000000000..c8d72e1c24 --- /dev/null +++ b/skyrl-tx/tx/tinker/db_models.py @@ -0,0 +1,116 @@ +"""Database models for the Tinker API.""" + +from datetime import datetime, timezone +from enum import Enum + +from sqlmodel import SQLModel, Field, JSON +from sqlalchemy import DateTime +from sqlalchemy.engine import url as sqlalchemy_url + +from tx.tinker import types + + +def get_async_database_url(db_url: str) -> str: + """Get the async database URL. + + Args: + db_url: Optional database URL to use. + + Returns: + Async database URL string for SQLAlchemy. + + Raises: + ValueError: If the database scheme is not supported. + """ + parsed_url = sqlalchemy_url.make_url(db_url) + + match parsed_url.get_backend_name(): + case "sqlite": + async_url = parsed_url.set(drivername="sqlite+aiosqlite") + case "postgresql": + async_url = parsed_url.set(drivername="postgresql+asyncpg") + case _ if "+" in parsed_url.drivername: + # Already has an async driver specified, keep it + async_url = parsed_url + case backend_name: + raise ValueError(f"Unsupported database scheme: {backend_name}") + + return async_url.render_as_string(hide_password=False) + + +class RequestStatus(str, Enum): + """Status of a request.""" + + PENDING = "pending" + COMPLETED = "completed" + FAILED = "failed" + + +class CheckpointStatus(str, Enum): + """Status of a checkpoint.""" + + PENDING = "pending" + COMPLETED = "completed" + FAILED = "failed" + + +# SQLModel table definitions +class ModelDB(SQLModel, table=True): + __tablename__ = "models" + + model_id: str = Field(primary_key=True) + base_model: str + lora_config: dict[str, object] = Field(sa_type=JSON) + status: str = Field(index=True) + request_id: int + session_id: str = Field(foreign_key="sessions.session_id", index=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), sa_type=DateTime(timezone=True)) + + +class FutureDB(SQLModel, table=True): + __tablename__ = "futures" + + request_id: int | None = Field(default=None, primary_key=True, sa_column_kwargs={"autoincrement": True}) + request_type: types.RequestType + model_id: str | None = Field(default=None, index=True) + request_data: dict = Field(sa_type=JSON) # this is of type types.{request_type}Input + result_data: dict | None = Field(default=None, sa_type=JSON) # this is of type types.{request_type}Output + status: RequestStatus = Field(default=RequestStatus.PENDING, index=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), sa_type=DateTime(timezone=True)) + completed_at: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + + +class CheckpointDB(SQLModel, table=True): + __tablename__ = "checkpoints" + + model_id: str = Field(foreign_key="models.model_id", primary_key=True) + checkpoint_id: str = Field(primary_key=True) + checkpoint_type: types.CheckpointType = Field(primary_key=True) + status: CheckpointStatus + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), sa_type=DateTime(timezone=True)) + completed_at: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + error_message: str | None = None + + +class SessionDB(SQLModel, table=True): + __tablename__ = "sessions" + + session_id: str = Field(primary_key=True) + tags: list[str] = Field(default_factory=list, sa_type=JSON) + user_metadata: dict = Field(default_factory=dict, sa_type=JSON) + sdk_version: str + status: str = Field(default="active", index=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), sa_type=DateTime(timezone=True)) + last_heartbeat_at: datetime | None = Field(default=None, sa_type=DateTime(timezone=True), index=True) + heartbeat_count: int = 0 + + +class SamplingSessionDB(SQLModel, table=True): + __tablename__ = "sampling_sessions" + + sampling_session_id: str = Field(primary_key=True) + session_id: str = Field(foreign_key="sessions.session_id", index=True) + sampling_session_seq_id: int + base_model: str | None = None + model_path: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), sa_type=DateTime(timezone=True)) diff --git a/skyrl-tx/tx/tinker/engine.py b/skyrl-tx/tx/tinker/engine.py new file mode 100644 index 0000000000..b7f4b29178 --- /dev/null +++ b/skyrl-tx/tx/tinker/engine.py @@ -0,0 +1,648 @@ +"""Background engine for processing training requests.""" + +import argparse +import time +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Callable + +from cloudpathlib import AnyPath +from pydantic import BaseModel +from sqlmodel import create_engine, Session, select, update, func + +from tx.tinker.db_models import FutureDB, RequestStatus, CheckpointDB, CheckpointStatus, ModelDB, SessionDB +from tx.tinker import types +from tx.tinker.config import EngineConfig, add_model +from tx.tinker.backends.jax import JaxBackend, JaxBackendConfig +from tx.tinker.backends.utils import log_timing +from tx.tinker.loss_fns import LOSS_TYPES +from tx.utils.log import logger + + +def prepare_sample_batch( + requests: dict[str, tuple[str, types.SampleInput]], + checkpoints_base: AnyPath | None = None, +) -> types.PreparedSampleBatch: + """Prepare batch data for sample operations. + + Extracts prompts and sampling params from requests into lists + that the backend will convert to arrays. + + Args: + requests: Dict mapping request_id to (model_id, request_data) tuples (pre-validated) + checkpoints_base: Base path for checkpoints (optional, needed for LoRA sampling) + + Returns: + PreparedSampleBatch with all data extracted from requests + """ + all_prompts = [] + all_sampling_params = [] + all_model_ids = [] + all_checkpoint_ids = [] + all_checkpoint_paths = [] + request_batch_slices = [] + + needs_prompt_logprobs = any(request_data.prompt_logprobs for (_, request_data) in requests.values()) + + for request_id, (model_id, request_data) in requests.items(): + request_start = len(all_prompts) + + # Expand requests for num_samples + prompt_tokens = [token for chunk in request_data.prompt.chunks for token in chunk.tokens] + checkpoint_path = "" + if model_id and request_data.checkpoint_id and checkpoints_base: + checkpoint_path = str( + checkpoints_base / model_id / "sampler_weights" / f"{request_data.checkpoint_id}.tar.gz" + ) + for _ in range(request_data.num_samples): + all_prompts.append(prompt_tokens) + all_sampling_params.append(request_data.sampling_params) + all_model_ids.append(model_id) + all_checkpoint_ids.append(request_data.checkpoint_id) + all_checkpoint_paths.append(checkpoint_path) + + request_batch_slices.append( + (request_id, model_id, request_start, len(all_prompts), request_data.prompt_logprobs) + ) + + return types.PreparedSampleBatch( + all_prompts=all_prompts, + all_sampling_params=all_sampling_params, + all_model_ids=all_model_ids, + all_checkpoint_ids=all_checkpoint_ids, + all_checkpoint_paths=all_checkpoint_paths, + needs_prompt_logprobs=needs_prompt_logprobs, + request_batch_slices=request_batch_slices, + ) + + +def prepare_model_pass_batch( + requests: dict[str, tuple[str, types.ForwardBackwardInput]], +) -> types.PreparedModelPassBatch: + """Prepare batch data for forward/forward_backward operations. + + Extracts tokens, targets, and metadata from requests into lists + that the backend will convert to arrays. + + Args: + requests: Dict mapping request_id to (model_id, request_data) tuples (pre-validated) + + Returns: + PreparedModelPassBatch with all data extracted from requests + """ + all_input_ids = [] + all_targets = [] + all_token_weights = [] + all_model_ids = [] + all_sampling_logprobs = [] + all_advantages = [] + all_loss_fn_types = [] + request_batch_slices = [] + + for request_id, (model_id, request_data) in requests.items(): + loss_fn_type = LOSS_TYPES[request_data.loss_fn] + + request_start = len(all_input_ids) + for item in request_data.data: + tokens = [t for chunk in item.model_input.chunks for t in chunk.tokens] + all_input_ids.append(tokens) + loss_fn_inputs = item.loss_fn_inputs + all_targets.append(loss_fn_inputs.target_tokens.data) + all_token_weights.append(loss_fn_inputs.weights.data) + all_sampling_logprobs.append(loss_fn_inputs.logprobs.data) + all_advantages.append(loss_fn_inputs.advantages.data) + all_model_ids.append(model_id) + all_loss_fn_types.append(loss_fn_type) + + request_batch_slices.append((request_id, model_id, request_start, len(all_input_ids))) + + return types.PreparedModelPassBatch( + all_input_ids=all_input_ids, + all_targets=all_targets, + all_token_weights=all_token_weights, + all_sampling_logprobs=all_sampling_logprobs, + all_advantages=all_advantages, + all_model_ids=all_model_ids, + all_loss_fn_types=all_loss_fn_types, + request_batch_slices=request_batch_slices, + ) + + +BACKENDS = { + "jax": (JaxBackend, JaxBackendConfig), +} + + +class TinkerEngine: + """Background engine for processing training requests. + + The engine handles: + - Database operations (futures, checkpoints) + - Request finding/scheduling + - File I/O (download/upload checkpoints) + - Validating requests against loaded models + + Computation and model management are delegated to the backend. + """ + + def _filter_valid_requests( + self, + requests: dict[str, tuple[str, BaseModel]], + ) -> tuple[dict[str, types.ErrorResponse], dict[str, tuple[str, BaseModel]]]: + """Filter out requests with invalid model_ids and return error results for them. + + Args: + requests: Dict mapping request_id to (model_id, request_data) tuples + + Returns: + Tuple of (error_results, valid_requests) + """ + results = {} + valid_requests = {} + + for request_id, (model_id, request_data) in requests.items(): + error = None + if model_id and not self.backend.has_model(model_id): + error = f"Model {model_id} not loaded" + elif not model_id and isinstance(request_data, types.SampleInput): + if request_data.base_model != self.config.base_model: + error = f"Engine is configured for '{self.config.base_model}' but request specified '{request_data.base_model}'" + elif request_data.checkpoint_id: + error = "checkpoint_id must be empty for base model sampling" + + if error: + results[request_id] = types.ErrorResponse(error=error, status="failed") + else: + valid_requests[request_id] = (model_id, request_data) + + return results, valid_requests + + def __init__( + self, + config: EngineConfig, + ): + """Initialize the engine with a database connection and base model.""" + self.config = config + self.db_engine = create_engine(config.database_url, echo=False) + + # Initialize the backend (handles model state, computation, and adapter management) + if config.backend not in BACKENDS: + raise ValueError(f"Unknown backend: {config.backend}. Available backends: {list(BACKENDS.keys())}") + + backend_class, backend_config_class = BACKENDS[config.backend] + backend_config = backend_config_class(**config.backend_config) + self.backend = backend_class(config.base_model, backend_config) + + # Track last cleanup time for periodic stale session cleanup + self._last_cleanup_time: float = time.time() + + logger.info(f"Initialized TinkerEngine with backend={type(self.backend).__name__}") + + @property + def metrics(self) -> types.EngineMetrics: + """Pass-through to backend metrics for backwards compatibility.""" + return self.backend.metrics + + @contextmanager + def _checkpoint_status_context(self, model_id: str, checkpoint_id: str, checkpoint_type: types.CheckpointType): + """Context manager to handle checkpoint DB status updates. + + Fetches the checkpoint entry, yields it, and updates its status to COMPLETED + or FAILED based on whether an exception occurred. + """ + with Session(self.db_engine) as session: + checkpoint_db = session.get(CheckpointDB, (model_id, checkpoint_id, checkpoint_type)) + if checkpoint_db is None: + raise ValueError( + f"Checkpoint entry not found for model '{model_id}', checkpoint '{checkpoint_id}', type '{checkpoint_type}'" + ) + + try: + yield checkpoint_db + checkpoint_db.status = CheckpointStatus.COMPLETED + except Exception as e: + logger.exception(f"Error saving checkpoint for model {model_id}, checkpoint {checkpoint_id}: {e}") + checkpoint_db.status = CheckpointStatus.FAILED + checkpoint_db.error_message = str(e) + raise + finally: + checkpoint_db.completed_at = datetime.now(timezone.utc) + session.add(checkpoint_db) + session.commit() + + def find_batchable_model_passes( + self, session: Session, request_type: types.RequestType + ) -> dict[str, tuple[str, types.ForwardBackwardInput]]: + """Find all requests of the given type that come before any destructive update for their model. + + Uses look-ahead scheduling: for each model, only returns operations + that have no optim_step or load_weights blocking them in the queue. + + Args: + session: Database session + request_type: The type of request to find (e.g., FORWARD or FORWARD_BACKWARD) + + Returns: + Dict mapping request_id to (model_id, request_data) tuples + """ + # Find the earliest pending optim_step or load_weights per model (these act as barriers) + barriers_query = ( + select(FutureDB.model_id, func.min(FutureDB.request_id).label("barrier_id")) + .where( + (FutureDB.request_type == types.RequestType.OPTIM_STEP) + | (FutureDB.request_type == types.RequestType.LOAD_WEIGHTS) + ) + .where(FutureDB.status == RequestStatus.PENDING) + .group_by(FutureDB.model_id) + ) + barriers = dict(session.exec(barriers_query).all()) + + # Get all pending operations of the requested type ordered by request_id + query = ( + select(FutureDB) + .where(FutureDB.request_type == request_type) + .where(FutureDB.status == RequestStatus.PENDING) + .order_by(FutureDB.request_id) + ) + ops = session.exec(query).all() + + # Filter: only include ops that come before their model's barrier + batchable = [op for op in ops if op.model_id not in barriers or op.request_id < barriers[op.model_id]] + + return { + str(f.request_id): (f.model_id, types.ForwardBackwardInput.model_validate(f.request_data)) + for f in batchable + } + + def find_batchable_sample(self, session: Session) -> dict[str, tuple[str, types.SampleInput]]: + """Find all sample ops that can be safely batched together. + + Returns sample operations ensuring that each model_id has only one checkpoint_id + to avoid loading different checkpoints for the same model in a single batch. + + If sample_max_num_sequences is configured, limits to that many requests so we don't + produce partial batches in process_sample_batch. If num_samples > 1 for some requests, + this may not be perfect, but it's good until we implement continuous batching. + + Args: + session: Database session + + Returns: + Dict mapping request_id to (model_id, request_data) tuples + """ + sample_query = ( + select(FutureDB) + .where(FutureDB.request_type == types.RequestType.SAMPLE) + .where(FutureDB.status == RequestStatus.PENDING) + .order_by(FutureDB.request_id) + ) + sample_ops = session.exec(sample_query).all() + + batchable = [] + model_checkpoints = {} # Map from model_id to checkpoint_id of first request to that model + for op in sample_ops: + checkpoint_id = op.request_data["checkpoint_id"] + # Base model requests (empty checkpoint_id) are always compatible, otherwise only + # take only requests with one checkpoint_id for a given model_id + if checkpoint_id == "" or model_checkpoints.setdefault(op.model_id, checkpoint_id) == checkpoint_id: + batchable.append(op) + + # TODO: This leaks the abstraction by accessing backend-specific config. + # We should find a better way to handle this going forward. + if isinstance(self.backend, JaxBackend) and self.backend.config.sample_max_num_sequences > 0: + batchable = batchable[: self.backend.config.sample_max_num_sequences] + + return {str(f.request_id): (f.model_id, types.SampleInput.model_validate(f.request_data)) for f in batchable} + + def find_single_requests(self, session: Session) -> dict[str, tuple[str, types.RequestType, dict]]: + """Find all requests that need to be processed individually (not batchable). + + Args: + session: Database session + + Returns: + Dict mapping request_id to (model_id, request_type, request_data) tuples + """ + statement = ( + select(FutureDB) + .where(FutureDB.status == RequestStatus.PENDING) + .where(FutureDB.request_type != types.RequestType.FORWARD_BACKWARD) + .where(FutureDB.request_type != types.RequestType.FORWARD) + .where(FutureDB.request_type != types.RequestType.SAMPLE) + .where(FutureDB.request_type != types.RequestType.EXTERNAL) + .order_by(FutureDB.request_id) + ) + other_futures = session.exec(statement).all() + + return {str(f.request_id): (f.model_id, f.request_type, f.request_data) for f in other_futures} + + def process_create_model(self, model_id: str, request_data: types.CreateModelInput) -> types.CreateModelOutput: + """Create and initialize a model.""" + # Create model in backend (allocates adapter_index, creates optimizer, and configures adapter) + self.backend.create_model(model_id, request_data.lora_config) + + logger.info(f"Created LoRA model {model_id}") + + return types.CreateModelOutput( + model_id=model_id, + base_model=self.config.base_model, + lora_config=request_data.lora_config, + ) + + def process_unload_model(self, model_id: str, request_data: types.UnloadModelInput) -> types.UnloadModelOutput: + """Unload a model and free all resources.""" + if not self.backend.has_model(model_id): + logger.warning(f"Ignoring unload request for model {model_id} that is not loaded.") + else: + self.backend.delete_model(model_id) + + # Update model status in DB + with Session(self.db_engine) as session: + _ = session.exec(update(ModelDB).where(ModelDB.model_id == model_id).values(status="unloaded")) + session.commit() + + logger.info(f"Unloaded model {model_id}") + + return types.UnloadModelOutput(model_id=model_id, status="unloaded") + + def cleanup_stale_sessions(self) -> int: + """Cleanup sessions with no recent heartbeat and unload their models. + + Returns: + Number of models unloaded + """ + cutoff = datetime.now(timezone.utc) - timedelta(seconds=self.config.session_timeout_sec) + unloaded_count = 0 + + with Session(self.db_engine) as session: + # Find stale sessions (active sessions with heartbeat older than cutoff) + stale_sessions = session.exec( + select(SessionDB).where( + SessionDB.status == "active", + SessionDB.last_heartbeat_at < cutoff, + ) + ).all() + + if not stale_sessions: + return 0 + + stale_session_ids = {s.session_id for s in stale_sessions} + + # Find all models for all stale sessions in one query + models_to_process = session.exec( + select(ModelDB).where( + ModelDB.session_id.in_(stale_session_ids), + ModelDB.status != "unloaded", + ) + ).all() + + sessions_with_failed_unloads = set() + for model in models_to_process: + if self.backend.has_model(model.model_id): + try: + self.backend.delete_model(model.model_id) + model.status = "unloaded" + unloaded_count += 1 + logger.info(f"Auto-unloaded stale model {model.model_id} from session {model.session_id}") + except Exception as e: + logger.error(f"Failed to auto-unload model {model.model_id}: {e}") + sessions_with_failed_unloads.add(model.session_id) + else: + # Model not in backend but status not unloaded - fix DB state + model.status = "unloaded" + + for sess in stale_sessions: + if sess.session_id not in sessions_with_failed_unloads: + sess.status = "expired" + logger.info(f"Expired stale session {sess.session_id} (last heartbeat: {sess.last_heartbeat_at})") + + session.commit() + + return unloaded_count + + def process_optim_step(self, model_id: str, request_data: types.OptimStepInput) -> types.OptimStepOutput: + """Process an optim_step request and apply accumulated gradients.""" + if not self.backend.has_model(model_id): + raise ValueError(f"Model {model_id} not loaded") + + return self.backend.optim_step(model_id, request_data) + + def process_forward_backward(self, requests: dict[str, tuple[str, types.ForwardBackwardInput]]) -> dict: + """Run forward and backward pass on a batch of requests.""" + prepared = prepare_model_pass_batch(requests) + return self.backend.forward_backward(prepared) + + def process_forward(self, requests: dict[str, tuple[str, types.ForwardBackwardInput]]) -> dict: + """Run forward-only pass on a batch of requests.""" + prepared = prepare_model_pass_batch(requests) + return self.backend.forward(prepared) + + def process_sample(self, requests: dict[str, tuple[str, types.SampleInput]]) -> dict: + """Generate samples for a batch of requests.""" + prepared = prepare_sample_batch(requests, self.config.checkpoints_base) + return self.backend.sample(prepared) + + def process_load_weights(self, model_id: str, request_data: types.LoadWeightsInput) -> types.LoadWeightsOutput: + """Loads a clean, trimmed training checkpoint.""" + if not self.backend.has_model(model_id): + raise ValueError("Model not loaded. Create the model before loading a checkpoint.") + + checkpoint_path = ( + self.config.checkpoints_base / request_data.source_model_id / f"{request_data.checkpoint_id}.tar.gz" + ) + + self.backend.load_checkpoint(checkpoint_path, model_id) + + return types.LoadWeightsOutput(type="load_weights") + + def process_save_weights(self, model_id: str, request_data: types.SaveWeightsInput) -> types.SaveWeightsOutput: + """ + Saves a clean training checkpoint by converting the trimmed NNX graph + to a pure dictionary before serialization, following official Flax docs. + """ + if not self.backend.has_model(model_id): + raise ValueError(f"Model {model_id} not loaded") + + checkpoint_id = request_data.path + output_path = self.config.checkpoints_base / model_id / f"{checkpoint_id}.tar.gz" + + with self._checkpoint_status_context(model_id, checkpoint_id, types.CheckpointType.TRAINING): + self.backend.save_checkpoint(output_path, model_id) + logger.info(f"Saved trimmed training checkpoint for model {model_id} to {output_path}") + + return types.SaveWeightsOutput( + path=f"tinker://{model_id}/weights/{checkpoint_id}", + type="save_weights", + ) + + def process_save_weights_for_sampler( + self, model_id: str, request_data: types.SaveWeightsForSamplerInput + ) -> types.SaveWeightsForSamplerOutput: + """Process a save_weights_for_sampler request and save model weights.""" + if not self.backend.has_model(model_id): + raise ValueError(f"Model {model_id} not loaded") + + # Make sure the user cannot store checkpoints in places like ../../ + checkpoint_id = Path(request_data.path).name + output_path = self.config.checkpoints_base / model_id / "sampler_weights" / f"{checkpoint_id}.tar.gz" + + with self._checkpoint_status_context(model_id, checkpoint_id, types.CheckpointType.SAMPLER): + self.backend.save_sampler_checkpoint(output_path, model_id) + logger.info(f"Saved LoRA adapter weights for model {model_id} to {output_path}") + + # Return path=None when using sampling_session_seq_id and seq_id (SDK expects this) + if request_data.sampling_session_seq_id is not None and request_data.seq_id is not None: + output_path_str = None + else: + output_path_str = f"tinker://{model_id}/{checkpoint_id}" + + return types.SaveWeightsForSamplerOutput( + path=output_path_str, + type="save_weights_for_sampler", + sampling_session_id=request_data.sampling_session_id, + ) + + def _complete_futures(self, results: dict[str, BaseModel]): + """Helper method to complete multiple futures in the database. + + Args: + results: Dict mapping request_id to result (Pydantic BaseModel) + """ + completed_at = datetime.now(timezone.utc) + params = [ + { + "request_id": int(request_id), + "result_data": result.model_dump(), + "status": RequestStatus.FAILED if isinstance(result, types.ErrorResponse) else RequestStatus.COMPLETED, + "completed_at": completed_at, + } + for request_id, result in results.items() + ] + + with Session(self.db_engine) as session: + session.execute(update(FutureDB), params) + session.commit() + + def process_single_request(self, request_type: types.RequestType, model_id: str, request_data: dict) -> BaseModel: + match request_type: + case types.RequestType.CREATE_MODEL: + return self.process_create_model(model_id, types.CreateModelInput.model_validate(request_data)) + case types.RequestType.OPTIM_STEP: + return self.process_optim_step(model_id, types.OptimStepInput.model_validate(request_data)) + case types.RequestType.SAVE_WEIGHTS_FOR_SAMPLER: + return self.process_save_weights_for_sampler( + model_id, types.SaveWeightsForSamplerInput.model_validate(request_data) + ) + case types.RequestType.SAVE_WEIGHTS: + return self.process_save_weights(model_id, types.SaveWeightsInput.model_validate(request_data)) + case types.RequestType.LOAD_WEIGHTS: + return self.process_load_weights(model_id, types.LoadWeightsInput.model_validate(request_data)) + case types.RequestType.UNLOAD_MODEL: + return self.process_unload_model(model_id, types.UnloadModelInput.model_validate(request_data)) + case _: + raise ValueError(f"Unknown request type: {request_type}") + + def process_single_requests(self, requests: dict[str, tuple[str, types.RequestType, dict]]): + """Process a collection of single (non-batchable) requests. + + Args: + requests: Dict mapping request_id to (model_id, request_type, request_data) tuples + """ + if not requests: + return + results = {} + for request_id, (model_id, request_type, request_data) in requests.items(): + with log_timing(f"process_single_request({request_type.value})"): + try: + result = self.process_single_request(request_type, model_id, request_data) + except Exception as e: + logger.exception(f"Error processing request {request_id}: {e}") + result = types.ErrorResponse(error=str(e), status="failed") + results[request_id] = result + self._complete_futures(results) + + def process_batch_requests( + self, + requests: dict[str, tuple[str, BaseModel]], + processor: Callable[[dict[str, tuple[str, BaseModel]]], dict[str, BaseModel]], + name: str, + ): + """Process a batch of requests with error handling and future completion. + + Args: + requests: Dict mapping request_id to (model_id, request_data) tuples + processor: Function that processes requests and returns results dict + name: Name for logging + """ + if not requests: + return + with log_timing(f"process_batch_requests({name}, n={len(requests)})"): + try: + error_results, valid_requests = self._filter_valid_requests(requests) + if valid_requests: + results = processor(valid_requests) + results.update(error_results) + else: + results = error_results + except Exception as e: + logger.exception(f"Error processing batch: {e}") + results = {request_id: types.ErrorResponse(error=str(e), status="failed") for request_id in requests} + self._complete_futures(results) + + def process_pending_requests(self): + """Main loop to process pending requests.""" + while True: + # Query for pending requests and extract data within session context + with Session(self.db_engine) as session: + # Use look-ahead scheduling to find batchable forward_backward and forward model passes + forward_backward_requests = self.find_batchable_model_passes( + session, types.RequestType.FORWARD_BACKWARD + ) + forward_requests = self.find_batchable_model_passes(session, types.RequestType.FORWARD) + # Find pending sample requests that can be batched + sample_requests = self.find_batchable_sample(session) + # Get other pending requests (non forward_backward and non sampling) + other_requests = self.find_single_requests(session) + + # Process batches outside of session context + self.process_batch_requests(forward_backward_requests, self.process_forward_backward, "forward_backward") + self.process_batch_requests(forward_requests, self.process_forward, "forward") + self.process_batch_requests(sample_requests, self.process_sample, "sample") + + # Process other request types individually (in the future we can also batch independent optim_steps) + self.process_single_requests(other_requests) + + # Periodically cleanup stale sessions (disabled if either config is negative) + cleanup_enabled = self.config.session_cleanup_interval_sec >= 0 and self.config.session_timeout_sec >= 0 + if cleanup_enabled and time.time() - self._last_cleanup_time > self.config.session_cleanup_interval_sec: + _ = self.cleanup_stale_sessions() + self._last_cleanup_time = time.time() + + # Poll every 100ms + time.sleep(0.1) + + def run(self): + """Entry point to start the engine.""" + logger.info("Starting background engine...") + self.process_pending_requests() + + +def main(): + """Entry point for the background engine.""" + # Create argument parser and add Pydantic model fields + parser = argparse.ArgumentParser(description="SkyRL tx tinker engine for processing requests") + add_model(parser, EngineConfig) + + # Parse command-line arguments + args = parser.parse_args() + + # Create EngineConfig from parsed arguments + config = EngineConfig.model_validate(vars(args)) + + # Initialize and run the engine + TinkerEngine(config).run() + + +if __name__ == "__main__": + main() diff --git a/skyrl-tx/tx/tinker/extra/__init__.py b/skyrl-tx/tx/tinker/extra/__init__.py new file mode 100644 index 0000000000..9d4472d73f --- /dev/null +++ b/skyrl-tx/tx/tinker/extra/__init__.py @@ -0,0 +1,3 @@ +from tx.tinker.extra.external_inference import ExternalInferenceClient + +__all__ = ["ExternalInferenceClient"] diff --git a/skyrl-tx/tx/tinker/extra/external_inference.py b/skyrl-tx/tx/tinker/extra/external_inference.py new file mode 100644 index 0000000000..c9e45fc306 --- /dev/null +++ b/skyrl-tx/tx/tinker/extra/external_inference.py @@ -0,0 +1,102 @@ +import httpx +from datetime import datetime, timezone +from sqlmodel.ext.asyncio.session import AsyncSession + +from tx.tinker import types +from tx.tinker.config import EngineConfig +from tx.tinker.db_models import FutureDB, RequestStatus +from tx.utils.log import logger +from tx.utils.storage import download_and_unpack + + +class ExternalInferenceClient: + """Client for calling external inference engines (e.g., vLLM).""" + + def __init__(self, engine_config: EngineConfig, db_engine): + self.base_url = f"{engine_config.external_inference_url}/v1" + self.api_key = engine_config.external_inference_api_key + self.checkpoints_base = engine_config.checkpoints_base + self.lora_base_dir = engine_config.external_inference_lora_base + self.db_engine = db_engine + + async def call_and_store_result( + self, + request_id: int, + sample_req, + model_id: str, + checkpoint_id: str, + ): + """Background task to call external engine and store result in database.""" + try: + async with httpx.AsyncClient( + base_url=self.base_url, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=httpx.Timeout(300.0, connect=10.0), # 5 minutes for inference, 10s for connect + ) as http_client: + result = await self._forward_to_engine(sample_req, model_id, checkpoint_id, http_client) + result_data = result.model_dump() + status = RequestStatus.COMPLETED + except Exception as e: + logger.exception("External engine error") + result_data = {"error": str(e), "status": "failed"} + status = RequestStatus.FAILED + + async with AsyncSession(self.db_engine) as session: + future = await session.get(FutureDB, request_id) + future.result_data = result_data + future.status = status + future.completed_at = datetime.now(timezone.utc) + await session.commit() + + async def _forward_to_engine( + self, request, model_id: str, checkpoint_id: str, http_client: httpx.AsyncClient + ) -> types.SampleOutput: + """Forward request to vLLM with dynamic LoRA loading. + + Extracts the checkpoint to the configured external_inference_lora_base and references it by a model name + that vLLM can dynamically load via the lora_filesystem_resolver plugin. + """ + prompt_tokens = [token for chunk in request.prompt.chunks for token in chunk.tokens] + checkpoint_path = self.checkpoints_base / model_id / "sampler_weights" / f"{checkpoint_id}.tar.gz" + model_name = f"{model_id}_{checkpoint_id}" + target_dir = self.lora_base_dir / model_name + target_dir.parent.mkdir(parents=True, exist_ok=True) + + # Extract the checkpoint if it doesn't already exist + if not target_dir.exists(): + try: + with download_and_unpack(checkpoint_path) as extracted_path: + extracted_path.rename(target_dir) + except FileExistsError: + # This could happen if two processes try to download the file. + # In that case the other process won the race and created target_dir. + pass + + payload = { + "model": model_name, + "prompt": prompt_tokens, + "max_tokens": request.sampling_params.max_tokens, + "temperature": request.sampling_params.temperature, + "top_p": request.sampling_params.top_p, + "top_k": request.sampling_params.top_k, + "logprobs": True, + "stream": False, + "return_token_ids": True, + } + + response = await http_client.post("/completions", json=payload) + response.raise_for_status() + result = response.json() + + sequences = [] + for choice in result["choices"]: + lp = choice["logprobs"] + sequences.append( + types.GeneratedSequence( + tokens=choice["token_ids"], + logprobs=lp["token_logprobs"], + stop_reason=choice["finish_reason"], + ) + ) + + return types.SampleOutput(sequences=sequences, prompt_logprobs=[]) diff --git a/skyrl-tx/tx/tinker/loss_fns.py b/skyrl-tx/tx/tinker/loss_fns.py new file mode 100644 index 0000000000..6aa7c98db0 --- /dev/null +++ b/skyrl-tx/tx/tinker/loss_fns.py @@ -0,0 +1,50 @@ +"""Loss functions for training.""" + +import jax +import jax.numpy as jnp + + +def safe_loss_mask(loss_output: jax.Array, loss_mask: jax.Array) -> jax.Array: + "Strongly mask the loss_output to 0.0 if the loss_mask is zero." + return jnp.where(loss_mask != 0.0, loss_mask * loss_output, jnp.zeros_like(loss_output)) + + +def cross_entropy_loss( + target_logprobs: jax.Array, loss_mask: jax.Array, sampling_logprobs: jax.Array, advantages: jax.Array +) -> jax.Array: + "Standard cross-entropy loss (i.e., negative log-likelihood)." + return -safe_loss_mask(target_logprobs, loss_mask) + + +def importance_sampling_loss( + target_logprobs: jax.Array, loss_mask: jax.Array, sampling_logprobs: jax.Array, advantages: jax.Array +) -> jax.Array: + "Importance sampling loss with target_logprobs from learner policy and sampling_logprobs from sampling policy." + prob_ratio = jnp.exp(target_logprobs - sampling_logprobs) + return -safe_loss_mask(prob_ratio * advantages, loss_mask) + + +def ppo_loss( + target_logprobs: jax.Array, loss_mask: jax.Array, sampling_logprobs: jax.Array, advantages: jax.Array +) -> jax.Array: + "PPO style clipped version of the importance sampling loss." + prob_ratio = jnp.exp(target_logprobs - sampling_logprobs) + clipped_ratio = jnp.clip(prob_ratio, 0.8, 1.2) + unclipped = prob_ratio * advantages + clipped = clipped_ratio * advantages + return -safe_loss_mask(jnp.minimum(unclipped, clipped), loss_mask) + + +# Map from string names to loss functions +# The ordering of this map determines the indices used in jax.lax.switch +LOSS_FUNCTION_MAP = { + "cross_entropy": cross_entropy_loss, + "importance_sampling": importance_sampling_loss, + "ppo": ppo_loss, +} + +# Map from loss function name to index (for jax.lax.switch) +LOSS_TYPES = {name: idx for idx, name in enumerate(LOSS_FUNCTION_MAP.keys())} + +# List of loss functions in order (for jax.lax.switch) +LOSS_FUNCTIONS = list(LOSS_FUNCTION_MAP.values()) diff --git a/skyrl-tx/tx/tinker/types.py b/skyrl-tx/tx/tinker/types.py new file mode 100644 index 0000000000..adbe1a8c2d --- /dev/null +++ b/skyrl-tx/tx/tinker/types.py @@ -0,0 +1,258 @@ +# These are the types we use to represent the data internally. +# They have some commonalities with the API request and response +# types as well as the database models, but are distinct. For +# example, usually we try to avoid optional values in these types. + +from __future__ import annotations + +from enum import Enum +from typing import Literal +from urllib.parse import urlparse + +from pydantic import BaseModel + + +class RequestType(str, Enum): + """Types of requests that can be processed.""" + + CREATE_MODEL = "create_model" + FORWARD_BACKWARD = "forward_backward" + FORWARD = "forward" + OPTIM_STEP = "optim_step" + SAVE_WEIGHTS_FOR_SAMPLER = "save_weights_for_sampler" + SAVE_WEIGHTS = "save_weights" + LOAD_WEIGHTS = "load_weights" + SAMPLE = "sample" + UNLOAD_MODEL = "unload_model" + + # External request that should not be processed by the engine + EXTERNAL = "external" + + +class CheckpointType(str, Enum): + """Type of checkpoint.""" + + TRAINING = "training" + SAMPLER = "sampler" + + +class TinkerPath(BaseModel): + primary_id: str + kind: str + secondary_id: str + + @classmethod + def parse(cls, url: str) -> TinkerPath | None: + """Parse a URL string into a TinkerPath object.""" + parsed = urlparse(url) + + match (parsed.scheme, *parsed.path.split("/")): + case ("tinker", "", secondary_id): + return cls(primary_id=parsed.netloc, kind="", secondary_id=secondary_id) + case ("tinker", "", kind, secondary_id): + return cls(primary_id=parsed.netloc, kind=kind, secondary_id=secondary_id) + case _: + return None + + +class AdamParams(BaseModel): + learning_rate: float + beta1: float + beta2: float + eps: float + weight_decay: float + + +class LoraConfig(BaseModel): + rank: int + alpha: float + seed: int + train_attn: bool = True + train_mlp: bool = True + train_unembed: bool = False + + +class CreateModelInput(BaseModel): + lora_config: LoraConfig + + +class CreateModelOutput(BaseModel): + model_id: str + base_model: str + lora_config: LoraConfig + + +class UnloadModelInput(BaseModel): + pass + + +class UnloadModelOutput(BaseModel): + model_id: str + status: str + type: str = "unload_model" + + +class ModelInputChunk(BaseModel): + tokens: list[int] + + +class ModelInput(BaseModel): + chunks: list[ModelInputChunk] + + +class TensorData(BaseModel): + data: list[int] | list[float] + + +class LossFnInputs(BaseModel): + target_tokens: TensorData + weights: TensorData + advantages: TensorData + logprobs: TensorData + + +class Datum(BaseModel): + loss_fn_inputs: LossFnInputs + model_input: ModelInput + + +class ForwardBackwardInput(BaseModel): + data: list[Datum] + loss_fn: Literal["cross_entropy", "importance_sampling", "ppo"] + + +class ForwardBackwardOutput(BaseModel): + loss_fn_output_type: str + loss_fn_outputs: list[dict] + metrics: dict + + +class ErrorResponse(BaseModel): + error: str + status: str + + +class OptimStepInput(BaseModel): + adam_params: AdamParams + + +class OptimStepOutput(BaseModel): + pass + + +class SaveWeightsForSamplerInput(BaseModel): + path: str | None = None + sampling_session_seq_id: int | None = None + seq_id: int | None = None + sampling_session_id: str | None = None + + +class SaveWeightsForSamplerOutput(BaseModel): + path: str | None = None + type: str + sampling_session_id: str | None = None + + +class SaveWeightsInput(BaseModel): + path: str + + +class SaveWeightsOutput(BaseModel): + path: str + type: str + + +class LoadWeightsInput(BaseModel): + source_model_id: str + checkpoint_id: str + + +class LoadWeightsOutput(BaseModel): + type: str + + +class SamplingParams(BaseModel): + temperature: float + max_tokens: int + seed: int + stop_tokens: list[int] | None = None + stop_strings: list[str] | None = None + top_k: int = -1 # -1 for no limit + top_p: float = 1.0 # 1.0 for no filtering + + +class ModelMetadata(BaseModel): + adapter_index: int + lora_config: LoraConfig + loaded_checkpoint_id: str | None = None + + +class SampleInput(BaseModel): + base_model: str | None = None + prompt: ModelInput + sampling_params: SamplingParams + num_samples: int + checkpoint_id: str + prompt_logprobs: bool + + +class GeneratedSequence(BaseModel): + stop_reason: Literal["length", "stop"] + tokens: list[int] + logprobs: list[float] + + +class SampleOutput(BaseModel): + sequences: list[GeneratedSequence] + prompt_logprobs: list[float] | None = None + + +# Metrics tracked in the engine +class EngineMetrics(BaseModel): + train_seq_len_jit_times: dict[int, float] = {} + sample_seq_len_jit_times: dict[int, float] = {} + + +# Prepared batch data for backend processing +# These are prepared by the engine and passed to the backend + + +class PreparedModelPassBatch(BaseModel): + """Prepared batch data for forward/forward_backward operations. + + Engine extracts this from requests, backend converts to JAX arrays and computes. + """ + + # Per-example data (list of lists) + all_input_ids: list[list[int]] + all_targets: list[list[int]] + all_token_weights: list[list[float]] + all_sampling_logprobs: list[list[float]] + all_advantages: list[list[float]] + + # Per-example scalars + all_model_ids: list[str] + all_loss_fn_types: list[int] + + # Mapping from examples back to requests: (request_id, model_id, start_idx, end_idx) + request_batch_slices: list[tuple[str, str, int, int]] + + +class PreparedSampleBatch(BaseModel): + """Prepared batch data for sample operations. + + Engine extracts this from requests, backend converts to JAX arrays and computes. + """ + + # Per-sample data + all_prompts: list[list[int]] + all_sampling_params: list[SamplingParams] + all_model_ids: list[str] + all_checkpoint_ids: list[str] + all_checkpoint_paths: list[str] + + # Whether any request needs prompt logprobs + needs_prompt_logprobs: bool + + # Mapping from samples back to requests: (request_id, model_id, start_idx, end_idx, prompt_logprobs_requested) + request_batch_slices: list[tuple[str, str, int, int, bool]] diff --git a/skyrl-tx/tx/utils/generator.py b/skyrl-tx/tx/utils/generator.py new file mode 100644 index 0000000000..15dcdf7505 --- /dev/null +++ b/skyrl-tx/tx/utils/generator.py @@ -0,0 +1,403 @@ +"""Generator mixin for autoregressive text generation with KV caching.""" + +from __future__ import annotations +from dataclasses import dataclass +import functools + +import jax +import jax.numpy as jnp +from tokenizers.decoders import DecodeStream +import tx.utils.models +from tx.tinker import types + + +@jax.tree_util.register_dataclass +@dataclass +class KVCache: + """Key-value cache for all layers, each entry in the list corresponds to one layer.""" + + keys: list[jax.Array] + values: list[jax.Array] + cache_position: int + + def pad_to_length(self, max_length: int) -> KVCache: + """Pad KV cache to a specified maximum length. + + Args: + max_length: Target length to pad the cache to. + + Returns: + New KVCache with padded keys and values. + """ + # k and v have shape [B, T, num_heads, head_dim] + cache_pad_length = max_length - self.keys[0].shape[1] + pad_spec = ((0, 0), (0, cache_pad_length), (0, 0), (0, 0)) + return KVCache( + keys=[jnp.pad(k, pad_spec) for k in self.keys], + values=[jnp.pad(v, pad_spec) for v in self.values], + cache_position=self.cache_position, + ) + + +@jax.tree_util.register_dataclass +@dataclass +class DecodeState: + """State of the decode loop.""" + + kv_cache: KVCache + rngs: jax.Array # of shape [B, key_dim] + attention_mask: jax.Array + last_positions: jax.Array + logits: jax.Array + stop_pos: jax.Array # Position where stop token was found + + +@dataclass +class GenerateOutput: + """Result from autoregressive text generation. + + Attributes: + generated_ids: List of token ID lists, one for each request (excluding the prompt). + stop_reasons: Reason for stopping generation for each sequence ('stop' or 'length'). + logprobs: Log probabilities for each sampled token. + """ + + generated_ids: list[list[int]] + stop_reasons: list[str] + logprobs: list[list[float]] + prompt_logprobs: list[list[float]] | None = None + + +def compute_positions(attention_mask: jax.Array) -> jax.Array: + """Compute positions from attention mask. + + Positions start at 0 from the first non-zero value in the attention mask + and increment sequentially. + """ + first_token_idx = jnp.argmax(attention_mask, axis=1, keepdims=True) + return jnp.arange(attention_mask.shape[1])[None, :] - first_token_idx + + +def find_string_stop_position( + tokens: list[int], + tokenizer, + stop_strings: list[str], +) -> int | None: + """Find the token position where a stop string first appears. + + Incrementally decodes tokens and checks for stop string matches. + Uses the tokenizers DecodeStream for efficient incremental decoding. + + Args: + tokens: List of generated token IDs + tokenizer: HuggingFace tokenizer instance + stop_strings: List of stop strings to search for + + Returns: + Token index to truncate to (exclusive), or None if no stop found. + """ + if not stop_strings or not tokens: + return None + + # Incremental decode using DecodeStream + stream = DecodeStream(skip_special_tokens=False) + text = "" + for i, token in enumerate(tokens): + chunk = stream.step(tokenizer._tokenizer, token) + if chunk is not None: + text += chunk + for stop_string in stop_strings: + if stop_string in text: + return i + 1 + + return None + + +def compute_prompt_logprobs(prefill_logits: jax.Array, input_ids: jax.Array) -> jax.Array: + """Compute log probabilities of prompt tokens from prefill logits""" + # TODO: Optimize memory usage by avoiding allocation of full vocab dimension. + logits_for_prompt = prefill_logits[:, :-1, :] + log_probs = jax.nn.log_softmax(logits_for_prompt, axis=-1) + prompt_tokens = input_ids[:, 1:] + prompt_logprobs = jnp.take_along_axis(log_probs, prompt_tokens[..., None], axis=-1).squeeze(-1) + return prompt_logprobs + + +class GeneratorMixin: + """Adds autoregressive generation with KV caching to causal language models.""" + + @staticmethod + @functools.partial( + jax.jit, static_argnames=("max_length", "max_new_tokens", "max_top_k", "use_top_p", "prompt_logprobs") + ) + def _prefill_and_decode( + model, + input_ids: jax.Array, + attention_mask: jax.Array, + max_length: int, + max_new_tokens: int, + adapter_indices: jax.Array | None, + temperatures: jax.Array, + rngs: jax.Array, + stop_tokens: jax.Array, + top_k_values: jax.Array, + top_p_values: jax.Array, + max_top_k: int, + use_top_p: bool, + prompt_logprobs: bool = False, + ): + """JIT-compiled prefill + decode loop. Fuses everything for maximum efficiency.""" + # Compute positions from attention mask + positions = compute_positions(attention_mask) + + # Prefill: process full prompt + outputs = model(input_ids, attention_mask=attention_mask, positions=positions, adapter_indices=adapter_indices) + + # Compute prompt logprobs if requested + prompt_logprobs_array = compute_prompt_logprobs(outputs.logits, input_ids) if prompt_logprobs else None + + # Pad KV cache and attention mask + kv_cache = outputs.kv_cache.pad_to_length(max_length) + decode_attention_mask = jnp.pad(attention_mask, ((0, 0), (0, max_length - attention_mask.shape[1]))) + + def decode_fn(s: DecodeState, step: jax.Array) -> tuple[DecodeState, tuple[jax.Array, jax.Array]]: + """Decode one token step. Returns (state, (token, logprob)) for scan accumulation.""" + # Sample next token + split_keys = jax.vmap(jax.random.split)(s.rngs) + rngs, sample_keys = split_keys[:, 0], split_keys[:, 1] + + zero_temp_mask = temperatures == 0.0 + scaled_logits = s.logits / jnp.where(zero_temp_mask, 1.0, temperatures)[:, None] + + # Apply top_k and top_p filtering + if max_top_k > 0: + scaled_logits = apply_top_k_batch(scaled_logits, top_k_values, max_top_k) + if use_top_p: + scaled_logits = apply_top_p_batch(scaled_logits, top_p_values) + + sampled = jax.vmap(lambda key, logit: jax.random.categorical(key, logit, axis=-1))( + sample_keys, scaled_logits + ) + greedy = jnp.argmax(s.logits, axis=-1) + next_token = jnp.where(zero_temp_mask[:, None], greedy[:, None], sampled[:, None]) + log_probs = jax.nn.log_softmax(s.logits, axis=-1) + sampled_logprob = jnp.take_along_axis(log_probs, next_token, axis=-1) + + # Track first stop token position (-1 means not stopped yet) + is_stop = jnp.any(next_token == stop_tokens, axis=1) + stop_pos = jnp.where((s.stop_pos == -1) & is_stop, step + 1, s.stop_pos) + + # Update attention mask: set next position to 1 + next_attention_mask = s.attention_mask.at[:, s.kv_cache.cache_position].set(1) + + outputs = model( + next_token, + attention_mask=next_attention_mask, + positions=s.last_positions + 1, + kv_cache=s.kv_cache, + adapter_indices=adapter_indices, + ) + next_state = DecodeState( + kv_cache=outputs.kv_cache, + rngs=rngs, + attention_mask=next_attention_mask, + last_positions=s.last_positions + 1, + logits=outputs.logits[:, -1, :], + stop_pos=stop_pos, + ) + return next_state, (next_token, sampled_logprob) + + initial_state = DecodeState( + kv_cache=kv_cache, + rngs=rngs, + attention_mask=decode_attention_mask, + last_positions=positions[:, -1:], + logits=outputs.logits[:, -1, :], + stop_pos=jnp.full((input_ids.shape[0],), -1), + ) + + final_state, (tokens_stacked, logprobs_stacked) = jax.lax.scan( + decode_fn, initial_state, xs=jnp.arange(max_new_tokens) + ) + + # Post-process: transpose scan outputs from [Steps, Batch, 1] to [Batch, Steps] + new_tokens = jnp.swapaxes(tokens_stacked, 0, 1).squeeze(-1) + new_logprobs = jnp.swapaxes(logprobs_stacked, 0, 1).squeeze(-1) + + return new_tokens, new_logprobs, final_state.stop_pos, prompt_logprobs_array + + def generate( + self, + input_ids: jax.Array, + attention_mask: jax.Array, + *, + sampling_params: list[types.SamplingParams], + adapter_indices: jax.Array | None = None, + prompt_logprobs: bool = False, + tokenizer=None, + ) -> GenerateOutput: + """Generate text autoregressively with KV caching. + + Args: + tokenizer: Optional tokenizer for string stop sequence detection. + Required if any sampling_params has stop_strings set. + + Returns: + GenerateOutput containing generated_ids, stop_reasons, and optionally logprobs. + """ + batch_size, prompt_length = input_ids.shape + assert len(sampling_params) == batch_size + max_new_tokens = max(sampling_param.max_tokens for sampling_param in sampling_params) + max_length = tx.utils.models.round_up_seq_len(prompt_length + max_new_tokens) + temperatures = jnp.array([sampling_param.temperature for sampling_param in sampling_params]) + top_k_values = jnp.array([sampling_param.top_k for sampling_param in sampling_params], dtype=jnp.int32) + top_p_values = jnp.array([sampling_param.top_p for sampling_param in sampling_params], dtype=jnp.float32) + + # One PRNGKey per provided seed + seeds = [sampling_param.seed for sampling_param in sampling_params] + rngs = jax.vmap(jax.random.PRNGKey)(jnp.array(seeds)) + + # Extract stop tokens and pad to same length + max_stop_tokens = max(len(sp.stop_tokens) if sp.stop_tokens else 0 for sp in sampling_params) + stop_tokens = [] + for sp in sampling_params: + stop = sp.stop_tokens or [] + stop_tokens.append(stop + [-1] * (max_stop_tokens - len(stop))) + stop_tokens = jnp.array(stop_tokens, dtype=jnp.int32) + + # Capture prompt lengths for prompt_logprobs if requested + prompt_lengths = attention_mask.sum(axis=1) if prompt_logprobs else None + + # Compute static flags for top_k and top_p filtering + max_top_k = max((sp.top_k for sp in sampling_params if sp.top_k > 0), default=0) + use_top_p = any(sp.top_p < 1.0 for sp in sampling_params) + + new_tokens, new_logprobs, stop_pos, prompt_logprobs_array = self._prefill_and_decode( + self, + input_ids, + attention_mask, + max_length, + max_new_tokens, + adapter_indices, + temperatures, + rngs, + stop_tokens, + top_k_values, + top_p_values, + max_top_k, + use_top_p, + prompt_logprobs=prompt_logprobs, + ) + + max_tokens = jnp.array([sp.max_tokens for sp in sampling_params]) + # stop_pos is -1 if no stop token found; has_stop is true only if found within limit + has_stop = (stop_pos != -1) & (stop_pos <= max_tokens) + end_positions = jnp.where(has_stop, stop_pos, max_tokens) + + # In multi-host mode, gather all shards before device_get + if jax.process_count() > 1: + from jax.experimental import multihost_utils + + (new_tokens, has_stop, new_logprobs, end_positions, prompt_logprobs_array, prompt_lengths) = jax.tree.map( + lambda x: multihost_utils.process_allgather(x, tiled=True), + (new_tokens, has_stop, new_logprobs, end_positions, prompt_logprobs_array, prompt_lengths), + ) + + # Single device-to-host transfer + ( + new_tokens_host, + has_stop_host, + new_logprobs_host, + end_positions_host, + prompt_logprobs_host, + prompt_lengths_host, + ) = jax.device_get((new_tokens, has_stop, new_logprobs, end_positions, prompt_logprobs_array, prompt_lengths)) + + # Build output lists, applying string stop detection where needed + generated_ids = [] + stop_reasons = [] + logprobs_out = [] + + for i in range(batch_size): + tokens = new_tokens_host[i][: end_positions_host[i]].tolist() + token_logprobs = new_logprobs_host[i][: end_positions_host[i]].tolist() + stop_reason = "stop" if has_stop_host[i] else "length" + + # Apply string stop detection if stop_strings specified + if sampling_params[i].stop_strings: + assert tokenizer is not None, "tokenizer is required when stop_strings is specified" + assert stop_reason == "length", "stop_tokens cannot be specified when stop_strings is specified" + string_stop_pos = find_string_stop_position(tokens, tokenizer, sampling_params[i].stop_strings) + if string_stop_pos is not None: + tokens = tokens[:string_stop_pos] + token_logprobs = token_logprobs[:string_stop_pos] + stop_reason = "stop" + + generated_ids.append(tokens) + stop_reasons.append(stop_reason) + logprobs_out.append(token_logprobs) + + return GenerateOutput( + generated_ids=generated_ids, + stop_reasons=stop_reasons, + logprobs=logprobs_out, + prompt_logprobs=( + [prompt_logprobs_host[i, : prompt_lengths_host[i] - 1].tolist() for i in range(batch_size)] + if prompt_logprobs + else None + ), + ) + + +def apply_top_k_batch(logits: jax.Array, k_values: jax.Array, max_k: int) -> jax.Array: + """Keep only top-k logits per example, set rest to -inf. + + Args: + logits: Logits tensor of shape [batch_size, vocab_size] + k_values: Per-example k values of shape [batch_size]. If k <= 0, no filtering. + max_k: Static maximum k value, must be > 0. + + Returns: + Filtered logits with the same shape. + """ + assert max_k > 0 + top_values, top_indices = jax.lax.top_k(logits, max_k) + + # Keep only first k values per example + keep = jnp.arange(max_k) < k_values[:, None] + top_values = jnp.where(keep, top_values, -jnp.inf) + + # Scatter back to original positions + batch_idx = jnp.arange(logits.shape[0])[:, None] + result = jnp.full_like(logits, -jnp.inf).at[batch_idx, top_indices].set(top_values) + + return jnp.where(k_values[:, None] <= 0, logits, result) + + +def apply_top_p_batch(logits: jax.Array, p_values: jax.Array) -> jax.Array: + """Keep only tokens with cumulative probability up to p, set rest to -inf. + + Args: + logits: Logits tensor of shape [batch_size, vocab_size] + p_values: Per-example p values of shape [batch_size]. If p >= 1.0, no filtering. + + Returns: + Filtered logits with the same shape. + """ + # Sort by logits (equivalent to sorting by probs since softmax is monotonic) + sorted_indices = jnp.argsort(-logits, axis=-1) + sorted_logits = jnp.take_along_axis(logits, sorted_indices, axis=-1) + sorted_probs = jax.nn.softmax(sorted_logits, axis=-1) + + # Exclusive cumsum: cumsum[i] - prob[i] gives sum of probs *before* position i + cumsum_exclusive = jnp.cumsum(sorted_probs, axis=-1) - sorted_probs + + keep_mask = cumsum_exclusive < p_values[:, None] + keep_mask = keep_mask.at[:, 0].set(True) # Always keep top token + filtered_sorted_logits = jnp.where(keep_mask, sorted_logits, -jnp.inf) + + # Scatter back to original positions + batch_idx = jnp.arange(logits.shape[0])[:, None] + result = jnp.empty_like(logits).at[batch_idx, sorted_indices].set(filtered_sorted_logits) + + return jnp.where(p_values[:, None] >= 1.0, logits, result) diff --git a/skyrl-tx/tx/utils/log.py b/skyrl-tx/tx/utils/log.py new file mode 100644 index 0000000000..0bedc75470 --- /dev/null +++ b/skyrl-tx/tx/utils/log.py @@ -0,0 +1,94 @@ +from enum import Enum +import logging +import os +from pathlib import Path +from typing import Any + +from rich.console import Console + +try: + import wandb # type: ignore[import-not-found] +except ImportError: + wandb = None # type: ignore[assignment] + + +def _setup_root_logger() -> None: + logger = logging.getLogger("tx") + logger.setLevel(logging.DEBUG) + logger.propagate = False # Prevent propagation to root logger + console = Console(highlight=True, markup=True) + + class RichStreamHandler(logging.Handler): + def emit(self, record): + msg = self.format(record) + console.print(msg, highlight=True) + + handler = RichStreamHandler() + handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(handler) + + +def add_file_handler(path: Path | str, level: int = logging.DEBUG, *, print_path: bool = True) -> None: + logger = logging.getLogger("tx") + handler = logging.FileHandler(path) + handler.setLevel(level) + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + if print_path: + print(f"Logging to '{path}'") + + +_setup_root_logger() +logger = logging.getLogger("tx") + + +class ExperimentTracker(str, Enum): + wandb = "wandb" + + +class Tracker: + + def __init__(self, config: dict[str, Any], **kwargs): + logger.info(f"model config: {config}") + + def log(self, metrics: dict[str, Any], step: int | None = None) -> None: + data = metrics if step is None else {"step": step, **metrics} + logger.info( + ", ".join( + f"{key}: {value:.3e}" if isinstance(value, float) else f"{key}: {value}" for key, value in data.items() + ) + ) + + +class WandbTracker(Tracker): + + def __init__(self, config: dict[str, Any], **kwargs): + super().__init__(config, **kwargs) + if wandb is None: + raise RuntimeError("wandb not installed") + if not os.environ.get("WANDB_API_KEY"): + raise ValueError("WANDB_API_KEY environment variable not set") + self.run = wandb.init(config=config, **kwargs) # type: ignore[union-attr] + + def log(self, metrics: dict[str, Any], step: int | None = None) -> None: + super().log(metrics, step) + if wandb is not None: + wandb.log(metrics, step=step) # type: ignore[union-attr] + + def __del__(self): + if wandb is not None: + wandb.finish() # type: ignore[union-attr] + + +def get_tracker(tracker: ExperimentTracker | None, config: dict[str, Any], **kwargs) -> Tracker: + match tracker: + case None: + return Tracker(config, **kwargs) + case ExperimentTracker.wandb: + return WandbTracker(config, **kwargs) + case _: + raise ValueError(f"Unsupported experiment tracker: {tracker}") + + +__all__ = ["logger"] diff --git a/skyrl-tx/tx/utils/models.py b/skyrl-tx/tx/utils/models.py new file mode 100644 index 0000000000..faf1a9634a --- /dev/null +++ b/skyrl-tx/tx/utils/models.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +from enum import Enum +import os +from pathlib import Path +from typing import Callable, TYPE_CHECKING + +from cloudpathlib import CloudPath +from flax import nnx +from huggingface_hub import snapshot_download +import jax +import jax.numpy as jnp +import numpy as np +import optax +import safetensors.numpy +from transformers import PretrainedConfig +import peft + +from tx.utils.log import logger +from tx.utils.storage import download_and_unpack, pack_and_upload +from tx.tinker.types import LoraConfig + +if TYPE_CHECKING: + import torch + + +def resolve_model_path(model_name_or_path: str) -> str: + """Resolve a model name or path to a local directory path. + + If the model_name_or_path points to an existing local directory, it will be + used directly. Otherwise, the model will be downloaded from HuggingFace Hub. + + Args: + model_name_or_path: Either a local path to a model directory or a + HuggingFace model identifier (e.g., "Qwen/Qwen3-0.6B"). + + Returns: + Path to the local directory containing model config and weights. + """ + local_path = Path(model_name_or_path).expanduser() + if local_path.is_dir(): + logger.info(f"Using local model at {local_path}") + return str(local_path) + return snapshot_download(model_name_or_path, allow_patterns=["*.safetensors", "*.json"]) + + +def get_dtype(dtype: str | torch.dtype) -> jnp.dtype: + "Convert torch dtype to jax dtype." + + match str(dtype): + case "torch.float32" | "float32": + return jnp.float32 + case "torch.bfloat16" | "bfloat16": + return jnp.bfloat16 + case "torch.float16" | "float16": + return jnp.float16 + case _: + raise ValueError(f"Unsupported torch dtype: {dtype}") + + +def get_model_class(config: PretrainedConfig) -> Callable[..., nnx.Module]: + "Get the correct model class based on the config." + import tx.models.llama3 + import tx.models.qwen3 + + for architecture in config.architectures or []: + if hasattr(tx.models.llama3, architecture): + return getattr(tx.models.llama3, architecture) + if hasattr(tx.models.qwen3, architecture): + return getattr(tx.models.qwen3, architecture) + + raise ValueError(f"None of the architectures {config.architectures} is currently supported.") + + +def get_param_key(path: tuple, prefix: str = "") -> str: + "Get the safetensors key for a given model path." + if path[-1] in {"embedding", "kernel"}: + path = (*path[:-1], "weight") + elif path[-1] in {"lora_A", "lora_B"}: + path = (*path, "weight") + return prefix + ".".join(map(str, path)) + + +def get_expert_key(path: tuple, expert_idx: int) -> str: + "Get the safetensors key for an expert weight model path." + path = tuple(s if s != "experts" else f"experts.{expert_idx}" for s in path) + return ".".join(map(str, path)) + + +def load_safetensors( + checkpoint_dir: str | os.PathLike, + config: PretrainedConfig, + model: nnx.Module, + skip_lora: bool = True, + prefix: str = "", + filter_fn: Callable[[tuple], bool] | None = None, +) -> None: + tensors = {} + for file in Path(checkpoint_dir).glob("*.safetensors"): + tensors.update(safetensors.numpy.load_file(file)) + tensors = {k.removeprefix(prefix): v for k, v in tensors.items()} + + model_params = nnx.to_flat_state(nnx.state(model)) + updates = [] + for path, param in model_params: + if filter_fn is not None and not filter_fn(path): + continue + key = get_param_key(path) + # Skip LoRA parameters if requested + if skip_lora and ("lora_A" in path or "lora_B" in path or "lora_scaling" in path or "lora_ranks" in path): + continue + if "experts" in path: + tensors[key] = np.stack([tensors[get_expert_key(path, i)].T for i in range(config.num_experts)], axis=0) + else: + tensors[key] = tensors[key] if "embed_tokens" in path else tensors[key].T + if path[-2] in {"q_proj", "k_proj", "v_proj", "o_proj"}: + tensors[key] = tensors[key].reshape(param.shape) + assert param.shape == tensors[key].shape, f"shape mismatch for {key}" + sharded_tensor = jax.device_put(tensors[key].astype(param.dtype), param.sharding) + updates.append((path, sharded_tensor)) + nnx.update(model, nnx.from_flat_state(updates)) + + +def save_safetensors( + config: PretrainedConfig, + model: nnx.Module, + filename: Path, + prefix: str = "", + filter_fn: Callable[[tuple], bool] | None = None, +) -> None: + model_params = nnx.to_flat_state(nnx.state(model)) + tensors = {} + for path, param in model_params: + if "rngs" in path: + continue + if filter_fn is not None and not filter_fn(path): + continue + key = get_param_key(path, prefix=prefix) + if "experts" in path: + for i in range(config.num_experts): + tensors[get_expert_key(path, i)] = param[i, :, :].T + continue + if "q_proj" in path or "k_proj" in path or "v_proj" in path: + param = param.reshape(param.shape[0], -1) + elif "o_proj" in path: + param = param.reshape(-1, param.shape[-1]) + tensors[key] = param if "embed_tokens" in path else param.T + + # In multi-host mode, gather all shards and only save from rank 0 + if jax.process_count() > 1: + from jax.experimental import multihost_utils + + tensors = {k: multihost_utils.process_allgather(v, tiled=True) for k, v in tensors.items()} + + if jax.process_index() == 0: + safetensors.numpy.save_file({k: np.asarray(v) for k, v in tensors.items()}, filename) + + +def filter_lora(adapter_config: LoraConfig, path: tuple[str, ...]) -> bool: + if not adapter_config.train_attn and "self_attn" in path: + return False + if not adapter_config.train_mlp and ("mlp" in path or "experts" in path): + return False + if not adapter_config.train_unembed and ("embed_tokens" in path or "lm_head" in path): + return False + return True + + +def load_lora_checkpoint( + model: nnx.Module, adapter_config: LoraConfig, adapter_index: int, checkpoint_path: Path | CloudPath +) -> None: + """Load LoRA adapter weights from a sampling checkpoint into the model. + + Args: + model: The Qwen3ForCausalLM model to load the adapter into + adapter_config: LoRA adapter configuration + adapter_index: Index of the adapter to load into + checkpoint_path: Path to the checkpoint tar.gz file + """ + _, lora_params, _ = nnx.split(model, model.is_lora_param, ...) + + adapter_lora_params = extract_adapter_state(adapter_index, lora_params, adapter_config.rank) + + with download_and_unpack(checkpoint_path) as temp_dir: + load_safetensors( + temp_dir, + model.config, + adapter_lora_params, + skip_lora=False, + prefix="base_model.model.", + filter_fn=lambda path: filter_lora(adapter_config, path), + ) + insert_adapter_state(adapter_index, lora_params, adapter_lora_params, adapter_config.rank) + + +def save_lora_checkpoint( + model: nnx.Module, + base_model_name: str, + adapter_config: LoraConfig, + adapter_index: int, + output_path: Path | CloudPath, +): + """Save a LoRA checkpoint as a tar.gz archive. + + Args: + model: The Qwen3ForCausalLM model to extract LoRA parameters from + adapter_config: LoRA adapter configuration + adapter_index: Index of the adapter to save + output_path: Path to save the checkpoint tar.gz file + """ + _, lora_params, _ = nnx.split(model, model.is_lora_param, ...) + + adapter_lora_params = extract_adapter_state(adapter_index, lora_params, adapter_config.rank) + + peft_config = peft.LoraConfig( + base_model_name_or_path=base_model_name, r=adapter_config.rank, lora_alpha=adapter_config.alpha + ) + + with pack_and_upload(output_path) as temp_dir: + save_safetensors( + model.config, + adapter_lora_params, + temp_dir / "adapter_model.safetensors", + prefix="base_model.model.", + filter_fn=lambda path: filter_lora(adapter_config, path), + ) + peft_config.save_pretrained(temp_dir) + + +class OptimizerName(str, Enum): + adamw = "adamw" + + +def get_optimizer(optimizer_name: OptimizerName, optimizer_args: dict) -> optax.GradientTransformation: + match (optimizer_name, optimizer_args): + case (OptimizerName.adamw, {"learning_rate": lr, **kwargs}): + return optax.adamw(lr, **kwargs) + case (_, {"learning_rate": _}): + raise ValueError(f"Unsupported optimizer: {optimizer_name}") + case _: + raise ValueError("The 'learning_rate' key must be provided in optimizer_args.") + + +@nnx.jit(static_argnames=("adapter_index", "rank")) +def extract_adapter_state(adapter_index: int, lora_params: nnx.GraphState, rank: int) -> nnx.GraphState: + "Helper function to extract the adapter parameters for a specific adapter index." + + def extract_state(path: tuple, p: jnp.ndarray): + if path[-2].key not in {"lora_A", "lora_B"}: + return p + assert p.ndim in {3, 4}, f"LoRA parameters must have 3 or 4 dimensions, got shape {p.shape}" + if path[-2].key == "lora_A": + return p[adapter_index, ..., :, :rank] + if path[-2].key == "lora_B": + return p[adapter_index, ..., :rank, :] + + return jax.tree.map_with_path(extract_state, lora_params) + + +# We need to use nnx.jit here instead of jax.jit so the nnx.update will be handled correctly +@nnx.jit(static_argnames=("adapter_index", "rank")) +def insert_adapter_state( + adapter_index: int, lora_params: nnx.GraphState, new_params: nnx.GraphState, rank: int +) -> None: + "Helper function to insert the adapter parameters for a specific adapter index (inverse of extract_adapter_state)." + + def insert_state(path: tuple, p: jax.Array, new: jax.Array): + if path[-2].key not in {"lora_A", "lora_B"}: + return new + assert p.ndim in {3, 4}, f"LoRA parameters must have 3 or 4 dimensions, got shape {p.shape}" + if path[-2].key == "lora_A": + return p.at[adapter_index, ..., :, :rank].set(new) + elif path[-2].key == "lora_B": + return p.at[adapter_index, ..., :rank, :].set(new) + + updated = jax.tree.map_with_path(insert_state, lora_params, new_params) + nnx.update(lora_params, updated) + + +def round_up_seq_len(seq_len: int) -> int: + """ + Rounds a sequence length up to roughly two significant binary digits. + We do this to pad sequences, so the Jax JIT compiler needs to + compile fewer different shapes. + """ + if seq_len <= 32: + return 32 + + # Find the position of the most significant bit. + msb_pos = seq_len.bit_length() - 1 + # Create a mask for the two most significant bits. + mask = (1 << msb_pos) | (1 << (msb_pos - 1)) + # Round down to the nearest value with at most two significant bits. + result = seq_len & mask + + # If we rounded down, round up to the next bucket boundary. + if result < seq_len: + result += 1 << (msb_pos - 1) + + return result diff --git a/skyrl-tx/tx/utils/storage.py b/skyrl-tx/tx/utils/storage.py new file mode 100644 index 0000000000..74d6e7ee77 --- /dev/null +++ b/skyrl-tx/tx/utils/storage.py @@ -0,0 +1,63 @@ +from contextlib import contextmanager +import io +import gzip +from pathlib import Path +import tarfile +from tempfile import TemporaryDirectory +from typing import Generator +from cloudpathlib import AnyPath + + +@contextmanager +def pack_and_upload(dest: AnyPath) -> Generator[Path, None, None]: + """Give the caller a temp directory that gets uploaded as a tar.gz archive on exit. + + Args: + dest: Destination path for the tar.gz file + """ + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + + yield tmp_path + + dest.parent.mkdir(parents=True, exist_ok=True) + + with dest.open("wb") as f: + # Use compresslevel=0 to prioritize speed, as checkpoint files don't compress well. + with gzip.GzipFile(fileobj=f, mode="wb", compresslevel=0) as gz_stream: + with tarfile.open(fileobj=gz_stream, mode="w:") as tar: + tar.add(tmp_path, arcname="") + + +@contextmanager +def download_and_unpack(source: AnyPath) -> Generator[Path, None, None]: + """Download and extract a tar.gz archive and give the content to the caller in a temp directory. + + Args: + source: Source path for the tar.gz file + """ + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + + # Download and extract tar archive (handles both local and cloud storage) + with source.open("rb") as f: + with tarfile.open(fileobj=f, mode="r:gz") as tar: + tar.extractall(tmp_path, filter="data") + + yield tmp_path + + +def download_file(source: AnyPath) -> io.BytesIO: + """Download a file from storage and return it as a BytesIO object. + + Args: + source: Source path for the file (local or cloud) + + Returns: + BytesIO object containing the file contents + """ + buffer = io.BytesIO() + with source.open("rb") as f: + buffer.write(f.read()) + buffer.seek(0) + return buffer diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 7a7aadbc9d..0000000000 --- a/tests/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/tests/checkpoint/run_deepseek_megatron_ckpt.sh b/tests/checkpoint/run_deepseek_megatron_ckpt.sh deleted file mode 100644 index 0f36b9f48c..0000000000 --- a/tests/checkpoint/run_deepseek_megatron_ckpt.sh +++ /dev/null @@ -1,104 +0,0 @@ -set -x - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -mkdir -p $HOME/models - -huggingface-cli download deepseek-ai/deepseek-coder-1.3b-instruct --local-dir $HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.checkpoint.contents=['model','hf_model','optimizer','extra'] \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=2 \ - critic.checkpoint.contents=['model','hf_model','optimizer','extra'] \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='deepseek_megatron_checkpoint_saveload' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=1 \ - trainer.test_freq=1 \ - trainer.resume_mode=disable \ - trainer.total_epochs=15 \ - trainer.total_training_steps=1 $@ - - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=2 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='deepseek_megatron_checkpoint_saveload' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.resume_mode=auto \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=1 $@ \ No newline at end of file diff --git a/tests/checkpoint/run_qwen_megatron_ckpt.sh b/tests/checkpoint/run_qwen_megatron_ckpt.sh deleted file mode 100644 index 125f6952bb..0000000000 --- a/tests/checkpoint/run_qwen_megatron_ckpt.sh +++ /dev/null @@ -1,104 +0,0 @@ -set -x - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -mkdir -p $HOME/models - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ - actor_rollout_ref.actor.checkpoint.contents=['model','hf_model','optimizer','extra'] \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=2 \ - critic.checkpoint.contents=['model','hf_model','optimizer','extra'] \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='qwen2_5_0b5_megatron_saveload' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=1 \ - trainer.test_freq=1 \ - trainer.resume_mode=disable \ - trainer.total_epochs=15 \ - trainer.total_training_steps=1 $@ - - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=2 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='qwen2_5_0b5_megatron_saveload' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.resume_mode=auto \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=1 $@ \ No newline at end of file diff --git a/tests/checkpoint/test_fsdp_ckpt.py b/tests/checkpoint/test_fsdp_ckpt.py deleted file mode 100644 index c8e0bcc18a..0000000000 --- a/tests/checkpoint/test_fsdp_ckpt.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import tempfile -import shutil -import torch -import copy -import torch.distributed -from torch.distributed import init_device_mesh -from verl.utils.distributed import initialize_global_process_group -from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager -from transformers import AutoModelForCausalLM, AutoTokenizer -from transformers import Qwen2Config - -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy, MixedPrecision, \ - CPUOffload - - -def test_fsdp_ckpt(): - assert torch.cuda.device_count() >= 2, "need at least 2 gpus for test" - local_rank, rank, world_size = initialize_global_process_group() - device_mesh = init_device_mesh('cuda', mesh_shape=(world_size,), mesh_dim_names=('dp',)) - - model_name = 'Qwen/Qwen2.5-0.5B-Instruct' - config = Qwen2Config(num_hidden_layers=1) - - with torch.device('cuda'): - model = AutoModelForCausalLM.from_config(config=config, - torch_dtype=torch.bfloat16, - attn_implementation='flash_attention_2') - model = model.to(device='cuda') - - # Wrap model with FSDP - mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32) - - model = FSDP(model, - use_orig_params=False, - device_id=torch.cuda.current_device(), - sharding_strategy=ShardingStrategy.FULL_SHARD, - mixed_precision=mixed_precision, - device_mesh=device_mesh) - - optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4) - lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.9) - - # Create checkpoint manager - tokenizer = AutoTokenizer.from_pretrained(model_name) - checkpoint_manager = FSDPCheckpointManager(model=model, - optimizer=optimizer, - lr_scheduler=lr_scheduler, - tokenizer=tokenizer) - - # Generate sample input - batch_size = 2 - seq_len = 32 - vocab_size = 32000 - # First input for initial update - input_ids1 = torch.randint(0, vocab_size, (batch_size, seq_len), device='cuda') - attention_mask1 = torch.ones_like(input_ids1) - - # Second input for verification - input_ids2 = torch.randint(0, vocab_size, (batch_size, seq_len), device='cuda') - attention_mask2 = torch.ones_like(input_ids2) - - # Step 1: Initial update and save checkpoint - outputs1 = model(input_ids=input_ids1, attention_mask=attention_mask1) - loss1 = outputs1.logits.mean() - loss1.backward() - optimizer.step() - lr_scheduler.step() - optimizer.zero_grad() - - # Save checkpoint after first update - temp_dir = tempfile.mkdtemp() - checkpoint_path = os.path.join(temp_dir, 'checkpoint') - checkpoint_manager.save_checkpoint(local_path=checkpoint_path, hdfs_path=None, global_step=0) - - # Step 2: Second update and forward pass - outputs2 = model(input_ids=input_ids2, attention_mask=attention_mask2) - loss2 = outputs2.logits.mean() - loss2.backward() - optimizer.step() - lr_scheduler.step() - optimizer.zero_grad() - - # Record logits after second update - with torch.no_grad(): - logits_before_load = model(input_ids=input_ids2, attention_mask=attention_mask2).logits - - # Step 3: Load checkpoint and repeat second update - checkpoint_manager.load_checkpoint(checkpoint_path) - - # Repeat the second update with same input - outputs3 = model(input_ids=input_ids2, attention_mask=attention_mask2) - loss3 = outputs3.logits.mean() - loss3.backward() - optimizer.step() - lr_scheduler.step() - optimizer.zero_grad() - - # Record logits after loaded checkpoint and update - with torch.no_grad(): - logits_after_load = model(input_ids=input_ids2, attention_mask=attention_mask2).logits - - # Step 4: Verify outputs match - torch.testing.assert_close(logits_before_load, logits_after_load, atol=0.0, rtol=0.0) - print("Checkpoint save/load test passed!") - - # Cleanup - shutil.rmtree(temp_dir) - torch.distributed.barrier() - - -if __name__ == '__main__': - test_fsdp_ckpt() diff --git a/tests/distributed/run_all.sh b/tests/distributed/run_all.sh deleted file mode 100644 index f8654d7b57..0000000000 --- a/tests/distributed/run_all.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/usr/bin/env bash - -set -e -x -torchrun --nproc-per-node=4 --standalone tests/distributed/test_tensor_dict.py \ No newline at end of file diff --git a/tests/distributed/test_tensor_dict.py b/tests/distributed/test_tensor_dict.py deleted file mode 100644 index 68a6a0d4d7..0000000000 --- a/tests/distributed/test_tensor_dict.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -os.environ['NCCL_DEBUG'] = 'WARN' - -from verl.protocol import all_gather_data_proto, DataProto -from verl.utils.distributed import initialize_global_process_group -import torch -import torch.distributed -import numpy as np - - -def test_all_gather_data_proto(): - device_mesh = torch.distributed.device_mesh.init_device_mesh('cuda', mesh_shape=[2, 2], mesh_dim_names=['dp', 'tp']) - - global_rank = torch.distributed.get_rank() - - obs = torch.tensor([[1 * global_rank, 2 * global_rank + 1], [3 * global_rank, 4 * global_rank + 1]]) - - labels = ['a', 'b'] if global_rank % 2 == 0 else ['b', 'a'] - labels = np.array(labels, dtype=object) - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'info': 'test_info'}) - - all_gather_data_proto(data=data, process_group=device_mesh.get_group('dp')) - - if global_rank == 0: - expected_obs = torch.tensor([[0, 1], [0, 1], [2, 5], [6, 9]], device='cuda') - expected_labels = ['a', 'b', 'a', 'b'] - elif global_rank == 1: - expected_obs = torch.tensor([[1, 3], [3, 5], [3, 7], [9, 13]], device='cuda') - expected_labels = ['b', 'a', 'b', 'a'] - elif global_rank == 2: - expected_obs = torch.tensor([[0, 1], [0, 1], [2, 5], [6, 9]], device='cuda') - expected_labels = ['a', 'b', 'a', 'b'] - elif global_rank == 3: - expected_obs = torch.tensor([[1, 3], [3, 5], [3, 7], [9, 13]], device='cuda') - expected_labels = ['b', 'a', 'b', 'a'] - - torch.testing.assert_close(data.batch['obs'], expected_obs, atol=0, rtol=0) - assert (data.non_tensor_batch['labels'] == expected_labels).all() - assert data.meta_info == {'info': 'test_info'} - - -if __name__ == '__main__': - local_rank, rank, world_size = initialize_global_process_group() - test_all_gather_data_proto() diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/tests/e2e/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/e2e/arithmetic_sequence/data/create_dataset.py b/tests/e2e/arithmetic_sequence/data/create_dataset.py deleted file mode 100644 index e023a29175..0000000000 --- a/tests/e2e/arithmetic_sequence/data/create_dataset.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tests.e2e.envs.digit_completion import DigitCompletion, generate_ground_truth_response -from torch.utils import data -import os - -if __name__ == '__main__': - simple_task = DigitCompletion(max_number=9, max_diff=9, max_num_in_response=9) - all_prompts = simple_task.get_all_prompts() - - # 21 * 6 * 4 - train_data, test_data = data.random_split(all_prompts, lengths=[0.8, 0.2]) - train_data = list(train_data) - test_data = list(test_data) - - train_data = [[{'role': 'user', 'content': str(item)}] \ - for item in train_data] - test_data = [[{'role': 'user', 'content': str(item)}] \ - for item in test_data] - - print(f'Size of train: {len(train_data)}, size of test: {len(test_data)}') - - train_data = {'prompt': train_data} - test_data = {'prompt': test_data} - - model_folder = os.path.join(os.path.dirname(os.path.abspath(__file__))) - - import pandas as pd - - train_data_frame = pd.DataFrame(train_data) - test_data_frame = pd.DataFrame(test_data) - - train_data_frame.to_parquet(os.path.join(model_folder, 'train.parquet')) - test_data_frame.to_parquet(os.path.join(model_folder, 'test.parquet')) diff --git a/tests/e2e/arithmetic_sequence/data/test.parquet b/tests/e2e/arithmetic_sequence/data/test.parquet deleted file mode 100644 index d0729dc3d6..0000000000 Binary files a/tests/e2e/arithmetic_sequence/data/test.parquet and /dev/null differ diff --git a/tests/e2e/arithmetic_sequence/data/train.parquet b/tests/e2e/arithmetic_sequence/data/train.parquet deleted file mode 100644 index 0a03a61a81..0000000000 Binary files a/tests/e2e/arithmetic_sequence/data/train.parquet and /dev/null differ diff --git a/tests/e2e/arithmetic_sequence/model/config.json b/tests/e2e/arithmetic_sequence/model/config.json deleted file mode 100644 index 87944c51fc..0000000000 --- a/tests/e2e/arithmetic_sequence/model/config.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "architectures": [ - "LlamaForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": null, - "eos_token_id": 1, - "hidden_act": "silu", - "hidden_size": 128, - "initializer_range": 0.02, - "intermediate_size": 344, - "max_position_embeddings": 2048, - "mlp_bias": false, - "model_type": "llama", - "num_attention_heads": 4, - "num_hidden_layers": 4, - "num_key_value_heads": 4, - "pad_token_id": 2, - "pretraining_tp": 1, - "rms_norm_eps": 1e-06, - "rope_scaling": null, - "rope_theta": 10000.0, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.43.3", - "use_cache": true, - "vocab_size": 16 -} diff --git a/tests/e2e/arithmetic_sequence/model/create_model_tokenizer.py b/tests/e2e/arithmetic_sequence/model/create_model_tokenizer.py deleted file mode 100644 index 9a3135f0c5..0000000000 --- a/tests/e2e/arithmetic_sequence/model/create_model_tokenizer.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Create a random model and tokenizer for PPO training -""" - -import torch -import os -from transformers import AutoModelForCausalLM, LlamaConfig, AutoTokenizer - -from tests.e2e.envs.digit_completion import CharTokenizer - -tokenizer = CharTokenizer( - characters=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ',', ':'], - model_max_length=2048, - chat_template= - "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set role = message['role'] %}{{ message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ sep_token }}{% endif %}" -) - -config = LlamaConfig(vocab_size=(tokenizer.vocab_size + 16 - 1) // 16 * 16, - hidden_size=128, - intermediate_size=344, - num_hidden_layers=4, - num_attention_heads=4, - num_key_value_heads=4, - pad_token_id=tokenizer.pad_token_id, - bos_token_id=tokenizer.bos_token_id, - eos_token_id=tokenizer.eos_token_id) - -model = AutoModelForCausalLM.from_config(config, torch_dtype=torch.bfloat16) - -model_folder = os.path.join(os.path.dirname(os.path.abspath(__file__))) -os.makedirs(model_folder, exist_ok=True) - -model.save_pretrained(model_folder) - -tokenizer_folder = model_folder -tokenizer.save_pretrained(tokenizer_folder) - -load_tokenizer = AutoTokenizer.from_pretrained(tokenizer_folder) - -chat = [{'role': 'user', 'content': '1,0:2,3'}] - -load_tokenizer.padding_side = 'left' -print( - load_tokenizer.apply_chat_template(chat, - tokenize=True, - add_generation_prompt=True, - max_length=10, - padding='max_length')) diff --git a/tests/e2e/arithmetic_sequence/model/generation_config.json b/tests/e2e/arithmetic_sequence/model/generation_config.json deleted file mode 100644 index 578d375057..0000000000 --- a/tests/e2e/arithmetic_sequence/model/generation_config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "_from_model_config": true, - "eos_token_id": 1, - "pad_token_id": 2, - "transformers_version": "4.43.3" -} diff --git a/tests/e2e/arithmetic_sequence/model/model.safetensors b/tests/e2e/arithmetic_sequence/model/model.safetensors deleted file mode 100644 index 509e6e97c5..0000000000 Binary files a/tests/e2e/arithmetic_sequence/model/model.safetensors and /dev/null differ diff --git a/tests/e2e/arithmetic_sequence/model/tokenizer_config.json b/tests/e2e/arithmetic_sequence/model/tokenizer_config.json deleted file mode 100644 index d01bf75f1d..0000000000 --- a/tests/e2e/arithmetic_sequence/model/tokenizer_config.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "char_ords": [ - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 44, - 58 - ], - "model_max_length": 2048, - "chat_template": "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set role = message['role'] %}{{ message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ sep_token }}{% endif %}" -} \ No newline at end of file diff --git a/tests/e2e/arithmetic_sequence/rl/README.md b/tests/e2e/arithmetic_sequence/rl/README.md deleted file mode 100644 index a9b3f9e2e3..0000000000 --- a/tests/e2e/arithmetic_sequence/rl/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Digit completion - -This is an example of solving a digit completion problem. The problem is defined as below: - -The prompt is a sequence of numbers with fixed difference. The agent's goal is to complete the next N numbers. -If the max number is reached, the next number should be modulo with max number. - -For example, -- prompt = [1, 2, 3] -- N = 5 -- max_number = 6 - -The response should be [4, 5, 6, 7%6, 8%6] = [4, 5, 6, 0, 1]. - -# Environment definition - -The core definition of the task is defined in verl/envs/digit_completion/task.py - -It is highly recommended to take a look at it for better understanding. - - - -# Run experiments - -The users are required to specify the config path and config name (and the relative model config path to the current working directory) - -```bash -# cd examples/arithmetic_sequence/rl - -# Specify the config path and config name (current working dir) -python3 -m verl.trainer.ppo.ray_megatron_train_synchronous --config-path=$(pwd)/config --config-name='ray_megatron' - -# The default relative path of model config is 'config/model_config', if you want to change it, you can rewrite it in ray_megatron.yaml or using: -python3 -m verl.trainer.ppo.ray_megatron_train_synchronous --config-path=$(pwd)/config --config-name='ray_megatron' ++model.base_path=config/model_config - -``` - diff --git a/tests/e2e/arithmetic_sequence/rl/main_trainer.py b/tests/e2e/arithmetic_sequence/rl/main_trainer.py deleted file mode 100644 index c1861b9879..0000000000 --- a/tests/e2e/arithmetic_sequence/rl/main_trainer.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Using FSDPTrainer -""" -import os -import hydra -import ray -import torch -from transformers import PreTrainedTokenizer, AutoTokenizer - -from verl import DataProto -from verl.trainer.ppo.ray_trainer import RayPPOTrainer -from verl.utils.fs import copy_to_local -from tests.e2e.envs.digit_completion import CharTokenizer - - -def make_reward_function(tokenizer, num_examine): - - def arithmetic_sequence_reward_function(data: DataProto, return_dict: bool = False): - from tests.e2e.envs.digit_completion.task import compute_reward - reward_tensor = torch.zeros_like(data.batch['responses'], dtype=torch.float32) - - for i in range(data.batch.batch_size[0]): - data_item = data[i] # DataProtoItem - - prompt_ids = data_item.batch['prompts'] - - prompt_length = prompt_ids.shape[-1] - - # extract raw prompt - valid_prompt_length = data_item.batch['attention_mask'][:prompt_length].sum() - valid_prompt_ids = prompt_ids[-valid_prompt_length:] - - # extract response - response_ids = data_item.batch['responses'] - response_length = response_ids.shape[-1] - response_mask = data.batch['attention_mask'][i][-response_length:] - valid_response_length = data_item.batch['attention_mask'][prompt_length:].sum() - valid_response_ids = response_ids[:valid_response_length] - - # decode - prompt = tokenizer.decode(valid_prompt_ids) - response = tokenizer.decode(valid_response_ids) - # remove bos and eos - prompt = prompt.replace(tokenizer.sep_token, '') - response = response.replace(tokenizer.eos_token, '') - if i < num_examine: - print(prompt, response) - - reward_output = compute_reward(prompt, response) - dense_reward = reward_output[0].tolist() - ground_truth_response = reward_output[1]['ground_truth_response'] - if len(dense_reward) > 0: - last_reward = dense_reward[-1] - else: - if len(ground_truth_response) == 0: - last_reward = 1 - else: - last_reward = 0 - - # pad to response_length - for _ in range(reward_tensor.shape[-1] - len(dense_reward)): - dense_reward.append(last_reward) - - dense_reward = torch.as_tensor(dense_reward, dtype=torch.float32, device=reward_tensor.device) - reward_tensor[i] = dense_reward * response_mask - - if return_dict: - return {"reward_tensor": reward_tensor} - else: - return reward_tensor - - return arithmetic_sequence_reward_function - - -@hydra.main(config_path='../../../../verl/trainer/config', config_name='ppo_trainer', version_base=None) -def main(config): - ray.init( - runtime_env={ - 'env_vars': { - 'MEGATRON_USE_CUDA_TIMER': '0', - 'MEGATRON_START_PROCESS_TIMER': 'False', - 'TOKENIZERS_PARALLELISM': 'true', - 'NCCL_DEBUG': 'WARN' - } - }) - - # print initial config - from pprint import pprint - from omegaconf import OmegaConf - pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values - - # print the config - # print initial config - print('Config after normalizing batch_size') - pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values - - # download the checkpoint from hdfs - local_path = copy_to_local(config.actor_rollout_ref.model.path) - local_path = os.path.expanduser(local_path) - # instantiate tokenizern - tokenizer = AutoTokenizer.from_pretrained(local_path) - print(f'Tokenizer vocab_size: {tokenizer.vocab_size}') - - # define worker classes - from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker - from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role - - role_worker_mapping = { - Role.ActorRollout: ray.remote(ActorRolloutRefWorker), - Role.Critic: ray.remote(CriticWorker), - } - - global_pool_id = 'global_pool' - resource_pool_spec = { - global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, - } - mapping = { - Role.ActorRollout: global_pool_id, - Role.Critic: global_pool_id, - } - - # use reward model - if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: - role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) - mapping[Role.RefPolicy] = global_pool_id - - reward_fn = make_reward_function(tokenizer=tokenizer, num_examine=1) - - resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) - - trainer = RayPPOTrainer(config=config, - tokenizer=tokenizer, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - reward_fn=reward_fn, - val_reward_fn=reward_fn) - trainer.init_workers() - trainer.fit() - - -if __name__ == '__main__': - main() diff --git a/tests/e2e/check_custom_rwd_fn.py b/tests/e2e/check_custom_rwd_fn.py deleted file mode 100644 index 4cabc6a2c2..0000000000 --- a/tests/e2e/check_custom_rwd_fn.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - - -def check_congratulations_in_file(output_file): - with open(output_file, 'r') as f: - output = f.read() - - success_message = "Congratulations!!! You have called my_reward_function successfully!!!" - assert success_message in output, f'Success message of my_reward_function not found in {output_file}' - print("Check passes") - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--output_file', required=True, type=str) - - args = parser.parse_args() - - check_congratulations_in_file(args.output_file) diff --git a/tests/e2e/check_results.py b/tests/e2e/check_results.py deleted file mode 100644 index e20ee2a569..0000000000 --- a/tests/e2e/check_results.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - -import numpy as np - - -def extract_reward_from_line(line): - # TODO: this function needs error handling - try: - key_vals = line.split(' - ') - for key_val in key_vals: - key, val = key_val.split(':') - if key == 'critic/rewards/mean': - reward = float(val) - return reward - return -np.inf - except Exception: - return -np.inf - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--output_file', required=True, type=str) - parser.add_argument('--target', type=float, default=0.2, help='target reward score') - - args = parser.parse_args() - - with open(args.output_file, 'r') as f: - output = f.read().split('\n') - - best_reward = -np.inf - for line in output: - if line.startswith('step'): - reward = extract_reward_from_line(line) - if reward > best_reward: - best_reward = reward - - print(f'Best reward is {best_reward}') - assert best_reward > args.target, f'Best reward must be greater than {args.target}. best_reward: {best_reward}' - print('Check passes') diff --git a/tests/e2e/envs/__init__.py b/tests/e2e/envs/__init__.py deleted file mode 100644 index 7d39147942..0000000000 --- a/tests/e2e/envs/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .digit_completion import DigitCompletion - -__all__ = ['DigitCompletion'] \ No newline at end of file diff --git a/tests/e2e/envs/digit_completion/__init__.py b/tests/e2e/envs/digit_completion/__init__.py deleted file mode 100644 index 3e3aa76da5..0000000000 --- a/tests/e2e/envs/digit_completion/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .task import DigitCompletion, generate_ground_truth_response -from .tokenizer import CharTokenizer - -from transformers import AutoTokenizer, LlamaConfig - -AutoTokenizer.register(LlamaConfig, CharTokenizer, exist_ok=True) - -__all__ = ['DigitCompletion', 'generate_ground_truth_response', 'CharTokenizer'] \ No newline at end of file diff --git a/tests/e2e/envs/digit_completion/task.py b/tests/e2e/envs/digit_completion/task.py deleted file mode 100644 index 7322027c78..0000000000 --- a/tests/e2e/envs/digit_completion/task.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Task and environment definition for digit completion.""" - -import numpy as np - - -class DigitCompletion(object): - """ - The implementation of a simple digit completion task. - The prompt is a sequence of numbers with fixed difference. The task is to complete the next N numbers. - If the max number is reached, the next number should be modulo with max number. - - For example, - - prompt = [1, 2, 3] - - N = 5 - - max_number = 6 - - the response should be [4, 5, 6, 7%6, 8%6] = [4, 5, 6, 0, 1] - - Note that the tokenizer is char-level to increase the difficulty. - """ - - def __init__(self, max_number: int, max_diff: int, max_num_in_response: int, seed=0): - """ - - Args: - max_number: the maximum number allowed in the arithmetic sequence - max_diff: the maximum diff. The actual common diff will be sampled from [0, max_diff] - max_num_in_response: the maximum number in the response - """ - super().__init__() - self.max_number = max_number - self.max_diff = max_diff - self.max_num_in_response = max_num_in_response - assert self.max_num_in_response < 10 - assert self.max_number > 0 - assert self.max_diff > 0 - self.max_number_length = len(str(max_number)) - # {num1},{num2}:{max_num_in_response},{max_number} - self._prompt_length = self.max_number_length * 2 + 4 + self.max_number_length # no negative is allowed - - self.np_rng = np.random.default_rng(seed=seed) - - def __str__(self): - return f'Prompt length: {self.prompt_length}. Response length: {self.response_length}, ' \ - f'Max number: {self.max_number}. Max diff: {self.max_diff}, ' \ - f'Max number in response: {self.max_num_in_response}' - - def get_state(self): - return {'rng': self.np_rng} - - def set_state(self, state): - assert 'rng' in state, 'rng must be inside state' - self.np_rng = state['rng'] - - @property - def prompt_length(self): - return self._prompt_length - - @property - def response_length(self): - # number length + comma length + [EOS] - # The actual number times 1.5 to allow 'U' - return (self.max_num_in_response * self.max_number_length + (self.max_num_in_response - 1) + 1) * 2 - - def add(self, a, b): - return (a + b) % self.max_number - - def get_all_prompts(self): - all_prompts = [] - for first_num in range(self.max_number + 1): - for diff in range(0, self.max_diff + 1): - second_num = self.add(first_num, diff) - for num_to_complete in range(self.max_num_in_response + 1): - prompt = str(first_num) + ',' + str(second_num) + f':{self.max_number},{num_to_complete}' - all_prompts.append(prompt) - return all_prompts - - def sample_str_prompts(self): - # step 1: sample initial numbers - first_num = self.np_rng.integers(self.max_number + 1) - diff = self.np_rng.integers(self.max_diff + 1) - second_num = self.add(first_num, diff) - num_to_complete = self.np_rng.integers(self.max_num_in_response + 1) - prompt = str(first_num) + ',' + str(second_num) + f':{self.max_number},{num_to_complete}' - return prompt - - def sample_batch_str_prompts(self, batch_size): - str_prompts = [] - for _ in range(batch_size): - str_prompts.append(self.sample_str_prompts()) - return str_prompts - - -def compute_attention_mask(prompts, pad_token_id): - mask = np.ones_like(prompts) - mask[prompts == pad_token_id] = 0 - return mask - - -def compute_position_id_with_mask(mask): - return np.clip(np.cumsum(mask, axis=-1) - 1, a_min=0, a_max=None) - - -def generate_ground_truth_response(prompt: str): - """Generate ground truth response given a prompt.""" - num, info = prompt.split(':') - num1, num2 = num.split(',') - max_number, num_to_gen = info.split(',') - num1 = int(num1) - num2 = int(num2) - max_number = int(max_number) - num_to_gen = int(num_to_gen) - diff = (num2 - num1) % max_number - results = [] - last_num = num2 - for _ in range(num_to_gen): - curr = (last_num + diff) % max_number - results.append(str(curr)) - last_num = curr - response = ','.join(results) - return response - - -def compute_reward(prompt: str, response: str, sequence_reward=1.): - """We compute dense reward here so that we can directly train RL without SFT""" - response_length = len(response) - ground_truth_response = generate_ground_truth_response(prompt) - per_token_reward = sequence_reward / (len(ground_truth_response) + 1) # including [EOS] - - # pad - reward = np.zeros(response_length, dtype=np.float32) # this assumes that each char is a token - # assign reward until mismatches - ground_truth_idx = 0 - for i in range(response_length): - if ground_truth_idx == len(ground_truth_response): - break - - ground_truth_response_token = ground_truth_response[ground_truth_idx] - response_token = response[i] - if ground_truth_response_token == response_token: - reward[i] = per_token_reward - ground_truth_idx += 1 - else: - # no matches - break - - return reward, {'ground_truth_response': ground_truth_response} - - -if __name__ == '__main__': - task = DigitCompletion(max_number=20, max_diff=3, max_num_in_response=5) - print(task.sample_str_prompts()) - - prompt = '7,8:20,0' - response = '' - print(compute_reward(prompt, response)) - - prompt = '7,8:20,0' - response = 'E000' - print(compute_reward(prompt, response)) - - prompt = '9,10:20,2' - response = '11,12,13' - print(compute_reward(prompt, response)) diff --git a/tests/e2e/envs/digit_completion/tokenizer.py b/tests/e2e/envs/digit_completion/tokenizer.py deleted file mode 100644 index 1b8d94ee34..0000000000 --- a/tests/e2e/envs/digit_completion/tokenizer.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Copied from https://github.com/dariush-bahrami/character-tokenizer/blob/master/charactertokenizer/core.py - -CharacterTokenzier for Hugging Face Transformers. - -This is heavily inspired from CanineTokenizer in transformers package. -""" - -import json -import os -from pathlib import Path -from typing import Dict, List, Optional, Sequence, Union - -from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer - - -class CharTokenizer(PreTrainedTokenizer): - - def __init__(self, characters: Sequence[str], model_max_length: int, chat_template, **kwargs): - """Character tokenizer for Hugging Face transformers. - - Args: - characters (Sequence[str]): List of desired characters. Any character which - is not included in this list will be replaced by a special token called - [UNK] with id=6. Following are list of all of the special tokens with - their corresponding ids: - "[CLS]": 0 - "[SEP]": 1 - "[BOS]": 2 - "[MASK]": 3 - "[PAD]": 4 - "[RESERVED]": 5 - "[UNK]": 6 - an id (starting at 7) will be assigned to each character. - - model_max_length (int): Model maximum sequence length. - """ - eos_token_str = 'E' - sep_token_str = 'S' - pad_token_str = 'P' - unk_token_str = 'U' - - self.characters = characters - self.model_max_length = model_max_length - eos_token = AddedToken(eos_token_str, lstrip=False, rstrip=False) - sep_token = AddedToken(sep_token_str, lstrip=False, rstrip=False) - pad_token = AddedToken(pad_token_str, lstrip=False, rstrip=False) - unk_token = AddedToken(unk_token_str, lstrip=False, rstrip=False) - - self._vocab_str_to_int = { - sep_token_str: 0, - eos_token_str: 1, - pad_token_str: 2, - unk_token_str: 3, - **{ - ch: i + 4 for i, ch in enumerate(characters) - }, - } - self._vocab_int_to_str = {v: k for k, v in self._vocab_str_to_int.items()} - - super().__init__( - eos_token=eos_token, - sep_token=sep_token, - pad_token=pad_token, - unk_token=unk_token, - add_prefix_space=False, - model_max_length=model_max_length, - **kwargs, - ) - - self.chat_template = chat_template - - @property - def vocab_size(self) -> int: - return len(self._vocab_str_to_int) - - def get_vocab(self): - return self._vocab_str_to_int - - def _tokenize(self, text: str) -> List[str]: - return list(text) - - def _convert_token_to_id(self, token: str) -> int: - return self._vocab_str_to_int.get(token, self._vocab_str_to_int["U"]) - - def _convert_id_to_token(self, index: int) -> str: - return self._vocab_int_to_str[index] - - def convert_tokens_to_string(self, tokens): - return "".join(tokens) - - def build_inputs_with_special_tokens(self, - token_ids_0: List[int], - token_ids_1: Optional[List[int]] = None) -> List[int]: - sep = [self.sep_token_id] - cls = [self.cls_token_id] - result = cls + token_ids_0 + sep - if token_ids_1 is not None: - result += token_ids_1 + sep - return result - - def get_special_tokens_mask( - self, - token_ids_0: List[int], - token_ids_1: Optional[List[int]] = None, - already_has_special_tokens: bool = False, - ) -> List[int]: - if already_has_special_tokens: - return super().get_special_tokens_mask( - token_ids_0=token_ids_0, - token_ids_1=token_ids_1, - already_has_special_tokens=True, - ) - - result = [1] + ([0] * len(token_ids_0)) + [1] - if token_ids_1 is not None: - result += ([0] * len(token_ids_1)) + [1] - return result - - def get_config(self) -> Dict: - return { - "char_ords": [ord(ch) for ch in self.characters], - "model_max_length": self.model_max_length, - "chat_template": self.chat_template - } - - @classmethod - def from_config(cls, config: Dict) -> "DigitCompletionTokenizer": - cfg = {} - cfg["characters"] = [chr(i) for i in config["char_ords"]] - cfg["model_max_length"] = config["model_max_length"] - cfg["chat_template"] = config["chat_template"] - return cls(**cfg) - - def save_pretrained(self, save_directory: Union[str, os.PathLike], **kwargs): - cfg_file = Path(save_directory) / "tokenizer_config.json" - cfg = self.get_config() - with open(cfg_file, "w") as f: - json.dump(cfg, f, indent=4) - - @classmethod - def from_pretrained(cls, save_directory: Union[str, os.PathLike], **kwargs): - cfg_file = Path(save_directory) / "tokenizer_config.json" - with open(cfg_file) as f: - cfg = json.load(f) - return cls.from_config(cfg) diff --git a/tests/e2e/run_deepseek_grpo.sh b/tests/e2e/run_deepseek_grpo.sh deleted file mode 100644 index 097d8207cd..0000000000 --- a/tests/e2e/run_deepseek_grpo.sh +++ /dev/null @@ -1,44 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -huggingface-cli download deepseek-ai/deepseek-coder-1.3b-instruct --local-dir $HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=80 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=160 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=160 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=2 $@ \ No newline at end of file diff --git a/tests/e2e/run_deepseek_grpo_megatron.sh b/tests/e2e/run_deepseek_grpo_megatron.sh deleted file mode 100644 index 40b01748da..0000000000 --- a/tests/e2e/run_deepseek_grpo_megatron.sh +++ /dev/null @@ -1,44 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -huggingface-cli download deepseek-ai/deepseek-coder-1.3b-instruct --local-dir $HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='deepseek_llm_7b_function_rm_math_megatron' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=2 $@ \ No newline at end of file diff --git a/tests/e2e/run_deepseek_megatron.sh b/tests/e2e/run_deepseek_megatron.sh deleted file mode 100644 index 5169c93ad2..0000000000 --- a/tests/e2e/run_deepseek_megatron.sh +++ /dev/null @@ -1,47 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -mkdir -p $HOME/models -huggingface-cli download deepseek-ai/deepseek-coder-1.3b-instruct --local-dir $HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.tensor_model_parallel_size=2 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='deepseek_llm_1b3_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=3 $@ diff --git a/tests/e2e/run_deepseek_megatron_parallelism.sh b/tests/e2e/run_deepseek_megatron_parallelism.sh deleted file mode 100644 index cb3ad6ccb8..0000000000 --- a/tests/e2e/run_deepseek_megatron_parallelism.sh +++ /dev/null @@ -1,53 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -mkdir -p $HOME/models -huggingface-cli download deepseek-ai/deepseek-coder-1.3b-instruct --local-dir $HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/deepseek-ai/deepseek-coder-1.3b-instruct \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=2 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='deepseek_llm_1b3_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=3 $@ diff --git a/tests/e2e/run_qwen2vl_geo3k_function_rm.sh b/tests/e2e/run_qwen2vl_geo3k_function_rm.sh deleted file mode 100644 index d8ec1e4430..0000000000 --- a/tests/e2e/run_qwen2vl_geo3k_function_rm.sh +++ /dev/null @@ -1,41 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2-VL-2B-Instruct --local-dir $HOME/models/Qwen/Qwen2-VL-2B-Instruct - -python3 -m verl.trainer.main_ppo \ - data.train_files=$HOME/data/geo3k/train.parquet \ - data.val_files=$HOME/data/geo3k/test.parquet \ - data.train_batch_size=128 \ - data.max_prompt_length=1536 \ - data.max_response_length=1536 \ - data.image_key=images \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2-VL-2B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=128 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - actor_rollout_ref.rollout.enable_chunked_prefill=False \ - actor_rollout_ref.rollout.enforce_eager=True \ - actor_rollout_ref.rollout.free_cache_engine=False \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - algorithm.adv_estimator=grpo \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_example_geo3k' \ - trainer.experiment_name='qwen2vl_e2e_ci_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_grpo.sh b/tests/e2e/run_qwen_grpo.sh deleted file mode 100644 index 7a1b6215cb..0000000000 --- a/tests/e2e/run_qwen_grpo.sh +++ /dev/null @@ -1,44 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=True \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=2 $@ \ No newline at end of file diff --git a/tests/e2e/run_qwen_grpo_megatron.sh b/tests/e2e/run_qwen_grpo_megatron.sh deleted file mode 100644 index 899f5c8056..0000000000 --- a/tests/e2e/run_qwen_grpo_megatron.sh +++ /dev/null @@ -1,44 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=grpo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.rollout.n=5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_grpo_example_gsm8k' \ - trainer.experiment_name='qwen2_7b_function_rm_megatron' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=5 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=2 $@ \ No newline at end of file diff --git a/tests/e2e/run_qwen_gsm8k_custom_function_rm.sh b/tests/e2e/run_qwen_gsm8k_custom_function_rm.sh deleted file mode 100644 index 08ba5c323c..0000000000 --- a/tests/e2e/run_qwen_gsm8k_custom_function_rm.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -set -e -x -FILE="$(pwd)/my_reward_function.py" -rm -rf $FILE -cat < "$FILE" -def my_reward_function(data_source, solution_str, ground_truth, extra_info=None): - print(f"Congratulations!!! You have called my_reward_function successfully!!!") - return 0.1 -EOF - - -OUTPUT_FILE="$(pwd)/output_custom_reward.txt" -FUNCTION_NAME="my_reward_function" -rm -rf $OUTPUT_FILE - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - custom_reward_function.path=$FILE\ - custom_reward_function.name=$FUNCTION_NAME\ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen_e2e_ci_custom_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.default_local_dir=$HOME/ckpt/ \ - trainer.total_training_steps=2 | tee $OUTPUT_FILE; - -python3 tests/e2e/check_custom_rwd_fn.py --output_file=$OUTPUT_FILE -rm -rf $FILE -rm -rf $OUTPUT_FILE \ No newline at end of file diff --git a/tests/e2e/run_qwen_gsm8k_dapo.sh b/tests/e2e/run_qwen_gsm8k_dapo.sh deleted file mode 100644 index 463b4d693d..0000000000 --- a/tests/e2e/run_qwen_gsm8k_dapo.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -adv_estimator=grpo - -kl_coef=0.0 -use_kl_in_reward=False -use_kl_loss=False -kl_loss_coef=0.0 - -clip_ratio_low=0.2 -clip_ratio_high=0.28 - -max_prompt_length=512 -max_response_length=512 -enable_overlong_buffer=True -overlong_buffer_len=128 -overlong_penalty_factor=1.0 - -loss_agg_mode="token-mean" - -enable_filter_groups=True -filter_groups_metric=seq_reward -max_num_gen_batches=10 -train_prompt_bsz=32 -train_prompt_mini_bsz=$((train_prompt_bsz / 2)) -gen_prompt_bsz=$((train_prompt_bsz * 3)) -n_resp_per_prompt=4 - -python3 -m recipe.dapo.src.main_dapo \ - data.train_files="$HOME/data/gsm8k/train.parquet" \ - data.val_files="$HOME/data/gsm8k/test.parquet" \ - reward_model.reward_manager=dapo \ - algorithm.adv_estimator=${adv_estimator} \ - algorithm.use_kl_in_reward=${use_kl_in_reward} \ - algorithm.kl_ctrl.kl_coef=${kl_coef} \ - actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ - actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ - actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ - actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ - data.max_prompt_length=${max_prompt_length} \ - data.max_response_length=${max_response_length} \ - reward_model.overlong_buffer.enable=${enable_overlong_buffer} \ - reward_model.overlong_buffer.len=${overlong_buffer_len} \ - reward_model.overlong_buffer.penalty_factor=${overlong_penalty_factor} \ - actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ - data.train_batch_size=${train_prompt_bsz} \ - data.gen_batch_size=${gen_prompt_bsz} \ - algorithm.filter_groups.enable=${enable_filter_groups} \ - algorithm.filter_groups.metric=${filter_groups_metric} \ - algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ - actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - trainer.logger=['console'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen2.5_0.5b_e2e_ci_dapo' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_function_rm.sh b/tests/e2e/run_qwen_gsm8k_function_rm.sh deleted file mode 100644 index 5b98cf8eb7..0000000000 --- a/tests/e2e/run_qwen_gsm8k_function_rm.sh +++ /dev/null @@ -1,41 +0,0 @@ -set -x -ENGINE=${1:-vllm} - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=$ENGINE \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen_e2e_ci_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=1 \ - trainer.resume_mode=disable \ - trainer.total_training_steps=1 diff --git a/tests/e2e/run_qwen_gsm8k_function_rm_both_kl.sh b/tests/e2e/run_qwen_gsm8k_function_rm_both_kl.sh deleted file mode 100644 index 3267d9906a..0000000000 --- a/tests/e2e/run_qwen_gsm8k_function_rm_both_kl.sh +++ /dev/null @@ -1,46 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.model.path=Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen_e2e_ci_function_rm_both_kl' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.default_local_dir=$HOME/ckpt/ \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_function_rm_grpo.sh b/tests/e2e/run_qwen_gsm8k_function_rm_grpo.sh deleted file mode 100644 index 2349866d50..0000000000 --- a/tests/e2e/run_qwen_gsm8k_function_rm_grpo.sh +++ /dev/null @@ -1,36 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=False \ - algorithm.adv_estimator=grpo \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen_e2e_ci_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_function_rm_no_rmpad.sh b/tests/e2e/run_qwen_gsm8k_function_rm_no_rmpad.sh deleted file mode 100644 index b5b5cea3a1..0000000000 --- a/tests/e2e/run_qwen_gsm8k_function_rm_no_rmpad.sh +++ /dev/null @@ -1,40 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=False \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=False \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.val_before_train=False \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen_e2e_ci_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_function_rm_remax.sh b/tests/e2e/run_qwen_gsm8k_function_rm_remax.sh deleted file mode 100644 index 8ce3250802..0000000000 --- a/tests/e2e/run_qwen_gsm8k_function_rm_remax.sh +++ /dev/null @@ -1,36 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - algorithm.adv_estimator=remax \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_example_gsm8k' \ - trainer.experiment_name='qwen_e2e_ci_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_model_rm.sh b/tests/e2e/run_qwen_gsm8k_model_rm.sh deleted file mode 100644 index 45c332cc2d..0000000000 --- a/tests/e2e/run_qwen_gsm8k_model_rm.sh +++ /dev/null @@ -1,48 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.optim.lr_warmup_steps_ratio=0.05 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - reward_model.enable=True \ - reward_model.model.path=$HOME/models/Qwen/Qwen2.5-0.5B\ - reward_model.model.use_remove_padding=True \ - reward_model.model.fsdp_config.param_offload=True \ - reward_model.micro_batch_size_per_gpu=16 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.val_before_train=False \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-0.5B-ci_hybrid_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_model_rm_liger_kernel.sh b/tests/e2e/run_qwen_gsm8k_model_rm_liger_kernel.sh deleted file mode 100644 index dbd85d2e55..0000000000 --- a/tests/e2e/run_qwen_gsm8k_model_rm_liger_kernel.sh +++ /dev/null @@ -1,49 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - +actor_rollout_ref.model.use_liger=True \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size=32 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size=128 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.optim.lr_warmup_steps_ratio=0.05 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size=32 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - reward_model.enable=True \ - reward_model.model.path=$HOME/models/Qwen/Qwen2.5-0.5B\ - reward_model.model.use_remove_padding=True \ - reward_model.model.fsdp_config.param_offload=True \ - reward_model.micro_batch_size=16 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.val_before_train=False \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-0.5B-ci_hybrid_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_model_rm_no_rmpad.sh b/tests/e2e/run_qwen_gsm8k_model_rm_no_rmpad.sh deleted file mode 100644 index e75fd2451f..0000000000 --- a/tests/e2e/run_qwen_gsm8k_model_rm_no_rmpad.sh +++ /dev/null @@ -1,48 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=False \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=False \ - critic.optim.lr_warmup_steps_ratio=0.05 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - reward_model.enable=True \ - reward_model.model.path=$HOME/models/Qwen/Qwen2.5-0.5B\ - reward_model.model.use_remove_padding=False \ - reward_model.model.fsdp_config.param_offload=True \ - reward_model.micro_batch_size_per_gpu=16 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.val_before_train=False \ - trainer.logger=['console'] \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-0.5B-ci_hybrid_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_model_rm_seq_balance.sh b/tests/e2e/run_qwen_gsm8k_model_rm_seq_balance.sh deleted file mode 100644 index fa25d5850b..0000000000 --- a/tests/e2e/run_qwen_gsm8k_model_rm_seq_balance.sh +++ /dev/null @@ -1,51 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=12000 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=12000 \ - critic.optim.lr=1e-5 \ - critic.model.use_remove_padding=True \ - critic.optim.lr_warmup_steps_ratio=0.05 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.use_dynamic_bsz=True \ - critic.ppo_max_token_len_per_gpu=98304 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - reward_model.enable=True \ - reward_model.model.path=$HOME/models/Qwen/Qwen2.5-0.5B\ - reward_model.model.use_remove_padding=True \ - reward_model.model.fsdp_config.param_offload=True \ - reward_model.use_dynamic_bsz=True \ - reward_model.forward_max_token_len_per_gpu=98304 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.val_before_train=False \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-0.5B-ci_hybrid_rm_seq_balance' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_model_rm_ulysses.sh b/tests/e2e/run_qwen_gsm8k_model_rm_ulysses.sh deleted file mode 100644 index f7fd224a42..0000000000 --- a/tests/e2e/run_qwen_gsm8k_model_rm_ulysses.sh +++ /dev/null @@ -1,53 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.1 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.fsdp_config.fsdp_size=4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ - critic.optim.lr=1e-5 \ - critic.ulysses_sequence_parallel_size=2 \ - critic.model.use_remove_padding=True \ - critic.optim.lr_warmup_steps_ratio=0.05 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.model.fsdp_config.param_offload=False \ - critic.model.fsdp_config.optimizer_offload=False \ - critic.model.fsdp_config.fsdp_size=4 \ - reward_model.enable=True \ - reward_model.ulysses_sequence_parallel_size=2 \ - reward_model.model.path=$HOME/models/Qwen/Qwen2.5-0.5B\ - reward_model.model.use_remove_padding=True \ - reward_model.model.fsdp_config.param_offload=True \ - reward_model.micro_batch_size_per_gpu=16 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.val_before_train=False \ - trainer.logger=['console'] \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-0.5B-ci_hybrid_rm_sp2' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_gsm8k_prime.sh b/tests/e2e/run_qwen_gsm8k_prime.sh deleted file mode 100644 index 4fea00fa96..0000000000 --- a/tests/e2e/run_qwen_gsm8k_prime.sh +++ /dev/null @@ -1,52 +0,0 @@ -set -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 -m recipe.prime.main_prime \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=32 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - data.filter_accuracy=True \ - data.accuracy_lower_bound=0.2 \ - data.accuracy_upper_bound=0.8 \ - data.oversample_factor=4 \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=5e-7 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=32 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.model.enable_gradient_checkpointing=False \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.n=4 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - algorithm.adv_estimator=rloo \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - reward_model.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - reward_model.micro_batch_size_per_gpu=1 \ - reward_model.model.update=before \ - reward_model.model.beta_train=0.05 \ - reward_model.model.optim.lr=1e-6 \ - reward_model.model.optim.grad_clip=10.0 \ - reward_model.model.input_tokenizer=null \ - reward_model.mini_batch_size=32 \ - reward_model.reward_manager=prime \ - trainer.val_before_train=False \ - trainer.logger=['console'] \ - trainer.project_name='verl_example' \ - trainer.experiment_name='Qwen2.5-0.5B-PRIME' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.total_training_steps=1 $@ diff --git a/tests/e2e/run_qwen_megatron.sh b/tests/e2e/run_qwen_megatron.sh deleted file mode 100644 index 9c3fe6d336..0000000000 --- a/tests/e2e/run_qwen_megatron.sh +++ /dev/null @@ -1,45 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -mkdir -p $HOME/models -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.tensor_model_parallel_size=2 \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='qwen2_5_0b5_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=3 $@ diff --git a/tests/e2e/run_qwen_megatron_parallelism.sh b/tests/e2e/run_qwen_megatron_parallelism.sh deleted file mode 100644 index 480afe3dcd..0000000000 --- a/tests/e2e/run_qwen_megatron_parallelism.sh +++ /dev/null @@ -1,56 +0,0 @@ -set -x - -export VLLM_ATTENTION_BACKEND=XFORMERS - -# the config file used: verl/trainer/main_ppo/config/ppo_megatron_trainer.yaml - -mkdir -p $HOME/models - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -export VLLM_ATTENTION_BACKEND=XFORMERS - -python3 -m verl.trainer.main_ppo --config-path=config \ - --config-name='ppo_megatron_trainer.yaml'\ - algorithm.adv_estimator=gae \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.train_batch_size=1024 \ - data.max_prompt_length=512 \ - data.max_response_length=512 \ - actor_rollout_ref.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - actor_rollout_ref.actor.optim.lr=2e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ - actor_rollout_ref.actor.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.actor.megatron.tensor_model_parallel_size=2 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=16 \ - actor_rollout_ref.ref.megatron.pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.virtual_pipeline_model_parallel_size=2 \ - actor_rollout_ref.ref.megatron.tensor_model_parallel_size=2 \ - critic.optim.lr=2e-5 \ - critic.model.path=$HOME/models/Qwen/Qwen2.5-0.5B \ - critic.model.enable_gradient_checkpointing=False \ - critic.ppo_micro_batch_size_per_gpu=4 \ - critic.megatron.pipeline_model_parallel_size=2 \ - critic.megatron.virtual_pipeline_model_parallel_size=2 \ - critic.megatron.tensor_model_parallel_size=2 \ - algorithm.use_kl_in_reward=True \ - algorithm.kl_penalty=kl \ - algorithm.kl_ctrl.kl_coef=0.001 \ - trainer.critic_warmup=0 \ - trainer.logger=['console'] \ - trainer.project_name='verl_megatron_gsm8k_examples' \ - trainer.experiment_name='qwen2_5_0b5_function_rm' \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=1 \ - trainer.total_epochs=15 \ - trainer.total_training_steps=3 $@ diff --git a/tests/e2e/run_r1_distill_qwen_aime24_eval.sh b/tests/e2e/run_r1_distill_qwen_aime24_eval.sh deleted file mode 100644 index 61f5ff3110..0000000000 --- a/tests/e2e/run_r1_distill_qwen_aime24_eval.sh +++ /dev/null @@ -1,30 +0,0 @@ -set -x - -huggingface-cli download deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \ - --local-dir $HOME/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B - -python3 -m verl.trainer.main_generation \ - trainer.nnodes=1 \ - trainer.n_gpus_per_node=8 \ - data.path=$HOME/data/r1/test.parquet \ - data.prompt_key=prompt \ - data.batch_size=1024 \ - data.n_samples=1 \ - data.output_path=$HOME/data/r1/test-output-k1.parquet \ - model.path=$HOME/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \ - rollout.temperature=0.6 \ - rollout.top_p=0.95 \ - rollout.prompt_length=1024 \ - rollout.response_length=32768 \ - rollout.tensor_model_parallel_size=1 \ - rollout.gpu_memory_utilization=0.95 \ - rollout.max_num_batched_tokens=65536 \ - rollout.enforce_eager=False \ - rollout.free_cache_engine=False - -python3 -m recipe.r1.main_eval \ - data.path=$HOME/data/r1/test-output-k1.parquet \ - data.prompt_key=prompt \ - data.response_key=responses \ - custom_reward_function.path=recipe/r1/reward_score.py \ - custom_reward_function.name=reward_func \ No newline at end of file diff --git a/tests/e2e/run_ray_trainer.sh b/tests/e2e/run_ray_trainer.sh deleted file mode 100644 index a6190d121f..0000000000 --- a/tests/e2e/run_ray_trainer.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -e -x - -OUTPUT_FILE="/tmp/output_ray_trainer.txt" - -export PATH=$PATH:~/.local/bin - -rm -rf $OUTPUT_FILE -python3 tests/e2e/arithmetic_sequence/rl/main_trainer.py \ - algorithm.adv_estimator=gae \ - data.train_files=tests/e2e/arithmetic_sequence/data/train.parquet \ - data.val_files=tests/e2e/arithmetic_sequence/data/test.parquet \ - data.train_batch_size=800 \ - data.max_prompt_length=16 \ - data.max_response_length=32 \ - data.return_raw_input_ids=True \ - actor_rollout_ref.model.path=tests/e2e/arithmetic_sequence/model \ - actor_rollout_ref.model.external_lib=tests.e2e.envs.digit_completion \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=200 \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=1e-4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=200 \ - actor_rollout_ref.rollout.name=hf \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - critic.ppo_micro_batch_size_per_gpu=200 \ - critic.model.path=tests/e2e/arithmetic_sequence/model \ - critic.optim.lr=1e-3 \ - algorithm.use_kl_in_reward=False \ - trainer.total_epochs=200 \ - trainer.experiment_name=arithmetic_sequences \ - trainer.logger=['console'] \ - trainer.n_gpus_per_node=1 \ - trainer.test_freq=1 \ - trainer.save_freq=110 | tee $OUTPUT_FILE; - -python3 tests/e2e/check_results.py --output_file=$OUTPUT_FILE -rm -rf $OUTPUT_FILE diff --git a/tests/e2e/run_ray_trainer_fire_sampling.sh b/tests/e2e/run_ray_trainer_fire_sampling.sh deleted file mode 100644 index a9286445f5..0000000000 --- a/tests/e2e/run_ray_trainer_fire_sampling.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -set -e -x - -OUTPUT_FILE="/tmp/output_ray_trainer.txt" - -export PATH=$PATH:~/.local/bin - -rm -rf $OUTPUT_FILE -python3 tests/e2e/arithmetic_sequence/rl/main_trainer.py \ - algorithm.adv_estimator=gae \ - data.train_files=tests/e2e/arithmetic_sequence/data/train.parquet \ - data.val_files=tests/e2e/arithmetic_sequence/data/test.parquet \ - data.train_batch_size=800 \ - data.val_batch_size=200 \ - data.max_prompt_length=16 \ - data.max_response_length=32 \ - data.return_raw_input_ids=True \ - actor_rollout_ref.model.path=tests/e2e/arithmetic_sequence/model \ - actor_rollout_ref.model.external_lib=tests.e2e.envs.digit_completion \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=200 \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.actor.optim.lr=1e-4 \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=200 \ - actor_rollout_ref.rollout.name=hf \ - actor_rollout_ref.rollout.use_fire_sampling=True \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - critic.ppo_micro_batch_size_per_gpu=200 \ - critic.model.path=tests/e2e/arithmetic_sequence/model \ - critic.optim.lr=1e-3 \ - algorithm.use_kl_in_reward=False \ - trainer.total_epochs=200 \ - trainer.experiment_name=arithmetic_sequences \ - trainer.logger=['console'] \ - trainer.n_gpus_per_node=1 \ - trainer.test_freq=1 \ - trainer.save_freq=110 | tee $OUTPUT_FILE; - -python3 tests/e2e/check_results.py --output_file=$OUTPUT_FILE --target 0.19 -rm -rf $OUTPUT_FILE diff --git a/tests/e2e/run_ray_trainer_rmpad.sh b/tests/e2e/run_ray_trainer_rmpad.sh deleted file mode 100644 index e4ca687d02..0000000000 --- a/tests/e2e/run_ray_trainer_rmpad.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -e -x - -huggingface-cli download Qwen/Qwen2.5-0.5B --local-dir $HOME/models/Qwen/Qwen2.5-0.5B - -python3 tests/e2e/arithmetic_sequence/rl/main_trainer.py \ - algorithm.adv_estimator=gae \ - data.train_files=tests/e2e/arithmetic_sequence/data/train.parquet \ - data.val_files=tests/e2e/arithmetic_sequence/data/test.parquet \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.model.path=tests/e2e/arithmetic_sequence/model \ - actor_rollout_ref.rollout.name=vllm \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.model.tokenizer_path=tests/e2e/arithmetic_sequence/model \ - critic.model.path=Qwen/Qwen2.5-0.5B \ - critic.model.use_remove_padding=True \ - algorithm.use_kl_in_reward=False \ - trainer.total_epochs=1 \ No newline at end of file diff --git a/tests/generation/run_gen_qwen05.sh b/tests/generation/run_gen_qwen05.sh deleted file mode 100755 index d428f465ed..0000000000 --- a/tests/generation/run_gen_qwen05.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Tested with 1 & 4 GPUs -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_gen_qwen05.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 -infer_tp=${3:-2} # Default tensor parallel size to 2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -huggingface-cli download Qwen/Qwen2.5-0.5B-Instruct --local-dir $HOME/models/Qwen/Qwen2.5-0.5B-Instruct - -python3 -m verl.trainer.main_generation \ - trainer.nnodes=1 \ - trainer.n_gpus_per_node=$nproc_per_node \ - data.path=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=prompt \ - data.n_samples=1 \ - data.output_path=$save_path \ - model.path=$HOME/models/Qwen/Qwen2.5-0.5B-Instruct \ - +model.trust_remote_code=True \ - rollout.temperature=1.0 \ - rollout.top_k=50 \ - rollout.top_p=0.7 \ - rollout.prompt_length=2048 \ - rollout.response_length=1024 \ - rollout.tensor_model_parallel_size=$infer_tp \ - rollout.gpu_memory_utilization=0.8 diff --git a/tests/gpu_utility/test_memory_buffers.py b/tests/gpu_utility/test_memory_buffers.py deleted file mode 100644 index 0116c8be02..0000000000 --- a/tests/gpu_utility/test_memory_buffers.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Test memory buffers -- We start with two models with the same weights -- We use Memory buffer to make one of the models and then compare the parameters -""" - -import torch -import gc - -from transformers import LlamaModel, LlamaConfig -from verl.utils.memory_buffer import MemoryBufferModuleWrapper - - -def test_memory_buffers(): - llama_config = LlamaConfig(vocab_size=256, - hidden_size=4096, - intermediate_size=11008, - num_hidden_layers=2, - num_attention_heads=16, - num_key_value_heads=16) - - model = LlamaModel(config=llama_config).cuda() - model_copy = LlamaModel(config=llama_config).cuda() - model_copy.load_state_dict(model.state_dict()) - - model_named_params = dict(model.named_parameters()) - model_copy_named_params = dict(model_copy.named_parameters()) - - norm_factor = 1024**3 - - t_before = torch.cuda.get_device_properties(0).total_memory / norm_factor - r_before = torch.cuda.memory_reserved(0) / norm_factor - a_before = torch.cuda.memory_allocated(0) / norm_factor - - print(f'Before Total memory: {t_before} GB, reserved: {r_before} GB, allocated: {a_before} GB') - - model_wrapper = MemoryBufferModuleWrapper(model) - - t = torch.cuda.get_device_properties(0).total_memory / norm_factor - r = torch.cuda.memory_reserved(0) / norm_factor - a = torch.cuda.memory_allocated(0) / norm_factor - - gc.collect() - torch.cuda.empty_cache() - - print(f'After Total memory: {t} GB, reserved: {r} GB, allocated: {a} GB') - - change_ratio = (a - a_before) / a_before - assert change_ratio < 0.01, f'make sure the allocated change is less than 1%, Got {change_ratio}' - - for (name1, param1), (name2, param2) in zip(model.named_parameters(), model_copy.named_parameters()): - assert name1 == name2 - assert torch.eq(param1.data, param2.data).all(), f'{param1.data}, {param2.data}, {name1}' - - -if __name__ == '__main__': - test_memory_buffers() diff --git a/tests/gpu_utility/test_ops.py b/tests/gpu_utility/test_ops.py deleted file mode 100644 index b7663cfd2e..0000000000 --- a/tests/gpu_utility/test_ops.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def test_flash_attn_cross_entropy(): - from verl.utils.torch_functional import logprobs_from_logits_naive - - from verl.utils.debug import log_gpu_memory_usage - - from flash_attn.ops.triton.cross_entropy import cross_entropy_loss - - import torch - from torch import nn - - log_gpu_memory_usage('At start') - - hidden_states = torch.randn(size=(2048, 5120), device='cuda', requires_grad=True, dtype=torch.bfloat16) - - linear = nn.Linear(in_features=5120, out_features=155136, bias=False, device='cuda', dtype=torch.bfloat16) - - logits = linear(hidden_states) - - # logits = logits.float() - labels = torch.randint(low=0, high=155136, size=(2048,), device='cuda') - - log_gpu_memory_usage('before computation') - # output = checkpoint.checkpoint(logprobs_from_logits, logits, labels, use_reentrant=True) - output = -cross_entropy_loss(logits, labels)[0] - # output = logprobs_from_logits(logits, labels) - log_gpu_memory_usage('After forward') - output.sum().backward() - log_gpu_memory_usage('After backward') - - groundtruth = logprobs_from_logits_naive(logits.float(), labels) - - torch.testing.assert_close(output, groundtruth) diff --git a/tests/gpu_utility/test_torch_functional.py b/tests/gpu_utility/test_torch_functional.py deleted file mode 100644 index 6dfa2e8679..0000000000 --- a/tests/gpu_utility/test_torch_functional.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl.utils.model import create_random_mask -from flash_attn.bert_padding import unpad_input -import torch -import pytest - - -def test_log_probs_from_logits_response_rmpad(): - from verl.utils.torch_functional import log_probs_from_logits_response, log_probs_from_logits_response_rmpad - vocab_size = 32000 - batch_size = 2 - prompt_length = 256 - response_length = 256 - - input_ids = torch.randint(low=0, high=vocab_size, size=(batch_size, prompt_length + response_length), device='cuda') - attention_mask = create_random_mask(input_ids=input_ids, - max_ratio_of_left_padding=0.2, - max_ratio_of_valid_token=0.8, - min_ratio_of_valid_token=0.6) - - response_mask = attention_mask[:, -response_length:] - - assert torch.all(response_mask[:, 0] == 1) - - logits = torch.randn(batch_size, prompt_length + response_length, vocab_size, device='cuda') - logits_rmpad = unpad_input(logits, attention_mask)[0] - - expected_output = log_probs_from_logits_response(input_ids=input_ids, - logits=logits, - response_length=response_length) - actual_output = log_probs_from_logits_response_rmpad(input_ids=input_ids, - attention_mask=attention_mask, - logits_rmpad=logits_rmpad, - response_length=response_length) - - # This should bitwise align as only this operation only contains gather operators - assert torch.all(torch.eq(actual_output * response_mask, expected_output * response_mask)) - - -@pytest.mark.parametrize("dtype", [torch.float64, torch.float32, torch.float16, torch.bfloat16]) -def test_logprobs_from_logits_v2(dtype): - from verl.utils.torch_functional import logprobs_from_logits_v2, logprobs_from_logits_naive - vocab_size = 32000 - batch_size = 2 - seq_len = 512 - - labels = torch.randint(low=0, high=vocab_size, size=(batch_size, seq_len), device='cuda') - logits = torch.randn(batch_size, seq_len, vocab_size, device='cuda', dtype=dtype) - - expected_output = logprobs_from_logits_naive(labels=labels, logits=logits) - actual_output = logprobs_from_logits_v2(labels=labels, logits=logits) - - if dtype in [torch.float16, torch.bfloat16]: # float16 falls back to an exactly equivalent method - assert torch.equal(actual_output, expected_output) - else: # small numerical difference when using gather / logsumexp approach - torch.testing.assert_close(actual_output, expected_output, rtol=1e-5, atol=1e-5) - - -def test_lr_scheduler(): - from torch import nn - model = nn.Linear(10, 10) - optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) - - from verl.utils.torch_functional import get_constant_schedule_with_warmup - constant_lr = get_constant_schedule_with_warmup(optimizer=optimizer, num_warmup_steps=2) - - lr_lst = [] - - for _ in range(5): - lr_lst.append(constant_lr.get_last_lr()[0]) - constant_lr.step() - - torch.testing.assert_close(lr_lst, [0.0, 0.0005, 0.001, 0.001, 0.001]) - - from verl.utils.torch_functional import get_cosine_schedule_with_warmup - optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) - cosine_lr = get_cosine_schedule_with_warmup(optimizer=optimizer, - num_warmup_steps=2, - num_training_steps=5, - min_lr_ratio=0.1) - - lr_lst = [] - - for _ in range(5): - lr_lst.append(cosine_lr.get_last_lr()[0]) - cosine_lr.step() - - torch.testing.assert_close(lr_lst, [0.0, 0.0005, 0.001, 0.0007750000000000002, 0.0003250000000000002]) diff --git a/tests/kill_github_tests.sh b/tests/kill_github_tests.sh deleted file mode 100644 index 5c76d7658d..0000000000 --- a/tests/kill_github_tests.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 YOUR_GITHUB_TOKEN" - echo "Please provide exactly one input argument for your github token." - exit 1 -fi - -# Set your GitHub repository details -OWNER="volcengine" -REPO="verl" -TOKEN=$1 - -# API URL for workflow runs -API_URL="https://api.github.com/repos/$OWNER/$REPO/actions/runs?status=queued" - -# Check required commands -command -v jq >/dev/null 2>&1 || { echo "jq is required but not installed. Aborting."; exit 1; } - -# Get queued workflow runs -response=$(curl -s -H "Authorization: token $TOKEN" -H "Accept: application/vnd.github.v3+json" "$API_URL") - -# Run this for debugging -# echo $response - -# Extract run IDs -queued_run_ids=$(echo "$response" | jq -r '.workflow_runs[] | .id') - -if [ -z "$queued_run_ids" ]; then - echo "No queued workflow runs found." - exit 0 -fi - -# Cancel each queued run -for run_id in $queued_run_ids; do - echo "Cancelling run $run_id" - cancel_url="https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/cancel" - curl -s -X POST -H "Authorization: token $TOKEN" -H "Accept: application/vnd.github.v3+json" "$cancel_url" -done - -echo "Cancelled all queued workflow runs." diff --git a/tests/model/test_transformer.py b/tests/model/test_transformer.py deleted file mode 100644 index 1dd7fcff14..0000000000 --- a/tests/model/test_transformer.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -from verl.utils.model import create_random_mask, compute_position_id_with_mask -from verl.utils.torch_functional import masked_mean, log_probs_from_logits_all_rmpad, logprobs_from_logits -from flash_attn.bert_padding import unpad_input, pad_input, index_first_axis, rearrange - -from transformers import LlamaConfig, MistralConfig, GemmaConfig, Qwen2Config -from transformers import AutoModelForCausalLM, AutoModelForTokenClassification, AutoModelForSequenceClassification -# TODO(sgm): add more models for test -# we only need one scale for each model -test_configs = [ - LlamaConfig(num_hidden_layers=1), - MistralConfig(num_hidden_layers=1), - GemmaConfig(num_hidden_layers=1), - Qwen2Config(num_hidden_layers=1) -] - - -def test_hf_casual_models(): - batch_size = 4 - seqlen = 128 - response_length = 127 - - for config in test_configs: - # config = AutoConfig.from_pretrained(test_case) - with torch.device('cuda'): - model = AutoModelForCausalLM.from_config(config=config, - torch_dtype=torch.bfloat16, - attn_implementation='flash_attention_2') - model = model.to(device='cuda') - input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device='cuda') - attention_mask = create_random_mask(input_ids=input_ids, - max_ratio_of_left_padding=0.1, - max_ratio_of_valid_token=0.8, - min_ratio_of_valid_token=0.5) - position_ids = compute_position_id_with_mask( - attention_mask) # TODO(sgm): we can construct the position_ids_rmpad here - - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - - # unpad the position_ids to align the rotary - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # input with input_ids_rmpad and postition_ids to enable flash attention varlen - logits_rmpad = model(input_ids_rmpad, position_ids=position_ids_rmpad, - use_cache=False).logits # (1, total_nnz, vocab_size) - - origin_logits = model(input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - use_cache=False).logits - origin_logits_rmpad, origin_logits_indices, *_ = unpad_input(origin_logits, attention_mask) - - logits_rmpad = logits_rmpad.squeeze(0) - log_probs = log_probs_from_logits_all_rmpad(input_ids_rmpad=input_ids_rmpad, - logits_rmpad=logits_rmpad, - indices=indices, - batch_size=batch_size, - seqlen=seqlen, - response_length=response_length) # (batch, seqlen) - origin_log_probs = log_probs_from_logits_all_rmpad(input_ids_rmpad=input_ids_rmpad, - logits_rmpad=origin_logits_rmpad, - indices=origin_logits_indices, - batch_size=batch_size, - seqlen=seqlen, - response_length=response_length) # (batch, seqlen) - - torch.testing.assert_close(masked_mean(log_probs, attention_mask[:, -response_length - 1:-1]), - masked_mean(origin_log_probs, attention_mask[:, -response_length - 1:-1]), - atol=1e-2, - rtol=1e-5) - print(f'Check pass') - - -def test_hf_value_models(): - batch_size = 4 - seqlen = 128 - - for config in test_configs: - # config = AutoConfig.from_pretrained(test_case) - config.num_labels = 1 - setattr(config, 'classifier_dropout', 0) - setattr(config, 'hidden_dropout', 0) - with torch.device('cuda'): - model = AutoModelForTokenClassification.from_config(config=config, - torch_dtype=torch.bfloat16, - attn_implementation='flash_attention_2') - model = model.to(device='cuda') - input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device='cuda') - attention_mask = create_random_mask(input_ids=input_ids, - max_ratio_of_left_padding=0.1, - max_ratio_of_valid_token=0.8, - min_ratio_of_valid_token=0.5) - position_ids = compute_position_id_with_mask( - attention_mask) # TODO(sgm): we can construct the position_ids_rmpad here - - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - - # unpad the position_ids to align the rotary - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - origin_logits = model(input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - use_cache=False).logits - - # input with input_ids_rmpad and postition_ids to enable flash attention varlen - rmpad_logits = model(input_ids_rmpad, position_ids=position_ids_rmpad, - use_cache=False).logits # (1, total_nnz, 1) - rmpad_logits = rmpad_logits.squeeze(0) - pad_logits = pad_input(rmpad_logits, indices, batch_size, seqlen=seqlen) - - torch.testing.assert_close(masked_mean(pad_logits, attention_mask[:, :, None]), - masked_mean(origin_logits, attention_mask[:, :, None]), - atol=1e-2, - rtol=1e-5) - print('Value model check pass') - - -if __name__ == '__main__': - test_hf_casual_models() - test_hf_value_models() diff --git a/tests/model/test_transformers_ulysses.py b/tests/model/test_transformers_ulysses.py deleted file mode 100644 index 2bbab1d50f..0000000000 --- a/tests/model/test_transformers_ulysses.py +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import time -import contextlib -from dataclasses import dataclass - -import pytest -import torch -import copy -import torch.distributed -from torch.distributed import init_device_mesh -from verl.utils.distributed import initialize_global_process_group -from verl.utils.model import create_random_mask, compute_position_id_with_mask -from verl.utils.ulysses import ulysses_pad_and_slice_inputs, gather_outpus_and_unpad -from verl.utils.ulysses import get_ulysses_sequence_parallel_world_size, set_ulysses_sequence_parallel_group -from verl.workers.sharding_manager import FSDPUlyssesShardingManager -from verl.protocol import DataProto -from flash_attn.bert_padding import unpad_input, index_first_axis, rearrange -from transformers import LlamaConfig, Qwen2Config, PretrainedConfig -from transformers import AutoModelForCausalLM -from verl.models.transformers.monkey_patch import apply_monkey_patch - -# TODO(sgm): add more models for test -# we only need one scale for each model - - -@dataclass -class SequenceParallelConfig: - config: PretrainedConfig - sp_size: int - is_valid: bool - - -def test_configs(): - return [ - SequenceParallelConfig(LlamaConfig(num_hidden_layers=2, num_attention_heads=32, num_key_value_heads=32), - sp_size=8, - is_valid=True), - SequenceParallelConfig(Qwen2Config(num_hidden_layers=2, - num_attention_heads=28, - num_key_value_heads=4, - hidden_size=3584), - sp_size=4, - is_valid=True), - SequenceParallelConfig(Qwen2Config(num_hidden_layers=2, - num_attention_heads=28, - num_key_value_heads=4, - hidden_size=3584), - sp_size=8, - is_valid=False), - SequenceParallelConfig(Qwen2Config(num_hidden_layers=2, num_attention_heads=32, num_key_value_heads=4), - sp_size=4, - is_valid=True), - SequenceParallelConfig(Qwen2Config(num_hidden_layers=2, num_attention_heads=32, num_key_value_heads=4), - sp_size=8, - is_valid=True), - ] - - -def sync_model_parameters_global(layer): - # synchronize weights - for p in layer.parameters(): - torch.distributed.broadcast(tensor=p.data, src=0) - - -@pytest.mark.parametrize("test_config", test_configs()) -def test_hf_casual_fwd_bwd(test_config): - if not torch.distributed.is_initialized(): - initialize_global_process_group() - - context = contextlib.nullcontext() if test_config.is_valid else pytest.raises(AssertionError) - with context: - world_size = torch.distributed.get_world_size() - _hf_casual_fwd_bwd(test_config.config, test_config.sp_size, world_size // test_config.sp_size) - - # TODO: seems not work, will cause `socketStartConnect: Connect to xxx failed : Software caused connection abort` - # torch.distributed.destroy_process_group() - - -def _hf_casual_fwd(config, sp_size, dp_size): - assert torch.cuda.device_count() >= 2, "need at least 2 gpus for test" - - ulysses_device_mesh = init_device_mesh(device_type='cuda', - mesh_shape=(dp_size, sp_size), - mesh_dim_names=('dp', 'sp')) - sharding_manager = FSDPUlyssesShardingManager(ulysses_device_mesh) - - batch_size = 1 - seqlen = 128 - response_length = 127 - - # patch before load - with torch.device('cuda'): - model = AutoModelForCausalLM.from_config(config=config, - torch_dtype=torch.bfloat16, - attn_implementation='flash_attention_2') - apply_monkey_patch(model, sp_size) - model = model.to(device='cuda') - sync_model_parameters_global(model) - - # different rank will generate different input_ids following fsdp - input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device='cuda') - attention_mask = create_random_mask(input_ids=input_ids, - max_ratio_of_left_padding=0, - max_ratio_of_valid_token=0.9, - min_ratio_of_valid_token=0.8) - position_ids = compute_position_id_with_mask( - attention_mask) # TODO(sgm): we can construct the position_ids_rmpad here - - model_inputs = { - 'input_ids': input_ids.cuda(), - 'attention_mask': attention_mask.cuda(), - 'position_ids': position_ids.int().cuda() - } - - model_inputs = DataProto.from_dict(model_inputs) - - # 1. perform ulysses forward - with sharding_manager: - model_inputs = sharding_manager.preprocess_data(model_inputs) - input_ids = model_inputs.batch['input_ids'] - attention_mask = model_inputs.batch['attention_mask'] - position_ids = model_inputs.batch['position_ids'] - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - # unpad the position_ids to align the rotary - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # slice input tensor for ulysses - # input_ids are padded and sliced - # postition_ids are only padded but not sliced - input_ids_rmpad_sliced, position_ids_rmpad_padded, pad_size = ulysses_pad_and_slice_inputs( - input_ids_rmpad, position_ids_rmpad, sp_size=get_ulysses_sequence_parallel_world_size()) - - # input with input_ids_rmpad and postition_ids to enable flash attention varlen - logits_split_in_seq = model(input_ids_rmpad_sliced, position_ids=position_ids_rmpad_padded, - use_cache=False).logits # (1, total_nnz/n, vocab_size) - - # all_gather output - logits_full = gather_outpus_and_unpad(logits_split_in_seq, gather_dim=1, unpad_dim=1, padding_size=pad_size) - - # 2. perform normal forward - set_ulysses_sequence_parallel_group(None) - logits_rmpad_local = model(input_ids_rmpad, position_ids=position_ids_rmpad, - use_cache=False).logits # (1, total_nnz, vocab_size) - - mean_local = logits_rmpad_local.mean() - mean_full = logits_full.mean() - torch.testing.assert_close(mean_local, mean_full, rtol=1e-2, atol=1e-5) - - -def _hf_casual_fwd_bwd(config, sp_size, dp_size): - assert torch.cuda.device_count() >= 2, "need at least 2 gpus for test" - - ulysses_device_mesh = init_device_mesh(device_type='cuda', - mesh_shape=(dp_size, sp_size), - mesh_dim_names=('dp', 'sp')) - sharding_manager = FSDPUlyssesShardingManager(ulysses_device_mesh) - - batch_size = 1 - seqlen = 128 - response_length = 127 - - # patch before load - with torch.device('cuda'): - model = AutoModelForCausalLM.from_config(config=config, - torch_dtype=torch.bfloat16, - attn_implementation='flash_attention_2') - apply_monkey_patch(model, sp_size) - model = model.to(device='cuda') - sync_model_parameters_global(model) - - # different rank will generate different input_ids following fsdp - input_ids = torch.randint(low=0, high=config.vocab_size, size=(batch_size, seqlen), device='cuda') - attention_mask = create_random_mask(input_ids=input_ids, - max_ratio_of_left_padding=0, - max_ratio_of_valid_token=0.9, - min_ratio_of_valid_token=0.8) - position_ids = compute_position_id_with_mask( - attention_mask) # TODO(sgm): we can construct the position_ids_rmpad here - - model_inputs = { - 'input_ids': input_ids.cuda(), - 'attention_mask': attention_mask.cuda(), - 'position_ids': position_ids.int().cuda() - } - - model_inputs = DataProto.from_dict(model_inputs) - - # 1. perform ulysses forward - with sharding_manager: - model_inputs = sharding_manager.preprocess_data(model_inputs) - input_ids = model_inputs.batch['input_ids'] - attention_mask = model_inputs.batch['attention_mask'] - position_ids = model_inputs.batch['position_ids'] - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - # unpad the position_ids to align the rotary - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # slice input tensor for ulysses - # input_ids are padded and sliced - # postition_ids are only padded but not sliced - input_ids_rmpad_sliced, position_ids_rmpad_padded, pad_size = ulysses_pad_and_slice_inputs( - input_ids_rmpad, position_ids_rmpad, sp_size=get_ulysses_sequence_parallel_world_size()) - - # input with input_ids_rmpad and postition_ids to enable flash attention varlen - logits_split_in_seq = model(input_ids_rmpad_sliced, position_ids=position_ids_rmpad_padded, - use_cache=False).logits # (1, total_nnz/n, vocab_size) - - # all_gather output - logits_full = gather_outpus_and_unpad(logits_split_in_seq, gather_dim=1, unpad_dim=1, padding_size=pad_size) - - # 2. perform normal forward - set_ulysses_sequence_parallel_group(None) - input_ids_full = copy.deepcopy(input_ids_rmpad) - position_ids_full = copy.deepcopy(position_ids_rmpad) - model_no_sp = copy.deepcopy(model) - logits_rmpad_local = model_no_sp(input_ids_full, position_ids=position_ids_full, - use_cache=False).logits # (1, total_nnz, vocab_size) - - mean_local = logits_rmpad_local.mean() - mean_full = logits_full.mean() - - mean_full.backward() - mean_local.backward() - - # 3. check the gradients - grad = model.model.layers[0].self_attn.q_proj.weight.grad - grad_full = model_no_sp.model.layers[0].self_attn.q_proj.weight.grad - torch.testing.assert_close(mean_local, mean_full, rtol=1e-2, atol=1e-5) - torch.testing.assert_close(grad, grad_full, atol=1e-2, rtol=1e-5) - - -if __name__ == '__main__': - pytest.main([__file__, "-svv"]) diff --git a/tests/ray/check_worker_alive/main.py b/tests/ray/check_worker_alive/main.py deleted file mode 100644 index fcebbfe20b..0000000000 --- a/tests/ray/check_worker_alive/main.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time -import sys -import os - -import ray - -from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup -from verl.single_controller.base.worker import Worker -from verl.single_controller.base.decorator import register, Dispatch - - -@ray.remote -class TestActor(Worker): - - def __init__(self) -> None: - super().__init__() - - @register(dispatch_mode=Dispatch.ONE_TO_ALL, blocking=False) - def foo(self, wait_time): - time.sleep(wait_time) - sys.exit(1) - - -if __name__ == "__main__": - wait_time = int(os.getenv("WAIT_TIME", "10")) - - ray.init() - - # test single-node-no-partition - print(f"test single-node-no-partition") - resource_pool = RayResourcePool([2], use_gpu=True) - class_with_args = RayClassWithInitArgs(cls=TestActor) - - print("create worker group") - wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="test") - - wg.start_worker_aliveness_check(1) - time.sleep(1) - - print(time.time(), "start foo") - - _ = wg.foo(wait_time) - print("foo started") - - print(time.time(), - f"wait 6x wait time {wait_time*6} to let signal returned to process but still not exceed process wait time") - time.sleep(wait_time * 6) - - ray.shutdown() diff --git a/tests/ray/detached_worker/README.md b/tests/ray/detached_worker/README.md deleted file mode 100644 index b06c4c6143..0000000000 --- a/tests/ray/detached_worker/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Detached Worker -## How to run (Only on a single node) -- Start a local ray cluster: -```bash -ray start --head --port=6379 -``` -- Run the server -```bash -python3 server.py -``` -- On another terminal, Run the client -```bash -python3 client.py -``` diff --git a/tests/ray/detached_worker/client.py b/tests/ray/detached_worker/client.py deleted file mode 100644 index 1773fffe70..0000000000 --- a/tests/ray/detached_worker/client.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -In client, we can get the server handler and send RPC request -""" - -import ray -import torch - -from verl import DataProto -from verl.single_controller.ray import RayClassWithInitArgs -from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup - -from tensordict import TensorDict - -from server import Trainer - - -def compute_position_id_with_mask(mask): - return torch.clip(torch.cumsum(mask, dim=-1) - 1, min=0, max=None) - - -if __name__ == '__main__': - - ray.init(address='auto', namespace='verl') - # get the worker group using names - worker_names = ['trainerTrainer_0:0', 'trainerTrainer_0:1'] - cls_with_init_args = RayClassWithInitArgs(cls=Trainer) - worker_group = NVMegatronRayWorkerGroup.from_detached(worker_names=worker_names, - ray_cls_with_init=cls_with_init_args) - - batch_size = 16 - sequence_length = 1024 - - # give Trainer some data to train - input_ids = torch.randint(low=0, high=256, size=(batch_size, sequence_length), dtype=torch.int64, device='cuda') - attention_mask = torch.ones_like(input_ids) - position_ids = compute_position_id_with_mask(attention_mask) - - data = DataProto(batch=TensorDict( - { - 'input_ids': input_ids, - 'attention_mask': attention_mask, - 'position_ids': position_ids - }, batch_size=batch_size), - meta_info={}) - - output = worker_group.train_model(data) - - print(output) diff --git a/tests/ray/detached_worker/run.sh b/tests/ray/detached_worker/run.sh deleted file mode 100644 index a3c6387933..0000000000 --- a/tests/ray/detached_worker/run.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -ray start --head --port=6379 -python3 server.py -python3 client.py -ray stop --force \ No newline at end of file diff --git a/tests/ray/detached_worker/server.py b/tests/ray/detached_worker/server.py deleted file mode 100644 index 2108f1ebd5..0000000000 --- a/tests/ray/detached_worker/server.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Server starts a Trainer. Client sends data to the server to train. -""" - -import os - -os.environ['MEGATRON_USE_CUDA_TIMER'] = '0' -os.environ['MEGATRON_START_PROCESS_TIMER'] = 'False' -os.environ['NCCL_DEBUG'] = 'WARN' - -import torch -from torch import nn - -import ray -from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool -from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup -from verl.single_controller.base.megatron.worker import MegatronWorker -from verl.single_controller.base.decorator import register, Dispatch -from verl import DataProto -from verl.models.llama.megatron import ParallelLlamaForCausalLMRmPadPP - -from megatron.core import parallel_state as mpu -from megatron.core.models.gpt.gpt_model import ModelType -from megatron.core import tensor_parallel -from verl.utils.megatron_utils import get_model, init_megatron_optim_config, init_model_parallel_config -from verl.utils.megatron.optimizer import get_megatron_optimizer - -from transformers import LlamaConfig - -from omegaconf import OmegaConf - -from tensordict import TensorDict - - -@ray.remote -class Trainer(MegatronWorker): - - def __init__(self): - super().__init__() - - if not torch.distributed.is_initialized(): - rank = int(os.environ['LOCAL_RANK']) - torch.distributed.init_process_group(backend="nccl") - torch.cuda.set_device(rank) - - os.environ['CUDA_DEVICE_MAX_CONNECTIONS'] = '1' - mpu.initialize_model_parallel( - tensor_model_parallel_size=2, - pipeline_model_parallel_size=1, - virtual_pipeline_model_parallel_size=None, - pipeline_model_parallel_split_rank=None, - use_sharp=False, - context_parallel_size=1, - expert_model_parallel_size=1, - nccl_communicator_config_path=None, - ) - tensor_parallel.model_parallel_cuda_manual_seed(10) - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - actor_model_config = LlamaConfig(vocab_size=256, - hidden_size=2048, - intermediate_size=5504, - num_hidden_layers=24, - num_attention_heads=16, - num_key_value_heads=16) - - megatron_config = OmegaConf.create({ - 'sequence_parallel_enabled': True, - 'param_dtype': 'bf16', - 'pipeline_model_parallel_rank': mpu.get_pipeline_model_parallel_rank(), - 'pipeline_model_parallel_size': mpu.get_pipeline_model_parallel_world_size(), - 'virtual_pipeline_model_parallel_rank': mpu.get_virtual_pipeline_model_parallel_rank(), - 'virtual_pipeline_model_parallel_size': mpu.get_virtual_pipeline_model_parallel_world_size() - }) - - megatron_config = init_model_parallel_config(megatron_config) - self.megatron_config = megatron_config - - def megatron_actor_model_provider(pre_process, post_process): - # vpp is not supported yet because it will hang for some reason. Need debugging - vpp_rank = mpu.get_virtual_pipeline_model_parallel_rank() # this will be set inside get_model - # this_megatron_config = copy.deepcopy(megatron_config) - # this_megatron_config.virtual_pipeline_model_parallel_rank = vpp_rank - parallel_model = ParallelLlamaForCausalLMRmPadPP(config=actor_model_config, - megatron_config=megatron_config, - pre_process=pre_process, - post_process=post_process) - parallel_model.cuda() - return parallel_model - - actor_module = get_model(model_provider_func=megatron_actor_model_provider, - model_type=ModelType.encoder_or_decoder, - wrap_with_ddp=True) - actor_module = nn.ModuleList(actor_module) - - optim_config = OmegaConf.create({'lr': 1e-6, 'clip_grad': 1.0}) - - optim_config = init_megatron_optim_config(optim_config) - self.optimizer_config = optim_config - actor_optimizer = get_megatron_optimizer(model=actor_module, config=optim_config) - - self.model = actor_module[0] - self.optimizer = actor_optimizer - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def train_model(self, data: DataProto) -> DataProto: - input_ids = data.batch['input_ids'] - attention_mask = data.batch['attention_mask'] - position_ids = data.batch['position_ids'] - - self.optimizer.zero_grad() - self.model.zero_grad_buffer( - zero_buffer=(not self.optimizer_config.use_distributed_optimizer - )) # use use_contiguous_buffers_in_local_ddp and no overlap_dp_param_comm - # update for 1 iteration - output = self.model(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids).logits - output.mean().backward() - - update_successful, grad_norm, num_zeros_in_grad = self.optimizer.step(self.megatron_config, - self.megatron_config.timers) - - return DataProto(batch=TensorDict({'loss': output.detach()}, batch_size=output.shape[0])) - - -if __name__ == '__main__': - ray.init(address='auto', namespace='verl') - - resource_pool = RayResourcePool(process_on_nodes=[2], detached=True) - cls_with_init_args = RayClassWithInitArgs(cls=Trainer) - worker_group = NVMegatronRayWorkerGroup( - resource_pool=resource_pool, - ray_cls_with_init=cls_with_init_args, - name_prefix='trainer', - detached=True, - ) - - worker_group.init_model() - - worker_names = worker_group.worker_names - print(worker_names) diff --git a/tests/ray/test_check_worker_alive.py b/tests/ray/test_check_worker_alive.py deleted file mode 100644 index 53b7f2aabc..0000000000 --- a/tests/ray/test_check_worker_alive.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time -import os -import subprocess - - -def test(): - wait_time = 10 - - my_env = os.environ.copy() - my_env["WAIT_TIME"] = str(wait_time) - - p = subprocess.Popen(["python3", "-u", "./check_worker_alive/main.py"], env=my_env, stdout=subprocess.PIPE) - - count = 0 - while b"foo started" not in p.stdout.read(): - time.sleep(1) - count += 1 - if count > 40: - raise RuntimeError("timeout for start foo in check_worker_alive/main.py") - - print( - time.time(), - f"wait 1.5 wait time {wait_time*1.5} to let signal returned to process but still not exceed process wait time") - time.sleep(wait_time * 1.5) - print(time.time(), f"start checking") - assert p.poll() is not None, f"process {p} still alive, expecting signal raised abort" - assert p.returncode != 0, f"process {p} exit with code 0, expecting not-zero exit code" - print(f"test passed") - - -if __name__ == "__main__": - test() diff --git a/tests/ray/test_colocated_workers.py b/tests/ray/test_colocated_workers.py deleted file mode 100644 index 96b859b4b2..0000000000 --- a/tests/ray/test_colocated_workers.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ray - -from verl.single_controller.base import Worker -from verl.single_controller.base.decorator import register, Dispatch -from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup, create_colocated_worker_cls - -from verl import DataProto - - -@ray.remote -class Actor(Worker): - - def __init__(self) -> None: - super().__init__() - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def add(self, data: DataProto): - data.batch['a'] += self.rank - return data - - -@ray.remote -class Critic(Worker): - - def __init__(self, config) -> None: - super().__init__() - self.config = config - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def sub(self, data: DataProto): - data.batch['a'] -= self.config['b'] - return data - - -def test_colocated_workers(): - ray.init() - - import torch - data = DataProto.from_dict({'a': torch.zeros(10)}) - # create separate workers on the same resource pool - actor_cls = RayClassWithInitArgs(cls=Actor) - critic_cls = RayClassWithInitArgs(cls=Critic, config={'b': 10}) - resource_pool = RayResourcePool(process_on_nodes=[2]) - - actor_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=actor_cls) - critic_wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=critic_cls) - - expected_actor_output = actor_wg.add(data) - expected_critic_output = critic_wg.sub(data) - - # create colocated workers - cls_dict = {'actor': actor_cls, 'critic': critic_cls} - ray_cls_with_init = create_colocated_worker_cls(cls_dict) - wg_dict = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) - spawn_wg = wg_dict.spawn(prefix_set=cls_dict.keys()) - - colocated_actor_wg = spawn_wg['actor'] - colocated_critic_wg = spawn_wg['critic'] - - actor_output = colocated_actor_wg.add(data) - critic_output = colocated_critic_wg.sub(data) - - torch.testing.assert_close(expected_actor_output.batch, actor_output.batch, atol=0, rtol=0) - torch.testing.assert_close(expected_critic_output.batch, critic_output.batch, atol=0, rtol=0) - - ray.shutdown() diff --git a/tests/ray/test_data_transfer.py b/tests/ray/test_data_transfer.py deleted file mode 100644 index a17affd309..0000000000 --- a/tests/ray/test_data_transfer.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -In this test, we instantiate a data parallel worker with 8 GPUs -""" - -from verl.single_controller.base import Worker -from verl.single_controller.ray import RayWorkerGroup, RayClassWithInitArgs, RayResourcePool - -from verl.single_controller.base.decorator import Dispatch, register - -import ray -import torch - -from torch import distributed as dist - -from verl import DataProto -from verl.utils.ray_utils import parallel_put - -from codetiming import Timer - -import tensordict - - -@ray.remote -class DummyWorker(Worker): - - def __init__(self): - super().__init__() - dist.init_process_group() - - @register(dispatch_mode=Dispatch.DP_COMPUTE, blocking=False) - def do_nothing(self, data): - for key in data.batch.keys(): - data.batch[key] += 1 - if tensordict.__version__ >= '0.5.0': - data.batch = data.batch.consolidate() - return data - - -def test_data_transfer(): - ray.init() - # construct resource pool - resource_pool = RayResourcePool([8]) - cls_with_init = RayClassWithInitArgs(cls=DummyWorker) - # construct worker group - wg = RayWorkerGroup(resource_pool, cls_with_init) - - # this is real dataset size - batch_size = 4096 - seqlen = 32768 - - data_dict = {} - - for i in range(2): - data_dict[str(i)] = torch.randint(0, 10000, (batch_size, seqlen)) - - data = DataProto.from_dict(tensors=data_dict) - - print(data) - - # we manually split data here and send to each worker - data_list = data.chunk(wg.world_size) - - for i in range(wg.world_size): - # consolidate is necessary - if tensordict.__version__ >= '0.5.0': - data_list[i].batch = data_list[i].batch.consolidate() - - with Timer(name='ray.pickle', initial_text=True): - for i in range(wg.world_size): - ray.cloudpickle.pickle.dumps(data_list[i]) - - with Timer(name='raw.pickle', initial_text=True): - import pickle - for i in range(wg.world_size): - pickle.dumps(data_list[i]) - - # we put in advance - with Timer(name='put', initial_text=True): - # takes around 40 seconds - data_list_ref = parallel_put(data_list) - # for i in range(wg.world_size): - # data_list[i] = ray.put(data_list[i]) - - with Timer(name='launch', initial_text=True): - output_ref = wg.do_nothing(data_list_ref) - - with Timer(name='get', initial_text=True): - # takes around 40 seconds - output_lst = ray.get(output_ref) - - for input_data, output_data in zip(data_list, output_lst): - for key in input_data.batch.keys(): - assert torch.all(torch.eq(input_data.batch[key] + 1, - output_data.batch[key])), (input_data.batch[key], output_data.batch[key], key) - - ray.shutdown() diff --git a/tests/ray/test_driverfunc_to_worker.py b/tests/ray/test_driverfunc_to_worker.py deleted file mode 100644 index ea253fd362..0000000000 --- a/tests/ray/test_driverfunc_to_worker.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import ray -import torch -from verl import DataProto -from tensordict import TensorDict - -from verl.single_controller.base.worker import Worker -from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs -from verl.single_controller.ray import RayWorkerGroup - -os.environ['RAY_DEDUP_LOGS'] = '0' -os.environ['NCCL_DEBUG'] = 'WARN' - - -@ray.remote -class ModelActor(Worker): - - def __init__(self): - pass - - -class HackSelf(): - - def __init__(self): - pass - - -def get_aux_metrics(self, test_proto): - sequence_ids = test_proto.batch["sequence_ids"] - decode_count = [] - for i in range(sequence_ids.size(0)): - decode_count.append(len(sequence_ids[i].tolist())) - ret_proto = DataProto(batch=TensorDict({ - "sequence_ids": sequence_ids, - "decode_count": torch.tensor(decode_count) - }, - batch_size=sequence_ids.size(0))) - return ret_proto - - -def test(): - # construct model - ray.init() - - # create 2 workers, each hold a GPU - resource_pool = RayResourcePool([2], use_gpu=True, name_prefix='a') - - class_with_args = RayClassWithInitArgs(cls=ModelActor) - shard_wg = RayWorkerGroup(resource_pool, class_with_args) - - test_bs = 8 - test_proto = DataProto(TensorDict({ - "sequence_ids": torch.ones([test_bs, 2048], dtype=torch.int64), - }, - batch_size=test_bs), - meta_info={"query_length": 1536}) - - # Sharding among different ranks - ret_proto1 = shard_wg.execute_with_func_generator(get_aux_metrics, test_proto) - - # compare execute on driver - hs = HackSelf() - ret_proto2 = get_aux_metrics(hs, test_proto) - - torch.testing.assert_close(ret_proto1.batch["decode_count"], ret_proto2.batch["decode_count"]) - - ray.shutdown() diff --git a/tests/ray/test_high_level_scheduling_api.py b/tests/ray/test_high_level_scheduling_api.py deleted file mode 100644 index 2d83206e5e..0000000000 --- a/tests/ray/test_high_level_scheduling_api.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -import ray - -from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup, merge_resource_pool -from verl.single_controller.base.worker import Worker - - -@ray.remote -class TestActor(Worker): - # TODO: pass *args and **kwargs is bug prone and not very convincing - def __init__(self, cuda_visible_devices=None) -> None: - super().__init__(cuda_visible_devices) - - def get_node_id(self): - return ray.get_runtime_context().get_node_id() - - -def test(): - ray.init() - - # test single-node-no-partition - print(f"test single-node-no-partition") - resource_pool = RayResourcePool([8], use_gpu=True) - - class_with_args = RayClassWithInitArgs(cls=TestActor) - - print("create actor worker group") - actor_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="high_level_api_actor") - print("create critic worker group") - critic_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="hight_level_api_critic") - print("create rm worker group") - rm_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="high_level_api_rm") - print("create ref worker group") - ref_wg = RayWorkerGroup(resource_pool, class_with_args, name_prefix="high_level_api_ref") - - assert actor_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] - assert critic_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] - assert rm_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] - assert ref_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] - - del actor_wg - del critic_wg - del rm_wg - del ref_wg - - [ray.util.remove_placement_group(pg) for pg in resource_pool.get_placement_groups()] - print("wait 5s to remove placemeng_group") - time.sleep(5) - # test single-node-multi-partition - - print(f"test single-node-multi-partition") - rm_resource_pool = RayResourcePool([4], use_gpu=True, name_prefix="rm") - ref_resource_pool = RayResourcePool([4], use_gpu=True, name_prefix="ref") - total_resource_pool = merge_resource_pool(rm_resource_pool, ref_resource_pool) - - assert rm_resource_pool.world_size == 4 - assert ref_resource_pool.world_size == 4 - assert total_resource_pool.world_size == 8 - - actor_wg = RayWorkerGroup(total_resource_pool, class_with_args, name_prefix="high_level_api_actor") - critic_wg = RayWorkerGroup(total_resource_pool, class_with_args, name_prefix="high_level_api_critic") - rm_wg = RayWorkerGroup(rm_resource_pool, class_with_args, name_prefix="high_level_api_rm") - ref_wg = RayWorkerGroup(ref_resource_pool, class_with_args, name_prefix="high_level_api_ref") - - assert actor_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] - assert critic_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(8)] - assert rm_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(4)] - assert ref_wg.execute_all_sync("get_cuda_visible_devices") == [str(i) for i in range(4, 8)] - - ray.shutdown() diff --git a/tests/ray/test_ray_local_envs.py b/tests/ray/test_ray_local_envs.py deleted file mode 100644 index 63102d0c67..0000000000 --- a/tests/ray/test_ray_local_envs.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -e2e test verl.single_controller.ray -""" -import os -import ray - -from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup -from verl.single_controller.base.worker import Worker -from verl.single_controller.base.decorator import register, Dispatch, collect_all_to_all, Execute - - -@ray.remote -class TestActor(Worker): - - def __init__(self) -> None: - super().__init__() - - def getenv(self, key): - val = os.getenv(key, f"{key} not set") - return val - - -def test_basics(): - ray.init() - - # create 4 workers, each hold a GPU - resource_pool = RayResourcePool([4], use_gpu=True) - class_with_args = RayClassWithInitArgs(cls=TestActor) - - worker_group = RayWorkerGroup(resource_pool=resource_pool, - ray_cls_with_init=class_with_args, - name_prefix="worker_group_basic") - - output = worker_group.execute_all_sync("getenv", key="RAY_LOCAL_WORLD_SIZE") - assert output == ["4", "4", "4", "4"] - - output = worker_group.execute_all_sync("getenv", key="RAY_LOCAL_RANK") - assert set(output) == set(["0", "1", "2", "3"]) - - ray.shutdown() - - -if __name__ == '__main__': - test_basics() diff --git a/tests/ray/test_rvdz.py b/tests/ray/test_rvdz.py deleted file mode 100644 index 9eec1d72c4..0000000000 --- a/tests/ray/test_rvdz.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ray - - -@ray.remote -class TestWorker: - - def __init__(self, rank, world_size, group_name): - self.rank = rank - self.world_size = world_size - self.group_name = group_name - self.communicator = None - - def init(self): - from verl.utils.rendezvous.ray_backend import create_nccl_communicator_in_ray - self.communicator = create_nccl_communicator_in_ray(self.rank, self.world_size, self.group_name) - - def test(self): - if self.communicator is None: - return None - return self.communicator.rank_id() - - -def test_rvdz(): - ray.init() - - group_name = "test_group" - world_size = 2 - - workers = [TestWorker.options(num_gpus=1).remote(rank, world_size, group_name) for rank in range(world_size)] - - ray.get([worker.init.remote() for worker in workers]) - - ranks = ray.get([worker.test.remote() for worker in workers]) - - assert ranks == [0, 1], f"expecting [0, 1], got {ranks}" - - ray.shutdown() diff --git a/tests/ray/test_worker_group_basics.py b/tests/ray/test_worker_group_basics.py deleted file mode 100644 index b4b633969f..0000000000 --- a/tests/ray/test_worker_group_basics.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -e2e test verl.single_controller.ray -""" - -import torch -import ray - -from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup -from verl.single_controller.base.worker import Worker -from verl.single_controller.base.decorator import register, Dispatch, collect_all_to_all, Execute - - -def two_to_all_dispatch_fn(worker_group, *args, **kwargs): - """ - Assume the input is a list of 2. Duplicate the input interleaved and pass to each worker. - """ - for arg in args: - assert len(arg) == 2 - for i in range(worker_group.world_size - 2): - arg.append(arg[i % 2]) - for k, v in kwargs.items(): - assert len(v) == 2 - for i in range(worker_group.world_size - 2): - v.append(v[i % 2]) - return args, kwargs - - -@ray.remote -class TestActor(Worker): - # TODO: pass *args and **kwargs is bug prone and not very convincing - def __init__(self, x) -> None: - super().__init__() - self._x = x - - def foo(self, y): - return self._x + y - - @register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.RANK_ZERO) - def foo_rank_zero(self, x, y): - return self._x + y + x - - @register(Dispatch.ONE_TO_ALL, blocking=False) - def foo_one_to_all(self, x, y): - return self._x + y + x - - @register(Dispatch.ALL_TO_ALL, blocking=False) - def foo_all_to_all(self, x, y): - return self._x + y + x - - @register(dispatch_mode={'dispatch_fn': two_to_all_dispatch_fn, 'collect_fn': collect_all_to_all}) - def foo_custom(self, x, y): - return self._x + y + x - - -@ray.remote(num_gpus=0.1) -def remote_call_wg(worker_names): - class_with_args = RayClassWithInitArgs(cls=TestActor, x=2) - worker_group = RayWorkerGroup.from_detached(worker_names=worker_names, ray_cls_with_init=class_with_args) - print(worker_group.worker_names) - - output_ref = worker_group.foo_custom(x=[1, 2], y=[5, 6]) - assert output_ref == [8, 10, 8, 10] - - output_ref = worker_group.foo_rank_zero(x=1, y=2) - assert output_ref == 5 - - return worker_group.worker_names - - -def add_one(data): - data = data.to("cuda") - data += 1 - data = data.to("cpu") - return data - - -def test_basics(): - ray.init() - - # create 4 workers, each hold a GPU - resource_pool = RayResourcePool([4], use_gpu=True) - class_with_args = RayClassWithInitArgs(cls=TestActor, x=2) - - worker_group = RayWorkerGroup(resource_pool=resource_pool, - ray_cls_with_init=class_with_args, - name_prefix="worker_group_basic") - - print(worker_group.worker_names) - - # this will wait for all the results - output = worker_group.execute_all_sync("foo", y=3) - assert output == [5, 5, 5, 5] - - # this is a list of object reference. It won't block. - output_ref = worker_group.execute_all_async("foo", y=4) - print(output_ref) - - assert ray.get(output_ref) == [6, 6, 6, 6] - - output_ref = worker_group.foo_one_to_all(x=1, y=2) - assert ray.get(output_ref) == [5, 5, 5, 5] - - output_ref = worker_group.foo_all_to_all(x=[1, 2, 3, 4], y=[5, 6, 7, 8]) - assert ray.get(output_ref) == [8, 10, 12, 14] - - print(ray.get(remote_call_wg.remote(worker_group.worker_names))) - - output = worker_group.execute_func_rank_zero(add_one, torch.ones(2, 2)) - torch.testing.assert_close(output, torch.ones(2, 2) + 1) - - ray.shutdown() - - -if __name__ == '__main__': - test_basics() diff --git a/tests/ray/test_worker_group_torch.py b/tests/ray/test_worker_group_torch.py deleted file mode 100644 index 13508ed309..0000000000 --- a/tests/ray/test_worker_group_torch.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -os.environ['RAY_DEDUP_LOGS'] = '0' -os.environ['NCCL_DEBUG'] = 'WARN' - -import torch -import torch.distributed -import ray - -from verl.single_controller.ray.base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup -from verl.single_controller.base.worker import Worker - - -@ray.remote -class TestAllGatherActor(Worker): - - def __init__(self, size) -> None: - super().__init__() - self.size = size - - def init(self): - torch.distributed.init_process_group() - self.tensor = torch.zeros(size=(self.size,), dtype=torch.int64, device='cuda') - self.tensor += self.rank - - def all_gather(self): - world_size = self._world_size - output = torch.zeros(size=(self.tensor.shape[0] * world_size,), - dtype=self.tensor.dtype, - device=self.tensor.device) - torch.distributed.all_gather_into_tensor(output, self.tensor, async_op=False) - return output - - -@ray.remote -class TestAllGatherActorV2(Worker): - - def __init__(self, size) -> None: - super().__init__() - self.size = size - - torch.distributed.init_process_group() - self.tensor = torch.zeros(size=(self.size,), dtype=torch.int64, device='cuda') - self.tensor += self.rank - - def all_gather(self): - world_size = self._world_size - output = torch.zeros(size=(self.tensor.shape[0] * world_size,), - dtype=self.tensor.dtype, - device=self.tensor.device) - torch.distributed.all_gather_into_tensor(output, self.tensor, async_op=False) - return output - - -def test_all_gather_torch(): - """ - In this test, we instantiate 4 GPUs in a group and test the all_gather - """ - ray.init() - - # create 4 workers, each hold a GPU - resource_pool = RayResourcePool([4], use_gpu=True) - class_with_args = RayClassWithInitArgs(cls=TestAllGatherActor, size=2) - - worker_group = RayWorkerGroup(resource_pool, class_with_args, name_prefix="worker_group_torch") - - worker_group.execute_all_sync('init') - output = worker_group.execute_all_sync('all_gather') - for i in range(1, len(output)): - assert torch.all(output[i] == output[0]) - - output = output[0].cpu() - print(output) - assert torch.all(output == torch.tensor([0, 0, 1, 1, 2, 2, 3, 3], dtype=torch.int64)) - - ray.shutdown() - - -def test_all_gather_torch_v2(): - """ - In this test, we instantiate 4 GPUs in a group and test the all_gather - """ - ray.init() - - # create 4 workers, each hold a GPU - resource_pool = RayResourcePool([4], use_gpu=True) - class_with_args = RayClassWithInitArgs(cls=TestAllGatherActorV2, size=2) - - worker_group = RayWorkerGroup(resource_pool, class_with_args, name_prefix="worker_group_torch") - - output = worker_group.execute_all_sync('all_gather') - for i in range(1, len(output)): - assert torch.all(output[i] == output[0]) - - output = output[0].cpu() - print(output) - assert torch.all(output == torch.tensor([0, 0, 1, 1, 2, 2, 3, 3], dtype=torch.int64)) - - ray.shutdown() diff --git a/tests/rollout/run_fsdp_vllm.py b/tests/rollout/run_fsdp_vllm.py deleted file mode 100644 index 93ebff295f..0000000000 --- a/tests/rollout/run_fsdp_vllm.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy, MixedPrecision, CPUOffload -from torch.distributed.fsdp.api import ShardingStrategy, ShardedStateDictConfig, StateDictType -import torch - -from verl.utils.distributed import initialize_global_process_group -from verl.third_party.vllm import LLM - -from vllm import SamplingParams - -import time -import torch.distributed as dist - - -def main(): - assert torch.cuda.is_available(), 'CUDA must be present to run FSDP vLLM example' - local_rank, rank, world_size = initialize_global_process_group() - - local_cache_path = '~/.cache/verl/rlhf' - local_cache_path = os.path.expanduser(local_cache_path) - hdfs_path = 'Qwen/Qwen2-7B-Instruct' - - from verl.utils.fs import copy_to_local - local_model_path = copy_to_local(src=hdfs_path, cache_dir=local_cache_path) - tokenizer = AutoTokenizer.from_pretrained(local_model_path, trust_remote_code=True) - actor_model_config = AutoConfig.from_pretrained(local_model_path, trust_remote_code=True) - with torch.device("cuda"): - actor_model = AutoModelForCausalLM.from_pretrained(local_model_path, trust_remote_code=True) - actor_model.to(torch.bfloat16) - - max_prompt_length = 16 - response_length = 32 - preencode_prompts = [ - "The president of the United States is", - "The capital of France is", - "The future of AI is", - ] - tokenizer.pad_token = tokenizer.eos_token - prompts = tokenizer(preencode_prompts, return_tensors='pt', padding=True) - input_ids = prompts['input_ids'] - attention_mask = prompts['attention_mask'] - from verl.utils.torch_functional import pad_sequence_to_length - input_ids = pad_sequence_to_length(input_ids, max_prompt_length, tokenizer.pad_token_id, left_pad=True).cuda() - attention_mask = pad_sequence_to_length(attention_mask, max_prompt_length, 0, left_pad=True).cuda() - - from transformers import GenerationConfig - generation_config = GenerationConfig(do_sample=False) - actor_model.cuda() - output = actor_model.generate( - input_ids=input_ids, - attention_mask=attention_mask, - max_new_tokens=32, - # max_length=max_length, - eos_token_id=tokenizer.eos_token_id, - pad_token_id=tokenizer.pad_token_id, - generation_config=generation_config, - # renormalize_logits=True, - output_scores=False, # this is potentially very large - return_dict_in_generate=True, - use_cache=False) # may OOM when use_cache = True - seq = output.sequences - response = seq[:, max_prompt_length:] - - print(f'hf response: {tokenizer.batch_decode(response)}') - - tensor_model_parallel_size = 4 - from torch.distributed.device_mesh import init_device_mesh - device_mesh = init_device_mesh('cuda', mesh_shape=(world_size,), mesh_dim_names=['fsdp']) - - mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32) - fsdp_model = FSDP(actor_model, - use_orig_params=True, - auto_wrap_policy=None, - device_id=torch.cuda.current_device(), - sharding_strategy=ShardingStrategy.FULL_SHARD, - mixed_precision=mixed_precision, - cpu_offload=CPUOffload(offload_params=False), - sync_module_states=False, - device_mesh=device_mesh) - - FSDP.set_state_dict_type(fsdp_model, - state_dict_type=StateDictType.SHARDED_STATE_DICT, - state_dict_config=ShardedStateDictConfig()) - - state_dict = fsdp_model.state_dict() - - sampling_params = SamplingParams(temperature=0, - top_p=1, - n=1, - max_tokens=response_length, - logprobs=1, - ignore_eos=True, - detokenize=False) - - print(actor_model_config) - llm = LLM(model=None, - tokenizer=tokenizer, - model_hf_config=actor_model_config, - tensor_parallel_size=tensor_model_parallel_size, - enforce_eager=True, - dtype='bfloat16', - load_format='dummy_dtensor', - gpu_memory_utilization=0.8, - trust_remote_code=True) - - # Warmup iterations - for _ in range(10): - torch.cuda.synchronize() - llm.sync_model_weights(actor_weights=state_dict, load_format='dtensor') - torch.cuda.synchronize() - dist.barrier() - - start_time = time.time() - llm.sync_model_weights(actor_weights=state_dict, load_format='dtensor') - torch.cuda.synchronize() - dist.barrier() - end_time = time.time() - - # Calculate elapsed time - elapsed_time = end_time - start_time - print(f"Time taken: {elapsed_time:.6f} seconds") - - input_ids = input_ids.cuda() - attention_mask = attention_mask.cuda() - idx_list = [] - batch_size = input_ids.shape[0] - - pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id - from verl.workers.rollout.vllm_rollout.vllm_rollout import _pre_process_inputs - for i in range(batch_size): - idx_list.append(_pre_process_inputs(pad_token_id, input_ids[i])) - print('start generation') - outputs = llm.generate(prompt_token_ids=idx_list, sampling_params=sampling_params, use_tqdm=False) - vllm_output = outputs[0].cuda() - if torch.distributed.get_rank() == 0: - print(f'hf response: {tokenizer.batch_decode(response)}') - print(f'vllm response: {tokenizer.batch_decode(vllm_output)}') - - -if __name__ == "__main__": - main() diff --git a/tests/rollout/test_sglang_spmd.py b/tests/rollout/test_sglang_spmd.py deleted file mode 100644 index da49459c88..0000000000 --- a/tests/rollout/test_sglang_spmd.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import torch -from torch.distributed.device_mesh import init_device_mesh - -from sglang.srt.entrypoints.verl_engine import VerlEngine - -from transformers import AutoTokenizer, AutoModelForCausalLM -from transformers import GenerationConfig - -from verl.utils.torch_functional import pad_sequence_to_length - - -def levenshtein(s1, s2): - m, n = len(s1), len(s2) - # Initialize matrix of zeros - dp = [[0] * (n + 1) for _ in range(m + 1)] - # Initialize first column and first row of the matrix - for i in range(m + 1): - dp[i][0] = i # Deletion from s1 to empty string - for j in range(n + 1): - dp[0][j] = j # Insertion to s1 from empty string - # Compute the Levenshtein distance matrix - for i in range(1, m + 1): - for j in range(1, n + 1): - cost = 0 if s1[i - 1] == s2[j - 1] else 1 # No cost if characters match - dp[i][j] = min( - dp[i - 1][j] + 1, # Deletion - dp[i][j - 1] + 1, # Insertion - dp[i - 1][j - 1] + cost # Substitution - ) - return dp[m][n] - - -def are_lists_similar(a, b): - if len(a) != len(b): - print("The lists are of different lengths.") - return False - - total_length = 0 - total_diff = 0 - - for s1, s2 in zip(a, b): - max_len = max(len(s1), len(s2)) - total_length += max_len - diff = levenshtein(s1, s2) - total_diff += diff - print(f"Comparing strings:\n{s1}\n{s2}\nDifference: {diff} characters\n") - - percentage_difference = (total_diff / total_length) * 100 - print(f"Total difference: {percentage_difference:.2f}%") - - return percentage_difference <= 10 - - -def initialize_global_process_group(timeout_second=36000): - from datetime import timedelta - - import torch.distributed - - # NOTE MODIFIED should provide backend=None to have nccl+gloo - # torch.distributed.init_process_group('nccl', timeout=timedelta(seconds=timeout_second)) - torch.distributed.init_process_group(timeout=timedelta(seconds=timeout_second)) - - local_rank = int(os.environ["LOCAL_RANK"]) - rank = int(os.environ["RANK"]) - world_size = int(os.environ["WORLD_SIZE"]) - - if torch.distributed.is_initialized(): - torch.cuda.set_device(local_rank) - return local_rank, rank, world_size - - -def test_sglang_spmd(): - assert torch.cuda.device_count() >= 2, 'At least 2 GPUs is required to run tp+dp tests.' - initialize_global_process_group() - # fill rollout config - max_prompt_length = 16 - max_response_length = 16 - - # Initialize model and token - local_cache_path = '~/.cache/verl/rlhf' - local_cache_path = os.path.expanduser(local_cache_path) - hdfs_path = 'Qwen/Qwen2-7B-Instruct' - from verl.utils.fs import copy_to_local - local_model_path = copy_to_local(src=hdfs_path, cache_dir=local_cache_path) - tokenizer = AutoTokenizer.from_pretrained(local_model_path, padding_side='left') - - preencode_prompts = [ - "Who won the Champions League in 2019?", - "The founder of Apple is", - "What's your name", - ] - tokenizer.pad_token = tokenizer.eos_token - prompts = tokenizer(preencode_prompts, return_tensors='pt', padding=True) - input_ids = prompts['input_ids'] - attention_mask = prompts['attention_mask'] - - input_ids = pad_sequence_to_length(input_ids, max_prompt_length, tokenizer.pad_token_id, left_pad=True) - attention_mask = pad_sequence_to_length(attention_mask, max_prompt_length, 0, left_pad=True) - - actor_model = AutoModelForCausalLM.from_pretrained(local_model_path) - actor_model.to(torch.bfloat16) - - sampling_params = dict(n=1, - temperature=0, - top_p=1, - top_k=-1, - max_new_tokens=max_response_length, - presence_penalty=0.0, - frequency_penalty=0.0, - repetition_penalty=1.0, - skip_special_tokens=True, - spaces_between_special_tokens=True, - ignore_eos=False) - - tensor_parallel_size = 4 - device_mesh_kwargs = dict(mesh_shape=(1, tensor_parallel_size, 1), mesh_dim_names=["dp", "tp", "pp"]) - inference_device_mesh_cpu = init_device_mesh("cpu", **device_mesh_kwargs) - - for k in ["TORCHELASTIC_USE_AGENT_STORE"]: - if k in os.environ: - del os.environ[k] - print('building sglang rollout engine') - llm = VerlEngine(model_path=local_model_path, - dtype="bfloat16", - mem_fraction_static=0.5, - device_mesh_cpu=inference_device_mesh_cpu["tp"], - base_gpu_id=0, - gpu_id_step=1) - - llm.release_memory_occupation() - print("start generation") - input_ids = input_ids.cuda() - attention_mask = attention_mask.cuda() - batch_size = input_ids.size(0) - - generation_config = GenerationConfig(do_sample=False) - actor_model.cuda() - output = actor_model.generate( - input_ids=input_ids, - attention_mask=attention_mask, - max_new_tokens=max_response_length, - # max_length=max_length, - eos_token_id=tokenizer.eos_token_id, - pad_token_id=tokenizer.pad_token_id, - generation_config=generation_config, - # renormalize_logits=True, - output_scores=False, # this is potentially very large - return_dict_in_generate=True, - use_cache=False) # may OOM when use_cache = True - seq = output.sequences - response = seq[:, max_prompt_length:] - - hf_response_tokens = tokenizer.batch_decode(response) - print(f"hf response: {hf_response_tokens}") - print(f"{sampling_params=}") - idx_list = [] - batch_size = input_ids.shape[0] - - pad_token_id = (tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id) - for i in range(batch_size): - idx_list.append(_pre_process_inputs(pad_token_id, input_ids[i])) - - outputs = llm.generate(input_ids=idx_list, sampling_params=sampling_params) - sglang_response_tokens = [] - - for output in outputs: - print(f"{output=}") - generated_text = output["text"] - sglang_response_tokens.append(generated_text) - - print(f"sglang response: {sglang_response_tokens}") - assert are_lists_similar(hf_response_tokens, sglang_response_tokens), \ - f"Strings differ more than 10%:\n" - print("Check Pass") - - -def _pre_process_inputs(pad_token_id, prompt_token_ids: torch.Tensor): - # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = prompt_token_ids[non_pad_index:].tolist() - return token_ids diff --git a/tests/rollout/test_vllm_hf_loader.py b/tests/rollout/test_vllm_hf_loader.py deleted file mode 100644 index 4466627bce..0000000000 --- a/tests/rollout/test_vllm_hf_loader.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import torch -import transformers - -from verl.third_party.vllm import LLM, vllm_version -from verl.utils.model import update_model_config -from vllm import SamplingParams -from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM - -from transformers import GenerationConfig - -from verl.utils.torch_functional import pad_sequence_to_length -from verl.workers.rollout.vllm_rollout.vllm_rollout import _pre_process_inputs - - -def levenshtein(s1, s2): - m, n = len(s1), len(s2) - # Initialize matrix of zeros - dp = [[0] * (n + 1) for _ in range(m + 1)] - # Initialize first column and first row of the matrix - for i in range(m + 1): - dp[i][0] = i # Deletion from s1 to empty string - for j in range(n + 1): - dp[0][j] = j # Insertion to s1 from empty string - # Compute the Levenshtein distance matrix - for i in range(1, m + 1): - for j in range(1, n + 1): - cost = 0 if s1[i - 1] == s2[j - 1] else 1 # No cost if characters match - dp[i][j] = min( - dp[i - 1][j] + 1, # Deletion - dp[i][j - 1] + 1, # Insertion - dp[i - 1][j - 1] + cost # Substitution - ) - return dp[m][n] - - -def are_lists_similar(a, b): - if len(a) != len(b): - print("The lists are of different lengths.") - return False - - total_length = 0 - total_diff = 0 - - for s1, s2 in zip(a, b): - max_len = max(len(s1), len(s2)) - total_length += max_len - diff = levenshtein(s1, s2) - total_diff += diff - print(f"Comparing strings:\n{s1}\n{s2}\nDifference: {diff} characters\n") - - percentage_difference = (total_diff / total_length) * 100 - print(f"Total difference: {percentage_difference:.2f}%") - - return percentage_difference <= 10 - - -def test_vllm_with_hf(): - assert torch.cuda.device_count() >= 2, 'At least 2 GPUs is required to run tp+dp tests.' - - # fill rollout config - max_prompt_length = 16 - max_response_length = 16 - - # Initialize model and token - local_cache_path = '~/.cache/verl/rlhf' - local_cache_path = os.path.expanduser(local_cache_path) - hdfs_path = 'deepseek-ai/deepseek-llm-7b-chat' - from verl.utils.fs import copy_to_local - local_model_path = copy_to_local(src=hdfs_path, cache_dir=local_cache_path) - tokenizer = AutoTokenizer.from_pretrained(local_model_path) - - preencode_prompts = [ - "Who won the Champions League in 2019?", - "The founder of Apple is", - "What's your name", - ] - tokenizer.pad_token = tokenizer.eos_token - prompts = tokenizer(preencode_prompts, return_tensors='pt', padding=True) - input_ids = prompts['input_ids'] - attention_mask = prompts['attention_mask'] - - input_ids = pad_sequence_to_length(input_ids, max_prompt_length, tokenizer.pad_token_id, left_pad=True) - attention_mask = pad_sequence_to_length(attention_mask, max_prompt_length, 0, left_pad=True) - - actor_model = AutoModelForCausalLM.from_pretrained(local_model_path) - actor_model.to(torch.bfloat16) - - actor_model_config = AutoConfig.from_pretrained(local_model_path) - - temperature = 0 - top_p = 1 - - kwargs = dict(n=1, - temperature=temperature, - top_p=top_p, - max_tokens=max_response_length, - logprobs=1, - ignore_eos=True) - - if vllm_version in ('0.4.2', '0.5.4', '0.6.3'): - kwargs['detokenize'] = False - sampling_params = SamplingParams(**kwargs) - - tensor_parallel_size = 2 - - llm = LLM(model=actor_model, - tokenizer=tokenizer, - model_hf_config=actor_model_config, - tensor_parallel_size=tensor_parallel_size, - dtype='bfloat16', - gpu_memory_utilization=0.1, - load_format='hf') - - print('start generation') - input_ids = input_ids.cuda() - attention_mask = attention_mask.cuda() - batch_size = input_ids.size(0) - - idx_list = [] - # parse idx from torch.Tensor to List[List[str]] - for i in range(batch_size): - idx_list.append(_pre_process_inputs(tokenizer.pad_token_id, input_ids[i])) - outputs = llm.generate(prompt_token_ids=idx_list, sampling_params=sampling_params, use_tqdm=False) - vllm_output = outputs[0].cuda() - llm.free_cache_engine() - llm = None - import gc - torch.cuda.empty_cache() - gc.collect() - - generation_config = GenerationConfig(do_sample=False) - actor_model.cuda() - output = actor_model.generate( - input_ids=input_ids, - attention_mask=attention_mask, - max_new_tokens=max_response_length, - # max_length=max_length, - eos_token_id=tokenizer.eos_token_id, - pad_token_id=tokenizer.pad_token_id, - generation_config=generation_config, - # renormalize_logits=True, - output_scores=False, # this is potentially very large - return_dict_in_generate=True, - use_cache=False) # may OOM when use_cache = True - seq = output.sequences - response = seq[:, max_prompt_length:] - - hf_response_tokens = tokenizer.batch_decode(response) - vllm_response_tokens = tokenizer.batch_decode(vllm_output) - - print(f'hf response: {hf_response_tokens}') - print(f'vllm response: {vllm_response_tokens}') - assert are_lists_similar(hf_response_tokens, vllm_response_tokens), \ - f'Strings differ more than 10%:\n' - print('Check Pass') - - -# if __name__ == "__main__": -# test_vllm_with_hf() diff --git a/tests/rollout/test_vllm_spmd.py b/tests/rollout/test_vllm_spmd.py deleted file mode 100644 index c2d2eb8a90..0000000000 --- a/tests/rollout/test_vllm_spmd.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import torch -import transformers -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy, MixedPrecision, CPUOffload -from torch.distributed.fsdp.api import ShardingStrategy, ShardedStateDictConfig, StateDictType - -from vllm import LLM, SamplingParams -from verl.utils.model import update_model_config -from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM - -from transformers import GenerationConfig -from verl.utils.distributed import initialize_global_process_group -from verl.utils.torch_functional import pad_sequence_to_length - - -def levenshtein(s1, s2): - m, n = len(s1), len(s2) - # Initialize matrix of zeros - dp = [[0] * (n + 1) for _ in range(m + 1)] - # Initialize first column and first row of the matrix - for i in range(m + 1): - dp[i][0] = i # Deletion from s1 to empty string - for j in range(n + 1): - dp[0][j] = j # Insertion to s1 from empty string - # Compute the Levenshtein distance matrix - for i in range(1, m + 1): - for j in range(1, n + 1): - cost = 0 if s1[i - 1] == s2[j - 1] else 1 # No cost if characters match - dp[i][j] = min( - dp[i - 1][j] + 1, # Deletion - dp[i][j - 1] + 1, # Insertion - dp[i - 1][j - 1] + cost # Substitution - ) - return dp[m][n] - - -def are_lists_similar(a, b): - if len(a) != len(b): - print("The lists are of different lengths.") - return False - - total_length = 0 - total_diff = 0 - - for s1, s2 in zip(a, b): - max_len = max(len(s1), len(s2)) - total_length += max_len - diff = levenshtein(s1, s2) - total_diff += diff - print(f"Comparing strings:\n{s1}\n{s2}\nDifference: {diff} characters\n") - - percentage_difference = (total_diff / total_length) * 100 - print(f"Total difference: {percentage_difference:.2f}%") - - return percentage_difference <= 15 - - -def test_vllm_spmd(): - assert torch.cuda.device_count() >= 2, 'At least 2 GPUs is required to run tp+dp tests.' - local_rank, rank, world_size = initialize_global_process_group() - - # Initialize model and token - local_cache_path = '~/.cache/verl/rlhf' - local_cache_path = os.path.expanduser(local_cache_path) - hdfs_path = 'Qwen/Qwen2-7B-Instruct' - from verl.utils.fs import copy_to_local - local_model_path = copy_to_local(src=hdfs_path, cache_dir=local_cache_path) - tokenizer = AutoTokenizer.from_pretrained(local_model_path, padding_side='left', trust_remote_code=True) - - actor_model = AutoModelForCausalLM.from_pretrained(local_model_path, trust_remote_code=True) - actor_model.to(torch.bfloat16) - - # fill rollout config - max_prompt_length = 16 - max_response_length = 32 - preencode_prompts = [ - "Who won the Champions League in 2019?", - "The founder of Apple is", - "What's your name", - ] - tokenizer.pad_token = tokenizer.eos_token - prompts = tokenizer(preencode_prompts, return_tensors='pt', padding=True) - input_ids = prompts['input_ids'] - attention_mask = prompts['attention_mask'] - - input_ids = pad_sequence_to_length(input_ids, max_prompt_length, tokenizer.pad_token_id, left_pad=True) - attention_mask = pad_sequence_to_length(attention_mask, max_prompt_length, 0, left_pad=True) - - print('start generation') - input_ids = input_ids.cuda() - attention_mask = attention_mask.cuda() - batch_size = input_ids.size(0) - - generation_config = GenerationConfig(do_sample=False) - actor_model.cuda() - output = actor_model.generate( - input_ids=input_ids, - attention_mask=attention_mask, - max_new_tokens=max_response_length, - eos_token_id=tokenizer.eos_token_id, - pad_token_id=tokenizer.pad_token_id, - generation_config=generation_config, - output_scores=False, # this is potentially very large - return_dict_in_generate=True, - use_cache=False) # may OOM when use_cache = True - seq = output.sequences - response = seq[:, max_prompt_length:] - - hf_response_tokens = tokenizer.batch_decode(response) - - temperature = 0 - top_p = 1 - kwargs = dict(n=1, - temperature=temperature, - top_p=top_p, - max_tokens=max_response_length, - logprobs=1, - ignore_eos=True) - - tensor_parallel_size = 4 - - from torch.distributed.device_mesh import init_device_mesh - device_mesh = init_device_mesh('cuda', mesh_shape=(world_size,), mesh_dim_names=['fsdp']) - - mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.float32, buffer_dtype=torch.float32) - - fsdp_model = FSDP(actor_model, - use_orig_params=True, - auto_wrap_policy=None, - device_id=torch.cuda.current_device(), - sharding_strategy=ShardingStrategy.FULL_SHARD, - mixed_precision=mixed_precision, - cpu_offload=CPUOffload(offload_params=False), - sync_module_states=False, - device_mesh=device_mesh) - - FSDP.set_state_dict_type(fsdp_model, - state_dict_type=StateDictType.SHARDED_STATE_DICT, - state_dict_config=ShardedStateDictConfig()) - - state_dict = fsdp_model.state_dict() - - sampling_params = SamplingParams(**kwargs) - llm = LLM( - model=local_model_path, - enable_sleep_mode=True, - tensor_parallel_size=tensor_parallel_size, - distributed_executor_backend="external_launcher", - dtype='bfloat16', - enforce_eager=True, - gpu_memory_utilization=0.8, - disable_custom_all_reduce=True, - disable_mm_preprocessor_cache=True, - skip_tokenizer_init=False, - enable_prefix_caching=True, - trust_remote_code=True, - ) - - # Warmup iterations - world_size = torch.distributed.get_world_size() - model = llm.llm_engine.model_executor.driver_worker.worker.model_runner.model - model.load_weights( - ((name, param.full_tensor() if world_size != 1 else param) for name, param in state_dict.items())) - - outputs = llm.generate(preencode_prompts, sampling_params=sampling_params, use_tqdm=False) - vllm_response_tokens = [] - for output in outputs: - prompt = output.prompt - generated_text = output.outputs[0].text - vllm_response_tokens.append(generated_text) - - if torch.distributed.get_rank() == 0: - print(f'hf response: {hf_response_tokens}') - print(f'vllm response: {vllm_response_tokens}') - assert are_lists_similar(hf_response_tokens, vllm_response_tokens), \ - f'Strings differ more than 10%:\n' - print('Check Pass') - torch.distributed.destroy_process_group() - - -if __name__ == "__main__": - test_vllm_spmd() diff --git a/tests/sandbox/test_sandbox.py b/tests/sandbox/test_sandbox.py deleted file mode 100644 index d60474e4a0..0000000000 --- a/tests/sandbox/test_sandbox.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -from verl.utils.reward_score import _default_compute_score -from verl.utils.reward_score.prime_code import apps_check_correctness -import asyncio -from verl.workers.reward_manager.prime import parallel_compute_score_async - -prime_math_answers = [ - """\\begin{bmatrix}\n -7 & 6 & -8 \\\\\n 11 & -9 & 12 \\\\\n 15 & -16 & 19 \n \\end{bmatrix}""", - """\\frac{\\sqrt{505}}{7}""", """x^2 + y^2 + 4x - 6y + 13""" -] -prime_math_gts = [ - """\\begin{pmatrix}\n -7 & 6 & -8 \\\\\n 11 & -9 & 12 \\\\\n 15 & -16 & 19\n \\end{pmatrix}""", # mat test - """\\frac{\\sqrt{505}}{7}""", # frac test - """(x + 2)^2 + (y - 3)^2 """ # symbolic test -] - -prime_code_answers = [ - """import sys -from collections import deque - -def main(): - data = sys.stdin.read().split() - it = iter(data) - - # Read start and target positions - x0, y0, x1, y1 = int(next(it)), int(next(it)), int(next(it)), int(next(it)) - - n = int(next(it)) - allowed = set() - # The total number of allowed cells is at most 10^5. - for _ in range(n): - r = int(next(it)) - a = int(next(it)) - b = int(next(it)) - for c in range(a, b + 1): - allowed.add((r, c)) - - # Directions for the king (8 neighboring cells) - directions = [(-1, -1), (-1, 0), (-1, 1), - (0, -1), (0, 1), - (1, -1), (1, 0), (1, 1)] - - start = (x0, y0) - target = (x1, y1) - - # BFS initialization - queue = deque() - queue.append((x0, y0, 0)) - # Mark the starting cell as visited by removing it from allowed set. - allowed.discard(start) - - while queue: - x, y, moves = queue.popleft() - if (x, y) == target: - print(moves) - return - for dx, dy in directions: - nx, ny = x + dx, y + dy - if (nx, ny) in allowed: - allowed.remove((nx, ny)) - queue.append((nx, ny, moves + 1)) - - print(-1) - -if __name__ == '__main__': - main() -""" -] * 2 -prime_code_gts = [ - """{\n \"inputs\": [\n \"5 7 6 11\\n3\\n5 3 8\\n6 7 11\\n5 2 5\\n\",\n \"3 4 3 10\\n3\\n3 1 4\\n4 5 9\\n3 10 10\\n\",\n \"1 1 2 10\\n2\\n1 1 3\\n2 6 10\\n\",\n \"9 8 7 8\\n9\\n10 6 6\\n10 6 6\\n7 7 8\\n9 5 6\\n8 9 9\\n9 5 5\\n9 8 8\\n8 5 6\\n9 10 10\\n\",\n \"6 15 7 15\\n9\\n6 15 15\\n7 14 14\\n6 15 15\\n9 14 14\\n7 14 16\\n6 15 15\\n6 15 15\\n7 14 14\\n8 15 15\\n\",\n \"13 16 20 10\\n18\\n13 16 16\\n20 10 10\\n19 10 10\\n12 15 15\\n20 10 10\\n18 11 11\\n19 10 10\\n19 10 10\\n20 10 10\\n19 10 10\\n20 10 10\\n20 10 10\\n19 10 10\\n18 11 11\\n13 16 16\\n12 15 15\\n19 10 10\\n19 10 10\\n\",\n \"89 29 88 30\\n16\\n87 31 31\\n14 95 95\\n98 88 89\\n96 88 88\\n14 97 97\\n13 97 98\\n100 88 88\\n88 32 32\\n99 88 89\\n90 29 29\\n87 31 31\\n15 94 96\\n89 29 29\\n88 32 32\\n97 89 89\\n88 29 30\\n\",\n \"30 14 39 19\\n31\\n35 7 11\\n37 11 12\\n32 13 13\\n37 5 6\\n46 13 13\\n37 14 14\\n31 13 13\\n43 13 19\\n45 15 19\\n46 13 13\\n32 17 17\\n41 14 19\\n30 14 14\\n43 13 17\\n34 16 18\\n44 11 19\\n38 13 13\\n40 12 20\\n37 16 18\\n46 16 18\\n34 10 14\\n36 9 10\\n36 15 19\\n38 15 19\\n42 13 19\\n33 14 15\\n35 15 19\\n33 17 18\\n39 12 20\\n36 5 7\\n45 12 12\\n\",\n \"2 1 1 1\\n2\\n1 1 2\\n2 1 2\\n\",\n \"1 1 1 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\",\n \"1 1 1000000000 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\"\n ],\n \"outputs\": [\n \"4\\n\",\n \"6\\n\",\n \"-1\\n\",\n \"2\\n\",\n \"1\\n\",\n \"-1\\n\",\n \"1\\n\",\n \"9\\n\",\n \"1\\n\",\n \"1\\n\",\n \"-1\\n\"\n ]\n}""", # A correct sample - """{\n \"inputs\": [\n \"5 7 6 11\\n3\\n5 3 8\\n6 7 11\\n5 2 5\\n\",\n \"3 4 3 10\\n3\\n3 1 4\\n4 5 9\\n3 10 10\\n\",\n \"1 1 2 10\\n2\\n1 1 3\\n2 6 10\\n\",\n \"9 8 7 8\\n9\\n10 6 6\\n10 6 6\\n7 7 8\\n9 5 6\\n8 9 9\\n9 5 5\\n9 8 8\\n8 5 6\\n9 10 10\\n\",\n \"6 15 7 15\\n9\\n6 15 15\\n7 14 14\\n6 15 15\\n9 14 14\\n7 14 16\\n6 15 15\\n6 15 15\\n7 14 14\\n8 15 15\\n\",\n \"13 16 20 10\\n18\\n13 16 16\\n20 10 10\\n19 10 10\\n12 15 15\\n20 10 10\\n18 11 11\\n19 10 10\\n19 10 10\\n20 10 10\\n19 10 10\\n20 10 10\\n20 10 10\\n19 10 10\\n18 11 11\\n13 16 16\\n12 15 15\\n19 10 10\\n19 10 10\\n\",\n \"89 29 88 30\\n16\\n87 31 31\\n14 95 95\\n98 88 89\\n96 88 88\\n14 97 97\\n13 97 98\\n100 88 88\\n88 32 32\\n99 88 89\\n90 29 29\\n87 31 31\\n15 94 96\\n89 29 29\\n88 32 32\\n97 89 89\\n88 29 30\\n\",\n \"30 14 39 19\\n31\\n35 7 11\\n37 11 12\\n32 13 13\\n37 5 6\\n46 13 13\\n37 14 14\\n31 13 13\\n43 13 19\\n45 15 19\\n46 13 13\\n32 17 17\\n41 14 19\\n30 14 14\\n43 13 17\\n34 16 18\\n44 11 19\\n38 13 13\\n40 12 20\\n37 16 18\\n46 16 18\\n34 10 14\\n36 9 10\\n36 15 19\\n38 15 19\\n42 13 19\\n33 14 15\\n35 15 19\\n33 17 18\\n39 12 20\\n36 5 7\\n45 12 12\\n\",\n \"2 1 1 1\\n2\\n1 1 2\\n2 1 2\\n\",\n \"1 1 1 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\",\n \"1 1 1000000000 2\\n5\\n1000000000 1 10000\\n19920401 1188 5566\\n1000000000 1 10000\\n1 1 10000\\n5 100 200\\n\"\n ],\n \"outputs\": [\n \"4\\n\",\n \"6\\n\",\n \"-1\\n\",\n \"-1\\n\",\n \"1\\n\",\n \"-1\\n\",\n \"1\\n\",\n \"9\\n\",\n \"1\\n\",\n \"1\\n\",\n \"-1\\n\"\n ]\n}""" -] # A failed sample with first several in-out passed - -prime_code_scores = [1.0, 0.9] - - -def test_parallelism(): - """ - Test if process pool works properly - """ - sequences_str = [] - ground_truth = [] - data_sources = [] - while len(sequences_str) < 32: - sequences_str.extend(prime_code_answers) - ground_truth.extend(prime_code_gts) - data_sources.extend(['codecontests'] * len(prime_code_answers)) - - sequences_str.extend(prime_math_answers) - ground_truth.extend(prime_math_gts) - data_sources.extend(['numina_aops_forum'] * len(prime_math_answers)) - - scores = asyncio.run( - parallel_compute_score_async(_default_compute_score, - sequences_str, - ground_truth, - data_sources, - num_processes=16)) - print(scores) - - -def test_prime_code(): - """ - Test PRIME code sandbox. - """ - data_source = 'codecontests' - for completion, ground_truth, score_ in zip(prime_code_answers, prime_code_gts, prime_code_scores): - score = _default_compute_score(data_source, completion, ground_truth) - assert float(score) == score_ - - -def test_check_correctness(): - completion = prime_code_answers[0] - ground_truth = json.loads(prime_code_gts[0]) - ground_truth_single = {'inputs': ground_truth['inputs'][:1], 'outputs': ground_truth['outputs'][:1]} - res, meta = apps_check_correctness(in_outs=ground_truth_single, generation=completion, timeout=5, debug=False) - print(res, meta) - - -def test_prime_math(): - data_source = 'numina_aops_forum' - for completion, ground_truth in zip(prime_math_answers, prime_math_gts): - score = _default_compute_score(data_source, completion, ground_truth) - assert float(score) == 1.0 diff --git a/tests/sanity/check_license.py b/tests/sanity/check_license.py deleted file mode 100644 index 076ca03f74..0000000000 --- a/tests/sanity/check_license.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -license_head_bytedance = "Copyright 2024 Bytedance Ltd. and/or its affiliates" -license_head_bytedance_25 = "Copyright 2025 Bytedance Ltd. and/or its affiliates" -# Add custom license headers below -license_head_prime = "Copyright 2024 PRIME team and/or its affiliates" - -license_headers = [license_head_bytedance, license_head_bytedance_25, license_head_prime] - -from pathlib import Path -from argparse import ArgumentParser - -if __name__ == '__main__': - parser = ArgumentParser() - parser.add_argument('--directory', '-d', required=True, type=str) - args = parser.parse_args() - directory_in_str = args.directory - - pathlist = Path(directory_in_str).glob('**/*.py') - for path in pathlist: - # because path is object not string - path_in_str = str(path.absolute()) - print(path_in_str) - with open(path_in_str, 'r', encoding='utf-8') as f: - file_content = f.read() - - has_license = False - for lh in license_headers: - if lh in file_content: - has_license = True - break - assert has_license, f'file {path_in_str} does not contain license' diff --git a/tests/sanity/test_import.py b/tests/sanity/test_import.py deleted file mode 100644 index 2adf63a15e..0000000000 --- a/tests/sanity/test_import.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def test_import(): - import verl - print(verl.__version__) - - -def test_single_controller_import(): - import verl.single_controller - print(verl.single_controller.__version__) diff --git a/tests/sft/run_sft.sh b/tests/sft/run_sft.sh deleted file mode 100644 index 891328324b..0000000000 --- a/tests/sft/run_sft.sh +++ /dev/null @@ -1,22 +0,0 @@ -# Tested with 2 & 4 GPUs - -set -x - -torchrun --standalone --nnodes=1 --nproc_per_node=8 \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size_per_gpu=32 \ - model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ - trainer.default_local_dir=$HOME/ckpts/ \ - trainer.project_name=qwen2.5-sft \ - trainer.experiment_name=gsm8k-sft-gemma-2b-it \ - trainer.total_training_steps=1 \ - trainer.logger=['console'] \ - trainer.default_hdfs_dir=null $@ - -rm -rf $HOME/ckpts/ \ No newline at end of file diff --git a/tests/sft/run_sft_qwen05_peft.sh b/tests/sft/run_sft_qwen05_peft.sh deleted file mode 100644 index bb7c692630..0000000000 --- a/tests/sft/run_sft_qwen05_peft.sh +++ /dev/null @@ -1,38 +0,0 @@ -# Tested with 2 & 4 GPUs - -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_sft_qwen05_peft.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - optim.lr=1e-4 \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size_per_gpu=4 \ - model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct \ - trainer.logger=['console'] \ - trainer.total_training_steps=1 \ - trainer.default_hdfs_dir=null $@ \ - model.lora_rank=32\ - model.lora_alpha=16 \ - model.target_modules=all-linear - - # Or you can do this: - # model.target_modules=[q_proj,v_proj] \ diff --git a/tests/sft/run_sft_qwen05_sp2_liger.sh b/tests/sft/run_sft_qwen05_sp2_liger.sh deleted file mode 100644 index 933b86f80e..0000000000 --- a/tests/sft/run_sft_qwen05_sp2_liger.sh +++ /dev/null @@ -1,33 +0,0 @@ -set -x - -if [ "$#" -lt 2 ]; then - echo "Usage: run_sft_qwen05_sp2_liger.sh [other_configs...]" - exit 1 -fi - -nproc_per_node=$1 -save_path=$2 - -# Shift the arguments so $@ refers to the rest -shift 2 - -torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \ - -m verl.trainer.fsdp_sft_trainer \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - optim.lr=1e-4 \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size=4 \ - model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ - model.use_liger=True \ - trainer.default_local_dir=$save_path \ - trainer.project_name=gsm8k-sft \ - trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct-sp2-liger \ - trainer.logger=['console'] \ - trainer.total_training_steps=1 \ - trainer.default_hdfs_dir=null $@ \ - ulysses_sequence_parallel_size=2 \ - use_remove_padding=true \ No newline at end of file diff --git a/tests/sft/run_sft_sp_loss_match.sh b/tests/sft/run_sft_sp_loss_match.sh deleted file mode 100644 index a63328ecbc..0000000000 --- a/tests/sft/run_sft_sp_loss_match.sh +++ /dev/null @@ -1,24 +0,0 @@ -# Tested with 2 & 4 GPUs - -set -x - -torchrun --standalone --nnodes=1 --nproc_per_node=8 \ - tests/sft/test_sp_loss_match.py \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - data.prompt_key=extra_info \ - data.response_key=extra_info \ - +data.prompt_dict_keys=['question'] \ - +data.response_dict_keys=['answer'] \ - data.micro_batch_size=32 \ - model.partial_pretrain=Qwen/Qwen2.5-0.5B-Instruct \ - ulysses_sequence_parallel_size=2 \ - use_remove_padding=True \ - trainer.default_local_dir=$HOME/ckpts/ \ - trainer.project_name=qwen2.5-sft \ - trainer.experiment_name=gsm8k-sft-gemma-2b-it \ - trainer.total_training_steps=1 \ - trainer.logger=['console'] \ - trainer.default_hdfs_dir=null $@ - -rm -rf $HOME/ckpts/ diff --git a/tests/sft/test_sp_loss_match.py b/tests/sft/test_sp_loss_match.py deleted file mode 100644 index 69223d3d38..0000000000 --- a/tests/sft/test_sp_loss_match.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.distributed -from tensordict import TensorDict -from verl.trainer.fsdp_sft_trainer import FSDPSFTTrainer -from torch.distributed.device_mesh import init_device_mesh -from verl.utils.distributed import initialize_global_process_group - - -def test_trainer_forward_consistency(trainer: FSDPSFTTrainer, total_steps: int = 4): - """Test consistency between original forward pass and SP+rmpad forward passes. - - Args: - trainer: The FSDPSFTTrainer instance to test - total_steps: Number of steps to test (default: 4) - """ - if trainer.device_mesh.get_rank() == 0: - print("\nStarting debug comparison between original and SP+rmpad forward passes...") - print(f"Sequence parallel size: {trainer.config.ulysses_sequence_parallel_size}") - print(f"Remove padding: {trainer.use_remove_padding}\n") - - steps_remaining = total_steps - - for epoch in range(1): # Just one epoch for testing - trainer.train_sampler.set_epoch(epoch=epoch) - for data in trainer.train_dataloader: - data = TensorDict(data, batch_size=trainer.config.data.train_batch_size).cuda() - trainer.fsdp_model.train() - micro_batches = data.split(trainer.config.data.micro_batch_size) - - for idx, micro_batch in enumerate(micro_batches): - if trainer.device_mesh.get_rank() == 0: - print(f"\nProcessing micro batch {idx + 1}/{len(micro_batches)}") - - # Compute losses using both methods - # Disable SP and rmpad - trainer.use_remove_padding = False - old_sp = trainer.config.ulysses_sequence_parallel_size - trainer.config.ulysses_sequence_parallel_size = 1 - loss_ref = trainer._compute_loss_and_backward(micro_batch.copy(), do_backward=False) - - # Do SP and rmpad - trainer.config.ulysses_sequence_parallel_size = old_sp - trainer.use_remove_padding = True - loss_sp = trainer._compute_loss_and_backward(micro_batch.copy(), do_backward=False) - - # Collect losses across all ranks - loss_ref_all = loss_ref.clone() - loss_sp_all = loss_sp.clone() - torch.distributed.all_reduce(loss_ref_all, op=torch.distributed.ReduceOp.AVG) - torch.distributed.all_reduce(loss_sp_all, op=torch.distributed.ReduceOp.AVG) - - # Calculate relative difference of averaged losses - rel_diff = torch.abs(loss_ref_all - loss_sp_all) / (torch.abs(loss_ref_all) + 1e-8) - - if trainer.device_mesh.get_rank() == 0: - print("\nComparison Results (Averaged across ranks):") - print(f"Reference Loss: {loss_ref_all.item():.6f}") - print(f"SP+rmpad Loss: {loss_sp_all.item():.6f}") - print(f"Relative Difference: {rel_diff.item():.6f}") - - assert rel_diff.item() < 1e-2, "Significant difference detected between averaged losses!" - print("Loss difference is within the acceptable range.") - - steps_remaining -= 1 - if steps_remaining == 0: - break - if steps_remaining == 0: - break - break - - if trainer.device_mesh.get_rank() == 0: - print("\nDebug comparison completed successfully.") - - -def create_trainer(config): - """Create and initialize a trainer instance with the given config. - - Args: - config: Configuration object with training parameters - - Returns: - FSDPSFTTrainer: Initialized trainer instance - """ - local_rank, rank, world_size = initialize_global_process_group() - - device_mesh = init_device_mesh(device_type='cuda', mesh_shape=(world_size,), mesh_dim_names=('fsdp',)) - - dp_size = world_size // config.ulysses_sequence_parallel_size - ulysses_device_mesh = init_device_mesh(device_type='cuda', - mesh_shape=(dp_size, config.ulysses_sequence_parallel_size), - mesh_dim_names=('dp', 'sp')) - - return FSDPSFTTrainer(config=config, device_mesh=device_mesh, ulysses_device_mesh=ulysses_device_mesh) - - -def main(config): - """Main function to run trainer tests. - - Args: - config: Configuration object with training parameters - """ - trainer = create_trainer(config) - test_trainer_forward_consistency(trainer) - - -if __name__ == '__main__': - import hydra - from omegaconf import DictConfig - - @hydra.main(config_path="../../verl/trainer/config", config_name="sft_trainer") - def hydra_entry(cfg: DictConfig) -> None: - main(cfg) - - hydra_entry() diff --git a/tests/test_reward.py b/tests/test_reward.py deleted file mode 100644 index fcf1f2c3fe..0000000000 --- a/tests/test_reward.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import ray - -os.environ["ALLHANDS_API_KEY"] = "sycao-sandbox-remote" -os.environ["SANDBOX_REMOTE_RUNTIME_API_URL"] = "http://146.235.215.207:8000" - -import torch -from tensordict import TensorDict - -from verl import DataProto -from verl.workers.reward_manager.swebench import SWEBenchRewardManager -import numpy as np - -incorrect_instance_id = "getmoto__moto-7456" -incorrect_patch = "diff --git a/models.py b/models.py\nnew file mode 100644\nindex 000000000..3319def44\n--- /dev/null\n+++ b/models.py\n@@ -0,0 +1,23 @@\n+from moto.core.models import BaseBackend\n+import boto3\n+\n+class ResilienceHubBackend(BaseBackend):\n+ def __init__(self, region_name):\n+ self.region_name = region_name\n+ self.policies = {}\n+\n+ def create_resiliency_policy(self, policy_name, policy_description, policy, tier):\n+ # Generate a unique ARN for the policy\n+ policy_arn = f\"arn:aws:resiliencehub:{self.region_name}:123456789012:resiliencypolicy/{policy_name}\"\n+ self.policies[policy_arn] = {\n+ \"policyName\": policy_name,\n+ \"policyDescription\": policy_description,\n+ \"policy\": policy,\n+ \"tier\": tier,\n+ \"policyArn\": policy_arn\n+ }\n+ return self.policies[policy_arn]\n+\n+\n+# Initialize the backend for us-east-1\n+resiliencehub_backends = {\"us-east-1\": ResilienceHubBackend(\"us-east-1\")}\n\\ No newline at end of file\ndiff --git a/moto/resiliencehub b/moto/resiliencehub\nnew file mode 100644\nindex 000000000..e69de29bb\ndiff --git a/reproduce_error.py b/reproduce_error.py\nnew file mode 100644\nindex 000000000..1ae93f615\n--- /dev/null\n+++ b/reproduce_error.py\n@@ -0,0 +1,33 @@\n+import boto3\n+from moto import mock_aws\n+import os\n+import pytest\n+\n+# Mocked AWS Credentials for moto\n+os.environ[\"AWS_ACCESS_KEY_ID\"] = \"testing\"\n+os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"testing\"\n+os.environ[\"AWS_SECURITY_TOKEN\"] = \"testing\"\n+os.environ[\"AWS_SESSION_TOKEN\"] = \"testing\"\n+os.environ[\"AWS_DEFAULT_REGION\"] = \"us-east-1\"\n+\n+def test_resiliencehub():\n+ with mock_aws():\n+ resiliencehub_client = boto3.client(\"resiliencehub\", region_name=\"us-east-2\")\n+ # Create a policy\n+ response = resiliencehub_client.create_resiliency_policy(\n+ policyName=\"mock-resilience-hub-basic-webapp\",\n+ policyDescription=\"Mocked resiliency policy\",\n+ policy={\n+ \"AZ\": {\"rpoInSecs\": 600, \"rtoInSecs\": 600},\n+ \"Hardware\": {\"rpoInSecs\": 600, \"rtoInSecs\": 600},\n+ \"Region\": {\"rpoInSecs\": 3600, \"rtoInSecs\": 14400},\n+ \"Software\": {\"rpoInSecs\": 600, \"rtoInSecs\": 600}\n+ },\n+ tier=\"Standard\",\n+ )\n+ print(\"create_policy response\")\n+ print(response)\n+ return response.get(\"policy\", {}).get(\"policyArn\")\n+\n+if __name__ == \"__main__\":\n+ test_resiliencehub()\n\\ No newline at end of file\ndiff --git a/moto/resiliencehub b/moto/resiliencehub\nnew file mode 100644\nindex 000000000..55fa1d462\n--- /dev/null\n+++ b/moto/resiliencehub\n@@ -0,0 +1,5 @@\n+__init__.py\n+exceptions.py\n+models.py\n+responses.py\n+urls.py\n\\ No newline at end of file\ndiff --git a/reproduce_error.py b/reproduce_error.py\nnew file mode 100644\nindex 000000000..87905c8e6\n--- /dev/null\n+++ b/reproduce_error.py\n@@ -0,0 +1,17 @@\n+import boto3\n+from moto import mock_aws\n+\n+@mock_aws\n+def test_resiliencehub():\n+ client = boto3.client(\"resiliencehub\", region_name=\"us-east-2\")\n+ response = client.create_resiliency_policy(\n+ policyName=\"mock-resilience-hub-basic-webapp\",\n+ policyDescription=\"Mocked resiliency policy\",\n+ policy={\"AZ\": {\"rpoInSecs\": 600, \"rtoInSecs\": 600}, \"Hardware\": {\"rpoInSecs\": 600, \"rtoInSecs\": 600}, \"Region\": {\"rpoInSecs\": 3600, \"rtoInSecs\": 14400}, \"Software\": {\"rpoInSecs\": 600, \"rtoInSecs\": 600}},\n+ tier=\"Standard\",\n+ )\n+ print(\"create_policy response\")\n+ print(response)\n+\n+if __name__ == \"__main__\":\n+ test_resiliencehub()\n\\ No newline at end of file" - -correct_instance_id = "getmoto__moto-4986" -correct_patch = "diff --git a/moto/eks/models.py b/moto/eks/models.py\nindex 89fff7930..924916695 100644\n--- a/moto/eks/models.py\n+++ b/moto/eks/models.py\n@@ -140,7 +140,7 @@ class Cluster:\n self.version = version or DEFAULT_KUBERNETES_VERSION\n\n self.client_request_token = client_request_token\n- self.encryption_config = encryption_config\n+ self.encryption_config = [encryption_config] if encryption_config else []\n self.name = name\n self.resources_vpc_config = resources_vpc_config\n self.role_arn = role_arn\n@@ -162,7 +162,7 @@ class Cluster:\n yield \"clientRequestToken\", self.client_request_token\n yield \"platformVersion\", self.platformVersion\n yield \"tags\", self.tags\n- yield \"encryptionConfig\", self.encryption_config\n+ yield \"encryptionConfig\", self.encryption_config if isinstance(self.encryption_config, list) else []\n\n def isActive(self):\n return self.status == \"ACTIVE\"\ndiff --git a/reproduce_issue.py b/reproduce_issue.py\nnew file mode 100644\nindex 000000000..d6fa9a4dd\n--- /dev/null\n+++ b/reproduce_issue.py\n@@ -0,0 +1,37 @@\n+import boto3\n+import json\n+from moto import mock_iam, mock_eks\n+import os\n+\n+# Mock AWS credentials\n+os.environ['AWS_ACCESS_KEY_ID'] = 'testing'\n+os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'\n+os.environ['AWS_SECURITY_TOKEN'] = 'testing'\n+os.environ['AWS_SESSION_TOKEN'] = 'testing'\n+\n+@mock_iam\n+@mock_eks\n+def test_create_eks_cluster():\n+ # Create IAM role\n+ iam = boto3.client('iam', region_name='us-east-1')\n+ response = iam.create_role(\n+ RoleName='test-cluster',\n+ AssumeRolePolicyDocument='{\"Version\": \"2012-10-17\", \"Statement\": [{\"Action\": \"sts:AssumeRole\", \"Effect\": \"Allow\", \"Principal\": {\"Service\": \"eks.amazonaws.com\"}}]}'\n+ )\n+ role_arn = response['Role']['Arn']\n+ print(f'Created IAM Role ARN: {role_arn}')\n+\n+ # Create EKS cluster and get encryptionConfig\n+ eks = boto3.client('eks', region_name='us-east-1')\n+ response = eks.create_cluster(\n+ name='test',\n+ roleArn=role_arn,\n+ resourcesVpcConfig={\n+ 'subnetIds': ['subnet-12345678'] # Assuming this subnet exists in the environment\n+ }\n+ )\n+ encryption_config = response['cluster']['encryptionConfig']\n+ print('Cluster encryptionConfig:', json.dumps(encryption_config))\n+\n+if __name__ == '__main__':\n+ test_create_eks_cluster()\n\\ No newline at end of file" - -from verl.utils import hf_tokenizer -tokenizer = hf_tokenizer("Qwen/Qwen2.5-Coder-7B-Instruct") - -n = 128 -data = DataProto( - batch=TensorDict({"responses": torch.randn(2*n)}, batch_size=2*n), - non_tensor_batch={ - "data_source": np.array(["swe-gym", "swe-gym"]*n, dtype=object), - "ability": np.array(["coding", "coding"]*n, dtype=object), - "instance": np.array([{"instance_id": incorrect_instance_id}, {"instance_id": correct_instance_id}]*n, dtype=object), - "git_patch": np.asarray([incorrect_patch, correct_patch]*n, dtype=object) - } -) - -@ray.remote(num_cpus=1) -def task(data): - manager = SWEBenchRewardManager(None, None, None) - score, metric = manager.verify_ray(data) - # print(score) - assert sum(score) == 1.0*n, "Not passed." - print(f"Pass reward manager test.") - -ray.get(task.remote(data)) \ No newline at end of file diff --git a/tests/utility/test_tensor_dict_utilities.py b/tests/utility/test_tensor_dict_utilities.py deleted file mode 100644 index dfd033c003..0000000000 --- a/tests/utility/test_tensor_dict_utilities.py +++ /dev/null @@ -1,301 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import random -import pytest -import torch -from tensordict import TensorDict - -from verl.protocol import union_tensor_dict, union_numpy_dict - -from verl import DataProto -import numpy as np - - -def test_union_tensor_dict(): - obs = torch.randn(100, 10) - - data1 = TensorDict({'obs': obs, 'act': torch.randn(100, 3)}, batch_size=[100]) - data2 = TensorDict({'obs': obs, 'next_obs': torch.randn(100, 10), 'rew': torch.randn(100)}, batch_size=[100]) - - data_with_copied_obs = TensorDict({ - 'obs': obs.clone(), - 'next_obs': torch.randn(100, 10), - 'rew': torch.randn(100) - }, - batch_size=[100]) - - data = union_tensor_dict(data1, data2) - with pytest.raises(AssertionError): - data = union_tensor_dict(data1, data_with_copied_obs) - - data = np.random.random(100) - data2 = [float('nan') for _ in range(99)] - data2.append('nan') - data2 = np.array(data2, dtype=object) - data3 = np.tile(data2, (2, 1)) - a = {'a': data, 'b': data2, 'c': data3} - b = {'a': data, 'b': data2, 'c': data3} - b_ = {'a': np.random.random(100)} - union_numpy_dict(a, b) - with pytest.raises(AssertionError): - union_numpy_dict(a, b_) - - -def test_tensor_dict_constructor(): - obs = torch.randn(100, 10) - act = torch.randn(100, 10, 3) - data = DataProto.from_dict(tensors={'obs': obs, 'act': act}) - - assert data.batch.batch_size == torch.Size([100]) - - with pytest.raises(AssertionError): - data = DataProto.from_dict(tensors={'obs': obs, 'act': act}, num_batch_dims=2) - - with pytest.raises(AssertionError): - data = DataProto.from_dict(tensors={'obs': obs, 'act': act}, num_batch_dims=3) - - -def test_tensor_dict_make_iterator(): - obs = torch.randn(100, 10) - labels = [random.choice(['abc', 'cde']) for _ in range(100)] - dataset = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}) - - data_iter_1 = dataset.make_iterator(mini_batch_size=10, epochs=2, seed=1) - data_list_1 = [] - for data in data_iter_1: - data_list_1.append(data) - - data_iter_2 = dataset.make_iterator(mini_batch_size=10, epochs=2, seed=1) - data_list_2 = [] - for data in data_iter_2: - data_list_2.append(data) - - for data1, data2 in zip(data_list_1, data_list_2): - assert isinstance(data1, DataProto) - assert isinstance(data2, DataProto) - result = torch.all(torch.eq(data1.batch['obs'], data2.batch['obs'])) - if not result.item(): - print(data1.batch['obs']) - print(data2.batch['obs']) - assert False - non_tensor_result = np.all(np.equal(data1.non_tensor_batch['labels'], data2.non_tensor_batch['labels'])) - if not non_tensor_result.item(): - print(data1.non_tensor_batch['labels']) - print(data2.non_tensor_batch['labels']) - - -def test_reorder(): - obs = torch.tensor([1, 2, 3, 4, 5, 6]) - labels = ['a', 'b', 'c', 'd', 'e', 'f'] - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'name': 'abdce'}) - data.reorder(torch.tensor([3, 4, 2, 0, 1, 5])) - - assert torch.all(torch.eq(data.batch['obs'], torch.tensor([4, 5, 3, 1, 2, 6]))) - assert np.all(data.non_tensor_batch['labels'] == np.array(['d', 'e', 'c', 'a', 'b', 'f'])) - assert data.meta_info == {'name': 'abdce'} - - -def test_chunk_concat(): - obs = torch.tensor([1, 2, 3, 4, 5, 6]) - labels = ['a', 'b', 'c', 'd', 'e', 'f'] - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'name': 'abdce'}) - - with pytest.raises(AssertionError): - data.chunk(5) - - data_split = data.chunk(2) - assert len(data_split) == 2 - assert torch.all(torch.eq(data_split[0].batch['obs'], torch.tensor([1, 2, 3]))) - assert np.all(data_split[0].non_tensor_batch['labels'] == np.array(['a', 'b', 'c'])) - assert data_split[0].meta_info == {'name': 'abdce'} - - assert torch.all(torch.eq(data_split[1].batch['obs'], torch.tensor([4, 5, 6]))) - assert np.all(data_split[1].non_tensor_batch['labels'] == np.array(['d', 'e', 'f'])) - assert data_split[1].meta_info == {'name': 'abdce'} - - concat_data = DataProto.concat(data_split) - assert torch.all(torch.eq(concat_data.batch['obs'], data.batch['obs'])) - assert np.all(concat_data.non_tensor_batch['labels'] == data.non_tensor_batch['labels']) - assert concat_data.meta_info == data.meta_info - - -def test_pop(): - obs = torch.randn(100, 10) - act = torch.randn(100, 3) - dataset = DataProto.from_dict({'obs': obs, 'act': act}, meta_info={'2': 2, '1': 1}) - poped_dataset = dataset.pop(batch_keys=['obs'], meta_info_keys=['2']) - - assert poped_dataset.batch.keys() == {'obs'} - assert poped_dataset.meta_info.keys() == {'2'} - - assert dataset.batch.keys() == {'act'} - assert dataset.meta_info.keys() == {'1'} - - -def test_repeat(): - # Create a DataProto object with some batch and non-tensor data - obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) - labels = ['a', 'b', 'c'] - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'info': 'test_info'}) - - # Test interleave=True - repeated_data_interleave = data.repeat(repeat_times=2, interleave=True) - expected_obs_interleave = torch.tensor([[1, 2], [1, 2], [3, 4], [3, 4], [5, 6], [5, 6]]) - expected_labels_interleave = ['a', 'a', 'b', 'b', 'c', 'c'] - - assert torch.all(torch.eq(repeated_data_interleave.batch['obs'], expected_obs_interleave)) - assert (repeated_data_interleave.non_tensor_batch['labels'] == expected_labels_interleave).all() - assert repeated_data_interleave.meta_info == {'info': 'test_info'} - - # Test interleave=False - repeated_data_no_interleave = data.repeat(repeat_times=2, interleave=False) - expected_obs_no_interleave = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2], [3, 4], [5, 6]]) - expected_labels_no_interleave = ['a', 'b', 'c', 'a', 'b', 'c'] - - assert torch.all(torch.eq(repeated_data_no_interleave.batch['obs'], expected_obs_no_interleave)) - assert (repeated_data_no_interleave.non_tensor_batch['labels'] == expected_labels_no_interleave).all() - assert repeated_data_no_interleave.meta_info == {'info': 'test_info'} - - -def test_dataproto_pad_unpad(): - obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) - labels = ['a', 'b', 'c'] - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'info': 'test_info'}) - - from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto - - padded_data, pad_size = pad_dataproto_to_divisor(data, size_divisor=2) - assert pad_size == 1 - - expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2]]) - expected_labels = ['a', 'b', 'c', 'a'] - - assert torch.all(torch.eq(padded_data.batch['obs'], expected_obs)) - assert (padded_data.non_tensor_batch['labels'] == expected_labels).all() - assert padded_data.meta_info == {'info': 'test_info'} - - unpadd_data = unpad_dataproto(padded_data, pad_size=pad_size) - assert torch.all(torch.eq(unpadd_data.batch['obs'], obs)) - assert (unpadd_data.non_tensor_batch['labels'] == labels).all() - assert unpadd_data.meta_info == {'info': 'test_info'} - - padded_data, pad_size = pad_dataproto_to_divisor(data, size_divisor=3) - assert pad_size == 0 - - expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) - expected_labels = ['a', 'b', 'c'] - - assert torch.all(torch.eq(padded_data.batch['obs'], expected_obs)) - assert (padded_data.non_tensor_batch['labels'] == expected_labels).all() - assert padded_data.meta_info == {'info': 'test_info'} - - unpadd_data = unpad_dataproto(padded_data, pad_size=pad_size) - assert torch.all(torch.eq(unpadd_data.batch['obs'], obs)) - assert (unpadd_data.non_tensor_batch['labels'] == labels).all() - assert unpadd_data.meta_info == {'info': 'test_info'} - - padded_data, pad_size = pad_dataproto_to_divisor(data, size_divisor=7) - assert pad_size == 4 - - expected_obs = torch.tensor([[1, 2], [3, 4], [5, 6], [1, 2], [3, 4], [5, 6], [1, 2]]) - expected_labels = ['a', 'b', 'c', 'a', 'b', 'c', 'a'] - assert torch.all(torch.eq(padded_data.batch['obs'], expected_obs)) - assert (padded_data.non_tensor_batch['labels'] == expected_labels).all() - assert padded_data.meta_info == {'info': 'test_info'} - - unpadd_data = unpad_dataproto(padded_data, pad_size=pad_size) - assert torch.all(torch.eq(unpadd_data.batch['obs'], obs)) - assert (unpadd_data.non_tensor_batch['labels'] == labels).all() - assert unpadd_data.meta_info == {'info': 'test_info'} - - -def test_dataproto_fold_unfold(): - from verl.protocol import fold_batch_dim, unfold_batch_dim, DataProto - - obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) - labels = ['a', 'b', 'c'] - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'info': 'test_info'}) - - data1 = data.repeat(repeat_times=2, interleave=True) - - data2 = fold_batch_dim(data1, new_batch_size=3) - - torch.testing.assert_close(data2.batch['obs'], torch.tensor([[[1, 2], [1, 2]], [[3, 4], [3, 4]], [[5, 6], [5, 6]]])) - assert (data2.non_tensor_batch['labels'] == [['a', 'a'], ['b', 'b'], ['c', 'c']]).all() - - data2.reorder(indices=torch.tensor([1, 2, 0])) - - data3 = unfold_batch_dim(data2, batch_dims=2) - - torch.testing.assert_close(data3.batch['obs'], torch.tensor([[3, 4], [3, 4], [5, 6], [5, 6], [1, 2], [1, 2]])) - assert (data3.non_tensor_batch['labels'] == ['b', 'b', 'c', 'c', 'a', 'a']).all() - assert data3.meta_info == {'info': 'test_info'} - - -def test_torch_save_data_proto(): - - obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) - labels = ['a', 'b', 'c'] - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'info': 'test_info'}) - data.save_to_disk('test_data.pt') - loaded_data = DataProto.load_from_disk('test_data.pt') - - assert torch.all(torch.eq(loaded_data.batch['obs'], data.batch['obs'])) - assert (loaded_data.non_tensor_batch['labels'] == data.non_tensor_batch['labels']).all() - assert loaded_data.meta_info == data.meta_info - - import os - os.remove('test_data.pt') - - -def test_len(): - obs = torch.tensor([[1, 2], [3, 4], [5, 6]]) - labels = np.array(['a', 'b', 'c'], dtype=object) - data = DataProto.from_dict(tensors={'obs': obs}, non_tensors={'labels': labels}, meta_info={'info': 'test_info'}) - - assert len(data) == 3 - - data = DataProto(batch=None, non_tensor_batch={'labels': labels}, meta_info={'info': 'test_info'}) - - assert len(data) == 3 - - data = DataProto(batch=None, non_tensor_batch={}, meta_info={'info': 'test_info'}) - - assert len(data) == 0 - - data = DataProto(batch=None, non_tensor_batch=None, meta_info={'info': 'test_info'}) - - assert len(data) == 0 - - -def test_seqlen_balancing(): - from verl.utils.seqlen_balancing import rearrange_micro_batches, get_reverse_idx - input_ids = torch.randint(low=0, high=10, size=(20, 100)) - from verl.utils.model import create_random_mask - attention_mask = create_random_mask(input_ids=input_ids, - max_ratio_of_left_padding=0.1, - max_ratio_of_valid_token=0.9, - min_ratio_of_valid_token=0.5) - data = {'input_ids': input_ids, 'attention_mask': attention_mask} - dataproto = DataProto.from_single_dict(data) - micro_batches, micro_bsz_idx_lst = rearrange_micro_batches(dataproto.batch, max_token_len=300) - batch = torch.cat(micro_batches) - micro_bsz_idx = [] - for idx in micro_bsz_idx_lst: - micro_bsz_idx.extend(idx) - reverse_idx_map = get_reverse_idx(micro_bsz_idx) - reverse_idx_map = torch.tensor(reverse_idx_map) - new_batch = batch[reverse_idx_map] - torch.testing.assert_close(new_batch, dataproto.batch) \ No newline at end of file diff --git a/tests/verl/utils/dataset/test_rl_dataset.py b/tests/verl/utils/dataset/test_rl_dataset.py deleted file mode 100644 index 9d3bba599f..0000000000 --- a/tests/verl/utils/dataset/test_rl_dataset.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import torch -from torch.utils.data import DataLoader -from transformers import AutoTokenizer - - -def get_gsm8k_data(): - # prepare test dataset - url = "https://github.com/eric-haibin-lin/verl-data/raw/refs/heads/main/gsm8k/train.parquet" - local_folder = os.path.expanduser('~/verl-data/gsm8k/') - local_path = os.path.join(local_folder, 'train.parquet') - os.makedirs(local_folder, exist_ok=True) - return local_path - - -def test_rl_dataset(): - from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn - from verl.utils import hf_tokenizer - tokenizer = hf_tokenizer('deepseek-ai/deepseek-coder-1.3b-instruct') - local_path = get_gsm8k_data() - dataset = RLHFDataset(parquet_files=local_path, tokenizer=tokenizer, prompt_key='prompt', max_prompt_length=256) - - dataloader = DataLoader(dataset=dataset, batch_size=16, shuffle=True, drop_last=True, collate_fn=collate_fn) - - a = next(iter(dataloader)) - - from verl import DataProto - - tensors = {} - non_tensors = {} - - for key, val in a.items(): - if isinstance(val, torch.Tensor): - tensors[key] = val - else: - non_tensors[key] = val - - data_proto = DataProto.from_dict(tensors=tensors, non_tensors=non_tensors) - - data = dataset[0]['input_ids'] - output = tokenizer.batch_decode([data])[0] - print(f'type: type{output}') - print(f'\n\noutput: {output}') diff --git a/tests/verl/utils/dataset/test_rm_dataset.py b/tests/verl/utils/dataset/test_rm_dataset.py deleted file mode 100644 index f40d4ac061..0000000000 --- a/tests/verl/utils/dataset/test_rm_dataset.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -from transformers import AutoTokenizer -from verl.utils import hf_tokenizer -from verl.utils.dataset.rm_dataset import RMDataset - - -def get_rm_data(): - # prepare test dataset - url = "https://github.com/eric-haibin-lin/verl-data/raw/refs/heads/main/full_hh_rlhf/rm/test.parquet" - local_folder = os.path.expanduser('~/verl-data/full_hh_rlhf/rm/') - local_path = os.path.join(local_folder, 'test.parquet') - os.makedirs(local_folder, exist_ok=True) - return local_path - - -def test_rm_dataset(): - tokenizer = hf_tokenizer("facebook/opt-1.3b") - local_path = get_rm_data() - dataset = RMDataset(parquet_files=local_path, tokenizer=tokenizer, max_length=512) - data = dataset[0]['input_ids'] - output = tokenizer.batch_decode(data) - assert len(output) > 1 - assert type(output[0]) == str diff --git a/tests/verl/utils/dataset/test_sft_dataset.py b/tests/verl/utils/dataset/test_sft_dataset.py deleted file mode 100644 index b58e9cfd40..0000000000 --- a/tests/verl/utils/dataset/test_sft_dataset.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -from transformers import AutoTokenizer -from verl.utils import hf_tokenizer -from verl.utils.dataset.sft_dataset import SFTDataset - - -def get_gsm8k_data(): - # prepare test dataset - url = "https://github.com/eric-haibin-lin/verl-data/raw/refs/heads/main/gsm8k/train.parquet" - local_folder = os.path.expanduser('~/verl-data/gsm8k/') - local_path = os.path.join(local_folder, 'train.parquet') - return local_path - - -def test_sft_cot_dataset(): - tokenizer = hf_tokenizer('deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct') - local_path = get_gsm8k_data() - from omegaconf import OmegaConf - dataset = SFTDataset(parquet_files=local_path, - tokenizer=tokenizer, - config=OmegaConf.create({ - 'prompt_key': 'prompt', - 'prompt_dict_keys': ['content'], - 'response_key': 'extra_info', - 'response_dict_keys': ['answer'], - 'max_length': 512, - })) - - data = dataset[0]['input_ids'] - output = tokenizer.batch_decode([data])[0] - assert len(output) > 1 - assert type(output) == str - - -def test_sft_dataset(): - tokenizer = hf_tokenizer('deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct') - local_path = get_gsm8k_data() - from omegaconf import OmegaConf - dataset = SFTDataset(parquet_files=local_path, - tokenizer=tokenizer, - config=OmegaConf.create({ - "prompt_key": 'extra_info', - 'prompt_dict_keys': ['question'], - 'response_key': 'extra_info', - 'response_dict_keys': ['answer'], - 'max_length': 512 - })) - - data = dataset[0]['input_ids'] - output = tokenizer.batch_decode([data])[0] - assert len(output) > 1 - assert type(output) == str diff --git a/uv.lock b/uv.lock deleted file mode 100644 index 18ef7cbb1f..0000000000 --- a/uv.lock +++ /dev/null @@ -1,5801 +0,0 @@ -version = 1 -revision = 2 -requires-python = "==3.12.*" -resolution-markers = [ - "platform_machine == 'x86_64' and sys_platform == 'linux'", - "platform_machine == 'aarch64' and sys_platform == 'linux'", - "platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux'", - "sys_platform == 'darwin'", - "sys_platform != 'darwin' and sys_platform != 'linux'", -] - -[manifest] -overrides = [ - { name = "flash-attn", url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl" }, - { name = "ray", specifier = "==2.43.0" }, - { name = "xgrammar", specifier = "==0.1.17" }, -] - -[[package]] -name = "accelerate" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "psutil" }, - { name = "pyyaml" }, - { name = "safetensors" }, - { name = "torch" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8a/6e/c29a1dcde7db07f47870ed63e5124086b11874ad52ccd533dc1ca2c799da/accelerate-1.6.0.tar.gz", hash = "sha256:28c1ef1846e690944f98b68dc7b8bb6c51d032d45e85dcbb3adb0c8b99dffb32", size = 363804, upload-time = "2025-04-01T11:53:03.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/b1/8198e3cdd11a426b1df2912e3381018c4a4a55368f6d0857ba3ca418ef93/accelerate-1.6.0-py3-none-any.whl", hash = "sha256:1aee717d3d3735ad6d09710a7c26990ee4652b79b4e93df46551551b5227c2aa", size = 354748, upload-time = "2025-04-01T11:53:01.298Z" }, -] - -[[package]] -name = "aiodocker" -version = "0.24.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f2/d7/30104dfac550ae6570d4dce24c2cbf2ddefd4937c9e861641314abfd8abb/aiodocker-0.24.0.tar.gz", hash = "sha256:661a6f9a479951f11f793031dcd5d55337e232c4ceaee69d51ceb885e5f16fac", size = 135482, upload-time = "2024-11-21T02:23:59.642Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/5b/2bb8b632041e314a0a917ade80382ca6a8f331f12c7eb409e59cd0485cc9/aiodocker-0.24.0-py3-none-any.whl", hash = "sha256:2199b7b01f8ce68f9cabab7910ecb26192f6f3494163f1ccffe527b4c3875689", size = 34849, upload-time = "2024-11-21T02:23:58.053Z" }, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, -] - -[[package]] -name = "aiohttp" -version = "3.11.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohappyeyeballs" }, - { name = "aiosignal" }, - { name = "attrs" }, - { name = "frozenlist" }, - { name = "multidict" }, - { name = "propcache" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f1/d9/1c4721d143e14af753f2bf5e3b681883e1f24b592c0482df6fa6e33597fa/aiohttp-3.11.16.tar.gz", hash = "sha256:16f8a2c9538c14a557b4d309ed4d0a7c60f0253e8ed7b6c9a2859a7582f8b1b8", size = 7676826, upload-time = "2025-04-02T02:17:44.74Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/38/100d01cbc60553743baf0fba658cb125f8ad674a8a771f765cdc155a890d/aiohttp-3.11.16-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:911a6e91d08bb2c72938bc17f0a2d97864c531536b7832abee6429d5296e5b27", size = 704881, upload-time = "2025-04-02T02:16:09.26Z" }, - { url = "https://files.pythonhosted.org/packages/21/ed/b4102bb6245e36591209e29f03fe87e7956e54cb604ee12e20f7eb47f994/aiohttp-3.11.16-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac13b71761e49d5f9e4d05d33683bbafef753e876e8e5a7ef26e937dd766713", size = 464564, upload-time = "2025-04-02T02:16:10.781Z" }, - { url = "https://files.pythonhosted.org/packages/3b/e1/a9ab6c47b62ecee080eeb33acd5352b40ecad08fb2d0779bcc6739271745/aiohttp-3.11.16-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb", size = 456548, upload-time = "2025-04-02T02:16:12.764Z" }, - { url = "https://files.pythonhosted.org/packages/80/ad/216c6f71bdff2becce6c8776f0aa32cb0fa5d83008d13b49c3208d2e4016/aiohttp-3.11.16-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d489d9778522fbd0f8d6a5c6e48e3514f11be81cb0a5954bdda06f7e1594b321", size = 1691749, upload-time = "2025-04-02T02:16:14.304Z" }, - { url = "https://files.pythonhosted.org/packages/bd/ea/7df7bcd3f4e734301605f686ffc87993f2d51b7acb6bcc9b980af223f297/aiohttp-3.11.16-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69a2cbd61788d26f8f1e626e188044834f37f6ae3f937bd9f08b65fc9d7e514e", size = 1736874, upload-time = "2025-04-02T02:16:16.538Z" }, - { url = "https://files.pythonhosted.org/packages/51/41/c7724b9c87a29b7cfd1202ec6446bae8524a751473d25e2ff438bc9a02bf/aiohttp-3.11.16-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd464ba806e27ee24a91362ba3621bfc39dbbb8b79f2e1340201615197370f7c", size = 1786885, upload-time = "2025-04-02T02:16:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/86/b3/f61f8492fa6569fa87927ad35a40c159408862f7e8e70deaaead349e2fba/aiohttp-3.11.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce63ae04719513dd2651202352a2beb9f67f55cb8490c40f056cea3c5c355ce", size = 1698059, upload-time = "2025-04-02T02:16:20.234Z" }, - { url = "https://files.pythonhosted.org/packages/ce/be/7097cf860a9ce8bbb0e8960704e12869e111abcd3fbd245153373079ccec/aiohttp-3.11.16-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09b00dd520d88eac9d1768439a59ab3d145065c91a8fab97f900d1b5f802895e", size = 1626527, upload-time = "2025-04-02T02:16:22.092Z" }, - { url = "https://files.pythonhosted.org/packages/1d/1d/aaa841c340e8c143a8d53a1f644c2a2961c58cfa26e7b398d6bf75cf5d23/aiohttp-3.11.16-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7f6428fee52d2bcf96a8aa7b62095b190ee341ab0e6b1bcf50c615d7966fd45b", size = 1644036, upload-time = "2025-04-02T02:16:23.707Z" }, - { url = "https://files.pythonhosted.org/packages/2c/88/59d870f76e9345e2b149f158074e78db457985c2b4da713038d9da3020a8/aiohttp-3.11.16-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:13ceac2c5cdcc3f64b9015710221ddf81c900c5febc505dbd8f810e770011540", size = 1685270, upload-time = "2025-04-02T02:16:25.874Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b1/c6686948d4c79c3745595efc469a9f8a43cab3c7efc0b5991be65d9e8cb8/aiohttp-3.11.16-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fadbb8f1d4140825069db3fedbbb843290fd5f5bc0a5dbd7eaf81d91bf1b003b", size = 1650852, upload-time = "2025-04-02T02:16:27.556Z" }, - { url = "https://files.pythonhosted.org/packages/fe/94/3e42a6916fd3441721941e0f1b8438e1ce2a4c49af0e28e0d3c950c9b3c9/aiohttp-3.11.16-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6a792ce34b999fbe04a7a71a90c74f10c57ae4c51f65461a411faa70e154154e", size = 1704481, upload-time = "2025-04-02T02:16:29.573Z" }, - { url = "https://files.pythonhosted.org/packages/b1/6d/6ab5854ff59b27075c7a8c610597d2b6c38945f9a1284ee8758bc3720ff6/aiohttp-3.11.16-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f4065145bf69de124accdd17ea5f4dc770da0a6a6e440c53f6e0a8c27b3e635c", size = 1735370, upload-time = "2025-04-02T02:16:31.191Z" }, - { url = "https://files.pythonhosted.org/packages/73/2a/08a68eec3c99a6659067d271d7553e4d490a0828d588e1daa3970dc2b771/aiohttp-3.11.16-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa73e8c2656a3653ae6c307b3f4e878a21f87859a9afab228280ddccd7369d71", size = 1697619, upload-time = "2025-04-02T02:16:32.873Z" }, - { url = "https://files.pythonhosted.org/packages/61/d5/fea8dbbfb0cd68fbb56f0ae913270a79422d9a41da442a624febf72d2aaf/aiohttp-3.11.16-cp312-cp312-win32.whl", hash = "sha256:f244b8e541f414664889e2c87cac11a07b918cb4b540c36f7ada7bfa76571ea2", size = 411710, upload-time = "2025-04-02T02:16:34.525Z" }, - { url = "https://files.pythonhosted.org/packages/33/fb/41cde15fbe51365024550bf77b95a4fc84ef41365705c946da0421f0e1e0/aiohttp-3.11.16-cp312-cp312-win_amd64.whl", hash = "sha256:23a15727fbfccab973343b6d1b7181bfb0b4aa7ae280f36fd2f90f5476805682", size = 438012, upload-time = "2025-04-02T02:16:36.103Z" }, -] - -[[package]] -name = "aiolimiter" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/23/b52debf471f7a1e42e362d959a3982bdcb4fe13a5d46e63d28868807a79c/aiolimiter-1.2.1.tar.gz", hash = "sha256:e02a37ea1a855d9e832252a105420ad4d15011505512a1a1d814647451b5cca9", size = 7185, upload-time = "2024-12-08T15:31:51.496Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/ba/df6e8e1045aebc4778d19b8a3a9bc1808adb1619ba94ca354d9ba17d86c3/aiolimiter-1.2.1-py3-none-any.whl", hash = "sha256:d3f249e9059a20badcb56b61601a83556133655c11d1eb3dd3e04ff069e5f3c7", size = 6711, upload-time = "2024-12-08T15:31:49.874Z" }, -] - -[[package]] -name = "aiosignal" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "frozenlist" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ba/b5/6d55e80f6d8a08ce22b982eafa278d823b541c925f11ee774b0b9c43473d/aiosignal-1.3.2.tar.gz", hash = "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54", size = 19424, upload-time = "2024-12-13T17:10:40.86Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", size = 7597, upload-time = "2024-12-13T17:10:38.469Z" }, -] - -[[package]] -name = "airportsdata" -version = "20250224" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/9a/e079f29ed111d9f2c28e00d36e791cf53f2f6d76873ed1821d2c2aff55ef/airportsdata-20250224.tar.gz", hash = "sha256:7f4538a613504444a13149be701aac5f9599b86da476d26b46aa24fd54714eda", size = 904203, upload-time = "2025-02-24T09:11:32.911Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/c3/3fc94ab580f50f56a8f68fd4e619730fbc8c079f0028cf37664c1c7411de/airportsdata-20250224-py3-none-any.whl", hash = "sha256:006128bca2cc1983dc5ed4fb1227e8df2289b5e95b8ab30d9bdd4eb7c6d2160d", size = 913673, upload-time = "2025-02-24T09:11:29.876Z" }, -] - -[[package]] -name = "altair" -version = "5.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "narwhals" }, - { name = "packaging" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/16/b1/f2969c7bdb8ad8bbdda031687defdce2c19afba2aa2c8e1d2a17f78376d8/altair-5.5.0.tar.gz", hash = "sha256:d960ebe6178c56de3855a68c47b516be38640b73fb3b5111c2a9ca90546dd73d", size = 705305, upload-time = "2024-11-23T23:39:58.542Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/f3/0b6ced594e51cc95d8c1fc1640d3623770d01e4969d29c0bd09945fafefa/altair-5.5.0-py3-none-any.whl", hash = "sha256:91a310b926508d560fe0148d02a194f38b824122641ef528113d029fcd129f8c", size = 731200, upload-time = "2024-11-23T23:39:56.4Z" }, -] - -[[package]] -name = "annotated-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, -] - -[[package]] -name = "anthropic" -version = "0.49.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/86/e3/a88c8494ce4d1a88252b9e053607e885f9b14d0a32273d47b727cbee4228/anthropic-0.49.0.tar.gz", hash = "sha256:c09e885b0f674b9119b4f296d8508907f6cff0009bc20d5cf6b35936c40b4398", size = 210016, upload-time = "2025-02-28T19:35:47.01Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/74/5d90ad14d55fbe3f9c474fdcb6e34b4bed99e3be8efac98734a5ddce88c1/anthropic-0.49.0-py3-none-any.whl", hash = "sha256:bbc17ad4e7094988d2fa86b87753ded8dce12498f4b85fe5810f208f454a8375", size = 243368, upload-time = "2025-02-28T19:35:44.963Z" }, -] - -[package.optional-dependencies] -vertex = [ - { name = "google-auth" }, -] - -[[package]] -name = "antlr4-python3-runtime" -version = "4.9.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } - -[[package]] -name = "anyio" -version = "4.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, -] - -[[package]] -name = "appnope" -version = "0.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, -] - -[[package]] -name = "argon2-cffi" -version = "23.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi-bindings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/31/fa/57ec2c6d16ecd2ba0cf15f3c7d1c3c2e7b5fcb83555ff56d7ab10888ec8f/argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08", size = 42798, upload-time = "2023-08-15T14:13:12.711Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/6a/e8a041599e78b6b3752da48000b14c8d1e8a04ded09c88c714ba047f34f5/argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea", size = 15124, upload-time = "2023-08-15T14:13:10.752Z" }, -] - -[[package]] -name = "argon2-cffi-bindings" -version = "21.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/184b8ccce6683b0aa2fbb7ba5683ea4b9c5763f1356347f1312c32e3c66e/argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3", size = 1779911, upload-time = "2021-12-01T08:52:55.68Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/13/838ce2620025e9666aa8f686431f67a29052241692a3dd1ae9d3692a89d3/argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367", size = 29658, upload-time = "2021-12-01T09:09:17.016Z" }, - { url = "https://files.pythonhosted.org/packages/b3/02/f7f7bb6b6af6031edb11037639c697b912e1dea2db94d436e681aea2f495/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d", size = 80583, upload-time = "2021-12-01T09:09:19.546Z" }, - { url = "https://files.pythonhosted.org/packages/ec/f7/378254e6dd7ae6f31fe40c8649eea7d4832a42243acaf0f1fff9083b2bed/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae", size = 86168, upload-time = "2021-12-01T09:09:21.445Z" }, - { url = "https://files.pythonhosted.org/packages/74/f6/4a34a37a98311ed73bb80efe422fed95f2ac25a4cacc5ae1d7ae6a144505/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c", size = 82709, upload-time = "2021-12-01T09:09:18.182Z" }, - { url = "https://files.pythonhosted.org/packages/74/2b/73d767bfdaab25484f7e7901379d5f8793cccbb86c6e0cbc4c1b96f63896/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86", size = 83613, upload-time = "2021-12-01T09:09:22.741Z" }, - { url = "https://files.pythonhosted.org/packages/4f/fd/37f86deef67ff57c76f137a67181949c2d408077e2e3dd70c6c42912c9bf/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f", size = 84583, upload-time = "2021-12-01T09:09:24.177Z" }, - { url = "https://files.pythonhosted.org/packages/6f/52/5a60085a3dae8fded8327a4f564223029f5f54b0cb0455a31131b5363a01/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e", size = 88475, upload-time = "2021-12-01T09:09:26.673Z" }, - { url = "https://files.pythonhosted.org/packages/8b/95/143cd64feb24a15fa4b189a3e1e7efbaeeb00f39a51e99b26fc62fbacabd/argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082", size = 27698, upload-time = "2021-12-01T09:09:27.87Z" }, - { url = "https://files.pythonhosted.org/packages/37/2c/e34e47c7dee97ba6f01a6203e0383e15b60fb85d78ac9a15cd066f6fe28b/argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f", size = 30817, upload-time = "2021-12-01T09:09:30.267Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e4/bf8034d25edaa495da3c8a3405627d2e35758e44ff6eaa7948092646fdcc/argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93", size = 53104, upload-time = "2021-12-01T09:09:31.335Z" }, -] - -[[package]] -name = "astor" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, -] - -[[package]] -name = "asttokens" -version = "3.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4a/e7/82da0a03e7ba5141f05cce0d302e6eed121ae055e0456ca228bf693984bc/asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", size = 61978, upload-time = "2024-11-30T04:30:14.439Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918, upload-time = "2024-11-30T04:30:10.946Z" }, -] - -[[package]] -name = "attrs" -version = "25.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, -] - -[[package]] -name = "bashlex" -version = "0.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/60/aae0bb54f9af5e0128ba90eb83d8d0d506ee8f0475c4fdda3deeda20b1d2/bashlex-0.18.tar.gz", hash = "sha256:5bb03a01c6d5676338c36fd1028009c8ad07e7d61d8a1ce3f513b7fff52796ee", size = 68742, upload-time = "2023-01-18T15:21:26.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/be/6985abb1011fda8a523cfe21ed9629e397d6e06fb5bae99750402b25c95b/bashlex-0.18-py2.py3-none-any.whl", hash = "sha256:91d73a23a3e51711919c1c899083890cdecffc91d8c088942725ac13e9dcfffa", size = 69539, upload-time = "2023-01-18T15:21:24.167Z" }, -] - -[[package]] -name = "beartype" -version = "0.12.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/78/bf7a20712dbf3a18e37f9ae1c2a1e04169c43b4628871f40f1db5f29c257/beartype-0.12.0.tar.gz", hash = "sha256:3b7545b3f333a6b07042b68b102141554c9add2e979dab7b0f8ed6378f7af7d7", size = 1029204, upload-time = "2023-01-17T08:36:34.367Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/1f/d066e5689e91e3c8845d275d85d25b23f2c7c78c9523d852b4b73be73875/beartype-0.12.0-py3-none-any.whl", hash = "sha256:3d9d5bec198bcf965c000d7b5120bebdd19a444ef6e39e97d0e93eb8832e10c8", size = 754537, upload-time = "2023-01-17T08:36:27.714Z" }, -] - -[[package]] -name = "beautifulsoup4" -version = "4.13.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f0/3c/adaf39ce1fb4afdd21b611e3d530b183bb7759c9b673d60db0e347fd4439/beautifulsoup4-4.13.3.tar.gz", hash = "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b", size = 619516, upload-time = "2025-02-04T20:05:01.681Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/49/6abb616eb3cbab6a7cca303dc02fdf3836de2e0b834bf966a7f5271a34d8/beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16", size = 186015, upload-time = "2025-02-04T20:05:03.729Z" }, -] - -[[package]] -name = "bidict" -version = "0.23.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" }, -] - -[[package]] -name = "binaryornot" -version = "0.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "chardet" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/fe/7ebfec74d49f97fc55cd38240c7a7d08134002b1e14be8c3897c0dd5e49b/binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061", size = 371054, upload-time = "2017-08-03T15:55:25.08Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/24/7e/f7b6f453e6481d1e233540262ccbfcf89adcd43606f44a028d7f5fae5eb2/binaryornot-0.4.4-py2.py3-none-any.whl", hash = "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4", size = 9006, upload-time = "2017-08-03T15:55:31.23Z" }, -] - -[[package]] -name = "blake3" -version = "1.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/12/c5886424124b2a70838731098c1cdefccaeea06c03b4094fc8aeee938069/blake3-1.0.4.tar.gz", hash = "sha256:09b2c66bc2c797e9d783521ec22b1e9a6c74e3ddb98bdd0dcd4fcc2213fb27ec", size = 115198, upload-time = "2025-01-24T13:08:01.231Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/fe/77bcfdf946da8952d3338a7ecb350edd4c2469bb6fba35853512b89d8f40/blake3-1.0.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1e3018d12e16faea2e08f210123a9c2e603de6c1b80b381624cffd536e1022d1", size = 338131, upload-time = "2025-01-24T13:06:14.214Z" }, - { url = "https://files.pythonhosted.org/packages/04/ba/2d6caaea87f5ad6526c5411868e231d73f52e6914f031baead849a597ccc/blake3-1.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:17fb8c25d62b3dc35c2c4d59f3b2f3234814b2aa374c0b9bea3d326184bf9268", size = 319326, upload-time = "2025-01-24T13:06:15.845Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8b/41e78c884fd18d382475ad315c8b7b9d4ca9bdd656bb4322abf0652b0f9b/blake3-1.0.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc9da486d47f399ac2aba8dfdfaf60cc7a507d8434623cee8f81f47852db594d", size = 362296, upload-time = "2025-01-24T13:06:17.272Z" }, - { url = "https://files.pythonhosted.org/packages/76/26/3f071b2131e51768520dc04927843d868b514c33b879ec6d0ee1266ceaaa/blake3-1.0.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0f5888e358ae4bba094d4595e1703dfc230d96dea6924e877c42c7a98beda7b5", size = 367445, upload-time = "2025-01-24T13:06:19.46Z" }, - { url = "https://files.pythonhosted.org/packages/7b/20/9e2be9622f8b0803a4cc60663988087694d458b6455b879fccfed0470942/blake3-1.0.4-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a1ab4bb7869fd38b7be2a88557d28cfe63d44b194bf2bf27e4ff08c5f2483ea", size = 436954, upload-time = "2025-01-24T13:06:20.982Z" }, - { url = "https://files.pythonhosted.org/packages/b6/6b/ae53e3b6e501b44ee6d1ac46bd559398755938ff0804e82707165c181c8b/blake3-1.0.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dbaf16fd19f93a2b5d2eadab82dca3161e2bf418606144df7edaf20bc38eda7c", size = 422046, upload-time = "2025-01-24T13:06:22.398Z" }, - { url = "https://files.pythonhosted.org/packages/df/7b/8d7b1fdd6471b93e3cbd287e3dcf76149ad9b8584902def9bea760eb80fa/blake3-1.0.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:222234ebea46d16ac981b0da528dd6e57e8ea37cef168e9f669894f660a18e09", size = 400855, upload-time = "2025-01-24T13:06:23.799Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0e/35182a7cf5ee2fb8f49296f34c2880bf99318df4fccb2c294485d4bc1bbb/blake3-1.0.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb2689cbef663d823011eeddec29c23d1c1f773ac867bfa854fb0590771a309d", size = 374969, upload-time = "2025-01-24T13:06:25.475Z" }, - { url = "https://files.pythonhosted.org/packages/c9/50/c1280aee0e2f8f297e49ce3c131e03436cab1579a2efa52e825851a27b1c/blake3-1.0.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1524b1cabb034f1c9dc2621f3c06c10d2a4608391cf04e5db182aa5d7a82fdbe", size = 529434, upload-time = "2025-01-24T13:06:27.172Z" }, - { url = "https://files.pythonhosted.org/packages/50/b2/2b196b9cae06f986b03832f164d58ef9ca039d7ed6ec4f8549f085aab85a/blake3-1.0.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7592124471fb1c8c67f94776c480743c182aff92952ceb5f5c793a632a1a1436", size = 537692, upload-time = "2025-01-24T13:06:28.874Z" }, - { url = "https://files.pythonhosted.org/packages/1f/ec/3e57c820c0f7ff78a7b4ea2f640d0cf2fe37053615e41314b5c8b881cd6e/blake3-1.0.4-cp312-cp312-win32.whl", hash = "sha256:78f4724d0a9f6bebd0fccf27e4afaed1ca4b6645740ee425d3621defe27c4e64", size = 228519, upload-time = "2025-01-24T13:06:30.495Z" }, - { url = "https://files.pythonhosted.org/packages/28/44/d70c4782b4e950887d9c5d77384fb54e84bd3a38472a6b6dc6792c9bf5df/blake3-1.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:af18fcd2a37aa51c24cedbb82f4934f39a9a4ea11a84d34c1ab63df94a28fdd1", size = 212489, upload-time = "2025-01-24T13:06:31.939Z" }, -] - -[[package]] -name = "blinker" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, -] - -[[package]] -name = "boto3" -version = "1.37.32" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/56/7a/be6dfbe66f3a04434240edbb5425c0756f848af40c52194109afc0d265e9/boto3-1.37.32.tar.gz", hash = "sha256:bc08c95a88ffeb51d78d25cb8bd72593b8cce1d8fdcc650030aff98c15437d04", size = 111354, upload-time = "2025-04-10T21:35:06.969Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/3c/27d76e8e2400e129d5253837841849b8f66cbdda9fbdeae61c3d6713c6a6/boto3-1.37.32-py3-none-any.whl", hash = "sha256:6f0d3863abfeed366b365fb3ad2fa508d7f2becd64ca712d4b70b593da7f9763", size = 139561, upload-time = "2025-04-10T21:35:03.895Z" }, -] - -[[package]] -name = "botocore" -version = "1.37.32" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8f/68/407e8a712694eab28ca95ca2f9135d66e92975a54a37dda1aeefd26f8cd5/botocore-1.37.32.tar.gz", hash = "sha256:3e5d097690b3423adeefdf257384e964d0ba7f9575d77bf3f8998273b92ef700", size = 13815793, upload-time = "2025-04-10T21:34:52.597Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/7a/f0813da18b6f194e994666e0fd772a8cd8bc46b3c71e93994fdd8f6e573a/botocore-1.37.32-py3-none-any.whl", hash = "sha256:c25989e09e29b382c1edcc994f795c4faadf2f30470269754024a55269a7a47d", size = 13481235, upload-time = "2025-04-10T21:34:46.814Z" }, -] - -[[package]] -name = "browsergym" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "browsergym-assistantbench" }, - { name = "browsergym-core" }, - { name = "browsergym-experiments" }, - { name = "browsergym-miniwob" }, - { name = "browsergym-visualwebarena" }, - { name = "browsergym-webarena" }, - { name = "browsergym-workarena" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a4/9e/f1f20eb0115b02778d91722883b2e90ea5e3cf9dd07286b0014a7c96dfae/browsergym-0.10.2.tar.gz", hash = "sha256:3cdd7520cca857421aa7ec0a965968df4bcef721299a424397f86d7cad078ab0", size = 1452, upload-time = "2024-10-24T15:37:34.223Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/2b/aad7f5edf9d395fda985c8005d7167d91eca543255b3bbb2dd5bae283e67/browsergym-0.10.2-py3-none-any.whl", hash = "sha256:9581d1d1f1fcd1cf35266cf30c881d60c147a0d374b3491eeaebb07d9690f868", size = 1394, upload-time = "2024-10-24T15:37:25.071Z" }, -] - -[[package]] -name = "browsergym-assistantbench" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "browsergym-core" }, - { name = "datasets" }, - { name = "numpy" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d9/ae/0c4b91beab9316662d3e10bc70142859a4ebb9f526c7b4ba3aae23405c1f/browsergym_assistantbench-0.10.2.tar.gz", hash = "sha256:de18eb7c010403d5d467b927b4713b56f6e97a59493bee4c42599d4d7cb54dce", size = 8122, upload-time = "2024-10-24T15:37:34.806Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/90/69bff6647419eb42575716368272da1a04779953b99b13d27eabd824107f/browsergym_assistantbench-0.10.2-py3-none-any.whl", hash = "sha256:af0d3a3e23686066b070feca38f8740262bed6d65ccf9098f393334a005987c0", size = 10413, upload-time = "2024-10-24T15:37:26.268Z" }, -] - -[[package]] -name = "browsergym-core" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "gymnasium" }, - { name = "lxml" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "playwright" }, - { name = "pyparsing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d5/86/6e9d4e17ad02ab96c2067917e9728ce81957768ccfbdaf85e77ed4d3e377/browsergym_core-0.10.2.tar.gz", hash = "sha256:7e93bad5cc3990badee77e9481413d625d2fce2ec8f7f9e195dbc194b6cfb4e9", size = 177176, upload-time = "2024-10-24T15:37:35.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/02/09310d81b74afe8685f454b01b02ee381d7b1c638a470ddfc235c3bd90df/browsergym_core-0.10.2-py3-none-any.whl", hash = "sha256:0686a8e2ee7244e33c97326193f54df0ad08d99aad9a4ed9ac28baba5ca26d18", size = 185196, upload-time = "2024-10-24T15:37:28.02Z" }, -] - -[[package]] -name = "browsergym-experiments" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "browsergym-core" }, - { name = "dataclasses-json" }, - { name = "tiktoken" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e6/82/2827bc0c1b5a33cafa48909c10138a51e7dc9b846f92590cfad17a9992c7/browsergym_experiments-0.10.2.tar.gz", hash = "sha256:b49bc27f315ad12014ff21580c7c7aca6489ca4106e7ab46502f716674efa236", size = 219986, upload-time = "2024-10-24T15:37:37.406Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/03/48f18ee18f202f03677b5e54db3506d73cf8116821093671a6f1b9528b91/browsergym_experiments-0.10.2-py3-none-any.whl", hash = "sha256:60a626b3159ef63b5ff72a6c8156c8f3cf82a9278dfc5a9d3ece39c2b1913595", size = 254727, upload-time = "2024-10-24T15:37:29.252Z" }, -] - -[[package]] -name = "browsergym-miniwob" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "browsergym-core" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/8d/6a507110c041ce261606fa768b38fe6e1c835d669811054732b47d29a1b2/browsergym_miniwob-0.10.2.tar.gz", hash = "sha256:9109b8122a61b27e227d923861055f220c6ddd60f34f877c3a30444c6f8a7b05", size = 8839, upload-time = "2024-10-24T15:37:38.257Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/ff/231bf2e042c0708af70b4ff8404c879cf064d79285982ce9b54efd1bea08/browsergym_miniwob-0.10.2-py3-none-any.whl", hash = "sha256:b11b04378868a8f5dee34f721134baed4780fd55ccaebf9db4de6fcac48f3190", size = 9119, upload-time = "2024-10-24T15:37:30.77Z" }, -] - -[[package]] -name = "browsergym-visualwebarena" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "browsergym-core" }, - { name = "libvisualwebarena" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/4c/4ac4e763fc34b535538cbe83d30dc7a973427767e59011ce0dcbd88e39bf/browsergym_visualwebarena-0.10.2.tar.gz", hash = "sha256:5f84a4f33a21106c9b650cecb0362b78af2546d9927255828c273fe800d776a1", size = 8130, upload-time = "2024-10-24T15:37:39.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/b5/84e7970deba8b4463b8444e3757bfbae637ce8ac0948e8a5a218ed6e0a3d/browsergym_visualwebarena-0.10.2-py3-none-any.whl", hash = "sha256:87c913ccd4d12a79c625b5c4d9ead7e0bc50b298d19e413204bb586a67736d83", size = 9045, upload-time = "2024-10-24T15:37:32.034Z" }, -] - -[[package]] -name = "browsergym-webarena" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "browsergym-core" }, - { name = "libwebarena" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1a/06/9ba229ca4bf7567b91b28fe234eee7539c75529a497949916a9592bced01/browsergym_webarena-0.10.2.tar.gz", hash = "sha256:b4b9a38f144b6aaa56bbbbce9dd2c5565a39a1b55e3647d61e02458ca3f5fd24", size = 6770, upload-time = "2024-10-24T15:37:40.959Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/e7/917353ae76c2b1aad43e44106a2b481f8e483f81b73c8bb6a838f020103f/browsergym_webarena-0.10.2-py3-none-any.whl", hash = "sha256:e9ca6d0ad263412ebb229fe1b66e1ab7f5841a3f838abedf3bf01b800a7c6597", size = 7112, upload-time = "2024-10-24T15:37:33.024Z" }, -] - -[[package]] -name = "browsergym-workarena" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "browsergym-core" }, - { name = "english-words" }, - { name = "faker" }, - { name = "numpy" }, - { name = "requests" }, - { name = "tenacity" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e2/b8/7541c4c42ca3b4b7a3708158a087f07be8d0b5c244965408d25776c50849/browsergym_workarena-0.4.1.tar.gz", hash = "sha256:ba2958d804b80836c7f81360d66b99c6c655c5070eddc5fae9c1c88306a23403", size = 6300893, upload-time = "2024-10-07T15:54:13.849Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/bb/815d32bb8917b2d71ec3b015ac80d88afd42f0012b962616b6ed15495363/browsergym_workarena-0.4.1-py3-none-any.whl", hash = "sha256:b8f04b2e3801fd32962b7d99f0685c507b258841e2b4bfdb46d041091d2f1b89", size = 6778925, upload-time = "2024-10-07T15:54:11.624Z" }, -] - -[[package]] -name = "cachetools" -version = "5.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, -] - -[[package]] -name = "certifi" -version = "2025.1.31" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", size = 167577, upload-time = "2025-01-31T02:16:47.166Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393, upload-time = "2025-01-31T02:16:45.015Z" }, -] - -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, -] - -[[package]] -name = "cfgv" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114, upload-time = "2023-08-12T20:38:17.776Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" }, -] - -[[package]] -name = "chardet" -version = "5.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", size = 123188, upload-time = "2024-12-24T18:12:35.43Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/9a/dd1e1cdceb841925b7798369a09279bd1cf183cef0f9ddf15a3a6502ee45/charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545", size = 196105, upload-time = "2024-12-24T18:10:38.83Z" }, - { url = "https://files.pythonhosted.org/packages/d3/8c/90bfabf8c4809ecb648f39794cf2a84ff2e7d2a6cf159fe68d9a26160467/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7", size = 140404, upload-time = "2024-12-24T18:10:44.272Z" }, - { url = "https://files.pythonhosted.org/packages/ad/8f/e410d57c721945ea3b4f1a04b74f70ce8fa800d393d72899f0a40526401f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757", size = 150423, upload-time = "2024-12-24T18:10:45.492Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b8/e6825e25deb691ff98cf5c9072ee0605dc2acfca98af70c2d1b1bc75190d/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa", size = 143184, upload-time = "2024-12-24T18:10:47.898Z" }, - { url = "https://files.pythonhosted.org/packages/3e/a2/513f6cbe752421f16d969e32f3583762bfd583848b763913ddab8d9bfd4f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d", size = 145268, upload-time = "2024-12-24T18:10:50.589Z" }, - { url = "https://files.pythonhosted.org/packages/74/94/8a5277664f27c3c438546f3eb53b33f5b19568eb7424736bdc440a88a31f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616", size = 147601, upload-time = "2024-12-24T18:10:52.541Z" }, - { url = "https://files.pythonhosted.org/packages/7c/5f/6d352c51ee763623a98e31194823518e09bfa48be2a7e8383cf691bbb3d0/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b", size = 141098, upload-time = "2024-12-24T18:10:53.789Z" }, - { url = "https://files.pythonhosted.org/packages/78/d4/f5704cb629ba5ab16d1d3d741396aec6dc3ca2b67757c45b0599bb010478/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d", size = 149520, upload-time = "2024-12-24T18:10:55.048Z" }, - { url = "https://files.pythonhosted.org/packages/c5/96/64120b1d02b81785f222b976c0fb79a35875457fa9bb40827678e54d1bc8/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a", size = 152852, upload-time = "2024-12-24T18:10:57.647Z" }, - { url = "https://files.pythonhosted.org/packages/84/c9/98e3732278a99f47d487fd3468bc60b882920cef29d1fa6ca460a1fdf4e6/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9", size = 150488, upload-time = "2024-12-24T18:10:59.43Z" }, - { url = "https://files.pythonhosted.org/packages/13/0e/9c8d4cb99c98c1007cc11eda969ebfe837bbbd0acdb4736d228ccaabcd22/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1", size = 146192, upload-time = "2024-12-24T18:11:00.676Z" }, - { url = "https://files.pythonhosted.org/packages/b2/21/2b6b5b860781a0b49427309cb8670785aa543fb2178de875b87b9cc97746/charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35", size = 95550, upload-time = "2024-12-24T18:11:01.952Z" }, - { url = "https://files.pythonhosted.org/packages/21/5b/1b390b03b1d16c7e382b561c5329f83cc06623916aab983e8ab9239c7d5c/charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f", size = 102785, upload-time = "2024-12-24T18:11:03.142Z" }, - { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767, upload-time = "2024-12-24T18:12:32.852Z" }, -] - -[[package]] -name = "click" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, -] - -[[package]] -name = "cloudpickle" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, -] - -[[package]] -name = "codetiming" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ad/4e/c40bf151af20ba2748bd6ea24e484d7b6196b1056ba3a1a4ee33b6939c37/codetiming-1.4.0.tar.gz", hash = "sha256:4937bf913a2814258b87eaaa43d9a1bb24711ffd3557a9ab6934fa1fe3ba0dbc", size = 14899, upload-time = "2022-11-08T07:12:29.361Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/91/e4a2b7c64e738beefddfa24b409d6eecb16c378bde01578918b6ea722a09/codetiming-1.4.0-py3-none-any.whl", hash = "sha256:3b80f409bef00941a9755c5524071ce2f72eaa4520f4bc35b33869cde024ccbd", size = 7165, upload-time = "2022-11-08T07:12:23.553Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "comm" -version = "0.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/a8/fb783cb0abe2b5fded9f55e5703015cdf1c9c85b3669087c538dd15a6a86/comm-0.2.2.tar.gz", hash = "sha256:3fd7a84065306e07bea1773df6eb8282de51ba82f77c72f9c85716ab11fe980e", size = 6210, upload-time = "2024-03-12T16:53:41.133Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/75/49e5bfe642f71f272236b5b2d2691cf915a7283cc0ceda56357b61daa538/comm-0.2.2-py3-none-any.whl", hash = "sha256:e6fb86cb70ff661ee8c9c14e7d36d6de3b4066f1441be4063df9c5009f0a64d3", size = 7180, upload-time = "2024-03-12T16:53:39.226Z" }, -] - -[[package]] -name = "commit0" -version = "0.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "datasets" }, - { name = "docker" }, - { name = "fastcore" }, - { name = "ghapi" }, - { name = "gitpython" }, - { name = "modal" }, - { name = "pre-commit" }, - { name = "pytest" }, - { name = "ruff" }, - { name = "strenum" }, - { name = "typer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a4/25/a45fdc46056f033ec9df463ace94f6815005a77c50bc5cf521a57b3e7da2/commit0-0.1.8.tar.gz", hash = "sha256:fd6402381df7daf29f3d5d423c1e1d7b26e8e9e9b506d5dfa7a3f0e724ec87fd", size = 25187644, upload-time = "2024-11-22T04:34:06.457Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/77/c75d378f784c9e6ecc488721841aac19596763d78370e5ff160409b29564/commit0-0.1.8-py3-none-any.whl", hash = "sha256:b5369d7c814d2f2b82f39da60849f7b8519f277a05c5ed91a591b800c4b4d431", size = 1027671, upload-time = "2024-11-22T04:34:01.125Z" }, -] - -[[package]] -name = "compressed-tensors" -version = "0.9.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "torch" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cf/43/2b5ada16e9e70c62dc24e30ef3a9f22782ab4130128b52b6345ead8d0de3/compressed_tensors-0.9.2.tar.gz", hash = "sha256:18c5627a7324a75cd4c7d984799269e0ddef592b6fb3b9a81c16754d5c4b56ff", size = 65839, upload-time = "2025-02-18T19:08:23.712Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/6e/dc0a80ce14802344e3f4d0520285e8773b83ec2fd864e7cab886718f55a9/compressed_tensors-0.9.2-py3-none-any.whl", hash = "sha256:fbc5d188ee43f93eccd6df566e8eccbb1eba907560b2b81ca85153335df55dd9", size = 97875, upload-time = "2025-02-18T19:08:20.596Z" }, -] - -[[package]] -name = "contourpy" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/c2/fc7193cc5383637ff390a712e88e4ded0452c9fbcf84abe3de5ea3df1866/contourpy-1.3.1.tar.gz", hash = "sha256:dfd97abd83335045a913e3bcc4a09c0ceadbe66580cf573fe961f4a825efa699", size = 13465753, upload-time = "2024-11-12T11:00:59.118Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/6b/175f60227d3e7f5f1549fcb374592be311293132207e451c3d7c654c25fb/contourpy-1.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0ffa84be8e0bd33410b17189f7164c3589c229ce5db85798076a3fa136d0e509", size = 271494, upload-time = "2024-11-12T10:54:23.6Z" }, - { url = "https://files.pythonhosted.org/packages/6b/6a/7833cfae2c1e63d1d8875a50fd23371394f540ce809d7383550681a1fa64/contourpy-1.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805617228ba7e2cbbfb6c503858e626ab528ac2a32a04a2fe88ffaf6b02c32bc", size = 255444, upload-time = "2024-11-12T10:54:28.267Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b3/7859efce66eaca5c14ba7619791b084ed02d868d76b928ff56890d2d059d/contourpy-1.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade08d343436a94e633db932e7e8407fe7de8083967962b46bdfc1b0ced39454", size = 307628, upload-time = "2024-11-12T10:54:33.418Z" }, - { url = "https://files.pythonhosted.org/packages/48/b2/011415f5e3f0a50b1e285a0bf78eb5d92a4df000553570f0851b6e309076/contourpy-1.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47734d7073fb4590b4a40122b35917cd77be5722d80683b249dac1de266aac80", size = 347271, upload-time = "2024-11-12T10:54:38.816Z" }, - { url = "https://files.pythonhosted.org/packages/84/7d/ef19b1db0f45b151ac78c65127235239a8cf21a59d1ce8507ce03e89a30b/contourpy-1.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ba94a401342fc0f8b948e57d977557fbf4d515f03c67682dd5c6191cb2d16ec", size = 318906, upload-time = "2024-11-12T10:54:44.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/99/6794142b90b853a9155316c8f470d2e4821fe6f086b03e372aca848227dd/contourpy-1.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efa874e87e4a647fd2e4f514d5e91c7d493697127beb95e77d2f7561f6905bd9", size = 323622, upload-time = "2024-11-12T10:54:48.788Z" }, - { url = "https://files.pythonhosted.org/packages/3c/0f/37d2c84a900cd8eb54e105f4fa9aebd275e14e266736778bb5dccbf3bbbb/contourpy-1.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1bf98051f1045b15c87868dbaea84f92408337d4f81d0e449ee41920ea121d3b", size = 1266699, upload-time = "2024-11-12T10:55:04.016Z" }, - { url = "https://files.pythonhosted.org/packages/3a/8a/deb5e11dc7d9cc8f0f9c8b29d4f062203f3af230ba83c30a6b161a6effc9/contourpy-1.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61332c87493b00091423e747ea78200659dc09bdf7fd69edd5e98cef5d3e9a8d", size = 1326395, upload-time = "2024-11-12T10:55:20.547Z" }, - { url = "https://files.pythonhosted.org/packages/1a/35/7e267ae7c13aaf12322ccc493531f1e7f2eb8fba2927b9d7a05ff615df7a/contourpy-1.3.1-cp312-cp312-win32.whl", hash = "sha256:e914a8cb05ce5c809dd0fe350cfbb4e881bde5e2a38dc04e3afe1b3e58bd158e", size = 175354, upload-time = "2024-11-12T10:55:24.377Z" }, - { url = "https://files.pythonhosted.org/packages/a1/35/c2de8823211d07e8a79ab018ef03960716c5dff6f4d5bff5af87fd682992/contourpy-1.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:08d9d449a61cf53033612cb368f3a1b26cd7835d9b8cd326647efe43bca7568d", size = 220971, upload-time = "2024-11-12T10:55:27.971Z" }, -] - -[[package]] -name = "cryptography" -version = "44.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/25/4ce80c78963834b8a9fd1cc1266be5ed8d1840785c0f2e1b73b8d128d505/cryptography-44.0.2.tar.gz", hash = "sha256:c63454aa261a0cf0c5b4718349629793e9e634993538db841165b3df74f37ec0", size = 710807, upload-time = "2025-03-02T00:01:37.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/ef/83e632cfa801b221570c5f58c0369db6fa6cef7d9ff859feab1aae1a8a0f/cryptography-44.0.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:efcfe97d1b3c79e486554efddeb8f6f53a4cdd4cf6086642784fa31fc384e1d7", size = 6676361, upload-time = "2025-03-02T00:00:06.528Z" }, - { url = "https://files.pythonhosted.org/packages/30/ec/7ea7c1e4c8fc8329506b46c6c4a52e2f20318425d48e0fe597977c71dbce/cryptography-44.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ecec49f3ba3f3849362854b7253a9f59799e3763b0c9d0826259a88efa02f1", size = 3952350, upload-time = "2025-03-02T00:00:09.537Z" }, - { url = "https://files.pythonhosted.org/packages/27/61/72e3afdb3c5ac510330feba4fc1faa0fe62e070592d6ad00c40bb69165e5/cryptography-44.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc821e161ae88bfe8088d11bb39caf2916562e0a2dc7b6d56714a48b784ef0bb", size = 4166572, upload-time = "2025-03-02T00:00:12.03Z" }, - { url = "https://files.pythonhosted.org/packages/26/e4/ba680f0b35ed4a07d87f9e98f3ebccb05091f3bf6b5a478b943253b3bbd5/cryptography-44.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c00b6b757b32ce0f62c574b78b939afab9eecaf597c4d624caca4f9e71e7843", size = 3958124, upload-time = "2025-03-02T00:00:14.518Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e8/44ae3e68c8b6d1cbc59040288056df2ad7f7f03bbcaca6b503c737ab8e73/cryptography-44.0.2-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7bdcd82189759aba3816d1f729ce42ffded1ac304c151d0a8e89b9996ab863d5", size = 3678122, upload-time = "2025-03-02T00:00:17.212Z" }, - { url = "https://files.pythonhosted.org/packages/27/7b/664ea5e0d1eab511a10e480baf1c5d3e681c7d91718f60e149cec09edf01/cryptography-44.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4973da6ca3db4405c54cd0b26d328be54c7747e89e284fcff166132eb7bccc9c", size = 4191831, upload-time = "2025-03-02T00:00:19.696Z" }, - { url = "https://files.pythonhosted.org/packages/2a/07/79554a9c40eb11345e1861f46f845fa71c9e25bf66d132e123d9feb8e7f9/cryptography-44.0.2-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4e389622b6927d8133f314949a9812972711a111d577a5d1f4bee5e58736b80a", size = 3960583, upload-time = "2025-03-02T00:00:22.488Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6d/858e356a49a4f0b591bd6789d821427de18432212e137290b6d8a817e9bf/cryptography-44.0.2-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f514ef4cd14bb6fb484b4a60203e912cfcb64f2ab139e88c2274511514bf7308", size = 4191753, upload-time = "2025-03-02T00:00:25.038Z" }, - { url = "https://files.pythonhosted.org/packages/b2/80/62df41ba4916067fa6b125aa8c14d7e9181773f0d5d0bd4dcef580d8b7c6/cryptography-44.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1bc312dfb7a6e5d66082c87c34c8a62176e684b6fe3d90fcfe1568de675e6688", size = 4079550, upload-time = "2025-03-02T00:00:26.929Z" }, - { url = "https://files.pythonhosted.org/packages/f3/cd/2558cc08f7b1bb40683f99ff4327f8dcfc7de3affc669e9065e14824511b/cryptography-44.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b721b8b4d948b218c88cb8c45a01793483821e709afe5f622861fc6182b20a7", size = 4298367, upload-time = "2025-03-02T00:00:28.735Z" }, - { url = "https://files.pythonhosted.org/packages/71/59/94ccc74788945bc3bd4cf355d19867e8057ff5fdbcac781b1ff95b700fb1/cryptography-44.0.2-cp37-abi3-win32.whl", hash = "sha256:51e4de3af4ec3899d6d178a8c005226491c27c4ba84101bfb59c901e10ca9f79", size = 2772843, upload-time = "2025-03-02T00:00:30.592Z" }, - { url = "https://files.pythonhosted.org/packages/ca/2c/0d0bbaf61ba05acb32f0841853cfa33ebb7a9ab3d9ed8bb004bd39f2da6a/cryptography-44.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:c505d61b6176aaf982c5717ce04e87da5abc9a36a5b39ac03905c4aafe8de7aa", size = 3209057, upload-time = "2025-03-02T00:00:33.393Z" }, - { url = "https://files.pythonhosted.org/packages/9e/be/7a26142e6d0f7683d8a382dd963745e65db895a79a280a30525ec92be890/cryptography-44.0.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e0ddd63e6bf1161800592c71ac794d3fb8001f2caebe0966e77c5234fa9efc3", size = 6677789, upload-time = "2025-03-02T00:00:36.009Z" }, - { url = "https://files.pythonhosted.org/packages/06/88/638865be7198a84a7713950b1db7343391c6066a20e614f8fa286eb178ed/cryptography-44.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81276f0ea79a208d961c433a947029e1a15948966658cf6710bbabb60fcc2639", size = 3951919, upload-time = "2025-03-02T00:00:38.581Z" }, - { url = "https://files.pythonhosted.org/packages/d7/fc/99fe639bcdf58561dfad1faa8a7369d1dc13f20acd78371bb97a01613585/cryptography-44.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1e657c0f4ea2a23304ee3f964db058c9e9e635cc7019c4aa21c330755ef6fd", size = 4167812, upload-time = "2025-03-02T00:00:42.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/7b/aafe60210ec93d5d7f552592a28192e51d3c6b6be449e7fd0a91399b5d07/cryptography-44.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6210c05941994290f3f7f175a4a57dbbb2afd9273657614c506d5976db061181", size = 3958571, upload-time = "2025-03-02T00:00:46.026Z" }, - { url = "https://files.pythonhosted.org/packages/16/32/051f7ce79ad5a6ef5e26a92b37f172ee2d6e1cce09931646eef8de1e9827/cryptography-44.0.2-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1c3572526997b36f245a96a2b1713bf79ce99b271bbcf084beb6b9b075f29ea", size = 3679832, upload-time = "2025-03-02T00:00:48.647Z" }, - { url = "https://files.pythonhosted.org/packages/78/2b/999b2a1e1ba2206f2d3bca267d68f350beb2b048a41ea827e08ce7260098/cryptography-44.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b042d2a275c8cee83a4b7ae30c45a15e6a4baa65a179a0ec2d78ebb90e4f6699", size = 4193719, upload-time = "2025-03-02T00:00:51.397Z" }, - { url = "https://files.pythonhosted.org/packages/72/97/430e56e39a1356e8e8f10f723211a0e256e11895ef1a135f30d7d40f2540/cryptography-44.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d03806036b4f89e3b13b6218fefea8d5312e450935b1a2d55f0524e2ed7c59d9", size = 3960852, upload-time = "2025-03-02T00:00:53.317Z" }, - { url = "https://files.pythonhosted.org/packages/89/33/c1cf182c152e1d262cac56850939530c05ca6c8d149aa0dcee490b417e99/cryptography-44.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c7362add18b416b69d58c910caa217f980c5ef39b23a38a0880dfd87bdf8cd23", size = 4193906, upload-time = "2025-03-02T00:00:56.49Z" }, - { url = "https://files.pythonhosted.org/packages/e1/99/87cf26d4f125380dc674233971069bc28d19b07f7755b29861570e513650/cryptography-44.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8cadc6e3b5a1f144a039ea08a0bdb03a2a92e19c46be3285123d32029f40a922", size = 4081572, upload-time = "2025-03-02T00:00:59.995Z" }, - { url = "https://files.pythonhosted.org/packages/b3/9f/6a3e0391957cc0c5f84aef9fbdd763035f2b52e998a53f99345e3ac69312/cryptography-44.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f101b1f780f7fc613d040ca4bdf835c6ef3b00e9bd7125a4255ec574c7916e4", size = 4298631, upload-time = "2025-03-02T00:01:01.623Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a5/5bc097adb4b6d22a24dea53c51f37e480aaec3465285c253098642696423/cryptography-44.0.2-cp39-abi3-win32.whl", hash = "sha256:3dc62975e31617badc19a906481deacdeb80b4bb454394b4098e3f2525a488c5", size = 2773792, upload-time = "2025-03-02T00:01:04.133Z" }, - { url = "https://files.pythonhosted.org/packages/33/cf/1f7649b8b9a3543e042d3f348e398a061923ac05b507f3f4d95f11938aa9/cryptography-44.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:5f6f90b72d8ccadb9c6e311c775c8305381db88374c65fa1a68250aa8a9cb3a6", size = 3210957, upload-time = "2025-03-02T00:01:06.987Z" }, -] - -[[package]] -name = "cuda-bindings" -version = "12.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/16/621f2ff6e4c6a0c1d57f5a0a373d1fb9d10eb9a7f05052cc64eba2e7dab2/cuda_bindings-12.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0865c9b75ee8f0535044c3f0f06ca34a37131192b573ab59e20a9e058da1ead4", size = 10904424, upload-time = "2025-01-25T03:27:52.688Z" }, - { url = "https://files.pythonhosted.org/packages/59/11/aee1afd60a5d6af67994dd88697912be22366a6e548e52e6cd2defdbe678/cuda_bindings-12.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0e6a889c87238e6cd55e9b25ce4fd1d90fe2d4169982860fed5f0bc3230795e", size = 11235285, upload-time = "2025-01-25T03:27:56.564Z" }, - { url = "https://files.pythonhosted.org/packages/c1/c7/eedad18aeb461e9a3c1f8e2ea856caa50202a572b024912cb561f847a054/cuda_bindings-12.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:d0123d841cb3053d227e18b08ea7680d0b5ca64fab4664a2b80b7c83c8edf1ee", size = 11224401, upload-time = "2025-01-25T03:28:00.327Z" }, -] - -[[package]] -name = "cuda-python" -version = "12.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/2c/02bb311b996ffb91d05f8c1fb79131bf50855f7410dd33d09f800fe78c58/cuda_python-12.8.0-py3-none-any.whl", hash = "sha256:3fca3a03c247d6aa1c414989dfe0dd21e9500307b8573f72216ed57d99344c5a", size = 11930, upload-time = "2025-01-25T03:30:07.301Z" }, -] - -[[package]] -name = "cycler" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, -] - -[[package]] -name = "dataclasses-json" -version = "0.6.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "marshmallow" }, - { name = "typing-inspect" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227, upload-time = "2024-06-09T16:20:19.103Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, -] - -[[package]] -name = "datasets" -version = "3.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "dill" }, - { name = "filelock" }, - { name = "fsspec", extra = ["http"] }, - { name = "huggingface-hub" }, - { name = "multiprocess" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "xxhash" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/1b/ce839a6a127935f8e87a8d5887e7dfa76cbdbe94ee7ac04c855221ea1e0a/datasets-3.0.1.tar.gz", hash = "sha256:40d63b09e76a3066c32e746d6fdc36fd3f29ed2acd49bf5b1a2100da32936511", size = 1875319, upload-time = "2024-09-26T08:25:47.995Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/3e/e58d4db4cfe71e3ed07d169af24db30cfd582e16f977378bd43fd7ec1998/datasets-3.0.1-py3-none-any.whl", hash = "sha256:db080aab41c8cc68645117a0f172e5c6789cbc672f066de0aa5a08fc3eebc686", size = 471557, upload-time = "2024-09-26T08:25:45.234Z" }, -] - -[[package]] -name = "debugpy" -version = "1.8.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/75/087fe07d40f490a78782ff3b0a30e3968936854105487decdb33446d4b0e/debugpy-1.8.14.tar.gz", hash = "sha256:7cd287184318416850aa8b60ac90105837bb1e59531898c07569d197d2ed5322", size = 1641444, upload-time = "2025-04-10T19:46:10.981Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/2a/ac2df0eda4898f29c46eb6713a5148e6f8b2b389c8ec9e425a4a1d67bf07/debugpy-1.8.14-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:8899c17920d089cfa23e6005ad9f22582fd86f144b23acb9feeda59e84405b84", size = 2501268, upload-time = "2025-04-10T19:46:26.044Z" }, - { url = "https://files.pythonhosted.org/packages/10/53/0a0cb5d79dd9f7039169f8bf94a144ad3efa52cc519940b3b7dde23bcb89/debugpy-1.8.14-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6bb5c0dcf80ad5dbc7b7d6eac484e2af34bdacdf81df09b6a3e62792b722826", size = 4221077, upload-time = "2025-04-10T19:46:27.464Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d5/84e01821f362327bf4828728aa31e907a2eca7c78cd7c6ec062780d249f8/debugpy-1.8.14-cp312-cp312-win32.whl", hash = "sha256:281d44d248a0e1791ad0eafdbbd2912ff0de9eec48022a5bfbc332957487ed3f", size = 5255127, upload-time = "2025-04-10T19:46:29.467Z" }, - { url = "https://files.pythonhosted.org/packages/33/16/1ed929d812c758295cac7f9cf3dab5c73439c83d9091f2d91871e648093e/debugpy-1.8.14-cp312-cp312-win_amd64.whl", hash = "sha256:5aa56ef8538893e4502a7d79047fe39b1dae08d9ae257074c6464a7b290b806f", size = 5297249, upload-time = "2025-04-10T19:46:31.538Z" }, - { url = "https://files.pythonhosted.org/packages/97/1a/481f33c37ee3ac8040d3d51fc4c4e4e7e61cb08b8bc8971d6032acc2279f/debugpy-1.8.14-py2.py3-none-any.whl", hash = "sha256:5cd9a579d553b6cb9759a7908a41988ee6280b961f24f63336835d9418216a20", size = 5256230, upload-time = "2025-04-10T19:46:54.077Z" }, -] - -[[package]] -name = "decorator" -version = "5.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, -] - -[[package]] -name = "decord" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/79/936af42edf90a7bd4e41a6cac89c913d4b47fa48a26b042d5129a9242ee3/decord-0.6.0-py3-none-manylinux2010_x86_64.whl", hash = "sha256:51997f20be8958e23b7c4061ba45d0efcd86bffd5fe81c695d0befee0d442976", size = 13602299, upload-time = "2021-06-14T21:30:55.486Z" }, - { url = "https://files.pythonhosted.org/packages/6c/be/e15b5b866da452e62635a7b27513f31cb581fa2ea9cc9b768b535d62a955/decord-0.6.0-py3-none-win_amd64.whl", hash = "sha256:02665d7c4f1193a330205a791bc128f7e108eb6ae5b67144437a02f700943bad", size = 24733380, upload-time = "2021-06-14T21:30:57.766Z" }, -] - -[[package]] -name = "deprecated" -version = "1.2.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, -] - -[[package]] -name = "depyf" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astor" }, - { name = "dill" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f9/ee/43a4cbba615abfc1eb2e5ff5eed3f80f38d58645b4d13d0ea06b9ca1909d/depyf-0.18.0.tar.gz", hash = "sha256:b99f0c383be949ae45d5d606fe444c71f375b55a57b8d6b20e7856670d52130d", size = 43050, upload-time = "2024-12-07T00:42:40.198Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/d8/efc291d5c69a9905515055d23977643dd0d482ebfeb0dbabef1947ee75d8/depyf-0.18.0-py3-none-any.whl", hash = "sha256:007294d5bac19a38a0767d747be0f49b9ffdcea0394a822644142df22b33a3e1", size = 38839, upload-time = "2024-12-07T00:42:38.83Z" }, -] - -[[package]] -name = "dill" -version = "0.3.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/17/4d/ac7ffa80c69ea1df30a8aa11b3578692a5118e7cd1aa157e3ef73b092d15/dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", size = 184847, upload-time = "2024-01-27T23:42:16.145Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, -] - -[[package]] -name = "dirhash" -version = "0.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "scantree" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/1f/c8bf92552b7f0a13b9f12b85e3de8df6d9814240e0f8ce8f37433df028b3/dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09", size = 13119, upload-time = "2024-08-03T22:14:11.688Z" }, -] - -[[package]] -name = "diskcache" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, -] - -[[package]] -name = "distlib" -version = "0.3.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0d/dd/1bec4c5ddb504ca60fc29472f3d27e8d4da1257a854e1d96742f15c1d02d/distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403", size = 613923, upload-time = "2024-10-09T18:35:47.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973, upload-time = "2024-10-09T18:35:44.272Z" }, -] - -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - -[[package]] -name = "dnspython" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/4a/263763cb2ba3816dd94b08ad3a33d5fdae34ecb856678773cc40a3605829/dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1", size = 345197, upload-time = "2024-10-05T20:14:59.362Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632, upload-time = "2024-10-05T20:14:57.687Z" }, -] - -[[package]] -name = "docker" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "requests" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, -] - -[[package]] -name = "docker-pycreds" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/e6/d1f6c00b7221e2d7c4b470132c931325c8b22c51ca62417e300f5ce16009/docker-pycreds-0.4.0.tar.gz", hash = "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", size = 8754, upload-time = "2018-11-29T03:26:50.996Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl", hash = "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49", size = 8982, upload-time = "2018-11-29T03:26:49.575Z" }, -] - -[[package]] -name = "docstring-parser" -version = "0.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/12/9c22a58c0b1e29271051222d8906257616da84135af9ed167c9e28f85cb3/docstring_parser-0.16.tar.gz", hash = "sha256:538beabd0af1e2db0146b6bd3caa526c35a34d61af9fd2887f3a8a27a739aa6e", size = 26565, upload-time = "2024-03-15T10:39:44.419Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/7c/e9fcff7623954d86bdc17782036cbf715ecab1bec4847c008557affe1ca8/docstring_parser-0.16-py3-none-any.whl", hash = "sha256:bf0a1387354d3691d102edef7ec124f219ef639982d096e26e3b60aeffa90637", size = 36533, upload-time = "2024-03-15T10:39:41.527Z" }, -] - -[[package]] -name = "e2b" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "httpcore" }, - { name = "httpx" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "python-dateutil" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/de/d051c7f6d762e2f743a4d5efebe6b30181c866bb499ec99ac06d3897ff6e/e2b-1.1.0.tar.gz", hash = "sha256:bd054fbaa9baed48919500ba853bdb72c750b04e0bac8365bde75cdfbdf80d18", size = 45977, upload-time = "2025-02-11T02:17:08.758Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/da/c0aea566943a85ab8660e1dd69668769c5a092ae69e957b682000fb63bde/e2b-1.1.0-py3-none-any.whl", hash = "sha256:5d99c675e155cf124f457d77f91c4cb32b286d241ca6cd37ac8d6c0711fc272e", size = 83132, upload-time = "2025-02-11T02:17:06.021Z" }, -] - -[[package]] -name = "einops" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e5/81/df4fbe24dff8ba3934af99044188e20a98ed441ad17a274539b74e82e126/einops-0.8.1.tar.gz", hash = "sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84", size = 54805, upload-time = "2025-02-09T03:17:00.434Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/62/9773de14fe6c45c23649e98b83231fffd7b9892b6cf863251dc2afa73643/einops-0.8.1-py3-none-any.whl", hash = "sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737", size = 64359, upload-time = "2025-02-09T03:17:01.998Z" }, -] - -[[package]] -name = "email-validator" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dnspython" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/ce/13508a1ec3f8bb981ae4ca79ea40384becc868bfae97fd1c942bb3a001b1/email_validator-2.2.0.tar.gz", hash = "sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7", size = 48967, upload-time = "2024-06-20T11:30:30.034Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/ee/bf0adb559ad3c786f12bcbc9296b3f5675f529199bef03e2df281fa1fadb/email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631", size = 33521, upload-time = "2024-06-20T11:30:28.248Z" }, -] - -[[package]] -name = "english-words" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/d1/78b51ad44e4a318ee4f6d32a0b344a918d5fd690de0b0ff6a116b1bc97cf/english-words-2.0.1.tar.gz", hash = "sha256:a4105c57493bb757a3d8973fcf8e1dc05e7ca09c836dff467c3fb445f84bc43d", size = 8151936, upload-time = "2023-05-24T15:10:50.093Z" } - -[[package]] -name = "evaluate" -version = "0.4.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "datasets" }, - { name = "dill" }, - { name = "fsspec", extra = ["http"] }, - { name = "huggingface-hub" }, - { name = "multiprocess" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "xxhash" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/a0/10a56e0939ece94c54276e81459cb4101f46f0e9a6f54fc31a35f64e8854/evaluate-0.4.3.tar.gz", hash = "sha256:3a5700cf83aabee9549264e1e5666f116367c61dbd4d38352015e859a5e2098d", size = 65679, upload-time = "2024-09-11T10:15:32Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/e7/cbca9e2d2590eb9b5aa8f7ebabe1beb1498f9462d2ecede5c9fd9735faaf/evaluate-0.4.3-py3-none-any.whl", hash = "sha256:47d8770bdea76e2c2ed0d40189273027d1a41ccea861bcc7ba12d30ec5d1e517", size = 84010, upload-time = "2024-09-11T10:15:30.018Z" }, -] - -[[package]] -name = "executing" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/50/a9d80c47ff289c611ff12e63f7c5d13942c65d68125160cefd768c73e6e4/executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755", size = 978693, upload-time = "2025-01-22T15:41:29.403Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702, upload-time = "2025-01-22T15:41:25.929Z" }, -] - -[[package]] -name = "faker" -version = "37.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ba/a6/b77f42021308ec8b134502343da882c0905d725a4d661c7adeaf7acaf515/faker-37.1.0.tar.gz", hash = "sha256:ad9dc66a3b84888b837ca729e85299a96b58fdaef0323ed0baace93c9614af06", size = 1875707, upload-time = "2025-03-24T16:14:02.958Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/a1/8936bc8e79af80ca38288dd93ed44ed1f9d63beb25447a4c59e746e01f8d/faker-37.1.0-py3-none-any.whl", hash = "sha256:dc2f730be71cb770e9c715b13374d80dbcee879675121ab51f9683d262ae9a1c", size = 1918783, upload-time = "2025-03-24T16:14:00.051Z" }, -] - -[[package]] -name = "farama-notifications" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/2c/8384832b7a6b1fd6ba95bbdcae26e7137bb3eedc955c42fd5cdcc086cfbf/Farama-Notifications-0.0.4.tar.gz", hash = "sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18", size = 2131, upload-time = "2023-02-27T18:28:41.047Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/2c/ffc08c54c05cdce6fbed2aeebc46348dbe180c6d2c541c7af7ba0aa5f5f8/Farama_Notifications-0.0.4-py3-none-any.whl", hash = "sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae", size = 2511, upload-time = "2023-02-27T18:28:39.447Z" }, -] - -[[package]] -name = "fastapi" -version = "0.115.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/55/ae499352d82338331ca1e28c7f4a63bfd09479b16395dce38cf50a39e2c2/fastapi-0.115.12.tar.gz", hash = "sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681", size = 295236, upload-time = "2025-03-23T22:55:43.822Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/b3/b51f09c2ba432a576fe63758bddc81f78f0c6309d9e5c10d194313bf021e/fastapi-0.115.12-py3-none-any.whl", hash = "sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d", size = 95164, upload-time = "2025-03-23T22:55:42.101Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "email-validator" }, - { name = "fastapi-cli", extra = ["standard"] }, - { name = "httpx" }, - { name = "jinja2" }, - { name = "python-multipart" }, - { name = "uvicorn", extra = ["standard"] }, -] - -[[package]] -name = "fastapi-cli" -version = "0.0.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "rich-toolkit" }, - { name = "typer" }, - { name = "uvicorn", extra = ["standard"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fe/73/82a5831fbbf8ed75905bacf5b2d9d3dfd6f04d6968b29fe6f72a5ae9ceb1/fastapi_cli-0.0.7.tar.gz", hash = "sha256:02b3b65956f526412515907a0793c9094abd4bfb5457b389f645b0ea6ba3605e", size = 16753, upload-time = "2024-12-15T14:28:10.028Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/e6/5daefc851b514ce2287d8f5d358ae4341089185f78f3217a69d0ce3a390c/fastapi_cli-0.0.7-py3-none-any.whl", hash = "sha256:d549368ff584b2804336c61f192d86ddea080c11255f375959627911944804f4", size = 10705, upload-time = "2024-12-15T14:28:06.18Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "uvicorn", extra = ["standard"] }, -] - -[[package]] -name = "fastcore" -version = "1.8.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1b/57/4f2f658ffa0641aa629909d850d7d67df11d4fc809ad9f197f19cc0eab19/fastcore-1.8.1.tar.gz", hash = "sha256:82494b84925fc80bbcfc21c65208dd4c125df70e3d465ef788d75d42cc2afc58", size = 75352, upload-time = "2025-04-05T16:07:27.125Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/9b/93bd0191c4dd1e0e378c08509f63d7694b32e99568209bd81b3ee632df9c/fastcore-1.8.1-py3-none-any.whl", hash = "sha256:51ae5d328d7864c0dd70081b32a852ff242b70af372b4dcec204969e92356afd", size = 78125, upload-time = "2025-04-05T16:07:25.105Z" }, -] - -[[package]] -name = "filelock" -version = "3.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2", size = 18075, upload-time = "2025-03-14T07:11:40.47Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215, upload-time = "2025-03-14T07:11:39.145Z" }, -] - -[[package]] -name = "flake8" -version = "7.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mccabe" }, - { name = "pycodestyle" }, - { name = "pyflakes" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e7/c4/5842fc9fc94584c455543540af62fd9900faade32511fab650e9891ec225/flake8-7.2.0.tar.gz", hash = "sha256:fa558ae3f6f7dbf2b4f22663e5343b6b6023620461f8d4ff2019ef4b5ee70426", size = 48177, upload-time = "2025-03-29T20:08:39.329Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/5c/0627be4c9976d56b1217cb5187b7504e7fd7d3503f8bfd312a04077bd4f7/flake8-7.2.0-py2.py3-none-any.whl", hash = "sha256:93b92ba5bdb60754a6da14fa3b93a9361fd00a59632ada61fd7b130436c40343", size = 57786, upload-time = "2025-03-29T20:08:37.902Z" }, -] - -[[package]] -name = "flash-attn" -version = "v2.7.4.post1+cu12torch2.5cxx11abifalse" -source = { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl" } -dependencies = [ - { name = "einops" }, - { name = "torch" }, -] -wheels = [ - { url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl", hash = "sha256:a496f383c843ed0cc6e01302556a0947f56828f1ec2b54e5297c4c7eaae39357" }, -] - -[package.metadata] -requires-dist = [ - { name = "einops" }, - { name = "torch" }, -] - -[[package]] -name = "flashinfer-python" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ninja" }, - { name = "numpy" }, - { name = "torch" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b6/10/2a63f1d09c5b337705236005dc9ccce513dcc08b7fd037cb40426f1695b1/flashinfer_python-0.2.3.tar.gz", hash = "sha256:f6e44bb80bff6f0a80046e6b357f1a66312b7fef65b79cb525ae1ff7d37abf6a", size = 1789786, upload-time = "2025-03-11T06:38:51.685Z" } - -[[package]] -name = "flask" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "blinker" }, - { name = "click" }, - { name = "itsdangerous" }, - { name = "jinja2" }, - { name = "werkzeug" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/50/dff6380f1c7f84135484e176e0cac8690af72fa90e932ad2a0a60e28c69b/flask-3.1.0.tar.gz", hash = "sha256:5f873c5184c897c8d9d1b05df1e3d01b14910ce69607a117bd3277098a5836ac", size = 680824, upload-time = "2024-11-13T18:24:38.127Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/47/93213ee66ef8fae3b93b3e29206f6b251e65c97bd91d8e1c5596ef15af0a/flask-3.1.0-py3-none-any.whl", hash = "sha256:d667207822eb83f1c4b50949b1623c8fc8d51f2341d65f72e1a1815397551136", size = 102979, upload-time = "2024-11-13T18:24:36.135Z" }, -] - -[[package]] -name = "fonttools" -version = "4.57.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/2d/a9a0b6e3a0cf6bd502e64fc16d894269011930cabfc89aee20d1635b1441/fonttools-4.57.0.tar.gz", hash = "sha256:727ece10e065be2f9dd239d15dd5d60a66e17eac11aea47d447f9f03fdbc42de", size = 3492448, upload-time = "2025-04-03T11:07:13.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/98/d4bc42d43392982eecaaca117d79845734d675219680cd43070bb001bc1f/fonttools-4.57.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:889e45e976c74abc7256d3064aa7c1295aa283c6bb19810b9f8b604dfe5c7f31", size = 2751824, upload-time = "2025-04-03T11:06:03.782Z" }, - { url = "https://files.pythonhosted.org/packages/1a/62/7168030eeca3742fecf45f31e63b5ef48969fa230a672216b805f1d61548/fonttools-4.57.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0425c2e052a5f1516c94e5855dbda706ae5a768631e9fcc34e57d074d1b65b92", size = 2283072, upload-time = "2025-04-03T11:06:05.533Z" }, - { url = "https://files.pythonhosted.org/packages/5d/82/121a26d9646f0986ddb35fbbaf58ef791c25b59ecb63ffea2aab0099044f/fonttools-4.57.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44c26a311be2ac130f40a96769264809d3b0cb297518669db437d1cc82974888", size = 4788020, upload-time = "2025-04-03T11:06:07.249Z" }, - { url = "https://files.pythonhosted.org/packages/5b/26/e0f2fb662e022d565bbe280a3cfe6dafdaabf58889ff86fdef2d31ff1dde/fonttools-4.57.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84c41ba992df5b8d680b89fd84c6a1f2aca2b9f1ae8a67400c8930cd4ea115f6", size = 4859096, upload-time = "2025-04-03T11:06:09.469Z" }, - { url = "https://files.pythonhosted.org/packages/9e/44/9075e323347b1891cdece4b3f10a3b84a8f4c42a7684077429d9ce842056/fonttools-4.57.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ea1e9e43ca56b0c12440a7c689b1350066595bebcaa83baad05b8b2675129d98", size = 4964356, upload-time = "2025-04-03T11:06:11.294Z" }, - { url = "https://files.pythonhosted.org/packages/48/28/caa8df32743462fb966be6de6a79d7f30393859636d7732e82efa09fbbb4/fonttools-4.57.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:84fd56c78d431606332a0627c16e2a63d243d0d8b05521257d77c6529abe14d8", size = 5226546, upload-time = "2025-04-03T11:06:13.6Z" }, - { url = "https://files.pythonhosted.org/packages/f6/46/95ab0f0d2e33c5b1a4fc1c0efe5e286ba9359602c0a9907adb1faca44175/fonttools-4.57.0-cp312-cp312-win32.whl", hash = "sha256:f4376819c1c778d59e0a31db5dc6ede854e9edf28bbfa5b756604727f7f800ac", size = 2146776, upload-time = "2025-04-03T11:06:15.643Z" }, - { url = "https://files.pythonhosted.org/packages/06/5d/1be5424bb305880e1113631f49a55ea7c7da3a5fe02608ca7c16a03a21da/fonttools-4.57.0-cp312-cp312-win_amd64.whl", hash = "sha256:57e30241524879ea10cdf79c737037221f77cc126a8cdc8ff2c94d4a522504b9", size = 2193956, upload-time = "2025-04-03T11:06:17.534Z" }, - { url = "https://files.pythonhosted.org/packages/90/27/45f8957c3132917f91aaa56b700bcfc2396be1253f685bd5c68529b6f610/fonttools-4.57.0-py3-none-any.whl", hash = "sha256:3122c604a675513c68bd24c6a8f9091f1c2376d18e8f5fe5a101746c81b3e98f", size = 1093605, upload-time = "2025-04-03T11:07:11.341Z" }, -] - -[[package]] -name = "frozenlist" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8f/ed/0f4cec13a93c02c47ec32d81d11c0c1efbadf4a471e3f3ce7cad366cbbd3/frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817", size = 39930, upload-time = "2024-10-23T09:48:29.903Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/73/fa6d1a96ab7fd6e6d1c3500700963eab46813847f01ef0ccbaa726181dd5/frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21", size = 94026, upload-time = "2024-10-23T09:46:58.601Z" }, - { url = "https://files.pythonhosted.org/packages/ab/04/ea8bf62c8868b8eada363f20ff1b647cf2e93377a7b284d36062d21d81d1/frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d", size = 54150, upload-time = "2024-10-23T09:46:59.608Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9a/8e479b482a6f2070b26bda572c5e6889bb3ba48977e81beea35b5ae13ece/frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e", size = 51927, upload-time = "2024-10-23T09:47:00.625Z" }, - { url = "https://files.pythonhosted.org/packages/e3/12/2aad87deb08a4e7ccfb33600871bbe8f0e08cb6d8224371387f3303654d7/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a", size = 282647, upload-time = "2024-10-23T09:47:01.992Z" }, - { url = "https://files.pythonhosted.org/packages/77/f2/07f06b05d8a427ea0060a9cef6e63405ea9e0d761846b95ef3fb3be57111/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a", size = 289052, upload-time = "2024-10-23T09:47:04.039Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9f/8bf45a2f1cd4aa401acd271b077989c9267ae8463e7c8b1eb0d3f561b65e/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee", size = 291719, upload-time = "2024-10-23T09:47:05.58Z" }, - { url = "https://files.pythonhosted.org/packages/41/d1/1f20fd05a6c42d3868709b7604c9f15538a29e4f734c694c6bcfc3d3b935/frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6", size = 267433, upload-time = "2024-10-23T09:47:07.807Z" }, - { url = "https://files.pythonhosted.org/packages/af/f2/64b73a9bb86f5a89fb55450e97cd5c1f84a862d4ff90d9fd1a73ab0f64a5/frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e", size = 283591, upload-time = "2024-10-23T09:47:09.645Z" }, - { url = "https://files.pythonhosted.org/packages/29/e2/ffbb1fae55a791fd6c2938dd9ea779509c977435ba3940b9f2e8dc9d5316/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9", size = 273249, upload-time = "2024-10-23T09:47:10.808Z" }, - { url = "https://files.pythonhosted.org/packages/2e/6e/008136a30798bb63618a114b9321b5971172a5abddff44a100c7edc5ad4f/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039", size = 271075, upload-time = "2024-10-23T09:47:11.938Z" }, - { url = "https://files.pythonhosted.org/packages/ae/f0/4e71e54a026b06724cec9b6c54f0b13a4e9e298cc8db0f82ec70e151f5ce/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784", size = 285398, upload-time = "2024-10-23T09:47:14.071Z" }, - { url = "https://files.pythonhosted.org/packages/4d/36/70ec246851478b1c0b59f11ef8ade9c482ff447c1363c2bd5fad45098b12/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631", size = 294445, upload-time = "2024-10-23T09:47:15.318Z" }, - { url = "https://files.pythonhosted.org/packages/37/e0/47f87544055b3349b633a03c4d94b405956cf2437f4ab46d0928b74b7526/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f", size = 280569, upload-time = "2024-10-23T09:47:17.149Z" }, - { url = "https://files.pythonhosted.org/packages/f9/7c/490133c160fb6b84ed374c266f42800e33b50c3bbab1652764e6e1fc498a/frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8", size = 44721, upload-time = "2024-10-23T09:47:19.012Z" }, - { url = "https://files.pythonhosted.org/packages/b1/56/4e45136ffc6bdbfa68c29ca56ef53783ef4c2fd395f7cbf99a2624aa9aaa/frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f", size = 51329, upload-time = "2024-10-23T09:47:20.177Z" }, - { url = "https://files.pythonhosted.org/packages/c6/c8/a5be5b7550c10858fcf9b0ea054baccab474da77d37f1e828ce043a3a5d4/frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3", size = 11901, upload-time = "2024-10-23T09:48:28.851Z" }, -] - -[[package]] -name = "fsspec" -version = "2024.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/90/b6/eba5024a9889fcfff396db543a34bef0ab9d002278f163129f9f01005960/fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49", size = 284584, upload-time = "2024-06-27T14:35:45.467Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/44/73bea497ac69bafde2ee4269292fa3b41f1198f4bb7bbaaabde30ad29d4a/fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e", size = 177561, upload-time = "2024-06-27T14:35:42.023Z" }, -] - -[package.optional-dependencies] -http = [ - { name = "aiohttp" }, -] - -[[package]] -name = "func-timeout" -version = "4.3.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b3/0d/bf0567477f7281d9a3926c582bfef21bff7498fc0ffd3e9de21811896a0b/func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd", size = 44264, upload-time = "2019-08-19T21:32:07.43Z" } - -[[package]] -name = "gdown" -version = "5.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "filelock" }, - { name = "requests", extra = ["socks"] }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/6a/37e6b70c5bda3161e40265861e63b64a86bfc6ca6a8f1c35328a675c84fd/gdown-5.2.0.tar.gz", hash = "sha256:2145165062d85520a3cd98b356c9ed522c5e7984d408535409fd46f94defc787", size = 284647, upload-time = "2024-05-12T06:45:12.725Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/70/e07c381e6488a77094f04c85c9caf1c8008cdc30778f7019bc52e5285ef0/gdown-5.2.0-py3-none-any.whl", hash = "sha256:33083832d82b1101bdd0e9df3edd0fbc0e1c5f14c9d8c38d2a35bf1683b526d6", size = 18235, upload-time = "2024-05-12T06:45:10.017Z" }, -] - -[[package]] -name = "gevent" -version = "24.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, - { name = "zope-event" }, - { name = "zope-interface" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/27/24/a3a7b713acfcf1177207f49ec25c665123f8972f42bee641bcc9f32961f4/gevent-24.2.1.tar.gz", hash = "sha256:432fc76f680acf7cf188c2ee0f5d3ab73b63c1f03114c7cd8a34cebbe5aa2056", size = 6147507, upload-time = "2024-02-14T11:31:10.128Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/72/eb98be1cec2a3d0f46d3af49b034deb48a6d6d9a1958ee110bc2e1e600ac/gevent-24.2.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:6c47ae7d1174617b3509f5d884935e788f325eb8f1a7efc95d295c68d83cce40", size = 3007004, upload-time = "2024-02-14T11:28:20.476Z" }, - { url = "https://files.pythonhosted.org/packages/f7/14/4cc83275fcdfa1977224cc266b710dc71b810d6760f575d259ca3be7b4dd/gevent-24.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7cac622e11b4253ac4536a654fe221249065d9a69feb6cdcd4d9af3503602e0", size = 5142074, upload-time = "2024-02-14T12:09:45.269Z" }, - { url = "https://files.pythonhosted.org/packages/56/ce/583d29e524c5666f7d66116e818449bee649bba8088d0ac48bec6c006215/gevent-24.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bf5b9c72b884c6f0c4ed26ef204ee1f768b9437330422492c319470954bc4cc7", size = 5307651, upload-time = "2024-02-14T12:07:36.645Z" }, - { url = "https://files.pythonhosted.org/packages/69/e7/072dfbf5c534516dcc91367d5dd5806ec8860b66c1df26b9d603493c1adb/gevent-24.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5de3c676e57177b38857f6e3cdfbe8f38d1cd754b63200c0615eaa31f514b4f", size = 5406093, upload-time = "2024-02-14T12:10:58.794Z" }, - { url = "https://files.pythonhosted.org/packages/d9/d3/f9d0f62cb6cb0421d0da2cffd10bad13b0f5d641c57ce35927bf8554661e/gevent-24.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4faf846ed132fd7ebfbbf4fde588a62d21faa0faa06e6f468b7faa6f436b661", size = 6730420, upload-time = "2024-02-14T11:54:02.399Z" }, - { url = "https://files.pythonhosted.org/packages/5b/eb/6b0e902e29283253324fe32317b805df289f05f0ef3e9859a721d403b71e/gevent-24.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:368a277bd9278ddb0fde308e6a43f544222d76ed0c4166e0d9f6b036586819d9", size = 6711332, upload-time = "2024-02-14T11:59:16.68Z" }, - { url = "https://files.pythonhosted.org/packages/0d/8b/02a07125324e23d64ec342ae7a4cff8dc7271114e787317a5f219027bf1b/gevent-24.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f8a04cf0c5b7139bc6368b461257d4a757ea2fe89b3773e494d235b7dd51119f", size = 5482031, upload-time = "2024-02-14T12:25:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/5f/fe/288ccd562ac20d5e4ae2624313b699ee35c76be1faa9104b414bfe714a67/gevent-24.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9d8d0642c63d453179058abc4143e30718b19a85cbf58c2744c9a63f06a1d388", size = 6812353, upload-time = "2024-02-14T12:01:19.819Z" }, - { url = "https://files.pythonhosted.org/packages/2e/90/d9fcdc22864d0cf471630071c264289b9a803892d6f55e895a69c2e3574b/gevent-24.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:94138682e68ec197db42ad7442d3cf9b328069c3ad8e4e5022e6b5cd3e7ffae5", size = 1523715, upload-time = "2024-02-14T11:31:09.195Z" }, -] - -[[package]] -name = "gguf" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pyyaml" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/c4/a159e9f842b0e8b8495b2689af6cf3426f002cf01207ca8134db82fc4088/gguf-0.10.0.tar.gz", hash = "sha256:52a30ef26328b419ffc47d9269fc580c238edf1c8a19b5ea143c323e04a038c1", size = 65704, upload-time = "2024-08-23T08:03:25.599Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/e4/c5f9bd71840ae9afb7e2b7c285ba209f2ef5e9cd83885f8c596c551d3026/gguf-0.10.0-py3-none-any.whl", hash = "sha256:706089fba756a06913227841b4a6c8398360fa991569fd974e663a92b224e33f", size = 71584, upload-time = "2024-08-23T08:03:24.088Z" }, -] - -[[package]] -name = "ghapi" -version = "1.0.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fastcore" }, - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f8/88/97e6b0c94885db3530d04ccab7016c606dcaf08bf0581ced1193b9668d06/ghapi-1.0.6.tar.gz", hash = "sha256:64fdd9f06d8e3373065c42c2a03e067e2bbb9ca18b583cd6e38a28aaad0224f6", size = 65518, upload-time = "2024-08-31T22:38:21.264Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/ad/f7204c0c38175f300621af7880737ca6379dd633e9b7d1c0a8fc2748f0dc/ghapi-1.0.6-py3-none-any.whl", hash = "sha256:b3d96bf18fcaa2cb7131bad9de2948e2a1c2bb226377a25826f6c80950c57854", size = 62391, upload-time = "2024-08-31T22:38:19.34Z" }, -] - -[[package]] -name = "gitdb" -version = "4.0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smmap" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, -] - -[[package]] -name = "gitpython" -version = "3.1.44" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "gitdb" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/89/37df0b71473153574a5cdef8f242de422a0f5d26d7a9e231e6f169b4ad14/gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269", size = 214196, upload-time = "2025-01-02T07:32:43.59Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/9a/4114a9057db2f1462d5c8f8390ab7383925fe1ac012eaa42402ad65c2963/GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110", size = 207599, upload-time = "2025-01-02T07:32:40.731Z" }, -] - -[[package]] -name = "google-ai-generativelanguage" -version = "0.6.15" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core", extra = ["grpc"] }, - { name = "google-auth" }, - { name = "proto-plus" }, - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/11/d1/48fe5d7a43d278e9f6b5ada810b0a3530bbeac7ed7fcbcd366f932f05316/google_ai_generativelanguage-0.6.15.tar.gz", hash = "sha256:8f6d9dc4c12b065fe2d0289026171acea5183ebf2d0b11cefe12f3821e159ec3", size = 1375443, upload-time = "2025-01-13T21:50:47.459Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/a3/67b8a6ff5001a1d8864922f2d6488dc2a14367ceb651bc3f09a947f2f306/google_ai_generativelanguage-0.6.15-py3-none-any.whl", hash = "sha256:5a03ef86377aa184ffef3662ca28f19eeee158733e45d7947982eb953c6ebb6c", size = 1327356, upload-time = "2025-01-13T21:50:44.174Z" }, -] - -[[package]] -name = "google-api-core" -version = "2.24.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/5c/085bcb872556934bb119e5e09de54daa07873f6866b8f0303c49e72287f7/google_api_core-2.24.2.tar.gz", hash = "sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696", size = 163516, upload-time = "2025-03-10T15:55:26.201Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/95/f472d85adab6e538da2025dfca9e976a0d125cc0af2301f190e77b76e51c/google_api_core-2.24.2-py3-none-any.whl", hash = "sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9", size = 160061, upload-time = "2025-03-10T15:55:24.386Z" }, -] - -[package.optional-dependencies] -grpc = [ - { name = "grpcio" }, - { name = "grpcio-status" }, -] - -[[package]] -name = "google-api-python-client" -version = "2.166.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-auth-httplib2" }, - { name = "httplib2" }, - { name = "uritemplate" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/c9/eac7b4e843039f0a54a563c2328d43de6f02e426a11b6a7e378996f667db/google_api_python_client-2.166.0.tar.gz", hash = "sha256:b8cf843bd9d736c134aef76cf1dc7a47c9283a2ef24267b97207b9dd43b30ef7", size = 12680525, upload-time = "2025-03-26T20:15:34.016Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/44/ae1528a6ca296d89704c8febb72b3e263c28b4e50ab29b9202df7a0f273d/google_api_python_client-2.166.0-py2.py3-none-any.whl", hash = "sha256:dd8cc74d9fc18538ab05cbd2e93cb4f82382f910c5f6945db06c91f1deae6e45", size = 13190078, upload-time = "2025-03-26T20:15:29.647Z" }, -] - -[[package]] -name = "google-auth" -version = "2.38.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "pyasn1-modules" }, - { name = "rsa" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/eb/d504ba1daf190af6b204a9d4714d457462b486043744901a6eeea711f913/google_auth-2.38.0.tar.gz", hash = "sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4", size = 270866, upload-time = "2025-01-23T01:05:29.119Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/47/603554949a37bca5b7f894d51896a9c534b9eab808e2520a748e081669d0/google_auth-2.38.0-py2.py3-none-any.whl", hash = "sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a", size = 210770, upload-time = "2025-01-23T01:05:26.572Z" }, -] - -[[package]] -name = "google-auth-httplib2" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "httplib2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/56/be/217a598a818567b28e859ff087f347475c807a5649296fb5a817c58dacef/google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05", size = 10842, upload-time = "2023-12-12T17:40:30.722Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/8a/fe34d2f3f9470a27b01c9e76226965863f153d5fbe276f83608562e49c04/google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d", size = 9253, upload-time = "2023-12-12T17:40:13.055Z" }, -] - -[[package]] -name = "google-auth-oauthlib" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "requests-oauthlib" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/0f/1772edb8d75ecf6280f1c7f51cbcebe274e8b17878b382f63738fd96cee5/google_auth_oauthlib-1.2.1.tar.gz", hash = "sha256:afd0cad092a2eaa53cd8e8298557d6de1034c6cb4a740500b5357b648af97263", size = 24970, upload-time = "2024-07-08T23:11:24.377Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/8e/22a28dfbd218033e4eeaf3a0533b2b54852b6530da0c0fe934f0cc494b29/google_auth_oauthlib-1.2.1-py2.py3-none-any.whl", hash = "sha256:2d58a27262d55aa1b87678c3ba7142a080098cbc2024f903c62355deb235d91f", size = 24930, upload-time = "2024-07-08T23:11:23.038Z" }, -] - -[[package]] -name = "google-cloud-aiplatform" -version = "1.88.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docstring-parser" }, - { name = "google-api-core", extra = ["grpc"] }, - { name = "google-auth" }, - { name = "google-cloud-bigquery" }, - { name = "google-cloud-resource-manager" }, - { name = "google-cloud-storage" }, - { name = "packaging" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "shapely" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/68/d0/4304a06cbbb2583f89c50f85b2a6069217019d457be3665ea79b42fbcf9b/google_cloud_aiplatform-1.88.0.tar.gz", hash = "sha256:07a06549f97a98e4d67d193ec225c581415f142b6a1d3aa4270187d429045d52", size = 9067321, upload-time = "2025-04-09T02:05:42.683Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/59/6a0921180a825e0bf034b39a591446ca17ff5fafa194b96cd0e502d0ca99/google_cloud_aiplatform-1.88.0-py2.py3-none-any.whl", hash = "sha256:7d632577b758021a1306e2c6d3e837e41da635b651776041d4a54cdbbf63f6f1", size = 7571904, upload-time = "2025-04-09T02:05:39.79Z" }, -] - -[[package]] -name = "google-cloud-bigquery" -version = "3.31.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core", extra = ["grpc"] }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-resumable-media" }, - { name = "packaging" }, - { name = "python-dateutil" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/91/4c7274f4d5faf13ac000b06353deaf3579575bf0e4bbad07fa68b9f09ba9/google_cloud_bigquery-3.31.0.tar.gz", hash = "sha256:b89dc716dbe4abdb7a4f873f7050100287bc98514e0614c5d54cd6a8e9fb0991", size = 479961, upload-time = "2025-03-25T18:54:40.43Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/bc/4cb8c61fc6dd817a4a390b745ec7b305f4578f547a16d09d54c8a790624b/google_cloud_bigquery-3.31.0-py3-none-any.whl", hash = "sha256:97f4a3219854ff01d6a3a57312feecb0b6e13062226b823f867e2d3619c4787b", size = 250099, upload-time = "2025-03-25T18:54:38.241Z" }, -] - -[[package]] -name = "google-cloud-core" -version = "2.4.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/b8/2b53838d2acd6ec6168fd284a990c76695e84c65deee79c9f3a4276f6b4f/google_cloud_core-2.4.3.tar.gz", hash = "sha256:1fab62d7102844b278fe6dead3af32408b1df3eb06f5c7e8634cbd40edc4da53", size = 35861, upload-time = "2025-03-10T21:05:38.948Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/86/bda7241a8da2d28a754aad2ba0f6776e35b67e37c36ae0c45d49370f1014/google_cloud_core-2.4.3-py2.py3-none-any.whl", hash = "sha256:5130f9f4c14b4fafdff75c79448f9495cfade0d8775facf1b09c3bf67e027f6e", size = 29348, upload-time = "2025-03-10T21:05:37.785Z" }, -] - -[[package]] -name = "google-cloud-resource-manager" -version = "1.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core", extra = ["grpc"] }, - { name = "google-auth" }, - { name = "grpc-google-iam-v1" }, - { name = "proto-plus" }, - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6e/ca/a4648f5038cb94af4b3942815942a03aa9398f9fb0bef55b3f1585b9940d/google_cloud_resource_manager-1.14.2.tar.gz", hash = "sha256:962e2d904c550d7bac48372607904ff7bb3277e3bb4a36d80cc9a37e28e6eb74", size = 446370, upload-time = "2025-03-17T11:35:56.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/ea/a92631c358da377af34d3a9682c97af83185c2d66363d5939ab4a1169a7f/google_cloud_resource_manager-1.14.2-py3-none-any.whl", hash = "sha256:d0fa954dedd1d2b8e13feae9099c01b8aac515b648e612834f9942d2795a9900", size = 394344, upload-time = "2025-03-17T11:35:54.722Z" }, -] - -[[package]] -name = "google-cloud-storage" -version = "2.19.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/76/4d965702e96bb67976e755bed9828fa50306dca003dbee08b67f41dd265e/google_cloud_storage-2.19.0.tar.gz", hash = "sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2", size = 5535488, upload-time = "2024-12-05T01:35:06.49Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/94/6db383d8ee1adf45dc6c73477152b82731fa4c4a46d9c1932cc8757e0fd4/google_cloud_storage-2.19.0-py2.py3-none-any.whl", hash = "sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba", size = 131787, upload-time = "2024-12-05T01:35:04.736Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/ae/87802e6d9f9d69adfaedfcfd599266bf386a54d0be058b532d04c794f76d/google_crc32c-1.7.1.tar.gz", hash = "sha256:2bff2305f98846f3e825dbeec9ee406f89da7962accdb29356e4eadc251bd472", size = 14495, upload-time = "2025-03-26T14:29:13.32Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/b7/787e2453cf8639c94b3d06c9d61f512234a82e1d12d13d18584bd3049904/google_crc32c-1.7.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2d73a68a653c57281401871dd4aeebbb6af3191dcac751a76ce430df4d403194", size = 30470, upload-time = "2025-03-26T14:34:31.655Z" }, - { url = "https://files.pythonhosted.org/packages/ed/b4/6042c2b0cbac3ec3a69bb4c49b28d2f517b7a0f4a0232603c42c58e22b44/google_crc32c-1.7.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:22beacf83baaf59f9d3ab2bbb4db0fb018da8e5aebdce07ef9f09fce8220285e", size = 30315, upload-time = "2025-03-26T15:01:54.634Z" }, - { url = "https://files.pythonhosted.org/packages/29/ad/01e7a61a5d059bc57b702d9ff6a18b2585ad97f720bd0a0dbe215df1ab0e/google_crc32c-1.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19eafa0e4af11b0a4eb3974483d55d2d77ad1911e6cf6f832e1574f6781fd337", size = 33180, upload-time = "2025-03-26T14:41:32.168Z" }, - { url = "https://files.pythonhosted.org/packages/3b/a5/7279055cf004561894ed3a7bfdf5bf90a53f28fadd01af7cd166e88ddf16/google_crc32c-1.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d86616faaea68101195c6bdc40c494e4d76f41e07a37ffdef270879c15fb65", size = 32794, upload-time = "2025-03-26T14:41:33.264Z" }, - { url = "https://files.pythonhosted.org/packages/0f/d6/77060dbd140c624e42ae3ece3df53b9d811000729a5c821b9fd671ceaac6/google_crc32c-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:b7491bdc0c7564fcf48c0179d2048ab2f7c7ba36b84ccd3a3e1c3f7a72d3bba6", size = 33477, upload-time = "2025-03-26T14:29:10.94Z" }, -] - -[[package]] -name = "google-generativeai" -version = "0.8.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-ai-generativelanguage" }, - { name = "google-api-core" }, - { name = "google-api-python-client" }, - { name = "google-auth" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/b0/6c6af327a8a6ef3be6fe79be1d6f1e2914d6c363aa6b081b93396f4460a7/google_generativeai-0.8.4-py3-none-any.whl", hash = "sha256:e987b33ea6decde1e69191ddcaec6ef974458864d243de7191db50c21a7c5b82", size = 175409, upload-time = "2025-01-21T00:51:50.361Z" }, -] - -[[package]] -name = "google-resumable-media" -version = "2.7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/5a/0efdc02665dca14e0837b62c8a1a93132c264bd02054a15abb2218afe0ae/google_resumable_media-2.7.2.tar.gz", hash = "sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0", size = 2163099, upload-time = "2024-08-07T22:20:38.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/35/b8d3baf8c46695858cb9d8835a53baa1eeb9906ddaf2f728a5f5b640fd1e/google_resumable_media-2.7.2-py2.py3-none-any.whl", hash = "sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa", size = 81251, upload-time = "2024-08-07T22:20:36.409Z" }, -] - -[[package]] -name = "googleapis-common-protos" -version = "1.69.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1b/d7/ee9d56af4e6dbe958562b5020f46263c8a4628e7952070241fc0e9b182ae/googleapis_common_protos-1.69.2.tar.gz", hash = "sha256:3e1b904a27a33c821b4b749fd31d334c0c9c30e6113023d495e48979a3dc9c5f", size = 144496, upload-time = "2025-03-17T11:40:16.583Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/53/d35476d547a286506f0a6a634ccf1e5d288fffd53d48f0bd5fef61d68684/googleapis_common_protos-1.69.2-py3-none-any.whl", hash = "sha256:0b30452ff9c7a27d80bfc5718954063e8ab53dd3697093d3bc99581f5fd24212", size = 293215, upload-time = "2025-03-17T11:40:15.234Z" }, -] - -[package.optional-dependencies] -grpc = [ - { name = "grpcio" }, -] - -[[package]] -name = "greenlet" -version = "3.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b6/02/47dbd5e1c9782e6d3f58187fa10789e308403f3fc3a490b3646b2bff6d9f/greenlet-3.0.0.tar.gz", hash = "sha256:19834e3f91f485442adc1ee440171ec5d9a4840a1f7bd5ed97833544719ce10b", size = 174704, upload-time = "2023-10-03T10:34:51.968Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/f9/40cad310a3242ca2c02c29fab6686dc4fe13e5cbd3c931441fa01ff19533/greenlet-3.0.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1482fba7fbed96ea7842b5a7fc11d61727e8be75a077e603e8ab49d24e234383", size = 263664, upload-time = "2023-10-04T18:40:23.849Z" }, - { url = "https://files.pythonhosted.org/packages/01/df/8ce8c5a549fcd4b61d9297dcbdb541745ca7a0b2b9b62eec48b8a3db1dc5/greenlet-3.0.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:be557119bf467d37a8099d91fbf11b2de5eb1fd5fc5b91598407574848dc910f", size = 216898, upload-time = "2023-10-02T21:29:37.73Z" }, - { url = "https://files.pythonhosted.org/packages/ac/27/42fcc07a3a9a2248990afd05fbee6b5c635d780b6229f34752721883c187/greenlet-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73b2f1922a39d5d59cc0e597987300df3396b148a9bd10b76a058a2f2772fc04", size = 655293, upload-time = "2023-10-02T19:42:07.638Z" }, - { url = "https://files.pythonhosted.org/packages/93/ee/55332b8273baa53eea54b81d6e80c30a7f59534df80fe6dec8355a30841f/greenlet-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1e22c22f7826096ad503e9bb681b05b8c1f5a8138469b255eb91f26a76634f2", size = 670844, upload-time = "2023-10-02T19:43:26.581Z" }, - { url = "https://files.pythonhosted.org/packages/ee/c8/4e9a97c796953c0bfc98eec624836ccd4c6e564077a944947277218ddb71/greenlet-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d363666acc21d2c204dd8705c0e0457d7b2ee7a76cb16ffc099d6799744ac99", size = 667148, upload-time = "2023-10-02T19:47:13.043Z" }, - { url = "https://files.pythonhosted.org/packages/f0/8a/b5ea17e21597c3325b3af8454fb002591c75553f1bf8a6322bf1c0debce5/greenlet-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334ef6ed8337bd0b58bb0ae4f7f2dcc84c9f116e474bb4ec250a8bb9bd797a66", size = 665483, upload-time = "2023-10-02T19:23:15.786Z" }, - { url = "https://files.pythonhosted.org/packages/b5/32/c638e4fe67e71677630535914ad9fe2d670b57a0a71595ca0cea1889fe98/greenlet-3.0.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6672fdde0fd1a60b44fb1751a7779c6db487e42b0cc65e7caa6aa686874e79fb", size = 620596, upload-time = "2023-10-02T19:23:14.524Z" }, - { url = "https://files.pythonhosted.org/packages/ee/b3/995949dbb0370006ceb3e56f3c5017585a07e2ec483167e2da12abafebfb/greenlet-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:952256c2bc5b4ee8df8dfc54fc4de330970bf5d79253c863fb5e6761f00dda35", size = 1149425, upload-time = "2023-10-02T19:45:47.968Z" }, - { url = "https://files.pythonhosted.org/packages/23/e7/be9bfb1ccc92572114cb090680424a1cc5a268dbfe37d56439f34452fec7/greenlet-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:269d06fa0f9624455ce08ae0179430eea61085e3cf6457f05982b37fd2cefe17", size = 1177675, upload-time = "2023-10-02T19:24:06.852Z" }, - { url = "https://files.pythonhosted.org/packages/d1/d0/390d8adbcf49693ff6465397b49ccc0b32058268d754da327cd3dfbfdbaf/greenlet-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9adbd8ecf097e34ada8efde9b6fec4dd2a903b1e98037adf72d12993a1c80b51", size = 288477, upload-time = "2023-10-02T19:43:59.328Z" }, -] - -[[package]] -name = "grep-ast" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pathspec" }, - { name = "tree-sitter-languages" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/51/31/728334e2f053b4a6c4cd2912b3e40218f9e12e158c2039a458814dd22e1f/grep_ast-0.3.3.tar.gz", hash = "sha256:42b8887d57301dc55634368f8d549e9c49c913dafb4d19c9b54c3ddb604fccf4", size = 11140, upload-time = "2024-07-30T15:36:25.493Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/9d/3db2dff3abe7706ba4c71fcb4f60ac5f8049a476c56392a8f6942de6c3fc/grep_ast-0.3.3-py3-none-any.whl", hash = "sha256:515cb889bffefefa26c4ab1377b9a75b3fc678aa5fa02bf9aa4f8f20999a83ad", size = 11833, upload-time = "2024-07-30T15:36:23.864Z" }, -] - -[[package]] -name = "grpc-google-iam-v1" -version = "0.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos", extra = ["grpc"] }, - { name = "grpcio" }, - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/4e/8d0ca3b035e41fe0b3f31ebbb638356af720335e5a11154c330169b40777/grpc_google_iam_v1-0.14.2.tar.gz", hash = "sha256:b3e1fc387a1a329e41672197d0ace9de22c78dd7d215048c4c78712073f7bd20", size = 16259, upload-time = "2025-03-17T11:40:23.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/6f/dd9b178aee7835b96c2e63715aba6516a9d50f6bebbd1cc1d32c82a2a6c3/grpc_google_iam_v1-0.14.2-py3-none-any.whl", hash = "sha256:a3171468459770907926d56a440b2bb643eec1d7ba215f48f3ecece42b4d8351", size = 19242, upload-time = "2025-03-17T11:40:22.648Z" }, -] - -[[package]] -name = "grpcio" -version = "1.71.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/95/aa11fc09a85d91fbc7dd405dcb2a1e0256989d67bf89fa65ae24b3ba105a/grpcio-1.71.0.tar.gz", hash = "sha256:2b85f7820475ad3edec209d3d89a7909ada16caab05d3f2e08a7e8ae3200a55c", size = 12549828, upload-time = "2025-03-10T19:28:49.203Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/83/bd4b6a9ba07825bd19c711d8b25874cd5de72c2a3fbf635c3c344ae65bd2/grpcio-1.71.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:0ff35c8d807c1c7531d3002be03221ff9ae15712b53ab46e2a0b4bb271f38537", size = 5184101, upload-time = "2025-03-10T19:24:54.11Z" }, - { url = "https://files.pythonhosted.org/packages/31/ea/2e0d90c0853568bf714693447f5c73272ea95ee8dad107807fde740e595d/grpcio-1.71.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:b78a99cd1ece4be92ab7c07765a0b038194ded2e0a26fd654591ee136088d8d7", size = 11310927, upload-time = "2025-03-10T19:24:56.1Z" }, - { url = "https://files.pythonhosted.org/packages/ac/bc/07a3fd8af80467390af491d7dc66882db43884128cdb3cc8524915e0023c/grpcio-1.71.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:dc1a1231ed23caac1de9f943d031f1bc38d0f69d2a3b243ea0d664fc1fbd7fec", size = 5654280, upload-time = "2025-03-10T19:24:58.55Z" }, - { url = "https://files.pythonhosted.org/packages/16/af/21f22ea3eed3d0538b6ef7889fce1878a8ba4164497f9e07385733391e2b/grpcio-1.71.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6beeea5566092c5e3c4896c6d1d307fb46b1d4bdf3e70c8340b190a69198594", size = 6312051, upload-time = "2025-03-10T19:25:00.682Z" }, - { url = "https://files.pythonhosted.org/packages/49/9d/e12ddc726dc8bd1aa6cba67c85ce42a12ba5b9dd75d5042214a59ccf28ce/grpcio-1.71.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5170929109450a2c031cfe87d6716f2fae39695ad5335d9106ae88cc32dc84c", size = 5910666, upload-time = "2025-03-10T19:25:03.01Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e9/38713d6d67aedef738b815763c25f092e0454dc58e77b1d2a51c9d5b3325/grpcio-1.71.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5b08d03ace7aca7b2fadd4baf291139b4a5f058805a8327bfe9aece7253b6d67", size = 6012019, upload-time = "2025-03-10T19:25:05.174Z" }, - { url = "https://files.pythonhosted.org/packages/80/da/4813cd7adbae6467724fa46c952d7aeac5e82e550b1c62ed2aeb78d444ae/grpcio-1.71.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f903017db76bf9cc2b2d8bdd37bf04b505bbccad6be8a81e1542206875d0e9db", size = 6637043, upload-time = "2025-03-10T19:25:06.987Z" }, - { url = "https://files.pythonhosted.org/packages/52/ca/c0d767082e39dccb7985c73ab4cf1d23ce8613387149e9978c70c3bf3b07/grpcio-1.71.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:469f42a0b410883185eab4689060a20488a1a0a00f8bbb3cbc1061197b4c5a79", size = 6186143, upload-time = "2025-03-10T19:25:08.877Z" }, - { url = "https://files.pythonhosted.org/packages/00/61/7b2c8ec13303f8fe36832c13d91ad4d4ba57204b1c723ada709c346b2271/grpcio-1.71.0-cp312-cp312-win32.whl", hash = "sha256:ad9f30838550695b5eb302add33f21f7301b882937460dd24f24b3cc5a95067a", size = 3604083, upload-time = "2025-03-10T19:25:10.736Z" }, - { url = "https://files.pythonhosted.org/packages/fd/7c/1e429c5fb26122055d10ff9a1d754790fb067d83c633ff69eddcf8e3614b/grpcio-1.71.0-cp312-cp312-win_amd64.whl", hash = "sha256:652350609332de6dac4ece254e5d7e1ff834e203d6afb769601f286886f6f3a8", size = 4272191, upload-time = "2025-03-10T19:25:13.12Z" }, -] - -[[package]] -name = "grpcio-status" -version = "1.62.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7c/d7/013ef01c5a1c2fd0932c27c904934162f69f41ca0f28396d3ffe4d386123/grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485", size = 13063, upload-time = "2024-08-06T00:37:08.003Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/40/972271de05f9315c0d69f9f7ebbcadd83bc85322f538637d11bb8c67803d/grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8", size = 14448, upload-time = "2024-08-06T00:30:15.702Z" }, -] - -[[package]] -name = "grpclib" -version = "0.4.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "h2" }, - { name = "multidict" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/b9/55936e462a5925190d7427e880b3033601d1effd13809b483d13a926061a/grpclib-0.4.7.tar.gz", hash = "sha256:2988ef57c02b22b7a2e8e961792c41ccf97efc2ace91ae7a5b0de03c363823c3", size = 61254, upload-time = "2023-12-24T17:47:41.572Z" } - -[[package]] -name = "gymnasium" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cloudpickle" }, - { name = "farama-notifications" }, - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/90/69/70cd29e9fc4953d013b15981ee71d4c9ef4d8b2183e6ef2fe89756746dce/gymnasium-1.1.1.tar.gz", hash = "sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d", size = 829326, upload-time = "2025-03-06T16:30:36.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/68/2bdc7b46b5f543dd865575f9d19716866bdb76e50dd33b71ed1a3dd8bb42/gymnasium-1.1.1-py3-none-any.whl", hash = "sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a", size = 965410, upload-time = "2025-03-06T16:30:34.443Z" }, -] - -[[package]] -name = "h11" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f5/38/3af3d3633a34a3316095b39c8e8fb4853a28a536e55d347bd8d8e9a14b03/h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", size = 100418, upload-time = "2022-09-25T15:40:01.519Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259, upload-time = "2022-09-25T15:39:59.68Z" }, -] - -[[package]] -name = "h2" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "hpack" }, - { name = "hyperframe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1b/38/d7f80fd13e6582fb8e0df8c9a653dcc02b03ca34f4d72f34869298c5baf8/h2-4.2.0.tar.gz", hash = "sha256:c8a52129695e88b1a0578d8d2cc6842bbd79128ac685463b887ee278126ad01f", size = 2150682, upload-time = "2025-02-02T07:43:51.815Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/9e/984486f2d0a0bd2b024bf4bc1c62688fcafa9e61991f041fb0e2def4a982/h2-4.2.0-py3-none-any.whl", hash = "sha256:479a53ad425bb29af087f3458a61d30780bc818e4ebcf01f0b536ba916462ed0", size = 60957, upload-time = "2025-02-01T11:02:26.481Z" }, -] - -[[package]] -name = "hf-transfer" -version = "0.1.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/eb/8fc64f40388c29ce8ce3b2b180a089d4d6b25b1d0d232d016704cb852104/hf_transfer-0.1.9.tar.gz", hash = "sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf", size = 25201, upload-time = "2025-01-07T10:05:12.947Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/f5/461d2e5f307e5048289b1168d5c642ae3bb2504e88dff1a38b92ed990a21/hf_transfer-0.1.9-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e66acf91df4a8b72f60223059df3003062a5ae111757187ed1a06750a30e911b", size = 1393046, upload-time = "2025-01-07T10:04:51.003Z" }, - { url = "https://files.pythonhosted.org/packages/41/ba/8d9fd9f1083525edfcb389c93738c802f3559cb749324090d7109c8bf4c2/hf_transfer-0.1.9-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:8669dbcc7a3e2e8d61d42cd24da9c50d57770bd74b445c65123291ca842a7e7a", size = 1348126, upload-time = "2025-01-07T10:04:45.712Z" }, - { url = "https://files.pythonhosted.org/packages/8e/a2/cd7885bc9959421065a6fae0fe67b6c55becdeda4e69b873e52976f9a9f0/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fd0167c4407a3bc4cdd0307e65ada2294ec04f1813d8a69a5243e379b22e9d8", size = 3728604, upload-time = "2025-01-07T10:04:14.173Z" }, - { url = "https://files.pythonhosted.org/packages/f6/2e/a072cf196edfeda3310c9a5ade0a0fdd785e6154b3ce24fc738c818da2a7/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f", size = 3064995, upload-time = "2025-01-07T10:04:18.663Z" }, - { url = "https://files.pythonhosted.org/packages/c2/84/aec9ef4c0fab93c1ea2b1badff38c78b4b2f86f0555b26d2051dbc920cde/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5828057e313de59300dd1abb489444bc452efe3f479d3c55b31a8f680936ba42", size = 3580908, upload-time = "2025-01-07T10:04:32.834Z" }, - { url = "https://files.pythonhosted.org/packages/29/63/b560d39651a56603d64f1a0212d0472a44cbd965db2fa62b99d99cb981bf/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d", size = 3400839, upload-time = "2025-01-07T10:04:26.122Z" }, - { url = "https://files.pythonhosted.org/packages/d6/d8/f87ea6f42456254b48915970ed98e993110521e9263472840174d32c880d/hf_transfer-0.1.9-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdca9bfb89e6f8f281890cc61a8aff2d3cecaff7e1a4d275574d96ca70098557", size = 3552664, upload-time = "2025-01-07T10:04:40.123Z" }, - { url = "https://files.pythonhosted.org/packages/d6/56/1267c39b65fc8f4e2113b36297320f102718bf5799b544a6cbe22013aa1d/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:89a23f58b7b7effbc047b8ca286f131b17728c99a9f972723323003ffd1bb916", size = 4073732, upload-time = "2025-01-07T10:04:55.624Z" }, - { url = "https://files.pythonhosted.org/packages/82/1a/9c748befbe3decf7cb415e34f8a0c3789a0a9c55910dea73d581e48c0ce5/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:dc7fff1345980d6c0ebb92c811d24afa4b98b3e07ed070c8e38cc91fd80478c5", size = 3390096, upload-time = "2025-01-07T10:04:59.98Z" }, - { url = "https://files.pythonhosted.org/packages/72/85/4c03da147b6b4b7cb12e074d3d44eee28604a387ed0eaf7eaaead5069c57/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:1a6bd16c667ebe89a069ca163060127a794fa3a3525292c900b8c8cc47985b0d", size = 3664743, upload-time = "2025-01-07T10:05:05.416Z" }, - { url = "https://files.pythonhosted.org/packages/e7/6e/e597b04f753f1b09e6893075d53a82a30c13855cbaa791402695b01e369f/hf_transfer-0.1.9-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d2fde99d502093ade3ab1b53f80da18480e9902aa960dab7f74fb1b9e5bc5746", size = 3695243, upload-time = "2025-01-07T10:05:11.411Z" }, - { url = "https://files.pythonhosted.org/packages/09/89/d4e234727a26b2546c8fb70a276cd924260d60135f2165bf8b9ed67bb9a4/hf_transfer-0.1.9-cp38-abi3-win32.whl", hash = "sha256:435cc3cdc8524ce57b074032b8fd76eed70a4224d2091232fa6a8cef8fd6803e", size = 1086605, upload-time = "2025-01-07T10:05:18.873Z" }, - { url = "https://files.pythonhosted.org/packages/a1/14/f1e15b851d1c2af5b0b1a82bf8eb10bda2da62d98180220ba6fd8879bb5b/hf_transfer-0.1.9-cp38-abi3-win_amd64.whl", hash = "sha256:16f208fc678911c37e11aa7b586bc66a37d02e636208f18b6bc53d29b5df40ad", size = 1160240, upload-time = "2025-01-07T10:05:14.324Z" }, -] - -[[package]] -name = "hpack" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, -] - -[[package]] -name = "html2text" -version = "2024.2.26" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/43/e1d53588561e533212117750ee79ad0ba02a41f52a08c1df3396bd466c05/html2text-2024.2.26.tar.gz", hash = "sha256:05f8e367d15aaabc96415376776cdd11afd5127a77fce6e36afc60c563ca2c32", size = 56527, upload-time = "2024-02-27T18:49:24.855Z" } - -[[package]] -name = "httpcore" -version = "1.0.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/45/ad3e1b4d448f22c0cff4f5692f5ed0666658578e358b8d58a19846048059/httpcore-1.0.8.tar.gz", hash = "sha256:86e94505ed24ea06514883fd44d2bc02d90e77e7979c8eb71b90f41d364a1bad", size = 85385, upload-time = "2025-04-11T14:42:46.661Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/8d/f052b1e336bb2c1fc7ed1aaed898aa570c0b61a09707b108979d9fc6e308/httpcore-1.0.8-py3-none-any.whl", hash = "sha256:5254cf149bcb5f75e9d1b2b9f729ea4a4b883d1ad7379fc632b727cec23674be", size = 78732, upload-time = "2025-04-11T14:42:44.896Z" }, -] - -[[package]] -name = "httplib2" -version = "0.22.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyparsing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81", size = 351116, upload-time = "2023-03-21T22:29:37.214Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", size = 96854, upload-time = "2023-03-21T22:29:35.683Z" }, -] - -[[package]] -name = "httptools" -version = "0.6.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/9a/ce5e1f7e131522e6d3426e8e7a490b3a01f39a6696602e1c4f33f9e94277/httptools-0.6.4.tar.gz", hash = "sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c", size = 240639, upload-time = "2024-10-16T19:45:08.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/0e/d0b71465c66b9185f90a091ab36389a7352985fe857e352801c39d6127c8/httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2", size = 200683, upload-time = "2024-10-16T19:44:30.175Z" }, - { url = "https://files.pythonhosted.org/packages/e2/b8/412a9bb28d0a8988de3296e01efa0bd62068b33856cdda47fe1b5e890954/httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44", size = 104337, upload-time = "2024-10-16T19:44:31.786Z" }, - { url = "https://files.pythonhosted.org/packages/9b/01/6fb20be3196ffdc8eeec4e653bc2a275eca7f36634c86302242c4fbb2760/httptools-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1", size = 508796, upload-time = "2024-10-16T19:44:32.825Z" }, - { url = "https://files.pythonhosted.org/packages/f7/d8/b644c44acc1368938317d76ac991c9bba1166311880bcc0ac297cb9d6bd7/httptools-0.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2", size = 510837, upload-time = "2024-10-16T19:44:33.974Z" }, - { url = "https://files.pythonhosted.org/packages/52/d8/254d16a31d543073a0e57f1c329ca7378d8924e7e292eda72d0064987486/httptools-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81", size = 485289, upload-time = "2024-10-16T19:44:35.111Z" }, - { url = "https://files.pythonhosted.org/packages/5f/3c/4aee161b4b7a971660b8be71a92c24d6c64372c1ab3ae7f366b3680df20f/httptools-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f", size = 489779, upload-time = "2024-10-16T19:44:36.253Z" }, - { url = "https://files.pythonhosted.org/packages/12/b7/5cae71a8868e555f3f67a50ee7f673ce36eac970f029c0c5e9d584352961/httptools-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970", size = 88634, upload-time = "2024-10-16T19:44:37.357Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "huggingface-hub" -version = "0.30.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/22/8eb91736b1dcb83d879bd49050a09df29a57cc5cd9f38e48a4b1c45ee890/huggingface_hub-0.30.2.tar.gz", hash = "sha256:9a7897c5b6fd9dad3168a794a8998d6378210f5b9688d0dfc180b1a228dc2466", size = 400868, upload-time = "2025-04-08T08:32:45.26Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/27/1fb384a841e9661faad1c31cbfa62864f59632e876df5d795234da51c395/huggingface_hub-0.30.2-py3-none-any.whl", hash = "sha256:68ff05969927058cfa41df4f2155d4bb48f5f54f719dd0390103eefa9b191e28", size = 481433, upload-time = "2025-04-08T08:32:43.305Z" }, -] - -[[package]] -name = "hydra-core" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "antlr4-python3-runtime" }, - { name = "omegaconf" }, - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/8e/07e42bc434a847154083b315779b0a81d567154504624e181caf2c71cd98/hydra-core-1.3.2.tar.gz", hash = "sha256:8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824", size = 3263494, upload-time = "2023-02-23T18:33:43.03Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/50/e0edd38dcd63fb26a8547f13d28f7a008bc4a3fd4eb4ff030673f22ad41a/hydra_core-1.3.2-py3-none-any.whl", hash = "sha256:fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b", size = 154547, upload-time = "2023-02-23T18:33:40.801Z" }, -] - -[[package]] -name = "hyperframe" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, -] - -[[package]] -name = "identify" -version = "2.6.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9b/98/a71ab060daec766acc30fb47dfca219d03de34a70d616a79a38c6066c5bf/identify-2.6.9.tar.gz", hash = "sha256:d40dfe3142a1421d8518e3d3985ef5ac42890683e32306ad614a29490abeb6bf", size = 99249, upload-time = "2025-03-08T15:54:13.632Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/ce/0845144ed1f0e25db5e7a79c2354c1da4b5ce392b8966449d5db8dca18f1/identify-2.6.9-py2.py3-none-any.whl", hash = "sha256:c98b4322da415a8e5a70ff6e51fbc2d2932c015532d77e9f8537b4ba7813b150", size = 99101, upload-time = "2025-03-08T15:54:12.026Z" }, -] - -[[package]] -name = "idna" -version = "3.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, -] - -[[package]] -name = "imageio" -version = "2.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pillow" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0c/47/57e897fb7094afb2d26e8b2e4af9a45c7cf1a405acdeeca001fdf2c98501/imageio-2.37.0.tar.gz", hash = "sha256:71b57b3669666272c818497aebba2b4c5f20d5b37c81720e5e1a56d59c492996", size = 389963, upload-time = "2025-01-20T02:42:37.089Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/bd/b394387b598ed84d8d0fa90611a90bee0adc2021820ad5729f7ced74a8e2/imageio-2.37.0-py3-none-any.whl", hash = "sha256:11efa15b87bc7871b61590326b2d635439acc321cf7f8ce996f812543ce10eed", size = 315796, upload-time = "2025-01-20T02:42:34.931Z" }, -] - -[[package]] -name = "importlib-metadata" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a0/fc/c4e6078d21fc4fa56300a241b87eae76766aa380a23fc450fc85bb7bf547/importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2", size = 52120, upload-time = "2024-03-20T19:51:32.429Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", hash = "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", size = 24409, upload-time = "2024-03-20T19:51:30.241Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, -] - -[[package]] -name = "interegular" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/9d/8b6dde58a028a3962ce17e84d5fe73758df61378e00ef8ac3d85da34b0ff/interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600", size = 24705, upload-time = "2024-01-06T23:01:22.372Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/01/72d6472f80651673716d1deda2a5bbb633e563ecf94f4479da5519d69d25/interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c", size = 23635, upload-time = "2024-01-06T23:01:20.829Z" }, -] - -[[package]] -name = "ipykernel" -version = "6.29.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "appnope", marker = "sys_platform == 'darwin'" }, - { name = "comm" }, - { name = "debugpy" }, - { name = "ipython" }, - { name = "jupyter-client" }, - { name = "jupyter-core" }, - { name = "matplotlib-inline" }, - { name = "nest-asyncio" }, - { name = "packaging" }, - { name = "psutil" }, - { name = "pyzmq" }, - { name = "tornado" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/5c/67594cb0c7055dc50814b21731c22a601101ea3b1b50a9a1b090e11f5d0f/ipykernel-6.29.5.tar.gz", hash = "sha256:f093a22c4a40f8828f8e330a9c297cb93dcab13bd9678ded6de8e5cf81c56215", size = 163367, upload-time = "2024-07-01T14:07:22.543Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/5c/368ae6c01c7628438358e6d337c19b05425727fbb221d2a3c4303c372f42/ipykernel-6.29.5-py3-none-any.whl", hash = "sha256:afdb66ba5aa354b09b91379bac28ae4afebbb30e8b39510c9690afb7a10421b5", size = 117173, upload-time = "2024-07-01T14:07:19.603Z" }, -] - -[[package]] -name = "ipython" -version = "9.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/70/9a/6b8984bedc990f3a4aa40ba8436dea27e23d26a64527de7c2e5e12e76841/ipython-9.1.0.tar.gz", hash = "sha256:a47e13a5e05e02f3b8e1e7a0f9db372199fe8c3763532fe7a1e0379e4e135f16", size = 4373688, upload-time = "2025-04-07T10:18:28.704Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/9d/4ff2adf55d1b6e3777b0303fdbe5b723f76e46cba4a53a32fe82260d2077/ipython-9.1.0-py3-none-any.whl", hash = "sha256:2df07257ec2f84a6b346b8d83100bcf8fa501c6e01ab75cd3799b0bb253b3d2a", size = 604053, upload-time = "2025-04-07T10:18:24.869Z" }, -] - -[[package]] -name = "ipython-pygments-lexers" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, -] - -[[package]] -name = "ipywidgets" -version = "8.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "comm" }, - { name = "ipython" }, - { name = "jupyterlab-widgets" }, - { name = "traitlets" }, - { name = "widgetsnbextension" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/aa/98/4074d9cb7e89f7ee387b41e9a4b74c8e0d6196e90b910af1cc674e1cdd3d/ipywidgets-8.1.6.tar.gz", hash = "sha256:d8ace49c66f14419fc66071371b99d01bed230bbc15d8a60233b18bfbd782851", size = 116764, upload-time = "2025-04-10T13:02:35.733Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/b8/62952729573d983d9433faacf62a52ee2e8cf46504418061ad1739967abe/ipywidgets-8.1.6-py3-none-any.whl", hash = "sha256:446e7630a1d025bdc7635e1169fcc06f2ce33b5bd41c2003edeb4a47c8d4bbb1", size = 139808, upload-time = "2025-04-10T13:02:33.904Z" }, -] - -[[package]] -name = "itsdangerous" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, -] - -[[package]] -name = "jedi" -version = "0.19.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parso" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "jiter" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1e/c2/e4562507f52f0af7036da125bb699602ead37a2332af0788f8e0a3417f36/jiter-0.9.0.tar.gz", hash = "sha256:aadba0964deb424daa24492abc3d229c60c4a31bfee205aedbf1acc7639d7893", size = 162604, upload-time = "2025-03-10T21:37:03.278Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/d7/c55086103d6f29b694ec79156242304adf521577530d9031317ce5338c59/jiter-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7b46249cfd6c48da28f89eb0be3f52d6fdb40ab88e2c66804f546674e539ec11", size = 309203, upload-time = "2025-03-10T21:35:44.852Z" }, - { url = "https://files.pythonhosted.org/packages/b0/01/f775dfee50beb420adfd6baf58d1c4d437de41c9b666ddf127c065e5a488/jiter-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:609cf3c78852f1189894383cf0b0b977665f54cb38788e3e6b941fa6d982c00e", size = 319678, upload-time = "2025-03-10T21:35:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/ab/b8/09b73a793714726893e5d46d5c534a63709261af3d24444ad07885ce87cb/jiter-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d726a3890a54561e55a9c5faea1f7655eda7f105bd165067575ace6e65f80bb2", size = 341816, upload-time = "2025-03-10T21:35:47.856Z" }, - { url = "https://files.pythonhosted.org/packages/35/6f/b8f89ec5398b2b0d344257138182cc090302854ed63ed9c9051e9c673441/jiter-0.9.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2e89dc075c1fef8fa9be219e249f14040270dbc507df4215c324a1839522ea75", size = 364152, upload-time = "2025-03-10T21:35:49.397Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/978cc3183113b8e4484cc7e210a9ad3c6614396e7abd5407ea8aa1458eef/jiter-0.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04e8ffa3c353b1bc4134f96f167a2082494351e42888dfcf06e944f2729cbe1d", size = 406991, upload-time = "2025-03-10T21:35:50.745Z" }, - { url = "https://files.pythonhosted.org/packages/13/3a/72861883e11a36d6aa314b4922125f6ae90bdccc225cd96d24cc78a66385/jiter-0.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:203f28a72a05ae0e129b3ed1f75f56bc419d5f91dfacd057519a8bd137b00c42", size = 395824, upload-time = "2025-03-10T21:35:52.162Z" }, - { url = "https://files.pythonhosted.org/packages/87/67/22728a86ef53589c3720225778f7c5fdb617080e3deaed58b04789418212/jiter-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fca1a02ad60ec30bb230f65bc01f611c8608b02d269f998bc29cca8619a919dc", size = 351318, upload-time = "2025-03-10T21:35:53.566Z" }, - { url = "https://files.pythonhosted.org/packages/69/b9/f39728e2e2007276806d7a6609cda7fac44ffa28ca0d02c49a4f397cc0d9/jiter-0.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:237e5cee4d5d2659aaf91bbf8ec45052cc217d9446070699441a91b386ae27dc", size = 384591, upload-time = "2025-03-10T21:35:54.95Z" }, - { url = "https://files.pythonhosted.org/packages/eb/8f/8a708bc7fd87b8a5d861f1c118a995eccbe6d672fe10c9753e67362d0dd0/jiter-0.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:528b6b71745e7326eed73c53d4aa57e2a522242320b6f7d65b9c5af83cf49b6e", size = 520746, upload-time = "2025-03-10T21:35:56.444Z" }, - { url = "https://files.pythonhosted.org/packages/95/1e/65680c7488bd2365dbd2980adaf63c562d3d41d3faac192ebc7ef5b4ae25/jiter-0.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9f48e86b57bc711eb5acdfd12b6cb580a59cc9a993f6e7dcb6d8b50522dcd50d", size = 512754, upload-time = "2025-03-10T21:35:58.789Z" }, - { url = "https://files.pythonhosted.org/packages/78/f3/fdc43547a9ee6e93c837685da704fb6da7dba311fc022e2766d5277dfde5/jiter-0.9.0-cp312-cp312-win32.whl", hash = "sha256:699edfde481e191d81f9cf6d2211debbfe4bd92f06410e7637dffb8dd5dfde06", size = 207075, upload-time = "2025-03-10T21:36:00.616Z" }, - { url = "https://files.pythonhosted.org/packages/cd/9d/742b289016d155f49028fe1bfbeb935c9bf0ffeefdf77daf4a63a42bb72b/jiter-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:099500d07b43f61d8bd780466d429c45a7b25411b334c60ca875fa775f68ccb0", size = 207999, upload-time = "2025-03-10T21:36:02.366Z" }, -] - -[[package]] -name = "jmespath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, -] - -[[package]] -name = "joblib" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/33/60135848598c076ce4b231e1b1895170f45fbcaeaa2c9d5e38b04db70c35/joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e", size = 2116621, upload-time = "2024-05-02T12:15:05.765Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/29/df4b9b42f2be0b623cbd5e2140cafcaa2bef0759a00b7b70104dcfe2fb51/joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6", size = 301817, upload-time = "2024-05-02T12:15:00.765Z" }, -] - -[[package]] -name = "json-repair" -version = "0.41.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/79/bfe51a451f6504e2fe40ccc01c59735736cd05cdcea8421fa2745049caca/json_repair-0.41.0.tar.gz", hash = "sha256:c9ee227d949d5da463024942b715e0e331f9f96cb7377e2f85a661b21596c50b", size = 31167, upload-time = "2025-04-10T09:29:47.208Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/f3/07804982ee14f80f480796949c9d218ac6385e35b2e5597ef4fe6e2bd51c/json_repair-0.41.0-py3-none-any.whl", hash = "sha256:e3be820988fa210fe046e54c8475eafa9e069f1b85d67725f842eb00d1b61f21", size = 21705, upload-time = "2025-04-10T09:29:45.946Z" }, -] - -[[package]] -name = "jsonschema" -version = "4.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/2e/03362ee4034a4c917f697890ccd4aec0800ccf9ded7f511971c75451deec/jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", size = 325778, upload-time = "2024-07-08T18:40:05.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/4a/4f9dbeb84e8850557c02365a0eee0649abe5eb1d84af92a25731c6c0f922/jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566", size = 88462, upload-time = "2024-07-08T18:40:00.165Z" }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2024.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/db/58f950c996c793472e336ff3655b13fbcf1e3b359dcf52dcf3ed3b52c352/jsonschema_specifications-2024.10.1.tar.gz", hash = "sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272", size = 15561, upload-time = "2024-10-08T12:29:32.068Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/0f/8910b19ac0670a0f80ce1008e5e751c4a57e14d2c4c13a482aa6079fa9d6/jsonschema_specifications-2024.10.1-py3-none-any.whl", hash = "sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf", size = 18459, upload-time = "2024-10-08T12:29:30.439Z" }, -] - -[[package]] -name = "jupyter-client" -version = "8.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-core" }, - { name = "python-dateutil" }, - { name = "pyzmq" }, - { name = "tornado" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/22/bf9f12fdaeae18019a468b68952a60fe6dbab5d67cd2a103cac7659b41ca/jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419", size = 342019, upload-time = "2024-09-17T10:44:17.613Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/85/b0394e0b6fcccd2c1eeefc230978a6f8cb0c5df1e4cd3e7625735a0d7d1e/jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f", size = 106105, upload-time = "2024-09-17T10:44:15.218Z" }, -] - -[[package]] -name = "jupyter-core" -version = "5.7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "platformdirs" }, - { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/00/11/b56381fa6c3f4cc5d2cf54a7dbf98ad9aa0b339ef7a601d6053538b079a7/jupyter_core-5.7.2.tar.gz", hash = "sha256:aa5f8d32bbf6b431ac830496da7392035d6f61b4f54872f15c4bd2a9c3f536d9", size = 87629, upload-time = "2024-03-12T12:37:35.652Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/fb/108ecd1fe961941959ad0ee4e12ee7b8b1477247f30b1fdfd83ceaf017f0/jupyter_core-5.7.2-py3-none-any.whl", hash = "sha256:4f7315d2f6b4bcf2e3e7cb6e46772eba760ae459cd1f59d29eb57b0a01bd7409", size = 28965, upload-time = "2024-03-12T12:37:32.36Z" }, -] - -[[package]] -name = "jupyterlab-widgets" -version = "3.0.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/94/766b8199e8a902a4c5ee12a9407a348bbabe9fa22400758576b153d17d8e/jupyterlab_widgets-3.0.14.tar.gz", hash = "sha256:bad03e59546869f026e537e0d170e454259e6dc7048e14041707ca31e523c8a1", size = 203815, upload-time = "2025-04-10T13:00:40.522Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/7a/f2479ba401e02f7fcbd3fc6af201eac888eaa188574b8e9df19452ab4972/jupyterlab_widgets-3.0.14-py3-none-any.whl", hash = "sha256:54c33e3306b7fca139d165d6190dc6c0627aafa5d14adfc974a4e9a3d26cb703", size = 213999, upload-time = "2025-04-10T13:00:38.626Z" }, -] - -[[package]] -name = "kiwisolver" -version = "1.4.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/59/7c91426a8ac292e1cdd53a63b6d9439abd573c875c3f92c146767dd33faf/kiwisolver-1.4.8.tar.gz", hash = "sha256:23d5f023bdc8c7e54eb65f03ca5d5bb25b601eac4d7f1a042888a1f45237987e", size = 97538, upload-time = "2024-12-24T18:30:51.519Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/aa/cea685c4ab647f349c3bc92d2daf7ae34c8e8cf405a6dcd3a497f58a2ac3/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6af5e8815fd02997cb6ad9bbed0ee1e60014438ee1a5c2444c96f87b8843502", size = 124152, upload-time = "2024-12-24T18:29:16.85Z" }, - { url = "https://files.pythonhosted.org/packages/c5/0b/8db6d2e2452d60d5ebc4ce4b204feeb16176a851fd42462f66ade6808084/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bade438f86e21d91e0cf5dd7c0ed00cda0f77c8c1616bd83f9fc157fa6760d31", size = 66555, upload-time = "2024-12-24T18:29:19.146Z" }, - { url = "https://files.pythonhosted.org/packages/60/26/d6a0db6785dd35d3ba5bf2b2df0aedc5af089962c6eb2cbf67a15b81369e/kiwisolver-1.4.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b83dc6769ddbc57613280118fb4ce3cd08899cc3369f7d0e0fab518a7cf37fdb", size = 65067, upload-time = "2024-12-24T18:29:20.096Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ed/1d97f7e3561e09757a196231edccc1bcf59d55ddccefa2afc9c615abd8e0/kiwisolver-1.4.8-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111793b232842991be367ed828076b03d96202c19221b5ebab421ce8bcad016f", size = 1378443, upload-time = "2024-12-24T18:29:22.843Z" }, - { url = "https://files.pythonhosted.org/packages/29/61/39d30b99954e6b46f760e6289c12fede2ab96a254c443639052d1b573fbc/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:257af1622860e51b1a9d0ce387bf5c2c4f36a90594cb9514f55b074bcc787cfc", size = 1472728, upload-time = "2024-12-24T18:29:24.463Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3e/804163b932f7603ef256e4a715e5843a9600802bb23a68b4e08c8c0ff61d/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b5637c3f316cab1ec1c9a12b8c5f4750a4c4b71af9157645bf32830e39c03a", size = 1478388, upload-time = "2024-12-24T18:29:25.776Z" }, - { url = "https://files.pythonhosted.org/packages/8a/9e/60eaa75169a154700be74f875a4d9961b11ba048bef315fbe89cb6999056/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:782bb86f245ec18009890e7cb8d13a5ef54dcf2ebe18ed65f795e635a96a1c6a", size = 1413849, upload-time = "2024-12-24T18:29:27.202Z" }, - { url = "https://files.pythonhosted.org/packages/bc/b3/9458adb9472e61a998c8c4d95cfdfec91c73c53a375b30b1428310f923e4/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc978a80a0db3a66d25767b03688f1147a69e6237175c0f4ffffaaedf744055a", size = 1475533, upload-time = "2024-12-24T18:29:28.638Z" }, - { url = "https://files.pythonhosted.org/packages/e4/7a/0a42d9571e35798de80aef4bb43a9b672aa7f8e58643d7bd1950398ffb0a/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:36dbbfd34838500a31f52c9786990d00150860e46cd5041386f217101350f0d3", size = 2268898, upload-time = "2024-12-24T18:29:30.368Z" }, - { url = "https://files.pythonhosted.org/packages/d9/07/1255dc8d80271400126ed8db35a1795b1a2c098ac3a72645075d06fe5c5d/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eaa973f1e05131de5ff3569bbba7f5fd07ea0595d3870ed4a526d486fe57fa1b", size = 2425605, upload-time = "2024-12-24T18:29:33.151Z" }, - { url = "https://files.pythonhosted.org/packages/84/df/5a3b4cf13780ef6f6942df67b138b03b7e79e9f1f08f57c49957d5867f6e/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a66f60f8d0c87ab7f59b6fb80e642ebb29fec354a4dfad687ca4092ae69d04f4", size = 2375801, upload-time = "2024-12-24T18:29:34.584Z" }, - { url = "https://files.pythonhosted.org/packages/8f/10/2348d068e8b0f635c8c86892788dac7a6b5c0cb12356620ab575775aad89/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858416b7fb777a53f0c59ca08190ce24e9abbd3cffa18886a5781b8e3e26f65d", size = 2520077, upload-time = "2024-12-24T18:29:36.138Z" }, - { url = "https://files.pythonhosted.org/packages/32/d8/014b89fee5d4dce157d814303b0fce4d31385a2af4c41fed194b173b81ac/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:085940635c62697391baafaaeabdf3dd7a6c3643577dde337f4d66eba021b2b8", size = 2338410, upload-time = "2024-12-24T18:29:39.991Z" }, - { url = "https://files.pythonhosted.org/packages/bd/72/dfff0cc97f2a0776e1c9eb5bef1ddfd45f46246c6533b0191887a427bca5/kiwisolver-1.4.8-cp312-cp312-win_amd64.whl", hash = "sha256:01c3d31902c7db5fb6182832713d3b4122ad9317c2c5877d0539227d96bb2e50", size = 71853, upload-time = "2024-12-24T18:29:42.006Z" }, - { url = "https://files.pythonhosted.org/packages/dc/85/220d13d914485c0948a00f0b9eb419efaf6da81b7d72e88ce2391f7aed8d/kiwisolver-1.4.8-cp312-cp312-win_arm64.whl", hash = "sha256:a3c44cb68861de93f0c4a8175fbaa691f0aa22550c331fefef02b618a9dcb476", size = 65424, upload-time = "2024-12-24T18:29:44.38Z" }, -] - -[[package]] -name = "lark" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, -] - -[[package]] -name = "latex2sympy2-extended" -version = "1.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "antlr4-python3-runtime" }, - { name = "sympy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/14/8d/de5cc53dcf8a98396094b81a6478eb7234ef2e5c5fa4900dcc335d3b7fc2/latex2sympy2_extended-1.10.1.tar.gz", hash = "sha256:e365f0cb8807296c9d1f1661be926859bebd922a6139ce4be1e6572ca183ca57", size = 205808, upload-time = "2025-02-27T16:02:48.124Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/eb/a5e8b06b924b4149cf498e1598116bad1e91ab23046c2dfc2c498154d393/latex2sympy2_extended-1.10.1-py3-none-any.whl", hash = "sha256:917a23e8f3b6edea88a56978fbbe87ed9fca4197f8277646be57b4660710347c", size = 207460, upload-time = "2025-02-27T16:02:45.941Z" }, -] - -[[package]] -name = "lazy-loader" -version = "0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/6b/c875b30a1ba490860c93da4cabf479e03f584eba06fe5963f6f6644653d8/lazy_loader-0.4.tar.gz", hash = "sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1", size = 15431, upload-time = "2024-04-05T13:03:12.261Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/60/d497a310bde3f01cb805196ac61b7ad6dc5dcf8dce66634dc34364b20b4f/lazy_loader-0.4-py3-none-any.whl", hash = "sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc", size = 12097, upload-time = "2024-04-05T13:03:10.514Z" }, -] - -[[package]] -name = "libtmux" -version = "0.39.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/c4/e35564de320c876c6c370b8b9303a31ca2fcd632051aa8ac14a2d4700b68/libtmux-0.39.0.tar.gz", hash = "sha256:59346aeef3c0d6017f3bc5e23248d43cdf50f32b775b9cb5d9ff5e2e5f3059f4", size = 314810, upload-time = "2024-11-26T12:53:25.72Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/f8/afdcf3715c7388962c1eef39c7ba415a170f6772808ad83f60f8fdfdc6d0/libtmux-0.39.0-py3-none-any.whl", hash = "sha256:6b6e338be2727f67aa6b7eb67fa134368fa3c3eac5df27565396467692891c1e", size = 56219, upload-time = "2024-11-26T12:53:22.561Z" }, -] - -[[package]] -name = "libvisualwebarena" -version = "0.0.14" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiolimiter" }, - { name = "beartype" }, - { name = "evaluate" }, - { name = "flask" }, - { name = "gymnasium" }, - { name = "nltk" }, - { name = "openai" }, - { name = "pillow" }, - { name = "playwright" }, - { name = "scikit-image" }, - { name = "text-generation" }, - { name = "tiktoken" }, - { name = "transformers" }, - { name = "types-tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/5f/f9f948eeffc803f41864169bdf162a36660cf3d0e4d322be5b757720e29c/libvisualwebarena-0.0.14.tar.gz", hash = "sha256:7e660179f60f1df8d884204f2b742a2117e7fe050823d839ca5744ea1c0709a7", size = 7545223, upload-time = "2024-10-04T19:37:54.529Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/c3/c1d94cf3a95e447c1254086aa35ea16ae6c2b18a66907826054286b278a2/libvisualwebarena-0.0.14-py3-none-any.whl", hash = "sha256:636b06ca1d52f1a363503b5b563492e83f2482efaf85bb26b69744565a499f0f", size = 7551762, upload-time = "2024-10-04T19:37:51.817Z" }, -] - -[[package]] -name = "libwebarena" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiolimiter" }, - { name = "beartype" }, - { name = "evaluate" }, - { name = "flask" }, - { name = "gymnasium" }, - { name = "nltk" }, - { name = "openai" }, - { name = "pillow" }, - { name = "playwright" }, - { name = "text-generation" }, - { name = "tiktoken" }, - { name = "transformers" }, - { name = "types-tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ea/a1/94238187b6c5225be340ebcaf86dd9c1befbc4836cdc7d20439727042237/libwebarena-0.0.3.tar.gz", hash = "sha256:3d05fae6749931aaf26e6c80fd665725dfeab41ac4848f168c407dbe3de89baf", size = 107054, upload-time = "2024-05-14T19:50:33.238Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/88/add70e890f192f280d39d823a2f026c2c0aae6806f8aa2512b065d00d4d2/libwebarena-0.0.3-py3-none-any.whl", hash = "sha256:aa0a0879486e5c90b2b2ec1c3bf309b0c7f13ee2bf7c8945447ac15f7027d248", size = 116251, upload-time = "2024-05-14T19:50:31.345Z" }, -] - -[[package]] -name = "liger-kernel" -version = "0.5.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "torch" }, - { name = "triton", version = "3.1.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/ca/75bfd08e935de343e4191ec374f7915cab0630f09d74f086bb0223916cb5/liger_kernel-0.5.6.tar.gz", hash = "sha256:be983852a0322a5df75ce2a7e9bd6a79d84fe440621082c0603a65f9ec03b788", size = 3586281, upload-time = "2025-04-02T21:55:36.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/45/3d1374bd8b5c4a058d00121ba114a192187dbd3003ef0d6e5da85d77b3a5/liger_kernel-0.5.6-py3-none-any.whl", hash = "sha256:7a1a952460622a5064e73f15e86822982aa20b05b30e4329a448491e5161c17d", size = 142529, upload-time = "2025-04-02T21:55:31.791Z" }, -] - -[[package]] -name = "litellm" -version = "1.65.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "click" }, - { name = "httpx" }, - { name = "importlib-metadata" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "openai" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "tiktoken" }, - { name = "tokenizers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7b/b2/847d00916c2961fa15a452fdfa17461ec4a969712b0144996c6fcbfa4406/litellm-1.65.7.tar.gz", hash = "sha256:e5ea827cc006be32f8ad977b5f8296eeed2ed42bdbff5539bb6814db068fae84", size = 7171383, upload-time = "2025-04-11T04:48:01.099Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/bc/491d078db53c349ca77758d188833b3ffd7a26fa87ea7e7042d6789dc36b/litellm-1.65.7-py3-none-any.whl", hash = "sha256:2a8a847ce6899b4096f7e25f808904e1abfde687eb62003dea7a56305c0ecfaf", size = 7519547, upload-time = "2025-04-11T04:47:57.925Z" }, -] - -[[package]] -name = "llguidance" -version = "0.7.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/aa/95d83dbb967c7149d3fd99825ab0cdb41087933df4932ab930ad258ffe7d/llguidance-0.7.14.tar.gz", hash = "sha256:f4608a98dc93f987a13cab4079940bfa8c4de7b6aec5900930871cc09626279a", size = 392530, upload-time = "2025-04-11T17:04:15.603Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/1d/d22042ef1263da4aba4964c916cd13a8f251ed1ecf7f2c1c52048508d02c/llguidance-0.7.14-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:738b525ed0ee694272f7995d46172ec6580cc25e95539af0cdbd69bca2ca24df", size = 3099455, upload-time = "2025-04-11T17:04:14.211Z" }, - { url = "https://files.pythonhosted.org/packages/f2/76/157bfa1bd9061c4eeeab91204a5569c1f8e2ee7df20b28fb57e16c19d7c8/llguidance-0.7.14-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ac8af50a294b9a6e5b3d8b11259340b9e3c8a39d6ce694f2107e419e3bee7f6d", size = 2999880, upload-time = "2025-04-11T17:04:12.294Z" }, - { url = "https://files.pythonhosted.org/packages/a5/5e/18994c432ecb27963e4226176e85be6990bf39e822b6293f262dde23014d/llguidance-0.7.14-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:823a67d88d17ecfd00e90c1711c3ac83b47785e7352dd4634f2297f5e4f89f28", size = 13781649, upload-time = "2025-04-11T17:04:04.121Z" }, - { url = "https://files.pythonhosted.org/packages/fe/dc/d1ba37ff9ffab8d3c011e53e247dbd910f8a7bdc85b94fea59f05f85be80/llguidance-0.7.14-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a69c5862c4de7df947fa7cf8bb7283736a28f5e41446c2ddadbcbafc45360278", size = 13905056, upload-time = "2025-04-11T17:04:06.989Z" }, - { url = "https://files.pythonhosted.org/packages/bb/0c/6b668e1daeb024c28a0d3d039df624bd1b4832594e1247356c80c0570bd5/llguidance-0.7.14-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc5d1ff55955e02a63904b59cf7cc6f633f327ca6bdf15a660b81b1666bc8adf", size = 13860881, upload-time = "2025-04-11T17:04:09.486Z" }, - { url = "https://files.pythonhosted.org/packages/3a/e2/0f3623c3047de99b287c93caafcd53e8d3af219b1c5875c6ae33176fe39f/llguidance-0.7.14-cp39-abi3-win32.whl", hash = "sha256:f48769a8273d0c74bc8ec7ce3be093f53266678163009b21ce264ea455660175", size = 2456784, upload-time = "2025-04-11T17:04:18.761Z" }, - { url = "https://files.pythonhosted.org/packages/14/01/8eb87fef2aa41aa95b60425b9bb3b7aad2f7580b052af35f7ae18dda1478/llguidance-0.7.14-cp39-abi3-win_amd64.whl", hash = "sha256:9a3ab48263c9517eae9fb7e2c94f88abd88ad399372fa1c04748c6c4cf49cac7", size = 2650606, upload-time = "2025-04-11T17:04:16.943Z" }, -] - -[[package]] -name = "llvmlite" -version = "0.43.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/3d/f513755f285db51ab363a53e898b85562e950f79a2e6767a364530c2f645/llvmlite-0.43.0.tar.gz", hash = "sha256:ae2b5b5c3ef67354824fb75517c8db5fbe93bc02cd9671f3c62271626bc041d5", size = 157069, upload-time = "2024-06-13T18:09:32.641Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/67/9443509e5d2b6d8587bae3ede5598fa8bd586b1c7701696663ea8af15b5b/llvmlite-0.43.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f99b600aa7f65235a5a05d0b9a9f31150c390f31261f2a0ba678e26823ec38f7", size = 31064409, upload-time = "2024-06-13T18:08:54.375Z" }, - { url = "https://files.pythonhosted.org/packages/a2/9c/24139d3712d2d352e300c39c0e00d167472c08b3bd350c3c33d72c88ff8d/llvmlite-0.43.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:35d80d61d0cda2d767f72de99450766250560399edc309da16937b93d3b676e7", size = 28793145, upload-time = "2024-06-13T18:08:57.953Z" }, - { url = "https://files.pythonhosted.org/packages/bf/f1/4c205a48488e574ee9f6505d50e84370a978c90f08dab41a42d8f2c576b6/llvmlite-0.43.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eccce86bba940bae0d8d48ed925f21dbb813519169246e2ab292b5092aba121f", size = 42857276, upload-time = "2024-06-13T18:09:02.067Z" }, - { url = "https://files.pythonhosted.org/packages/00/5f/323c4d56e8401c50185fd0e875fcf06b71bf825a863699be1eb10aa2a9cb/llvmlite-0.43.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df6509e1507ca0760787a199d19439cc887bfd82226f5af746d6977bd9f66844", size = 43871781, upload-time = "2024-06-13T18:09:06.667Z" }, - { url = "https://files.pythonhosted.org/packages/c6/94/dea10e263655ce78d777e78d904903faae39d1fc440762be4a9dc46bed49/llvmlite-0.43.0-cp312-cp312-win_amd64.whl", hash = "sha256:7a2872ee80dcf6b5dbdc838763d26554c2a18aa833d31a2635bff16aafefb9c9", size = 28107442, upload-time = "2024-06-13T18:09:10.709Z" }, -] - -[[package]] -name = "lm-format-enforcer" -version = "0.10.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "interegular" }, - { name = "packaging" }, - { name = "pydantic" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5b/cc/8a5bf6706385c89474161081d2eeec4dd9cef12dc29cca6acc872685ceb6/lm_format_enforcer-0.10.11.tar.gz", hash = "sha256:8ab371924e166a1df68f243aca73a8a647bea5909f37edd6a53a694e7e7c3274", size = 39390, upload-time = "2025-02-26T22:18:45.338Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/cb/bf172960241842e953b3354247f792aae2fc5221552a0741a1c98f35b6f7/lm_format_enforcer-0.10.11-py3-none-any.whl", hash = "sha256:563e0dbc930a6d50fb687951506c5de098c6e962601be0ce723f3b7d0b916a1b", size = 44229, upload-time = "2025-02-26T22:18:42.543Z" }, -] - -[[package]] -name = "lxml" -version = "5.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/61/d3dc048cd6c7be6fe45b80cedcbdd4326ba4d550375f266d9f4246d0f4bc/lxml-5.3.2.tar.gz", hash = "sha256:773947d0ed809ddad824b7b14467e1a481b8976e87278ac4a730c2f7c7fcddc1", size = 3679948, upload-time = "2025-04-05T18:31:58.757Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/7e/c749257a7fabc712c4df57927b0f703507f316e9f2c7e3219f8f76d36145/lxml-5.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:16b3897691ec0316a1aa3c6585f61c8b7978475587c5b16fc1d2c28d283dc1b0", size = 8193212, upload-time = "2025-04-05T18:26:42.692Z" }, - { url = "https://files.pythonhosted.org/packages/a8/50/17e985ba162c9f1ca119f4445004b58f9e5ef559ded599b16755e9bfa260/lxml-5.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a8d4b34a0eeaf6e73169dcfd653c8d47f25f09d806c010daf074fba2db5e2d3f", size = 4451439, upload-time = "2025-04-05T18:26:46.468Z" }, - { url = "https://files.pythonhosted.org/packages/c2/b5/4960ba0fcca6ce394ed4a2f89ee13083e7fcbe9641a91166e8e9792fedb1/lxml-5.3.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9cd7a959396da425022e1e4214895b5cfe7de7035a043bcc2d11303792b67554", size = 5052146, upload-time = "2025-04-05T18:26:49.737Z" }, - { url = "https://files.pythonhosted.org/packages/5f/d1/184b04481a5d1f5758916de087430752a7b229bddbd6c1d23405078c72bd/lxml-5.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cac5eaeec3549c5df7f8f97a5a6db6963b91639389cdd735d5a806370847732b", size = 4789082, upload-time = "2025-04-05T18:26:52.295Z" }, - { url = "https://files.pythonhosted.org/packages/7d/75/1a19749d373e9a3d08861addccdf50c92b628c67074b22b8f3c61997cf5a/lxml-5.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29b5f7d77334877c2146e7bb8b94e4df980325fab0a8af4d524e5d43cd6f789d", size = 5312300, upload-time = "2025-04-05T18:26:54.923Z" }, - { url = "https://files.pythonhosted.org/packages/fb/00/9d165d4060d3f347e63b219fcea5c6a3f9193e9e2868c6801e18e5379725/lxml-5.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13f3495cfec24e3d63fffd342cc8141355d1d26ee766ad388775f5c8c5ec3932", size = 4836655, upload-time = "2025-04-05T18:26:57.488Z" }, - { url = "https://files.pythonhosted.org/packages/b8/e9/06720a33cc155966448a19677f079100517b6629a872382d22ebd25e48aa/lxml-5.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e70ad4c9658beeff99856926fd3ee5fde8b519b92c693f856007177c36eb2e30", size = 4961795, upload-time = "2025-04-05T18:27:00.126Z" }, - { url = "https://files.pythonhosted.org/packages/2d/57/4540efab2673de2904746b37ef7f74385329afd4643ed92abcc9ec6e00ca/lxml-5.3.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:507085365783abd7879fa0a6fa55eddf4bdd06591b17a2418403bb3aff8a267d", size = 4779791, upload-time = "2025-04-05T18:27:03.061Z" }, - { url = "https://files.pythonhosted.org/packages/99/ad/6056edf6c9f4fa1d41e6fbdae52c733a4a257fd0d7feccfa26ae051bb46f/lxml-5.3.2-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:5bb304f67cbf5dfa07edad904732782cbf693286b9cd85af27059c5779131050", size = 5346807, upload-time = "2025-04-05T18:27:05.877Z" }, - { url = "https://files.pythonhosted.org/packages/a1/fa/5be91fc91a18f3f705ea5533bc2210b25d738c6b615bf1c91e71a9b2f26b/lxml-5.3.2-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:3d84f5c093645c21c29a4e972b84cb7cf682f707f8706484a5a0c7ff13d7a988", size = 4909213, upload-time = "2025-04-05T18:27:08.588Z" }, - { url = "https://files.pythonhosted.org/packages/f3/74/71bb96a3b5ae36b74e0402f4fa319df5559a8538577f8c57c50f1b57dc15/lxml-5.3.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bdc13911db524bd63f37b0103af014b7161427ada41f1b0b3c9b5b5a9c1ca927", size = 4987694, upload-time = "2025-04-05T18:27:11.66Z" }, - { url = "https://files.pythonhosted.org/packages/08/c2/3953a68b0861b2f97234b1838769269478ccf872d8ea7a26e911238220ad/lxml-5.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1ec944539543f66ebc060ae180d47e86aca0188bda9cbfadff47d86b0dc057dc", size = 4862865, upload-time = "2025-04-05T18:27:14.194Z" }, - { url = "https://files.pythonhosted.org/packages/e0/9a/52e48f7cfd5a5e61f44a77e679880580dfb4f077af52d6ed5dd97e3356fe/lxml-5.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:59d437cc8a7f838282df5a199cf26f97ef08f1c0fbec6e84bd6f5cc2b7913f6e", size = 5423383, upload-time = "2025-04-05T18:27:16.988Z" }, - { url = "https://files.pythonhosted.org/packages/17/67/42fe1d489e4dcc0b264bef361aef0b929fbb2b5378702471a3043bc6982c/lxml-5.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e275961adbd32e15672e14e0cc976a982075208224ce06d149c92cb43db5b93", size = 5286864, upload-time = "2025-04-05T18:27:19.703Z" }, - { url = "https://files.pythonhosted.org/packages/29/e4/03b1d040ee3aaf2bd4e1c2061de2eae1178fe9a460d3efc1ea7ef66f6011/lxml-5.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:038aeb6937aa404480c2966b7f26f1440a14005cb0702078c173c028eca72c31", size = 5056819, upload-time = "2025-04-05T18:27:22.814Z" }, - { url = "https://files.pythonhosted.org/packages/83/b3/e2ec8a6378e4d87da3af9de7c862bcea7ca624fc1a74b794180c82e30123/lxml-5.3.2-cp312-cp312-win32.whl", hash = "sha256:3c2c8d0fa3277147bff180e3590be67597e17d365ce94beb2efa3138a2131f71", size = 3486177, upload-time = "2025-04-05T18:27:25.078Z" }, - { url = "https://files.pythonhosted.org/packages/d5/8a/6a08254b0bab2da9573735725caab8302a2a1c9b3818533b41568ca489be/lxml-5.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:77809fcd97dfda3f399102db1794f7280737b69830cd5c961ac87b3c5c05662d", size = 3817134, upload-time = "2025-04-05T18:27:27.481Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "3.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, - { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, - { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, - { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" }, - { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" }, - { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" }, - { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" }, - { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" }, - { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" }, -] - -[[package]] -name = "marshmallow" -version = "3.26.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ab/5e/5e53d26b42ab75491cda89b871dab9e97c840bf12c63ec58a1919710cd06/marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6", size = 221825, upload-time = "2025-02-03T15:32:25.093Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878, upload-time = "2025-02-03T15:32:22.295Z" }, -] - -[[package]] -name = "math-verify" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "latex2sympy2-extended" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ab/59/7daf53e52593af0f7f455ea97c91dc34682f1c2e72f493490259a640836c/math_verify-0.7.0.tar.gz", hash = "sha256:b7459c7b9f1c503b7df32566a95f1e6a33a6cad8cbb759eaa6f100d401112024", size = 57582, upload-time = "2025-02-27T16:21:04.151Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/c5/e5de2e29f9db4c92956ce167e1aa534b1e1770ddc0b17600257cf5f8bd50/math_verify-0.7.0-py3-none-any.whl", hash = "sha256:bbdd491b511f6ceef27c5e08543affa1735807b6e3abb6f005bf493dc0eb485b", size = 28677, upload-time = "2025-02-27T16:20:56.612Z" }, -] - -[[package]] -name = "mathruler" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/0b/cb3d2332f2e821752782cecd4aab40c7c73d00738f3f783050bff873915c/mathruler-0.1.0.tar.gz", hash = "sha256:38f02b5c5c12efcf2f20f70a5fde1d74d177615f873b2b8719a18e1ca1e4fb04", size = 15251, upload-time = "2025-02-22T03:25:10.839Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/a5/91d16d65b66251c2423bfedb9219c2e803f11d91af9bfba75f05ddc33bfb/mathruler-0.1.0-py3-none-any.whl", hash = "sha256:e8ce0d9e07e892bf971fbff5e6eee330f6a66fd022666c99828a0000c788079f", size = 15190, upload-time = "2025-02-22T03:25:09.85Z" }, -] - -[[package]] -name = "matplotlib" -version = "3.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "contourpy" }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2f/08/b89867ecea2e305f408fbb417139a8dd941ecf7b23a2e02157c36da546f0/matplotlib-3.10.1.tar.gz", hash = "sha256:e8d2d0e3881b129268585bf4765ad3ee73a4591d77b9a18c214ac7e3a79fb2ba", size = 36743335, upload-time = "2025-02-27T19:19:51.038Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/1d/5e0dc3b59c034e43de16f94deb68f4ad8a96b3ea00f4b37c160b7474928e/matplotlib-3.10.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:66e907a06e68cb6cfd652c193311d61a12b54f56809cafbed9736ce5ad92f107", size = 8175488, upload-time = "2025-02-27T19:18:51.436Z" }, - { url = "https://files.pythonhosted.org/packages/7a/81/dae7e14042e74da658c3336ab9799128e09a1ee03964f2d89630b5d12106/matplotlib-3.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b4bb156abb8fa5e5b2b460196f7db7264fc6d62678c03457979e7d5254b7be", size = 8046264, upload-time = "2025-02-27T19:18:54.344Z" }, - { url = "https://files.pythonhosted.org/packages/21/c4/22516775dcde10fc9c9571d155f90710761b028fc44f660508106c363c97/matplotlib-3.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1985ad3d97f51307a2cbfc801a930f120def19ba22864182dacef55277102ba6", size = 8452048, upload-time = "2025-02-27T19:18:56.536Z" }, - { url = "https://files.pythonhosted.org/packages/63/23/c0615001f67ce7c96b3051d856baedc0c818a2ed84570b9bf9bde200f85d/matplotlib-3.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c96f2c2f825d1257e437a1482c5a2cf4fee15db4261bd6fc0750f81ba2b4ba3d", size = 8597111, upload-time = "2025-02-27T19:18:59.439Z" }, - { url = "https://files.pythonhosted.org/packages/ca/c0/a07939a82aed77770514348f4568177d7dadab9787ebc618a616fe3d665e/matplotlib-3.10.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:35e87384ee9e488d8dd5a2dd7baf471178d38b90618d8ea147aced4ab59c9bea", size = 9402771, upload-time = "2025-02-27T19:19:01.944Z" }, - { url = "https://files.pythonhosted.org/packages/a6/b6/a9405484fb40746fdc6ae4502b16a9d6e53282ba5baaf9ebe2da579f68c4/matplotlib-3.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:cfd414bce89cc78a7e1d25202e979b3f1af799e416010a20ab2b5ebb3a02425c", size = 8063742, upload-time = "2025-02-27T19:19:04.632Z" }, -] - -[[package]] -name = "matplotlib-inline" -version = "0.1.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/99/5b/a36a337438a14116b16480db471ad061c36c3694df7c2084a0da7ba538b7/matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90", size = 8159, upload-time = "2024-04-15T13:44:44.803Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899, upload-time = "2024-04-15T13:44:43.265Z" }, -] - -[[package]] -name = "mccabe" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] -name = "memory-profiler" -version = "0.61.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "psutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/88/e1907e1ca3488f2d9507ca8b0ae1add7b1cd5d3ca2bc8e5b329382ea2c7b/memory_profiler-0.61.0.tar.gz", hash = "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0", size = 35935, upload-time = "2022-11-15T17:57:28.994Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/26/aaca612a0634ceede20682e692a6c55e35a94c21ba36b807cc40fe910ae1/memory_profiler-0.61.0-py3-none-any.whl", hash = "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84", size = 31803, upload-time = "2022-11-15T17:57:27.031Z" }, -] - -[[package]] -name = "minio" -version = "7.2.15" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi" }, - { name = "certifi" }, - { name = "pycryptodome" }, - { name = "typing-extensions" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/68/86a1cef80396e6a35a6fc4fafee5d28578c1a137bddd3ca2aa86f9b26a22/minio-7.2.15.tar.gz", hash = "sha256:5247df5d4dca7bfa4c9b20093acd5ad43e82d8710ceb059d79c6eea970f49f79", size = 138040, upload-time = "2025-01-19T08:57:26.626Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/6f/3690028e846fe432bfa5ba724a0dc37ec9c914965b7733e19d8ca2c4c48d/minio-7.2.15-py3-none-any.whl", hash = "sha256:c06ef7a43e5d67107067f77b6c07ebdd68733e5aa7eed03076472410ca19d876", size = 95075, upload-time = "2025-01-19T08:57:24.169Z" }, -] - -[[package]] -name = "mistral-common" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jsonschema" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "sentencepiece" }, - { name = "tiktoken" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/75/31/0453db671b61d2716bd263aecd95c53fcc0aa4d32ab404fa7070909dc005/mistral_common-1.5.4.tar.gz", hash = "sha256:0af4124ab09d1409761e91ec61681476882d46f9418eea8908d39c01222e0f6b", size = 6269751, upload-time = "2025-03-16T21:05:45.102Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/80/7a/421819257cd642b33d71819e2ff259fb019a49ea48e830e5a32558c52cb7/mistral_common-1.5.4-py3-none-any.whl", hash = "sha256:acef3367a4386d5dd3d9e23330348bbebe90a5cbd2fc5587d8a8d13d9893e537", size = 6477779, upload-time = "2025-03-16T21:05:41.366Z" }, -] - -[package.optional-dependencies] -opencv = [ - { name = "opencv-python-headless" }, -] - -[[package]] -name = "modal" -version = "0.73.165" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "certifi" }, - { name = "click" }, - { name = "fastapi" }, - { name = "grpclib" }, - { name = "protobuf" }, - { name = "rich" }, - { name = "synchronicity" }, - { name = "toml" }, - { name = "typer" }, - { name = "types-certifi" }, - { name = "types-toml" }, - { name = "typing-extensions" }, - { name = "watchfiles" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/31/2dadeeafba3469df2250e6c988c2cc09e4c83b99ef9b383303b5d34225f8/modal-0.73.165.tar.gz", hash = "sha256:36750c63df36375f9f5d8a5ad49710c912a030033a0e1a342c90cf54868bbe9b", size = 491808, upload-time = "2025-04-11T11:57:24.645Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/c8/a0b8ef21c8920387c3fe588dde28c2cd7d6c9a544c696ab39ac8f05dfd23/modal-0.73.165-py3-none-any.whl", hash = "sha256:26903d090b37092ffa27d84fe8fca24aa98d36996ef867ba20825995098a2267", size = 558994, upload-time = "2025-04-11T11:57:21.858Z" }, -] - -[[package]] -name = "modelscope" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "tqdm" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/4b/1c11ca8bea237b49ad55889fd2387c012ce5d1fc574d4280cd593c32f41b/modelscope-1.25.0.tar.gz", hash = "sha256:b9a5ef2e68f17a63c8ea688104d82de1fc8b506c37ded1aeedfcf3eb6028b60a", size = 4388940, upload-time = "2025-04-10T10:46:48.542Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/8f/b051fc200ba09584f4d53c35da68e4e3d08a0a8688dab60d6fe726e290c8/modelscope-1.25.0-py3-none-any.whl", hash = "sha256:30f0b8c94502ca44c911a9ce5ff388b3de4276c5b5cba2f850bc370dbf8cac57", size = 5854156, upload-time = "2025-04-10T10:46:46.025Z" }, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - -[[package]] -name = "msgpack" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cb/d0/7555686ae7ff5731205df1012ede15dd9d927f6227ea151e901c7406af4f/msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e", size = 167260, upload-time = "2024-09-10T04:25:52.197Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/d6/716b7ca1dbde63290d2973d22bbef1b5032ca634c3ff4384a958ec3f093a/msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d", size = 152421, upload-time = "2024-09-10T04:25:49.63Z" }, - { url = "https://files.pythonhosted.org/packages/70/da/5312b067f6773429cec2f8f08b021c06af416bba340c912c2ec778539ed6/msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2", size = 85277, upload-time = "2024-09-10T04:24:48.562Z" }, - { url = "https://files.pythonhosted.org/packages/28/51/da7f3ae4462e8bb98af0d5bdf2707f1b8c65a0d4f496e46b6afb06cbc286/msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420", size = 82222, upload-time = "2024-09-10T04:25:36.49Z" }, - { url = "https://files.pythonhosted.org/packages/33/af/dc95c4b2a49cff17ce47611ca9ba218198806cad7796c0b01d1e332c86bb/msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2", size = 392971, upload-time = "2024-09-10T04:24:58.129Z" }, - { url = "https://files.pythonhosted.org/packages/f1/54/65af8de681fa8255402c80eda2a501ba467921d5a7a028c9c22a2c2eedb5/msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39", size = 401403, upload-time = "2024-09-10T04:25:40.428Z" }, - { url = "https://files.pythonhosted.org/packages/97/8c/e333690777bd33919ab7024269dc3c41c76ef5137b211d776fbb404bfead/msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f", size = 385356, upload-time = "2024-09-10T04:25:31.406Z" }, - { url = "https://files.pythonhosted.org/packages/57/52/406795ba478dc1c890559dd4e89280fa86506608a28ccf3a72fbf45df9f5/msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247", size = 383028, upload-time = "2024-09-10T04:25:17.08Z" }, - { url = "https://files.pythonhosted.org/packages/e7/69/053b6549bf90a3acadcd8232eae03e2fefc87f066a5b9fbb37e2e608859f/msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c", size = 391100, upload-time = "2024-09-10T04:25:08.993Z" }, - { url = "https://files.pythonhosted.org/packages/23/f0/d4101d4da054f04274995ddc4086c2715d9b93111eb9ed49686c0f7ccc8a/msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b", size = 394254, upload-time = "2024-09-10T04:25:06.048Z" }, - { url = "https://files.pythonhosted.org/packages/1c/12/cf07458f35d0d775ff3a2dc5559fa2e1fcd06c46f1ef510e594ebefdca01/msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b", size = 69085, upload-time = "2024-09-10T04:25:01.494Z" }, - { url = "https://files.pythonhosted.org/packages/73/80/2708a4641f7d553a63bc934a3eb7214806b5b39d200133ca7f7afb0a53e8/msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f", size = 75347, upload-time = "2024-09-10T04:25:33.106Z" }, -] - -[[package]] -name = "msgspec" -version = "0.19.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cf/9b/95d8ce458462b8b71b8a70fa94563b2498b89933689f3a7b8911edfae3d7/msgspec-0.19.0.tar.gz", hash = "sha256:604037e7cd475345848116e89c553aa9a233259733ab51986ac924ab1b976f8e", size = 216934, upload-time = "2024-12-27T17:40:28.597Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/5f/a70c24f075e3e7af2fae5414c7048b0e11389685b7f717bb55ba282a34a7/msgspec-0.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f98bd8962ad549c27d63845b50af3f53ec468b6318400c9f1adfe8b092d7b62f", size = 190485, upload-time = "2024-12-27T17:39:44.974Z" }, - { url = "https://files.pythonhosted.org/packages/89/b0/1b9763938cfae12acf14b682fcf05c92855974d921a5a985ecc197d1c672/msgspec-0.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:43bbb237feab761b815ed9df43b266114203f53596f9b6e6f00ebd79d178cdf2", size = 183910, upload-time = "2024-12-27T17:39:46.401Z" }, - { url = "https://files.pythonhosted.org/packages/87/81/0c8c93f0b92c97e326b279795f9c5b956c5a97af28ca0fbb9fd86c83737a/msgspec-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cfc033c02c3e0aec52b71710d7f84cb3ca5eb407ab2ad23d75631153fdb1f12", size = 210633, upload-time = "2024-12-27T17:39:49.099Z" }, - { url = "https://files.pythonhosted.org/packages/d0/ef/c5422ce8af73928d194a6606f8ae36e93a52fd5e8df5abd366903a5ca8da/msgspec-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d911c442571605e17658ca2b416fd8579c5050ac9adc5e00c2cb3126c97f73bc", size = 213594, upload-time = "2024-12-27T17:39:51.204Z" }, - { url = "https://files.pythonhosted.org/packages/19/2b/4137bc2ed45660444842d042be2cf5b18aa06efd2cda107cff18253b9653/msgspec-0.19.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:757b501fa57e24896cf40a831442b19a864f56d253679f34f260dcb002524a6c", size = 214053, upload-time = "2024-12-27T17:39:52.866Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e6/8ad51bdc806aac1dc501e8fe43f759f9ed7284043d722b53323ea421c360/msgspec-0.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5f0f65f29b45e2816d8bded36e6b837a4bf5fb60ec4bc3c625fa2c6da4124537", size = 219081, upload-time = "2024-12-27T17:39:55.142Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ef/27dd35a7049c9a4f4211c6cd6a8c9db0a50647546f003a5867827ec45391/msgspec-0.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:067f0de1c33cfa0b6a8206562efdf6be5985b988b53dd244a8e06f993f27c8c0", size = 187467, upload-time = "2024-12-27T17:39:56.531Z" }, -] - -[[package]] -name = "multidict" -version = "6.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/2c/e367dfb4c6538614a0c9453e510d75d66099edf1c4e69da1b5ce691a1931/multidict-6.4.3.tar.gz", hash = "sha256:3ada0b058c9f213c5f95ba301f922d402ac234f1111a7d8fd70f1b99f3c281ec", size = 89372, upload-time = "2025-04-10T22:20:17.956Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/bb/3abdaf8fe40e9226ce8a2ba5ecf332461f7beec478a455d6587159f1bf92/multidict-6.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1f1c2f58f08b36f8475f3ec6f5aeb95270921d418bf18f90dffd6be5c7b0e676", size = 64019, upload-time = "2025-04-10T22:18:23.174Z" }, - { url = "https://files.pythonhosted.org/packages/7e/b5/1b2e8de8217d2e89db156625aa0fe4a6faad98972bfe07a7b8c10ef5dd6b/multidict-6.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:26ae9ad364fc61b936fb7bf4c9d8bd53f3a5b4417142cd0be5c509d6f767e2f1", size = 37925, upload-time = "2025-04-10T22:18:24.834Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e2/3ca91c112644a395c8eae017144c907d173ea910c913ff8b62549dcf0bbf/multidict-6.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:659318c6c8a85f6ecfc06b4e57529e5a78dfdd697260cc81f683492ad7e9435a", size = 37008, upload-time = "2025-04-10T22:18:26.069Z" }, - { url = "https://files.pythonhosted.org/packages/60/23/79bc78146c7ac8d1ac766b2770ca2e07c2816058b8a3d5da6caed8148637/multidict-6.4.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1eb72c741fd24d5a28242ce72bb61bc91f8451877131fa3fe930edb195f7054", size = 224374, upload-time = "2025-04-10T22:18:27.714Z" }, - { url = "https://files.pythonhosted.org/packages/86/35/77950ed9ebd09136003a85c1926ba42001ca5be14feb49710e4334ee199b/multidict-6.4.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3cd06d88cb7398252284ee75c8db8e680aa0d321451132d0dba12bc995f0adcc", size = 230869, upload-time = "2025-04-10T22:18:29.162Z" }, - { url = "https://files.pythonhosted.org/packages/49/97/2a33c6e7d90bc116c636c14b2abab93d6521c0c052d24bfcc231cbf7f0e7/multidict-6.4.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4543d8dc6470a82fde92b035a92529317191ce993533c3c0c68f56811164ed07", size = 231949, upload-time = "2025-04-10T22:18:30.679Z" }, - { url = "https://files.pythonhosted.org/packages/56/ce/e9b5d9fcf854f61d6686ada7ff64893a7a5523b2a07da6f1265eaaea5151/multidict-6.4.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:30a3ebdc068c27e9d6081fca0e2c33fdf132ecea703a72ea216b81a66860adde", size = 231032, upload-time = "2025-04-10T22:18:32.146Z" }, - { url = "https://files.pythonhosted.org/packages/f0/ac/7ced59dcdfeddd03e601edb05adff0c66d81ed4a5160c443e44f2379eef0/multidict-6.4.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b038f10e23f277153f86f95c777ba1958bcd5993194fda26a1d06fae98b2f00c", size = 223517, upload-time = "2025-04-10T22:18:33.538Z" }, - { url = "https://files.pythonhosted.org/packages/db/e6/325ed9055ae4e085315193a1b58bdb4d7fc38ffcc1f4975cfca97d015e17/multidict-6.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c605a2b2dc14282b580454b9b5d14ebe0668381a3a26d0ac39daa0ca115eb2ae", size = 216291, upload-time = "2025-04-10T22:18:34.962Z" }, - { url = "https://files.pythonhosted.org/packages/fa/84/eeee6d477dd9dcb7691c3bb9d08df56017f5dd15c730bcc9383dcf201cf4/multidict-6.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8bd2b875f4ca2bb527fe23e318ddd509b7df163407b0fb717df229041c6df5d3", size = 228982, upload-time = "2025-04-10T22:18:36.443Z" }, - { url = "https://files.pythonhosted.org/packages/82/94/4d1f3e74e7acf8b0c85db350e012dcc61701cd6668bc2440bb1ecb423c90/multidict-6.4.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c2e98c840c9c8e65c0e04b40c6c5066c8632678cd50c8721fdbcd2e09f21a507", size = 226823, upload-time = "2025-04-10T22:18:37.924Z" }, - { url = "https://files.pythonhosted.org/packages/09/f0/1e54b95bda7cd01080e5732f9abb7b76ab5cc795b66605877caeb2197476/multidict-6.4.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66eb80dd0ab36dbd559635e62fba3083a48a252633164857a1d1684f14326427", size = 222714, upload-time = "2025-04-10T22:18:39.807Z" }, - { url = "https://files.pythonhosted.org/packages/e7/a2/f6cbca875195bd65a3e53b37ab46486f3cc125bdeab20eefe5042afa31fb/multidict-6.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c23831bdee0a2a3cf21be057b5e5326292f60472fb6c6f86392bbf0de70ba731", size = 233739, upload-time = "2025-04-10T22:18:41.341Z" }, - { url = "https://files.pythonhosted.org/packages/79/68/9891f4d2b8569554723ddd6154375295f789dc65809826c6fb96a06314fd/multidict-6.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1535cec6443bfd80d028052e9d17ba6ff8a5a3534c51d285ba56c18af97e9713", size = 230809, upload-time = "2025-04-10T22:18:42.817Z" }, - { url = "https://files.pythonhosted.org/packages/e6/72/a7be29ba1e87e4fc5ceb44dabc7940b8005fd2436a332a23547709315f70/multidict-6.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b73e7227681f85d19dec46e5b881827cd354aabe46049e1a61d2f9aaa4e285a", size = 226934, upload-time = "2025-04-10T22:18:44.311Z" }, - { url = "https://files.pythonhosted.org/packages/12/c1/259386a9ad6840ff7afc686da96808b503d152ac4feb3a96c651dc4f5abf/multidict-6.4.3-cp312-cp312-win32.whl", hash = "sha256:8eac0c49df91b88bf91f818e0a24c1c46f3622978e2c27035bfdca98e0e18124", size = 35242, upload-time = "2025-04-10T22:18:46.193Z" }, - { url = "https://files.pythonhosted.org/packages/06/24/c8fdff4f924d37225dc0c56a28b1dca10728fc2233065fafeb27b4b125be/multidict-6.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:11990b5c757d956cd1db7cb140be50a63216af32cd6506329c2c59d732d802db", size = 38635, upload-time = "2025-04-10T22:18:47.498Z" }, - { url = "https://files.pythonhosted.org/packages/96/10/7d526c8974f017f1e7ca584c71ee62a638e9334d8d33f27d7cdfc9ae79e4/multidict-6.4.3-py3-none-any.whl", hash = "sha256:59fe01ee8e2a1e8ceb3f6dbb216b09c8d9f4ef1c22c4fc825d045a147fa2ebc9", size = 10400, upload-time = "2025-04-10T22:20:16.445Z" }, -] - -[[package]] -name = "multiprocess" -version = "0.70.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dill" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" }, - { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" }, - { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" }, - { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" }, - { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, -] - -[[package]] -name = "mypy-extensions" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433, upload-time = "2023-02-04T12:11:27.157Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695, upload-time = "2023-02-04T12:11:25.002Z" }, -] - -[[package]] -name = "nanobind" -version = "2.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e9/38/c28a15cfc2f311e79ea01a5eba8960eb02f64d58b6f675b8a14ef4680dac/nanobind-2.6.1.tar.gz", hash = "sha256:e05c6816a844aa699e46408add3bff8c743af9fd3d38eafa307a73633fddf94e", size = 968121, upload-time = "2025-03-28T15:42:25.893Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/f9/0b11e9b99a35dbf91972845af6012c562f3f1845a33e5b4ea631568d2740/nanobind-2.6.1-py3-none-any.whl", hash = "sha256:678be25b6ffdb7e3776548c2c9f29072e7eb41219b72c51f8e14ac0fe5c8ae0b", size = 238966, upload-time = "2025-03-28T15:42:24.188Z" }, -] - -[[package]] -name = "narwhals" -version = "1.34.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f5/6d/0ad9e8fe41dba509f1c3241a46e52c9677eb9f67f7eb3c8018b1ae088469/narwhals-1.34.1.tar.gz", hash = "sha256:8941a08cf62f39ae67be945090b20a19c51fb9048ac909cad2f95aee23f9e302", size = 265602, upload-time = "2025-04-09T09:03:40.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/a2/2ea3f2058ae661b06fad1a14c0f29f1cc8e49478d02a2d68f0eac3872b86/narwhals-1.34.1-py3-none-any.whl", hash = "sha256:aee49a30d1624371ad975ede2bab16cd125ab5df0fd185e2f4757e103b074d27", size = 325548, upload-time = "2025-04-09T09:03:39.24Z" }, -] - -[[package]] -name = "nest-asyncio" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, -] - -[[package]] -name = "networkx" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263, upload-time = "2024-10-21T12:39:36.247Z" }, -] - -[[package]] -name = "ninja" -version = "1.11.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/d4/6b0324541018561c5e73e617bd16f20a4fc17d1179bb3b3520b6ca8beb7b/ninja-1.11.1.4.tar.gz", hash = "sha256:6aa39f6e894e0452e5b297327db00019383ae55d5d9c57c73b04f13bf79d438a", size = 201256, upload-time = "2025-03-22T06:46:43.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/b1/3a61b348936b62a386465b1937cd778fa3a5748582e26d832dbab844ff27/ninja-1.11.1.4-py3-none-macosx_10_9_universal2.whl", hash = "sha256:b33923c8da88e8da20b6053e38deb433f53656441614207e01d283ad02c5e8e7", size = 279071, upload-time = "2025-03-22T06:46:17.806Z" }, - { url = "https://files.pythonhosted.org/packages/12/42/4c94fdad51fcf1f039a156e97de9e4d564c2a8cc0303782d36f9bd893a4b/ninja-1.11.1.4-py3-none-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cede0af00b58e27b31f2482ba83292a8e9171cdb9acc2c867a3b6e40b3353e43", size = 472026, upload-time = "2025-03-22T06:46:19.974Z" }, - { url = "https://files.pythonhosted.org/packages/eb/7a/455d2877fe6cf99886849c7f9755d897df32eaf3a0fba47b56e615f880f7/ninja-1.11.1.4-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:096487995473320de7f65d622c3f1d16c3ad174797602218ca8c967f51ec38a0", size = 422814, upload-time = "2025-03-22T06:46:21.235Z" }, - { url = "https://files.pythonhosted.org/packages/e3/ad/fb6cca942528e25e8e0ab0f0cf98fe007319bf05cf69d726c564b815c4af/ninja-1.11.1.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3090d4488fadf6047d0d7a1db0c9643a8d391f0d94729554dbb89b5bdc769d7", size = 156965, upload-time = "2025-03-22T06:46:23.45Z" }, - { url = "https://files.pythonhosted.org/packages/a8/e7/d94a1b60031b115dd88526834b3da69eaacdc3c1a6769773ca8e2b1386b5/ninja-1.11.1.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ecce44a00325a93631792974659cf253a815cc6da4ec96f89742925dfc295a0d", size = 179937, upload-time = "2025-03-22T06:46:24.728Z" }, - { url = "https://files.pythonhosted.org/packages/08/cc/e9316a28235409e9363794fc3d0b3083e48dd80d441006de66421e55f364/ninja-1.11.1.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c29bb66d2aa46a2409ab369ea804c730faec7652e8c22c1e428cc09216543e5", size = 157020, upload-time = "2025-03-22T06:46:26.046Z" }, - { url = "https://files.pythonhosted.org/packages/e3/30/389b22300541aa5f2e9dad322c4de2f84be4e32aa4e8babd9160d620b5f1/ninja-1.11.1.4-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:055f386fb550c2c9d6157e45e20a84d29c47968876b9c5794ae2aec46f952306", size = 130389, upload-time = "2025-03-22T06:46:27.174Z" }, - { url = "https://files.pythonhosted.org/packages/a9/10/e27f35cb92813aabbb7ae771b1685b45be1cc8a0798ce7d4bfd08d142b93/ninja-1.11.1.4-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:f6186d7607bb090c3be1e10c8a56b690be238f953616626f5032238c66e56867", size = 372435, upload-time = "2025-03-22T06:46:28.637Z" }, - { url = "https://files.pythonhosted.org/packages/c2/26/e3559619756739aae124c6abf7fe41f7e546ab1209cfbffb13137bff2d2e/ninja-1.11.1.4-py3-none-musllinux_1_1_i686.whl", hash = "sha256:cf4453679d15babc04ba023d68d091bb613091b67101c88f85d2171c6621c6eb", size = 419300, upload-time = "2025-03-22T06:46:30.392Z" }, - { url = "https://files.pythonhosted.org/packages/35/46/809e4e9572570991b8e6f88f3583807d017371ab4cb09171cbc72a7eb3e4/ninja-1.11.1.4-py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:d4a6f159b08b0ac4aca5ee1572e3e402f969139e71d85d37c0e2872129098749", size = 420239, upload-time = "2025-03-22T06:46:32.442Z" }, - { url = "https://files.pythonhosted.org/packages/e6/64/5cb5710d15f844edf02ada577f8eddfdcd116f47eec15850f3371a3a4b33/ninja-1.11.1.4-py3-none-musllinux_1_1_s390x.whl", hash = "sha256:c3b96bd875f3ef1db782470e9e41d7508905a0986571f219d20ffed238befa15", size = 415986, upload-time = "2025-03-22T06:46:33.821Z" }, - { url = "https://files.pythonhosted.org/packages/95/b2/0e9ab1d926f423b12b09925f78afcc5e48b3c22e7121be3ddf6c35bf06a3/ninja-1.11.1.4-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:cf554e73f72c04deb04d0cf51f5fdb1903d9c9ca3d2344249c8ce3bd616ebc02", size = 379657, upload-time = "2025-03-22T06:46:36.166Z" }, - { url = "https://files.pythonhosted.org/packages/c8/3e/fd6d330d0434168e7fe070d414b57dd99c4c133faa69c05b42a3cbdc6c13/ninja-1.11.1.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:cfdd09776436a1ff3c4a2558d3fc50a689fb9d7f1bdbc3e6f7b8c2991341ddb3", size = 454466, upload-time = "2025-03-22T06:46:37.413Z" }, - { url = "https://files.pythonhosted.org/packages/e6/df/a25f3ad0b1c59d1b90564096e4fd89a6ca30d562b1e942f23880c3000b89/ninja-1.11.1.4-py3-none-win32.whl", hash = "sha256:2ab67a41c90bea5ec4b795bab084bc0b3b3bb69d3cd21ca0294fc0fc15a111eb", size = 255931, upload-time = "2025-03-22T06:46:39.171Z" }, - { url = "https://files.pythonhosted.org/packages/5b/10/9b8fe9ac004847490cc7b54896124c01ce2d87d95dc60aabd0b8591addff/ninja-1.11.1.4-py3-none-win_amd64.whl", hash = "sha256:4617b3c12ff64b611a7d93fd9e378275512bb36eff8babff7c83f5116b4f8d66", size = 296461, upload-time = "2025-03-22T06:46:40.532Z" }, - { url = "https://files.pythonhosted.org/packages/b9/58/612a17593c2d117f96c7f6b7f1e6570246bddc4b1e808519403a1417f217/ninja-1.11.1.4-py3-none-win_arm64.whl", hash = "sha256:5713cf50c5be50084a8693308a63ecf9e55c3132a78a41ab1363a28b6caaaee1", size = 271441, upload-time = "2025-03-22T06:46:42.147Z" }, -] - -[[package]] -name = "nltk" -version = "3.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "joblib" }, - { name = "regex" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3c/87/db8be88ad32c2d042420b6fd9ffd4a149f9a0d7f0e86b3f543be2eeeedd2/nltk-3.9.1.tar.gz", hash = "sha256:87d127bd3de4bd89a4f81265e5fa59cb1b199b27440175370f7417d2bc7ae868", size = 2904691, upload-time = "2024-08-18T19:48:37.769Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/66/7d9e26593edda06e8cb531874633f7c2372279c3b0f46235539fe546df8b/nltk-3.9.1-py3-none-any.whl", hash = "sha256:4fa26829c5b00715afe3061398a8989dc643b92ce7dd93fb4585a70930d168a1", size = 1505442, upload-time = "2024-08-18T19:48:21.909Z" }, -] - -[[package]] -name = "nodeenv" -version = "1.9.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, -] - -[[package]] -name = "numba" -version = "0.60.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3c/93/2849300a9184775ba274aba6f82f303343669b0592b7bb0849ea713dabb0/numba-0.60.0.tar.gz", hash = "sha256:5df6158e5584eece5fc83294b949fd30b9f1125df7708862205217e068aabf16", size = 2702171, upload-time = "2024-06-13T18:11:19.869Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/5c/b5ec752c475e78a6c3676b67c514220dbde2725896bbb0b6ec6ea54b2738/numba-0.60.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d7da4098db31182fc5ffe4bc42c6f24cd7d1cb8a14b59fd755bfee32e34b8404", size = 2647866, upload-time = "2024-06-13T18:10:54.453Z" }, - { url = "https://files.pythonhosted.org/packages/65/42/39559664b2e7c15689a638c2a38b3b74c6e69a04e2b3019b9f7742479188/numba-0.60.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38d6ea4c1f56417076ecf8fc327c831ae793282e0ff51080c5094cb726507b1c", size = 2650208, upload-time = "2024-06-13T18:10:56.779Z" }, - { url = "https://files.pythonhosted.org/packages/67/88/c4459ccc05674ef02119abf2888ccd3e2fed12a323f52255f4982fc95876/numba-0.60.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:62908d29fb6a3229c242e981ca27e32a6e606cc253fc9e8faeb0e48760de241e", size = 3466946, upload-time = "2024-06-13T18:10:58.961Z" }, - { url = "https://files.pythonhosted.org/packages/8b/41/ac11cf33524def12aa5bd698226ae196a1185831c05ed29dc0c56eaa308b/numba-0.60.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ebaa91538e996f708f1ab30ef4d3ddc344b64b5227b67a57aa74f401bb68b9d", size = 3761463, upload-time = "2024-06-13T18:11:01.657Z" }, - { url = "https://files.pythonhosted.org/packages/ca/bd/0fe29fcd1b6a8de479a4ed25c6e56470e467e3611c079d55869ceef2b6d1/numba-0.60.0-cp312-cp312-win_amd64.whl", hash = "sha256:f75262e8fe7fa96db1dca93d53a194a38c46da28b112b8a4aca168f0df860347", size = 2707588, upload-time = "2024-06-13T18:11:04.261Z" }, -] - -[[package]] -name = "numpy" -version = "1.26.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, - { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, - { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, - { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, - { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, -] - -[[package]] -name = "nvidia-cublas-cu12" -version = "12.4.5.8" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/71/1c91302526c45ab494c23f61c7a84aa568b8c1f9d196efa5993957faf906/nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b", size = 363438805, upload-time = "2024-04-03T20:57:06.025Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti-cu12" -version = "12.4.127" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/42/f4f60238e8194a3106d06a058d494b18e006c10bb2b915655bd9f6ea4cb1/nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb", size = 13813957, upload-time = "2024-04-03T20:55:01.564Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc-cu12" -version = "12.4.127" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/14/91ae57cd4db3f9ef7aa99f4019cfa8d54cb4caa7e00975df6467e9725a9f/nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338", size = 24640306, upload-time = "2024-04-03T20:56:01.463Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime-cu12" -version = "12.4.127" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/27/1795d86fe88ef397885f2e580ac37628ed058a92ed2c39dc8eac3adf0619/nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5", size = 883737, upload-time = "2024-04-03T20:54:51.355Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu12" -version = "9.1.0.70" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741, upload-time = "2024-04-22T15:24:15.253Z" }, -] - -[[package]] -name = "nvidia-cufft-cu12" -version = "11.2.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/94/3266821f65b92b3138631e9c8e7fe1fb513804ac934485a8d05776e1dd43/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9", size = 211459117, upload-time = "2024-04-03T20:57:40.402Z" }, -] - -[[package]] -name = "nvidia-curand-cu12" -version = "10.3.5.147" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/6d/44ad094874c6f1b9c654f8ed939590bdc408349f137f9b98a3a23ccec411/nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b", size = 56305206, upload-time = "2024-04-03T20:58:08.722Z" }, -] - -[[package]] -name = "nvidia-cusolver-cu12" -version = "11.6.1.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/e1/5b9089a4b2a4790dfdea8b3a006052cfecff58139d5a4e34cb1a51df8d6f/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260", size = 127936057, upload-time = "2024-04-03T20:58:28.735Z" }, -] - -[[package]] -name = "nvidia-cusparse-cu12" -version = "12.3.1.170" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/f7/97a9ea26ed4bbbfc2d470994b8b4f338ef663be97b8f677519ac195e113d/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1", size = 207454763, upload-time = "2024-04-03T20:58:59.995Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu12" -version = "0.6.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/a8/bcbb63b53a4b1234feeafb65544ee55495e1bb37ec31b999b963cbccfd1d/nvidia_cusparselt_cu12-0.6.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:df2c24502fd76ebafe7457dbc4716b2fec071aabaed4fb7691a201cde03704d9", size = 150057751, upload-time = "2024-07-23T02:35:53.074Z" }, -] - -[[package]] -name = "nvidia-ml-py" -version = "12.570.86" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ad/6e/7b0c9b88c7d520fb8639024a1a3b6dd1db03bf2c17ae85040c8758d2eb6f/nvidia_ml_py-12.570.86.tar.gz", hash = "sha256:0508d4a0c7b6d015cf574530b95a62ed4fc89da3b8b47e1aefe6777db170ec8b", size = 43147, upload-time = "2025-01-24T21:50:38.869Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/a8/ec37169be4e2b7063b9076ed3fe0661e87335fbca665eed3f48c415cb234/nvidia_ml_py-12.570.86-py3-none-any.whl", hash = "sha256:58907de35a845abd13dcb227f18298f3b5dd94a72d04c9e594e77711e95c0b51", size = 44442, upload-time = "2025-01-24T21:50:37.701Z" }, -] - -[[package]] -name = "nvidia-nccl-cu12" -version = "2.21.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/99/12cd266d6233f47d00daf3a72739872bdc10267d0383508b0b9c84a18bb6/nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0", size = 188654414, upload-time = "2024-04-03T15:32:57.427Z" }, -] - -[[package]] -name = "nvidia-nvjitlink-cu12" -version = "12.4.127" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/ff/847841bacfbefc97a00036e0fce5a0f086b640756dc38caea5e1bb002655/nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57", size = 21066810, upload-time = "2024-04-03T20:59:46.957Z" }, -] - -[[package]] -name = "nvidia-nvtx-cu12" -version = "12.4.127" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/20/199b8713428322a2f22b722c62b8cc278cc53dffa9705d744484b5035ee9/nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a", size = 99144, upload-time = "2024-04-03T20:56:12.406Z" }, -] - -[[package]] -name = "oauthlib" -version = "3.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/fa/fbf4001037904031639e6bfbfc02badfc7e12f137a8afa254df6c4c8a670/oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918", size = 177352, upload-time = "2022-10-17T20:04:27.471Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/80/cab10959dc1faead58dc8384a781dfbf93cb4d33d50988f7a69f1b7c9bbe/oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca", size = 151688, upload-time = "2022-10-17T20:04:24.037Z" }, -] - -[[package]] -name = "omegaconf" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "antlr4-python3-runtime" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, -] - -[[package]] -name = "openai" -version = "1.72.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/41/56/41de36c0e9f787c406211552ecf2ca4fba3db900207c5c158c4dc67263fc/openai-1.72.0.tar.gz", hash = "sha256:f51de971448905cc90ed5175a5b19e92fd94e31f68cde4025762f9f5257150db", size = 426061, upload-time = "2025-04-08T19:52:45.527Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/1c/a0870f31bd71244c8c3a82e171677d9a148a8ea1cb157308cb9e06a41a37/openai-1.72.0-py3-none-any.whl", hash = "sha256:34f5496ba5c8cb06c592831d69e847e2d164526a2fb92afdc3b5cf2891c328c3", size = 643863, upload-time = "2025-04-08T19:52:43.583Z" }, -] - -[[package]] -name = "opencv-python-headless" -version = "4.11.0.86" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929, upload-time = "2025-01-16T13:53:40.22Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/53/2c50afa0b1e05ecdb4603818e85f7d174e683d874ef63a6abe3ac92220c8/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:48128188ade4a7e517237c8e1e11a9cdf5c282761473383e77beb875bb1e61ca", size = 37326460, upload-time = "2025-01-16T13:52:57.015Z" }, - { url = "https://files.pythonhosted.org/packages/3b/43/68555327df94bb9b59a1fd645f63fafb0762515344d2046698762fc19d58/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81", size = 56723330, upload-time = "2025-01-16T13:55:45.731Z" }, - { url = "https://files.pythonhosted.org/packages/45/be/1438ce43ebe65317344a87e4b150865c5585f4c0db880a34cdae5ac46881/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6efabcaa9df731f29e5ea9051776715b1bdd1845d7c9530065c7951d2a2899eb", size = 29487060, upload-time = "2025-01-16T13:51:59.625Z" }, - { url = "https://files.pythonhosted.org/packages/dd/5c/c139a7876099916879609372bfa513b7f1257f7f1a908b0bdc1c2328241b/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b", size = 49969856, upload-time = "2025-01-16T13:53:29.654Z" }, - { url = "https://files.pythonhosted.org/packages/95/dd/ed1191c9dc91abcc9f752b499b7928aacabf10567bb2c2535944d848af18/opencv_python_headless-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b", size = 29324425, upload-time = "2025-01-16T13:52:49.048Z" }, - { url = "https://files.pythonhosted.org/packages/86/8a/69176a64335aed183529207ba8bc3d329c2999d852b4f3818027203f50e6/opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca", size = 39402386, upload-time = "2025-01-16T13:52:56.418Z" }, -] - -[[package]] -name = "openhands-aci" -version = "0.2.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "binaryornot" }, - { name = "cachetools" }, - { name = "charset-normalizer" }, - { name = "flake8" }, - { name = "gitpython" }, - { name = "grep-ast" }, - { name = "pydantic" }, - { name = "tree-sitter" }, - { name = "tree-sitter-javascript" }, - { name = "tree-sitter-python" }, - { name = "tree-sitter-ruby" }, - { name = "tree-sitter-typescript" }, - { name = "whatthepatch" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d7/de/e61507cfa98d8fc5d11d9f580c597bb8cbf18944257b3be56ed62b88e212/openhands_aci-0.2.9.tar.gz", hash = "sha256:ca899614a10ed4ded3858c854ab30514b69fe4b07b74969d6a89ea10af8ae480", size = 18785, upload-time = "2025-04-11T10:20:07.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/d4/913dfe8affa3aedf8b6c4e423c4998ae2a702846793f35341a74efc62b75/openhands_aci-0.2.9-py3-none-any.whl", hash = "sha256:5d9d9e67c9fb1485e111f594aa0ec145b57111b6dc5905b868c2af706ce348d6", size = 24404, upload-time = "2025-04-11T10:20:06.37Z" }, -] - -[[package]] -name = "openhands-ai" -version = "0.25.0" -source = { editable = "SkyRL-OpenHands" } -dependencies = [ - { name = "aiodocker" }, - { name = "anthropic", extra = ["vertex"] }, - { name = "bashlex" }, - { name = "boto3" }, - { name = "browsergym-core" }, - { name = "datasets" }, - { name = "dirhash" }, - { name = "docker" }, - { name = "e2b" }, - { name = "fastapi" }, - { name = "gevent" }, - { name = "google-api-python-client" }, - { name = "google-auth-httplib2" }, - { name = "google-auth-oauthlib" }, - { name = "google-cloud-aiplatform" }, - { name = "google-generativeai" }, - { name = "grep-ast" }, - { name = "html2text" }, - { name = "ipywidgets" }, - { name = "jinja2" }, - { name = "joblib" }, - { name = "json-repair" }, - { name = "libtmux" }, - { name = "litellm" }, - { name = "memory-profiler" }, - { name = "minio" }, - { name = "modal" }, - { name = "numpy" }, - { name = "openhands-aci" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "pandas" }, - { name = "pathspec" }, - { name = "pexpect" }, - { name = "protobuf" }, - { name = "psutil" }, - { name = "pyarrow" }, - { name = "pydantic-settings" }, - { name = "pygithub" }, - { name = "pyjwt" }, - { name = "pylatexenc" }, - { name = "pylcs" }, - { name = "pypdf2" }, - { name = "python-docx" }, - { name = "python-dotenv" }, - { name = "python-frontmatter" }, - { name = "python-multipart" }, - { name = "python-pptx" }, - { name = "python-socketio" }, - { name = "qtconsole" }, - { name = "redis" }, - { name = "runloop-api-client" }, - { name = "seaborn" }, - { name = "sse-starlette" }, - { name = "stripe" }, - { name = "tenacity" }, - { name = "termcolor" }, - { name = "toml" }, - { name = "tornado" }, - { name = "tree-sitter" }, - { name = "types-toml" }, - { name = "uvicorn" }, - { name = "whatthepatch" }, - { name = "zope-interface" }, -] - -[package.metadata] -requires-dist = [ - { name = "aiodocker" }, - { name = "anthropic", extras = ["vertex"] }, - { name = "bashlex", specifier = ">=0.18,<0.19" }, - { name = "boto3" }, - { name = "browsergym-core", specifier = "==0.10.2" }, - { name = "datasets" }, - { name = "dirhash" }, - { name = "docker" }, - { name = "e2b", specifier = ">=1.0.5,<1.2.0" }, - { name = "fastapi" }, - { name = "gevent", specifier = ">=24.2.1,<25.0.0" }, - { name = "google-api-python-client" }, - { name = "google-auth-httplib2" }, - { name = "google-auth-oauthlib" }, - { name = "google-cloud-aiplatform" }, - { name = "google-generativeai" }, - { name = "grep-ast", specifier = "==0.3.3" }, - { name = "html2text" }, - { name = "ipywidgets", specifier = ">=8.1.5,<9.0.0" }, - { name = "jinja2", specifier = ">=3.1.3,<4.0.0" }, - { name = "joblib" }, - { name = "json-repair" }, - { name = "libtmux", specifier = ">=0.37,<0.40" }, - { name = "litellm", specifier = ">=1.60.0,<2.0.0" }, - { name = "memory-profiler", specifier = ">=0.61.0,<0.62.0" }, - { name = "minio", specifier = ">=7.2.8,<8.0.0" }, - { name = "modal", specifier = ">=0.66.26,<0.74.0" }, - { name = "numpy" }, - { name = "openhands-aci", specifier = ">=0.2.5,<0.3.0" }, - { name = "opentelemetry-api", specifier = "==1.25.0" }, - { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = "==1.25.0" }, - { name = "pandas" }, - { name = "pathspec", specifier = ">=0.12.1,<0.13.0" }, - { name = "pexpect" }, - { name = "protobuf", specifier = ">=4.21.6,<5.0.0" }, - { name = "psutil" }, - { name = "pyarrow", specifier = "==19.0.1" }, - { name = "pydantic-settings" }, - { name = "pygithub", specifier = ">=2.5.0,<3.0.0" }, - { name = "pyjwt", specifier = ">=2.9.0,<3.0.0" }, - { name = "pylatexenc" }, - { name = "pylcs", specifier = ">=0.1.1,<0.2.0" }, - { name = "pypdf2" }, - { name = "python-docx" }, - { name = "python-dotenv" }, - { name = "python-frontmatter", specifier = ">=1.1.0,<2.0.0" }, - { name = "python-multipart" }, - { name = "python-pptx" }, - { name = "python-socketio", specifier = ">=5.11.4,<6.0.0" }, - { name = "qtconsole", specifier = ">=5.6.1,<6.0.0" }, - { name = "redis", specifier = ">=5.2.0,<6.0.0" }, - { name = "runloop-api-client", specifier = "==0.24.0" }, - { name = "seaborn" }, - { name = "sse-starlette", specifier = ">=2.1.3,<3.0.0" }, - { name = "stripe", specifier = ">=11.5.0,<12.0.0" }, - { name = "tenacity", specifier = ">=8.5,<10.0" }, - { name = "termcolor" }, - { name = "toml" }, - { name = "tornado" }, - { name = "tree-sitter", specifier = ">=0.24.0,<0.25.0" }, - { name = "types-toml" }, - { name = "uvicorn" }, - { name = "whatthepatch", specifier = ">=1.0.6,<2.0.0" }, - { name = "zope-interface", specifier = "==7.2" }, -] - -[[package]] -name = "opentelemetry-api" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "importlib-metadata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/0d/10357006dc10fc65f7c7b46c18232e466e355f9e606ac461cfc7193b4cbe/opentelemetry_api-1.25.0.tar.gz", hash = "sha256:77c4985f62f2614e42ce77ee4c9da5fa5f0bc1e1821085e9a47533a9323ae869", size = 60383, upload-time = "2024-05-31T01:40:38.766Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/b2/4bc5e52c9a23df0ac17dbb23923e609a8269cd67000a712b4f5bcfae1490/opentelemetry_api-1.25.0-py3-none-any.whl", hash = "sha256:757fa1aa020a0f8fa139f8959e53dec2051cc26b832e76fa839a6d76ecefd737", size = 59910, upload-time = "2024-05-31T01:40:00.911Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-proto" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/37/a7/85ffaaacd712e4634fa1c56cbf79a02cf90b8a178fe1eee2cabfb0b7f44d/opentelemetry_exporter_otlp_proto_common-1.25.0.tar.gz", hash = "sha256:c93f4e30da4eee02bacd1e004eb82ce4da143a2f8e15b987a9f603e0a85407d3", size = 17152, upload-time = "2024-05-31T01:40:42.259Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/02/74ac6619eec78c82a923324f916d3eccd2f2254cf4270b669e96b76bf717/opentelemetry_exporter_otlp_proto_common-1.25.0-py3-none-any.whl", hash = "sha256:15637b7d580c2675f70246563363775b4e6de947871e01d0f4e3881d1848d693", size = 17762, upload-time = "2024-05-31T01:40:13.172Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/76/94c72787f82130d1a24be64fad34eeaf7fa85cfe9d1740d096f577c5e57d/opentelemetry_exporter_otlp_proto_grpc-1.25.0.tar.gz", hash = "sha256:c0b1661415acec5af87625587efa1ccab68b873745ca0ee96b69bb1042087eac", size = 25278, upload-time = "2024-05-31T01:40:42.974Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/f3/e24294e7b3f6d2b9aafc97b9b82e214dfe9ffa152dfecbd897e7ffbf6844/opentelemetry_exporter_otlp_proto_grpc-1.25.0-py3-none-any.whl", hash = "sha256:3131028f0c0a155a64c430ca600fd658e8e37043cb13209f0109db5c1a3e4eb4", size = 18223, upload-time = "2024-05-31T01:40:14.844Z" }, -] - -[[package]] -name = "opentelemetry-proto" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c9/3c/28c9ce40eb8ab287471af81659089ca98ef4f7ce289669e23b19c29f24a8/opentelemetry_proto-1.25.0.tar.gz", hash = "sha256:35b6ef9dc4a9f7853ecc5006738ad40443701e52c26099e197895cbda8b815a3", size = 35062, upload-time = "2024-05-31T01:40:52.737Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/ae/d6b5f11ecbffafe8b6d54130fed0cc78aad3711e00074d63a7359d6dcf3b/opentelemetry_proto-1.25.0-py3-none-any.whl", hash = "sha256:f07e3341c78d835d9b86665903b199893befa5e98866f63d22b00d0b7ca4972f", size = 52450, upload-time = "2024-05-31T01:40:30.987Z" }, -] - -[[package]] -name = "opentelemetry-sdk" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/05/3c/77076b77f1d73141adc119f62370ec9456ef314ba0b4e7072e3775c36ef7/opentelemetry_sdk-1.25.0.tar.gz", hash = "sha256:ce7fc319c57707ef5bf8b74fb9f8ebdb8bfafbe11898410e0d2a761d08a98ec7", size = 141042, upload-time = "2024-05-31T01:40:53.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/b2/729a959a8aa032bce246c791f977161099ab60fb0188408ccec1bf283b00/opentelemetry_sdk-1.25.0-py3-none-any.whl", hash = "sha256:d97ff7ec4b351692e9d5a15af570c693b8715ad78b8aafbec5c7100fe966b4c9", size = 107028, upload-time = "2024-05-31T01:40:33.281Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.46b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4e/ea/a4a5277247b3d2ed2e23a58b0d509c2eafa4ebb56038ba5b23c0f9ea6242/opentelemetry_semantic_conventions-0.46b0.tar.gz", hash = "sha256:fbc982ecbb6a6e90869b15c1673be90bd18c8a56ff1cffc0864e38e2edffaefa", size = 80198, upload-time = "2024-05-31T01:40:54.722Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/41/28dae1ec1fe0151373f06bd06d9170ca14b52d5b3a6c2dc55f85bc219619/opentelemetry_semantic_conventions-0.46b0-py3-none-any.whl", hash = "sha256:6daef4ef9fa51d51855d9f8e0ccd3a1bd59e0e545abe99ac6203804e36ab3e07", size = 130549, upload-time = "2024-05-31T01:40:35.348Z" }, -] - -[[package]] -name = "orjson" -version = "3.10.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/c7/03913cc4332174071950acf5b0735463e3f63760c80585ef369270c2b372/orjson-3.10.16.tar.gz", hash = "sha256:d2aaa5c495e11d17b9b93205f5fa196737ee3202f000aaebf028dc9a73750f10", size = 5410415, upload-time = "2025-03-24T17:00:23.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/15/67ce9d4c959c83f112542222ea3b9209c1d424231d71d74c4890ea0acd2b/orjson-3.10.16-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6d3444abbfa71ba21bb042caa4b062535b122248259fdb9deea567969140abca", size = 249325, upload-time = "2025-03-24T16:59:19.784Z" }, - { url = "https://files.pythonhosted.org/packages/da/2c/1426b06f30a1b9ada74b6f512c1ddf9d2760f53f61cdb59efeb9ad342133/orjson-3.10.16-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:30245c08d818fdcaa48b7d5b81499b8cae09acabb216fe61ca619876b128e184", size = 133621, upload-time = "2025-03-24T16:59:21.207Z" }, - { url = "https://files.pythonhosted.org/packages/9e/88/18d26130954bc73bee3be10f95371ea1dfb8679e0e2c46b0f6d8c6289402/orjson-3.10.16-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0ba1d0baa71bf7579a4ccdcf503e6f3098ef9542106a0eca82395898c8a500a", size = 138270, upload-time = "2025-03-24T16:59:22.514Z" }, - { url = "https://files.pythonhosted.org/packages/4f/f9/6d8b64fcd58fae072e80ee7981be8ba0d7c26ace954e5cd1d027fc80518f/orjson-3.10.16-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb0beefa5ef3af8845f3a69ff2a4aa62529b5acec1cfe5f8a6b4141033fd46ef", size = 132346, upload-time = "2025-03-24T16:59:24.277Z" }, - { url = "https://files.pythonhosted.org/packages/16/3f/2513fd5bc786f40cd12af569c23cae6381aeddbefeed2a98f0a666eb5d0d/orjson-3.10.16-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6daa0e1c9bf2e030e93c98394de94506f2a4d12e1e9dadd7c53d5e44d0f9628e", size = 136845, upload-time = "2025-03-24T16:59:25.588Z" }, - { url = "https://files.pythonhosted.org/packages/6d/42/b0e7b36720f5ab722b48e8ccf06514d4f769358dd73c51abd8728ef58d0b/orjson-3.10.16-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9da9019afb21e02410ef600e56666652b73eb3e4d213a0ec919ff391a7dd52aa", size = 138078, upload-time = "2025-03-24T16:59:27.288Z" }, - { url = "https://files.pythonhosted.org/packages/a3/a8/d220afb8a439604be74fc755dbc740bded5ed14745ca536b304ed32eb18a/orjson-3.10.16-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:daeb3a1ee17b69981d3aae30c3b4e786b0f8c9e6c71f2b48f1aef934f63f38f4", size = 142712, upload-time = "2025-03-24T16:59:28.613Z" }, - { url = "https://files.pythonhosted.org/packages/8c/88/7e41e9883c00f84f92fe357a8371edae816d9d7ef39c67b5106960c20389/orjson-3.10.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fed80eaf0e20a31942ae5d0728849862446512769692474be5e6b73123a23b", size = 133136, upload-time = "2025-03-24T16:59:29.987Z" }, - { url = "https://files.pythonhosted.org/packages/e9/ca/61116095307ad0be828ea26093febaf59e38596d84a9c8d765c3c5e4934f/orjson-3.10.16-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73390ed838f03764540a7bdc4071fe0123914c2cc02fb6abf35182d5fd1b7a42", size = 135258, upload-time = "2025-03-24T16:59:31.339Z" }, - { url = "https://files.pythonhosted.org/packages/dc/1b/09493cf7d801505f094c9295f79c98c1e0af2ac01c7ed8d25b30fcb19ada/orjson-3.10.16-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a22bba012a0c94ec02a7768953020ab0d3e2b884760f859176343a36c01adf87", size = 412326, upload-time = "2025-03-24T16:59:32.709Z" }, - { url = "https://files.pythonhosted.org/packages/ea/02/125d7bbd7f7a500190ddc8ae5d2d3c39d87ed3ed28f5b37cfe76962c678d/orjson-3.10.16-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5385bbfdbc90ff5b2635b7e6bebf259652db00a92b5e3c45b616df75b9058e88", size = 152800, upload-time = "2025-03-24T16:59:34.134Z" }, - { url = "https://files.pythonhosted.org/packages/f9/09/7658a9e3e793d5b3b00598023e0fb6935d0e7bbb8ff72311c5415a8ce677/orjson-3.10.16-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:02c6279016346e774dd92625d46c6c40db687b8a0d685aadb91e26e46cc33e1e", size = 137516, upload-time = "2025-03-24T16:59:35.446Z" }, - { url = "https://files.pythonhosted.org/packages/29/87/32b7a4831e909d347278101a48d4cf9f3f25901b2295e7709df1651f65a1/orjson-3.10.16-cp312-cp312-win32.whl", hash = "sha256:7ca55097a11426db80f79378e873a8c51f4dde9ffc22de44850f9696b7eb0e8c", size = 141759, upload-time = "2025-03-24T16:59:37.509Z" }, - { url = "https://files.pythonhosted.org/packages/35/ce/81a27e7b439b807bd393585271364cdddf50dc281fc57c4feef7ccb186a6/orjson-3.10.16-cp312-cp312-win_amd64.whl", hash = "sha256:86d127efdd3f9bf5f04809b70faca1e6836556ea3cc46e662b44dab3fe71f3d6", size = 133944, upload-time = "2025-03-24T16:59:38.814Z" }, -] - -[[package]] -name = "outlines" -version = "0.1.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "airportsdata" }, - { name = "cloudpickle" }, - { name = "diskcache" }, - { name = "interegular" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "lark" }, - { name = "nest-asyncio" }, - { name = "numpy" }, - { name = "outlines-core" }, - { name = "pycountry" }, - { name = "pydantic" }, - { name = "referencing" }, - { name = "requests" }, - { name = "torch" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/d0/d59ae830bf7026425942899e3d48e77b58a713cff946a695e5405808da1b/outlines-0.1.11.tar.gz", hash = "sha256:0997bd9da1cc050e430bd08995dc7d4bd855918bafa4531e49d3f37110a23aba", size = 2488858, upload-time = "2024-12-13T07:24:08.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/b4/99ea4a122bef60e3fd6402d19665aff1f928e0daf8fac3044d0b73f72003/outlines-0.1.11-py3-none-any.whl", hash = "sha256:f5a5f2242ed9802d3aab7a92789bf4008d734c576be9258cc0a297f690124727", size = 87623, upload-time = "2024-12-13T07:24:05.817Z" }, -] - -[[package]] -name = "outlines-core" -version = "0.1.26" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "interegular" }, - { name = "jsonschema" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/f3/274d07f4702728b43581235a77e545ec602b25f9b0098b288a0f3052521d/outlines_core-0.1.26.tar.gz", hash = "sha256:481c4301341e77cc8f1832d616784adb4d461b4fec65878e7c0d2cba7163a189", size = 75139, upload-time = "2024-12-12T23:38:50.703Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/86/0fb40746e579db38d89f127122a3900d9e0350f76aae8cb61adeaff44cc2/outlines_core-0.1.26-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f54633bca50055d42ea4d94ae06dcbe52d3d76a9b621b75723b1177d0d952953", size = 321874, upload-time = "2024-12-12T23:38:26.834Z" }, - { url = "https://files.pythonhosted.org/packages/ab/0c/b91f7bc03843796c1d643ee030b6cd8fd5a8ba2cd4856c855f140c878976/outlines_core-0.1.26-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9525321b48700dcaaabf60bcdc951e45f9357ba3fb3e1bfc81b662d7d4170e7c", size = 301995, upload-time = "2024-12-12T23:38:29.625Z" }, - { url = "https://files.pythonhosted.org/packages/ad/db/fa91a2d54288b900de82d86eda3adb2417b3b5b2db6256854a5e8bc85c32/outlines_core-0.1.26-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f409f72c11f6ffadb57066950dd384d5388015028c1a1a615c9a64988dae3e", size = 321050, upload-time = "2024-12-12T23:38:32.274Z" }, - { url = "https://files.pythonhosted.org/packages/e2/1d/a36292b6198986bd9c3ff8c24355deb82ed5475403379ee40b5b5473e2e3/outlines_core-0.1.26-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86a1bb46adc5cbf6dfd7a7fe4105e0e2a4c6e041732a053126b41c521a1f223", size = 343201, upload-time = "2024-12-12T23:38:34.631Z" }, - { url = "https://files.pythonhosted.org/packages/08/63/5dd2b5a364412f674b6edcb59b0c21513bdb07cdcc7613b064c1a0660d01/outlines_core-0.1.26-cp312-cp312-win32.whl", hash = "sha256:19f462f6b00935708677ad27cb4df55e0e17f6ffe713ab750f5f2683b090f95d", size = 233970, upload-time = "2024-12-12T23:38:37.318Z" }, - { url = "https://files.pythonhosted.org/packages/a5/56/8adf0b7446d1e975c2314454813c59eb7b195889908a2932ed34148c113c/outlines_core-0.1.26-cp312-cp312-win_amd64.whl", hash = "sha256:9b36bff12779e58883747116893a17b3551bbd10865878b951b03a44d112229a", size = 243578, upload-time = "2024-12-12T23:38:39.964Z" }, -] - -[[package]] -name = "packaging" -version = "24.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950, upload-time = "2024-11-08T09:47:47.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451, upload-time = "2024-11-08T09:47:44.722Z" }, -] - -[[package]] -name = "pandas" -version = "2.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a3/fb2734118db0af37ea7433f57f722c0a56687e14b14690edff0cdb4b7e58/pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9", size = 12529893, upload-time = "2024-09-20T13:09:09.655Z" }, - { url = "https://files.pythonhosted.org/packages/e1/0c/ad295fd74bfac85358fd579e271cded3ac969de81f62dd0142c426b9da91/pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4", size = 11363475, upload-time = "2024-09-20T13:09:14.718Z" }, - { url = "https://files.pythonhosted.org/packages/c6/2a/4bba3f03f7d07207481fed47f5b35f556c7441acddc368ec43d6643c5777/pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3", size = 15188645, upload-time = "2024-09-20T19:02:03.88Z" }, - { url = "https://files.pythonhosted.org/packages/38/f8/d8fddee9ed0d0c0f4a2132c1dfcf0e3e53265055da8df952a53e7eaf178c/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319", size = 12739445, upload-time = "2024-09-20T13:09:17.621Z" }, - { url = "https://files.pythonhosted.org/packages/20/e8/45a05d9c39d2cea61ab175dbe6a2de1d05b679e8de2011da4ee190d7e748/pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8", size = 16359235, upload-time = "2024-09-20T19:02:07.094Z" }, - { url = "https://files.pythonhosted.org/packages/1d/99/617d07a6a5e429ff90c90da64d428516605a1ec7d7bea494235e1c3882de/pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a", size = 14056756, upload-time = "2024-09-20T13:09:20.474Z" }, - { url = "https://files.pythonhosted.org/packages/29/d4/1244ab8edf173a10fd601f7e13b9566c1b525c4f365d6bee918e68381889/pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13", size = 11504248, upload-time = "2024-09-20T13:09:23.137Z" }, -] - -[[package]] -name = "parso" -version = "0.8.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/94/68e2e17afaa9169cf6412ab0f28623903be73d1b32e208d9e8e541bb086d/parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d", size = 400609, upload-time = "2024-04-05T09:43:55.897Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" }, -] - -[[package]] -name = "partial-json-parser" -version = "0.2.1.1.post5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/9c/9c366aed65acb40a97842ce1375a87b27ea37d735fc9717f7729bae3cc00/partial_json_parser-0.2.1.1.post5.tar.gz", hash = "sha256:992710ac67e90b367921d52727698928040f7713ba7ecb33b96371ea7aec82ca", size = 10313, upload-time = "2025-01-08T15:44:02.147Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/ee/a9476f01f27c74420601be208c6c2c0dd3486681d515e9d765931b89851c/partial_json_parser-0.2.1.1.post5-py3-none-any.whl", hash = "sha256:627715aaa3cb3fb60a65b0d62223243acaa6c70846520a90326fef3a2f0b61ca", size = 10885, upload-time = "2025-01-08T15:44:00.987Z" }, -] - -[[package]] -name = "pathspec" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, -] - -[[package]] -name = "peft" -version = "0.15.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "accelerate" }, - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "psutil" }, - { name = "pyyaml" }, - { name = "safetensors" }, - { name = "torch" }, - { name = "tqdm" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/ef/c5c9f8dd6b8ceb080112f41037e9f0ce442cfd8937739cf3e5d8645de3d5/peft-0.15.1.tar.gz", hash = "sha256:e4c65af70683a9ef3baf1ab450710f1eb7181f369ef6172ca8bf15bf4ae6ff71", size = 471440, upload-time = "2025-03-27T15:45:41.002Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/47/f2938c20b1c6d7b738ca2a129cf73cd1453cea76d5c9a2f2bbf6199bf037/peft-0.15.1-py3-none-any.whl", hash = "sha256:5fb3960beb518f00668f2cdc53424a5cc495c78281697821ce24609c90ca0a10", size = 411012, upload-time = "2025-03-27T15:45:38.885Z" }, -] - -[[package]] -name = "pexpect" -version = "4.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ptyprocess" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, -] - -[[package]] -name = "pillow" -version = "11.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/af/c097e544e7bd278333db77933e535098c259609c4eb3b85381109602fb5b/pillow-11.1.0.tar.gz", hash = "sha256:368da70808b36d73b4b390a8ffac11069f8a5c85f29eff1f1b01bcf3ef5b2a20", size = 46742715, upload-time = "2025-01-02T08:13:58.407Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/20/9ce6ed62c91c073fcaa23d216e68289e19d95fb8188b9fb7a63d36771db8/pillow-11.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2062ffb1d36544d42fcaa277b069c88b01bb7298f4efa06731a7fd6cc290b81a", size = 3226818, upload-time = "2025-01-02T08:11:22.518Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d8/f6004d98579a2596c098d1e30d10b248798cceff82d2b77aa914875bfea1/pillow-11.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a85b653980faad27e88b141348707ceeef8a1186f75ecc600c395dcac19f385b", size = 3101662, upload-time = "2025-01-02T08:11:25.19Z" }, - { url = "https://files.pythonhosted.org/packages/08/d9/892e705f90051c7a2574d9f24579c9e100c828700d78a63239676f960b74/pillow-11.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9409c080586d1f683df3f184f20e36fb647f2e0bc3988094d4fd8c9f4eb1b3b3", size = 4329317, upload-time = "2025-01-02T08:11:30.371Z" }, - { url = "https://files.pythonhosted.org/packages/8c/aa/7f29711f26680eab0bcd3ecdd6d23ed6bce180d82e3f6380fb7ae35fcf3b/pillow-11.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fdadc077553621911f27ce206ffcbec7d3f8d7b50e0da39f10997e8e2bb7f6a", size = 4412999, upload-time = "2025-01-02T08:11:33.499Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c4/8f0fe3b9e0f7196f6d0bbb151f9fba323d72a41da068610c4c960b16632a/pillow-11.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:93a18841d09bcdd774dcdc308e4537e1f867b3dec059c131fde0327899734aa1", size = 4368819, upload-time = "2025-01-02T08:11:37.304Z" }, - { url = "https://files.pythonhosted.org/packages/38/0d/84200ed6a871ce386ddc82904bfadc0c6b28b0c0ec78176871a4679e40b3/pillow-11.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9aa9aeddeed452b2f616ff5507459e7bab436916ccb10961c4a382cd3e03f47f", size = 4496081, upload-time = "2025-01-02T08:11:39.598Z" }, - { url = "https://files.pythonhosted.org/packages/84/9c/9bcd66f714d7e25b64118e3952d52841a4babc6d97b6d28e2261c52045d4/pillow-11.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cdcdb0b896e981678eee140d882b70092dac83ac1cdf6b3a60e2216a73f2b91", size = 4296513, upload-time = "2025-01-02T08:11:43.083Z" }, - { url = "https://files.pythonhosted.org/packages/db/61/ada2a226e22da011b45f7104c95ebda1b63dcbb0c378ad0f7c2a710f8fd2/pillow-11.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36ba10b9cb413e7c7dfa3e189aba252deee0602c86c309799da5a74009ac7a1c", size = 4431298, upload-time = "2025-01-02T08:11:46.626Z" }, - { url = "https://files.pythonhosted.org/packages/e7/c4/fc6e86750523f367923522014b821c11ebc5ad402e659d8c9d09b3c9d70c/pillow-11.1.0-cp312-cp312-win32.whl", hash = "sha256:cfd5cd998c2e36a862d0e27b2df63237e67273f2fc78f47445b14e73a810e7e6", size = 2291630, upload-time = "2025-01-02T08:11:49.401Z" }, - { url = "https://files.pythonhosted.org/packages/08/5c/2104299949b9d504baf3f4d35f73dbd14ef31bbd1ddc2c1b66a5b7dfda44/pillow-11.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a697cd8ba0383bba3d2d3ada02b34ed268cb548b369943cd349007730c92bddf", size = 2626369, upload-time = "2025-01-02T08:11:52.02Z" }, - { url = "https://files.pythonhosted.org/packages/37/f3/9b18362206b244167c958984b57c7f70a0289bfb59a530dd8af5f699b910/pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5", size = 2375240, upload-time = "2025-01-02T08:11:56.193Z" }, -] - -[[package]] -name = "platformdirs" -version = "4.3.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b6/2d/7d512a3913d60623e7eb945c6d1b4f0bddf1d0b7ada5225274c87e5b53d1/platformdirs-4.3.7.tar.gz", hash = "sha256:eb437d586b6a0986388f0d6f74aa0cde27b48d0e3d66843640bfb6bdcdb6e351", size = 21291, upload-time = "2025-03-19T20:36:10.989Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/45/59578566b3275b8fd9157885918fcd0c4d74162928a5310926887b856a51/platformdirs-4.3.7-py3-none-any.whl", hash = "sha256:a03875334331946f13c549dbd8f4bac7a13a50a895a0eb1e8c6a8ace80d40a94", size = 18499, upload-time = "2025-03-19T20:36:09.038Z" }, -] - -[[package]] -name = "playwright" -version = "1.39.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet" }, - { name = "pyee" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/e1/25cb1f5b4279e542e2a36882856fa845b83bc454d1029bd71d936da97ff2/playwright-1.39.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:384e195a6d09343f319031cf552e9cd601ede78fe9c082b9fa197537c5cbfe7a", size = 32973954, upload-time = "2023-10-17T19:43:41.679Z" }, - { url = "https://files.pythonhosted.org/packages/ae/2e/02e6b2b7bb68c96d69b52c70029607e156507ac956e12370d082c22cfc2c/playwright-1.39.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d2c3634411828d9273196ed6f69f2fa7645c89732b3c982dcf09ab03ed4c5d2b", size = 31443801, upload-time = "2023-10-17T19:43:49.895Z" }, - { url = "https://files.pythonhosted.org/packages/1c/b3/7950e87f1d3898c4667d37277c5ceaf2f0a80fd3e467b4acc105246f15b1/playwright-1.39.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:d2fd90f370599cf9a2c6a041bd79a5eeec62baf0e943c7c5c2079b29be476d2a", size = 32973950, upload-time = "2023-10-17T19:43:54.183Z" }, - { url = "https://files.pythonhosted.org/packages/58/7c/fca2b190e04984439f0dc0c79cc71c49191ab4ceef5597bf4b2c1d52b655/playwright-1.39.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:699a8e707ca5f3567aa28223ee1be7e42d2bf25eda7d3d86babda71e36e5f16f", size = 35496523, upload-time = "2023-10-17T19:43:58.283Z" }, - { url = "https://files.pythonhosted.org/packages/78/f5/9d42131797f2bbad57cf029a76e1e9ba55227a7884da58cc6811d3a24509/playwright-1.39.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:654bb3ae0dc3c69ffddc0c38c127c3b8e93032d8cf3928e2c4f21890cb39514b", size = 35388063, upload-time = "2023-10-17T19:44:02.846Z" }, - { url = "https://files.pythonhosted.org/packages/cf/78/ca1493e84b3752c7bf7e0812a29425727b448b54719fd3b3e391372efbc6/playwright-1.39.0-py3-none-win32.whl", hash = "sha256:40ed7f2546c64f1bb3d22b2295b4d43ed5a2f0b7ea7599d93a72f723a1883e1e", size = 29190413, upload-time = "2023-10-17T19:44:06.66Z" }, - { url = "https://files.pythonhosted.org/packages/04/98/17748e2d8f50428772232befdf6d55edccb8606c427203b849cf2b05031c/playwright-1.39.0-py3-none-win_amd64.whl", hash = "sha256:a420d814e21b05e1156747e2a9fae6c3cca2b46bb4a0226fb26ee65538ce09c9", size = 29190414, upload-time = "2023-10-17T19:44:10.845Z" }, -] - -[[package]] -name = "pluggy" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955, upload-time = "2024-04-20T21:34:42.531Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556, upload-time = "2024-04-20T21:34:40.434Z" }, -] - -[[package]] -name = "pre-commit" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cfgv" }, - { name = "identify" }, - { name = "nodeenv" }, - { name = "pyyaml" }, - { name = "virtualenv" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/08/39/679ca9b26c7bb2999ff122d50faa301e49af82ca9c066ec061cfbc0c6784/pre_commit-4.2.0.tar.gz", hash = "sha256:601283b9757afd87d40c4c4a9b2b5de9637a8ea02eaff7adc2d0fb4e04841146", size = 193424, upload-time = "2025-03-18T21:35:20.987Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/74/a88bf1b1efeae488a0c0b7bdf71429c313722d1fc0f377537fbe554e6180/pre_commit-4.2.0-py2.py3-none-any.whl", hash = "sha256:a009ca7205f1eb497d10b845e52c838a98b6cdd2102a6c8e4540e94ee75c58bd", size = 220707, upload-time = "2025-03-18T21:35:19.343Z" }, -] - -[[package]] -name = "prometheus-client" -version = "0.21.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/62/14/7d0f567991f3a9af8d1cd4f619040c93b68f09a02b6d0b6ab1b2d1ded5fe/prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb", size = 78551, upload-time = "2024-12-03T14:59:12.164Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/c2/ab7d37426c179ceb9aeb109a85cda8948bb269b7561a0be870cc656eefe4/prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301", size = 54682, upload-time = "2024-12-03T14:59:10.935Z" }, -] - -[[package]] -name = "prometheus-fastapi-instrumentator" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "prometheus-client" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/69/6d/24d53033cf93826aa7857699a4450c1c67e5b9c710e925b1ed2b320c04df/prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e", size = 20220, upload-time = "2025-03-19T19:35:05.351Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/72/0824c18f3bc75810f55dacc2dd933f6ec829771180245ae3cc976195dec0/prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9", size = 19296, upload-time = "2025-03-19T19:35:04.323Z" }, -] - -[[package]] -name = "prompt-toolkit" -version = "3.0.50" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/e1/bd15cb8ffdcfeeb2bdc215de3c3cffca11408d829e4b8416dcfe71ba8854/prompt_toolkit-3.0.50.tar.gz", hash = "sha256:544748f3860a2623ca5cd6d2795e7a14f3d0e1c3c9728359013f79877fc89bab", size = 429087, upload-time = "2025-01-20T15:55:35.072Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/ea/d836f008d33151c7a1f62caf3d8dd782e4d15f6a43897f64480c2b8de2ad/prompt_toolkit-3.0.50-py3-none-any.whl", hash = "sha256:9b6427eb19e479d98acff65196a307c555eb567989e6d88ebbb1b509d9779198", size = 387816, upload-time = "2025-01-20T15:55:29.98Z" }, -] - -[[package]] -name = "propcache" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/07/c8/fdc6686a986feae3541ea23dcaa661bd93972d3940460646c6bb96e21c40/propcache-0.3.1.tar.gz", hash = "sha256:40d980c33765359098837527e18eddefc9a24cea5b45e078a7f3bb5b032c6ecf", size = 43651, upload-time = "2025-03-26T03:06:12.05Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/aa/ca78d9be314d1e15ff517b992bebbed3bdfef5b8919e85bf4940e57b6137/propcache-0.3.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f78eb8422acc93d7b69964012ad7048764bb45a54ba7a39bb9e146c72ea29723", size = 80430, upload-time = "2025-03-26T03:04:26.436Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d8/f0c17c44d1cda0ad1979af2e593ea290defdde9eaeb89b08abbe02a5e8e1/propcache-0.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:89498dd49c2f9a026ee057965cdf8192e5ae070ce7d7a7bd4b66a8e257d0c976", size = 46637, upload-time = "2025-03-26T03:04:27.932Z" }, - { url = "https://files.pythonhosted.org/packages/ae/bd/c1e37265910752e6e5e8a4c1605d0129e5b7933c3dc3cf1b9b48ed83b364/propcache-0.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:09400e98545c998d57d10035ff623266927cb784d13dd2b31fd33b8a5316b85b", size = 46123, upload-time = "2025-03-26T03:04:30.659Z" }, - { url = "https://files.pythonhosted.org/packages/d4/b0/911eda0865f90c0c7e9f0415d40a5bf681204da5fd7ca089361a64c16b28/propcache-0.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa8efd8c5adc5a2c9d3b952815ff8f7710cefdcaf5f2c36d26aff51aeca2f12f", size = 243031, upload-time = "2025-03-26T03:04:31.977Z" }, - { url = "https://files.pythonhosted.org/packages/0a/06/0da53397c76a74271621807265b6eb61fb011451b1ddebf43213df763669/propcache-0.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2fe5c910f6007e716a06d269608d307b4f36e7babee5f36533722660e8c4a70", size = 249100, upload-time = "2025-03-26T03:04:33.45Z" }, - { url = "https://files.pythonhosted.org/packages/f1/eb/13090e05bf6b963fc1653cdc922133ced467cb4b8dab53158db5a37aa21e/propcache-0.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ab8cf8cdd2194f8ff979a43ab43049b1df0b37aa64ab7eca04ac14429baeb7", size = 250170, upload-time = "2025-03-26T03:04:35.542Z" }, - { url = "https://files.pythonhosted.org/packages/3b/4c/f72c9e1022b3b043ec7dc475a0f405d4c3e10b9b1d378a7330fecf0652da/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25", size = 245000, upload-time = "2025-03-26T03:04:37.501Z" }, - { url = "https://files.pythonhosted.org/packages/e8/fd/970ca0e22acc829f1adf5de3724085e778c1ad8a75bec010049502cb3a86/propcache-0.3.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb6e0faf8cb6b4beea5d6ed7b5a578254c6d7df54c36ccd3d8b3eb00d6770277", size = 230262, upload-time = "2025-03-26T03:04:39.532Z" }, - { url = "https://files.pythonhosted.org/packages/c4/42/817289120c6b9194a44f6c3e6b2c3277c5b70bbad39e7df648f177cc3634/propcache-0.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1c5c7ab7f2bb3f573d1cb921993006ba2d39e8621019dffb1c5bc94cdbae81e8", size = 236772, upload-time = "2025-03-26T03:04:41.109Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9c/3b3942b302badd589ad6b672da3ca7b660a6c2f505cafd058133ddc73918/propcache-0.3.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:050b571b2e96ec942898f8eb46ea4bfbb19bd5502424747e83badc2d4a99a44e", size = 231133, upload-time = "2025-03-26T03:04:42.544Z" }, - { url = "https://files.pythonhosted.org/packages/98/a1/75f6355f9ad039108ff000dfc2e19962c8dea0430da9a1428e7975cf24b2/propcache-0.3.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e1c4d24b804b3a87e9350f79e2371a705a188d292fd310e663483af6ee6718ee", size = 230741, upload-time = "2025-03-26T03:04:44.06Z" }, - { url = "https://files.pythonhosted.org/packages/67/0c/3e82563af77d1f8731132166da69fdfd95e71210e31f18edce08a1eb11ea/propcache-0.3.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e4fe2a6d5ce975c117a6bb1e8ccda772d1e7029c1cca1acd209f91d30fa72815", size = 244047, upload-time = "2025-03-26T03:04:45.983Z" }, - { url = "https://files.pythonhosted.org/packages/f7/50/9fb7cca01532a08c4d5186d7bb2da6c4c587825c0ae134b89b47c7d62628/propcache-0.3.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:feccd282de1f6322f56f6845bf1207a537227812f0a9bf5571df52bb418d79d5", size = 246467, upload-time = "2025-03-26T03:04:47.699Z" }, - { url = "https://files.pythonhosted.org/packages/a9/02/ccbcf3e1c604c16cc525309161d57412c23cf2351523aedbb280eb7c9094/propcache-0.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ec314cde7314d2dd0510c6787326bbffcbdc317ecee6b7401ce218b3099075a7", size = 241022, upload-time = "2025-03-26T03:04:49.195Z" }, - { url = "https://files.pythonhosted.org/packages/db/19/e777227545e09ca1e77a6e21274ae9ec45de0f589f0ce3eca2a41f366220/propcache-0.3.1-cp312-cp312-win32.whl", hash = "sha256:7d2d5a0028d920738372630870e7d9644ce437142197f8c827194fca404bf03b", size = 40647, upload-time = "2025-03-26T03:04:50.595Z" }, - { url = "https://files.pythonhosted.org/packages/24/bb/3b1b01da5dd04c77a204c84e538ff11f624e31431cfde7201d9110b092b1/propcache-0.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:88c423efef9d7a59dae0614eaed718449c09a5ac79a5f224a8b9664d603f04a3", size = 44784, upload-time = "2025-03-26T03:04:51.791Z" }, - { url = "https://files.pythonhosted.org/packages/b8/d3/c3cb8f1d6ae3b37f83e1de806713a9b3642c5895f0215a62e1a4bd6e5e34/propcache-0.3.1-py3-none-any.whl", hash = "sha256:9a8ecf38de50a7f518c21568c80f985e776397b902f1ce0b01f799aba1608b40", size = 12376, upload-time = "2025-03-26T03:06:10.5Z" }, -] - -[[package]] -name = "proto-plus" -version = "1.26.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/ac/87285f15f7cce6d4a008f33f1757fb5a13611ea8914eb58c3d0d26243468/proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012", size = 56142, upload-time = "2025-03-10T15:54:38.843Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/6d/280c4c2ce28b1593a19ad5239c8b826871fc6ec275c21afc8e1820108039/proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", size = 50163, upload-time = "2025-03-10T15:54:37.335Z" }, -] - -[[package]] -name = "protobuf" -version = "4.25.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/48/d5/cccc7e82bbda9909ced3e7a441a24205ea07fea4ce23a772743c0c7611fa/protobuf-4.25.6.tar.gz", hash = "sha256:f8cfbae7c5afd0d0eaccbe73267339bff605a2315860bb1ba08eb66670a9a91f", size = 380631, upload-time = "2025-01-24T20:53:09.498Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/41/0ff3559d9a0fbdb37c9452f2b84e61f7784d8d7b9850182c7ef493f523ee/protobuf-4.25.6-cp310-abi3-win32.whl", hash = "sha256:61df6b5786e2b49fc0055f636c1e8f0aff263808bb724b95b164685ac1bcc13a", size = 392454, upload-time = "2025-01-24T20:52:51.08Z" }, - { url = "https://files.pythonhosted.org/packages/79/84/c700d6c3f3be770495b08a1c035e330497a31420e4a39a24c22c02cefc6c/protobuf-4.25.6-cp310-abi3-win_amd64.whl", hash = "sha256:b8f837bfb77513fe0e2f263250f423217a173b6d85135be4d81e96a4653bcd3c", size = 413443, upload-time = "2025-01-24T20:52:54.523Z" }, - { url = "https://files.pythonhosted.org/packages/b7/03/361e87cc824452376c2abcef0eabd18da78a7439479ec6541cf29076a4dc/protobuf-4.25.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6d4381f2417606d7e01750e2729fe6fbcda3f9883aa0c32b51d23012bded6c91", size = 394246, upload-time = "2025-01-24T20:52:56.694Z" }, - { url = "https://files.pythonhosted.org/packages/64/d5/7dbeb69b74fa88f297c6d8f11b7c9cef0c2e2fb1fdf155c2ca5775cfa998/protobuf-4.25.6-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:5dd800da412ba7f6f26d2c08868a5023ce624e1fdb28bccca2dc957191e81fb5", size = 293714, upload-time = "2025-01-24T20:52:57.992Z" }, - { url = "https://files.pythonhosted.org/packages/d4/f0/6d5c100f6b18d973e86646aa5fc09bc12ee88a28684a56fd95511bceee68/protobuf-4.25.6-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:4434ff8bb5576f9e0c78f47c41cdf3a152c0b44de475784cd3fd170aef16205a", size = 294634, upload-time = "2025-01-24T20:52:59.671Z" }, - { url = "https://files.pythonhosted.org/packages/71/eb/be11a1244d0e58ee04c17a1f939b100199063e26ecca8262c04827fe0bf5/protobuf-4.25.6-py3-none-any.whl", hash = "sha256:07972021c8e30b870cfc0863409d033af940213e0e7f64e27fe017b929d2c9f7", size = 156466, upload-time = "2025-01-24T20:53:08.287Z" }, -] - -[[package]] -name = "psutil" -version = "7.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003, upload-time = "2025-02-13T21:54:07.946Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051, upload-time = "2025-02-13T21:54:12.36Z" }, - { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535, upload-time = "2025-02-13T21:54:16.07Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004, upload-time = "2025-02-13T21:54:18.662Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986, upload-time = "2025-02-13T21:54:21.811Z" }, - { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544, upload-time = "2025-02-13T21:54:24.68Z" }, - { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053, upload-time = "2025-02-13T21:54:34.31Z" }, - { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885, upload-time = "2025-02-13T21:54:37.486Z" }, -] - -[[package]] -name = "ptyprocess" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, -] - -[[package]] -name = "pure-eval" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, -] - -[[package]] -name = "py" -version = "1.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/ff/fec109ceb715d2a6b4c4a85a61af3b40c723a961e8828319fbcb15b868dc/py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", size = 207796, upload-time = "2021-11-04T17:17:01.377Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/f0/10642828a8dfb741e5f3fbaac830550a518a775c7fff6f04a007259b0548/py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378", size = 98708, upload-time = "2021-11-04T17:17:00.152Z" }, -] - -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - -[[package]] -name = "py-spy" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7c/cd/9dacc04604dc4398ce5bed77ed59918ad0940f15165954d4aaa651cc640c/py_spy-0.4.0.tar.gz", hash = "sha256:806602ce7972782cc9c1e383f339bfc27bfb822d42485e6a3e0530ae5040e1f0", size = 253236, upload-time = "2024-11-01T19:08:51.487Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/7e/02ca3ee68507db47afce769504060d71b4dc1455f0f9faa8d32fc7762221/py_spy-0.4.0-py2.py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428", size = 3617847, upload-time = "2024-11-01T19:08:37.44Z" }, - { url = "https://files.pythonhosted.org/packages/65/7c/d9e26cc4c8e91f96a3a65de04d2e2e4131fbcaf6830d10917d4fab9d6788/py_spy-0.4.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:47cdda4c34d9b6cb01f3aaeceb2e88faf57da880207fe72ff6ff97e9bb6cc8a9", size = 1761955, upload-time = "2024-11-01T19:08:39.632Z" }, - { url = "https://files.pythonhosted.org/packages/d2/e4/8fbfd219b7f282b80e6b2e74c9197850d2c51db8555705567bb65507b060/py_spy-0.4.0-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab", size = 2059471, upload-time = "2024-11-01T19:08:41.818Z" }, - { url = "https://files.pythonhosted.org/packages/a7/1d/79a94a5ace810c13b730ce96765ca465c171b4952034f1be7402d8accbc1/py_spy-0.4.0-py2.py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c5f06ffce4c9c98b7fc9f5e67e5e7db591173f1351837633f3f23d9378b1d18a", size = 2067486, upload-time = "2024-11-01T19:08:43.673Z" }, - { url = "https://files.pythonhosted.org/packages/6d/90/fbbb038f826a83ed15ebc4ae606815d6cad6c5c6399c86c7ab96f6c60817/py_spy-0.4.0-py2.py3-none-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:87573e64dbfdfc89ba2e0f5e2f525aa84e0299c7eb6454b47ea335fde583a7a0", size = 2141433, upload-time = "2024-11-01T19:08:45.988Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c1/5e012669ebb687e546dc99fcfc4861ebfcf3a337b7a41af945df23140bb5/py_spy-0.4.0-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8bf2f3702cef367a489faa45177b41a6c31b2a3e5bd78c978d44e29340152f5a", size = 2732951, upload-time = "2024-11-01T19:08:48.109Z" }, - { url = "https://files.pythonhosted.org/packages/74/8b/dd8490660019a6b0be28d9ffd2bf1db967604b19f3f2719c0e283a16ac7f/py_spy-0.4.0-py2.py3-none-win_amd64.whl", hash = "sha256:77d8f637ade38367d944874776f45b703b7ac5938b1f7be8891f3a5876ddbb96", size = 1810770, upload-time = "2024-11-01T19:08:50.229Z" }, -] - -[[package]] -name = "pyarrow" -version = "19.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7f/09/a9046344212690f0632b9c709f9bf18506522feb333c894d0de81d62341a/pyarrow-19.0.1.tar.gz", hash = "sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e", size = 1129437, upload-time = "2025-02-18T18:55:57.027Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b4/94e828704b050e723f67d67c3535cf7076c7432cd4cf046e4bb3b96a9c9d/pyarrow-19.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b", size = 30670749, upload-time = "2025-02-18T18:53:00.062Z" }, - { url = "https://files.pythonhosted.org/packages/7e/3b/4692965e04bb1df55e2c314c4296f1eb12b4f3052d4cf43d29e076aedf66/pyarrow-19.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294", size = 32128007, upload-time = "2025-02-18T18:53:06.581Z" }, - { url = "https://files.pythonhosted.org/packages/22/f7/2239af706252c6582a5635c35caa17cb4d401cd74a87821ef702e3888957/pyarrow-19.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14", size = 41144566, upload-time = "2025-02-18T18:53:11.958Z" }, - { url = "https://files.pythonhosted.org/packages/fb/e3/c9661b2b2849cfefddd9fd65b64e093594b231b472de08ff658f76c732b2/pyarrow-19.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34", size = 42202991, upload-time = "2025-02-18T18:53:17.678Z" }, - { url = "https://files.pythonhosted.org/packages/fe/4f/a2c0ed309167ef436674782dfee4a124570ba64299c551e38d3fdaf0a17b/pyarrow-19.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6", size = 40507986, upload-time = "2025-02-18T18:53:26.263Z" }, - { url = "https://files.pythonhosted.org/packages/27/2e/29bb28a7102a6f71026a9d70d1d61df926887e36ec797f2e6acfd2dd3867/pyarrow-19.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832", size = 42087026, upload-time = "2025-02-18T18:53:33.063Z" }, - { url = "https://files.pythonhosted.org/packages/16/33/2a67c0f783251106aeeee516f4806161e7b481f7d744d0d643d2f30230a5/pyarrow-19.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960", size = 25250108, upload-time = "2025-02-18T18:53:38.462Z" }, -] - -[[package]] -name = "pyasn1" -version = "0.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/e9/01f1a64245b89f039897cb0130016d79f77d52669aae6ee7b159a6c4c018/pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034", size = 145322, upload-time = "2024-09-10T22:41:42.55Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - -[[package]] -name = "pybind11" -version = "2.13.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d2/c1/72b9622fcb32ff98b054f724e213c7f70d6898baa714f4516288456ceaba/pybind11-2.13.6.tar.gz", hash = "sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a", size = 218403, upload-time = "2024-09-14T00:35:22.606Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/2f/0f24b288e2ce56f51c920137620b4434a38fd80583dbbe24fc2a1656c388/pybind11-2.13.6-py3-none-any.whl", hash = "sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5", size = 243282, upload-time = "2024-09-14T00:35:20.361Z" }, -] - -[[package]] -name = "pycodestyle" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/6e/1f4a62078e4d95d82367f24e685aef3a672abfd27d1a868068fed4ed2254/pycodestyle-2.13.0.tar.gz", hash = "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae", size = 39312, upload-time = "2025-03-29T17:33:30.669Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/be/b00116df1bfb3e0bb5b45e29d604799f7b91dd861637e4d448b4e09e6a3e/pycodestyle-2.13.0-py2.py3-none-any.whl", hash = "sha256:35863c5974a271c7a726ed228a14a4f6daf49df369d8c50cd9a6f58a5e143ba9", size = 31424, upload-time = "2025-03-29T17:33:29.405Z" }, -] - -[[package]] -name = "pycountry" -version = "24.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/57/c389fa68c50590881a75b7883eeb3dc15e9e73a0fdc001cdd45c13290c92/pycountry-24.6.1.tar.gz", hash = "sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221", size = 6043910, upload-time = "2024-06-01T04:12:15.05Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/ec/1fb891d8a2660716aadb2143235481d15ed1cbfe3ad669194690b0604492/pycountry-24.6.1-py3-none-any.whl", hash = "sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f", size = 6335189, upload-time = "2024-06-01T04:11:49.711Z" }, -] - -[[package]] -name = "pycparser" -version = "2.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, -] - -[[package]] -name = "pycryptodome" -version = "3.22.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/44/e6/099310419df5ada522ff34ffc2f1a48a11b37fc6a76f51a6854c182dbd3e/pycryptodome-3.22.0.tar.gz", hash = "sha256:fd7ab568b3ad7b77c908d7c3f7e167ec5a8f035c64ff74f10d47a4edd043d723", size = 4917300, upload-time = "2025-03-15T23:03:36.506Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/65/a05831c3e4bcd1bf6c2a034e399f74b3d6f30bb4e37e36b9c310c09dc8c0/pycryptodome-3.22.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:009e1c80eea42401a5bd5983c4bab8d516aef22e014a4705622e24e6d9d703c6", size = 2490637, upload-time = "2025-03-15T23:02:43.111Z" }, - { url = "https://files.pythonhosted.org/packages/5c/76/ff3c2e7a60d17c080c4c6120ebaf60f38717cd387e77f84da4dcf7f64ff0/pycryptodome-3.22.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3b76fa80daeff9519d7e9f6d9e40708f2fce36b9295a847f00624a08293f4f00", size = 1635372, upload-time = "2025-03-15T23:02:45.564Z" }, - { url = "https://files.pythonhosted.org/packages/cc/7f/cc5d6da0dbc36acd978d80a72b228e33aadaec9c4f91c93221166d8bdc05/pycryptodome-3.22.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a31fa5914b255ab62aac9265654292ce0404f6b66540a065f538466474baedbc", size = 2177456, upload-time = "2025-03-15T23:02:47.688Z" }, - { url = "https://files.pythonhosted.org/packages/92/65/35f5063e68790602d892ad36e35ac723147232a9084d1999630045c34593/pycryptodome-3.22.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0092fd476701eeeb04df5cc509d8b739fa381583cda6a46ff0a60639b7cd70d", size = 2263744, upload-time = "2025-03-15T23:02:49.548Z" }, - { url = "https://files.pythonhosted.org/packages/cc/67/46acdd35b1081c3dbc72dc466b1b95b80d2f64cad3520f994a9b6c5c7d00/pycryptodome-3.22.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d5b0ddc7cf69231736d778bd3ae2b3efb681ae33b64b0c92fb4626bb48bb89", size = 2303356, upload-time = "2025-03-15T23:02:52.122Z" }, - { url = "https://files.pythonhosted.org/packages/3d/f9/a4f8a83384626098e3f55664519bec113002b9ef751887086ae63a53135a/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f6cf6aa36fcf463e622d2165a5ad9963b2762bebae2f632d719dfb8544903cf5", size = 2176714, upload-time = "2025-03-15T23:02:53.85Z" }, - { url = "https://files.pythonhosted.org/packages/88/65/e5f8c3a885f70a6e05c84844cd5542120576f4369158946e8cfc623a464d/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:aec7b40a7ea5af7c40f8837adf20a137d5e11a6eb202cde7e588a48fb2d871a8", size = 2337329, upload-time = "2025-03-15T23:02:56.11Z" }, - { url = "https://files.pythonhosted.org/packages/b8/2a/25e0be2b509c28375c7f75c7e8d8d060773f2cce4856a1654276e3202339/pycryptodome-3.22.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d21c1eda2f42211f18a25db4eaf8056c94a8563cd39da3683f89fe0d881fb772", size = 2262255, upload-time = "2025-03-15T23:02:58.055Z" }, - { url = "https://files.pythonhosted.org/packages/41/58/60917bc4bbd91712e53ce04daf237a74a0ad731383a01288130672994328/pycryptodome-3.22.0-cp37-abi3-win32.whl", hash = "sha256:f02baa9f5e35934c6e8dcec91fcde96612bdefef6e442813b8ea34e82c84bbfb", size = 1763403, upload-time = "2025-03-15T23:03:00.616Z" }, - { url = "https://files.pythonhosted.org/packages/55/f4/244c621afcf7867e23f63cfd7a9630f14cfe946c9be7e566af6c3915bcde/pycryptodome-3.22.0-cp37-abi3-win_amd64.whl", hash = "sha256:d086aed307e96d40c23c42418cbbca22ecc0ab4a8a0e24f87932eeab26c08627", size = 1794568, upload-time = "2025-03-15T23:03:03.189Z" }, -] - -[[package]] -name = "pydantic" -version = "2.11.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/2e/ca897f093ee6c5f3b0bee123ee4465c50e75431c3d5b6a3b44a47134e891/pydantic-2.11.3.tar.gz", hash = "sha256:7471657138c16adad9322fe3070c0116dd6c3ad8d649300e3cbdfe91f4db4ec3", size = 785513, upload-time = "2025-04-08T13:27:06.399Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/1d/407b29780a289868ed696d1616f4aad49d6388e5a77f567dcd2629dcd7b8/pydantic-2.11.3-py3-none-any.whl", hash = "sha256:a082753436a07f9ba1289c6ffa01cd93db3548776088aa917cc43b63f68fa60f", size = 443591, upload-time = "2025-04-08T13:27:03.789Z" }, -] - -[[package]] -name = "pydantic-core" -version = "2.33.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/17/19/ed6a078a5287aea7922de6841ef4c06157931622c89c2a47940837b5eecd/pydantic_core-2.33.1.tar.gz", hash = "sha256:bcc9c6fdb0ced789245b02b7d6603e17d1563064ddcfc36f046b61c0c05dd9df", size = 434395, upload-time = "2025-04-02T09:49:41.8Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/ce/3cb22b07c29938f97ff5f5bb27521f95e2ebec399b882392deb68d6c440e/pydantic_core-2.33.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1293d7febb995e9d3ec3ea09caf1a26214eec45b0f29f6074abb004723fc1de8", size = 2026640, upload-time = "2025-04-02T09:47:25.394Z" }, - { url = "https://files.pythonhosted.org/packages/19/78/f381d643b12378fee782a72126ec5d793081ef03791c28a0fd542a5bee64/pydantic_core-2.33.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:99b56acd433386c8f20be5c4000786d1e7ca0523c8eefc995d14d79c7a081498", size = 1852649, upload-time = "2025-04-02T09:47:27.417Z" }, - { url = "https://files.pythonhosted.org/packages/9d/2b/98a37b80b15aac9eb2c6cfc6dbd35e5058a352891c5cce3a8472d77665a6/pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35a5ec3fa8c2fe6c53e1b2ccc2454398f95d5393ab398478f53e1afbbeb4d939", size = 1892472, upload-time = "2025-04-02T09:47:29.006Z" }, - { url = "https://files.pythonhosted.org/packages/4e/d4/3c59514e0f55a161004792b9ff3039da52448f43f5834f905abef9db6e4a/pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b172f7b9d2f3abc0efd12e3386f7e48b576ef309544ac3a63e5e9cdd2e24585d", size = 1977509, upload-time = "2025-04-02T09:47:33.464Z" }, - { url = "https://files.pythonhosted.org/packages/a9/b6/c2c7946ef70576f79a25db59a576bce088bdc5952d1b93c9789b091df716/pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9097b9f17f91eea659b9ec58148c0747ec354a42f7389b9d50701610d86f812e", size = 2128702, upload-time = "2025-04-02T09:47:34.812Z" }, - { url = "https://files.pythonhosted.org/packages/88/fe/65a880f81e3f2a974312b61f82a03d85528f89a010ce21ad92f109d94deb/pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc77ec5b7e2118b152b0d886c7514a4653bcb58c6b1d760134a9fab915f777b3", size = 2679428, upload-time = "2025-04-02T09:47:37.315Z" }, - { url = "https://files.pythonhosted.org/packages/6f/ff/4459e4146afd0462fb483bb98aa2436d69c484737feaceba1341615fb0ac/pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3d15245b08fa4a84cefc6c9222e6f37c98111c8679fbd94aa145f9a0ae23d", size = 2008753, upload-time = "2025-04-02T09:47:39.013Z" }, - { url = "https://files.pythonhosted.org/packages/7c/76/1c42e384e8d78452ededac8b583fe2550c84abfef83a0552e0e7478ccbc3/pydantic_core-2.33.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ef99779001d7ac2e2461d8ab55d3373fe7315caefdbecd8ced75304ae5a6fc6b", size = 2114849, upload-time = "2025-04-02T09:47:40.427Z" }, - { url = "https://files.pythonhosted.org/packages/00/72/7d0cf05095c15f7ffe0eb78914b166d591c0eed72f294da68378da205101/pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fc6bf8869e193855e8d91d91f6bf59699a5cdfaa47a404e278e776dd7f168b39", size = 2069541, upload-time = "2025-04-02T09:47:42.01Z" }, - { url = "https://files.pythonhosted.org/packages/b3/69/94a514066bb7d8be499aa764926937409d2389c09be0b5107a970286ef81/pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:b1caa0bc2741b043db7823843e1bde8aaa58a55a58fda06083b0569f8b45693a", size = 2239225, upload-time = "2025-04-02T09:47:43.425Z" }, - { url = "https://files.pythonhosted.org/packages/84/b0/e390071eadb44b41f4f54c3cef64d8bf5f9612c92686c9299eaa09e267e2/pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ec259f62538e8bf364903a7d0d0239447059f9434b284f5536e8402b7dd198db", size = 2248373, upload-time = "2025-04-02T09:47:44.979Z" }, - { url = "https://files.pythonhosted.org/packages/d6/b2/288b3579ffc07e92af66e2f1a11be3b056fe1214aab314748461f21a31c3/pydantic_core-2.33.1-cp312-cp312-win32.whl", hash = "sha256:e14f369c98a7c15772b9da98987f58e2b509a93235582838bd0d1d8c08b68fda", size = 1907034, upload-time = "2025-04-02T09:47:46.843Z" }, - { url = "https://files.pythonhosted.org/packages/02/28/58442ad1c22b5b6742b992ba9518420235adced665513868f99a1c2638a5/pydantic_core-2.33.1-cp312-cp312-win_amd64.whl", hash = "sha256:1c607801d85e2e123357b3893f82c97a42856192997b95b4d8325deb1cd0c5f4", size = 1956848, upload-time = "2025-04-02T09:47:48.404Z" }, - { url = "https://files.pythonhosted.org/packages/a1/eb/f54809b51c7e2a1d9f439f158b8dd94359321abcc98767e16fc48ae5a77e/pydantic_core-2.33.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d13f0276806ee722e70a1c93da19748594f19ac4299c7e41237fc791d1861ea", size = 1903986, upload-time = "2025-04-02T09:47:49.839Z" }, -] - -[[package]] -name = "pydantic-settings" -version = "2.8.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/82/c79424d7d8c29b994fb01d277da57b0a9b09cc03c3ff875f9bd8a86b2145/pydantic_settings-2.8.1.tar.gz", hash = "sha256:d5c663dfbe9db9d5e1c646b2e161da12f0d734d422ee56f567d0ea2cee4e8585", size = 83550, upload-time = "2025-02-27T10:10:32.338Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/53/a64f03044927dc47aafe029c42a5b7aabc38dfb813475e0e1bf71c4a59d0/pydantic_settings-2.8.1-py3-none-any.whl", hash = "sha256:81942d5ac3d905f7f3ee1a70df5dfb62d5569c12f51a5a647defc1c3d9ee2e9c", size = 30839, upload-time = "2025-02-27T10:10:30.711Z" }, -] - -[[package]] -name = "pydeck" -version = "0.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/ca/40e14e196864a0f61a92abb14d09b3d3da98f94ccb03b49cf51688140dab/pydeck-0.9.1.tar.gz", hash = "sha256:f74475ae637951d63f2ee58326757f8d4f9cd9f2a457cf42950715003e2cb605", size = 3832240, upload-time = "2024-05-10T15:36:21.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/4c/b888e6cf58bd9db9c93f40d1c6be8283ff49d88919231afe93a6bcf61626/pydeck-0.9.1-py2.py3-none-any.whl", hash = "sha256:b3f75ba0d273fc917094fa61224f3f6076ca8752b93d46faf3bcfd9f9d59b038", size = 6900403, upload-time = "2024-05-10T15:36:17.36Z" }, -] - -[[package]] -name = "pyee" -version = "11.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/88/c9a028db29f009170db6337a5096476a12448ffce437bbf91458fd5b5767/pyee-11.0.1.tar.gz", hash = "sha256:a642c51e3885a33ead087286e35212783a4e9b8d6514a10a5db4e57ac57b2b29", size = 27877, upload-time = "2023-10-14T17:50:00.259Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/40/bfe7fd2cb55ca7dbb4c56e9f0060567c3e84dd9bdf8a782261f1d2d7c32f/pyee-11.0.1-py3-none-any.whl", hash = "sha256:9bcc9647822234f42c228d88de63d0f9ffa881e87a87f9d36ddf5211f6ac977d", size = 15249, upload-time = "2023-10-14T17:49:57.906Z" }, -] - -[[package]] -name = "pyflakes" -version = "3.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/cc/1df338bd7ed1fa7c317081dcf29bf2f01266603b301e6858856d346a12b3/pyflakes-3.3.2.tar.gz", hash = "sha256:6dfd61d87b97fba5dcfaaf781171ac16be16453be6d816147989e7f6e6a9576b", size = 64175, upload-time = "2025-03-31T13:21:20.34Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/40/b293a4fa769f3b02ab9e387c707c4cbdc34f073f945de0386107d4e669e6/pyflakes-3.3.2-py2.py3-none-any.whl", hash = "sha256:5039c8339cbb1944045f4ee5466908906180f13cc99cc9949348d10f82a5c32a", size = 63164, upload-time = "2025-03-31T13:21:18.503Z" }, -] - -[[package]] -name = "pygithub" -version = "2.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "pynacl" }, - { name = "requests" }, - { name = "typing-extensions" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/88/e08ab18dc74b2916f48703ed1a797d57cb64eca0e23b0a9254e13cfe3911/pygithub-2.6.1.tar.gz", hash = "sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf", size = 3659473, upload-time = "2025-02-21T13:45:58.262Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/fc/a444cd19ccc8c4946a512f3827ed0b3565c88488719d800d54a75d541c0b/PyGithub-2.6.1-py3-none-any.whl", hash = "sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3", size = 410451, upload-time = "2025-02-21T13:45:55.519Z" }, -] - -[[package]] -name = "pygments" -version = "2.19.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581, upload-time = "2025-01-06T17:26:30.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload-time = "2025-01-06T17:26:25.553Z" }, -] - -[[package]] -name = "pyjwt" -version = "2.10.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785, upload-time = "2024-11-28T03:43:29.933Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - -[[package]] -name = "pylatexenc" -version = "2.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5d/ab/34ec41718af73c00119d0351b7a2531d2ebddb51833a36448fc7b862be60/pylatexenc-2.10.tar.gz", hash = "sha256:3dd8fd84eb46dc30bee1e23eaab8d8fb5a7f507347b23e5f38ad9675c84f40d3", size = 162597, upload-time = "2021-04-06T07:56:07.854Z" } - -[[package]] -name = "pylcs" -version = "0.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pybind11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/7f/9ca900387de8f3d3658dbf7d0aba96aa2a69f4f5329c83af9be98dc7307d/pylcs-0.1.1.tar.gz", hash = "sha256:632c69235d77cda0ba524d82796878801d2f46131fc59e730c98767fc4ce1307", size = 11274, upload-time = "2023-10-24T23:56:24.576Z" } - -[[package]] -name = "pynacl" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/22/27582568be639dfe22ddb3902225f91f2f17ceff88ce80e4db396c8986da/PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", size = 3392854, upload-time = "2022-01-07T22:05:41.134Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/75/0b8ede18506041c0bf23ac4d8e2971b4161cd6ce630b177d0a08eb0d8857/PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", size = 349920, upload-time = "2022-01-07T22:05:49.156Z" }, - { url = "https://files.pythonhosted.org/packages/59/bb/fddf10acd09637327a97ef89d2a9d621328850a72f1fdc8c08bdf72e385f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", size = 601722, upload-time = "2022-01-07T22:05:50.989Z" }, - { url = "https://files.pythonhosted.org/packages/5d/70/87a065c37cca41a75f2ce113a5a2c2aa7533be648b184ade58971b5f7ccc/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", size = 680087, upload-time = "2022-01-07T22:05:52.539Z" }, - { url = "https://files.pythonhosted.org/packages/ee/87/f1bb6a595f14a327e8285b9eb54d41fef76c585a0edef0a45f6fc95de125/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", size = 856678, upload-time = "2022-01-07T22:05:54.251Z" }, - { url = "https://files.pythonhosted.org/packages/66/28/ca86676b69bf9f90e710571b67450508484388bfce09acf8a46f0b8c785f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", size = 1133660, upload-time = "2022-01-07T22:05:56.056Z" }, - { url = "https://files.pythonhosted.org/packages/3d/85/c262db650e86812585e2bc59e497a8f59948a005325a11bbbc9ecd3fe26b/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", size = 663824, upload-time = "2022-01-07T22:05:57.434Z" }, - { url = "https://files.pythonhosted.org/packages/fd/1a/cc308a884bd299b651f1633acb978e8596c71c33ca85e9dc9fa33a5399b9/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", size = 1117912, upload-time = "2022-01-07T22:05:58.665Z" }, - { url = "https://files.pythonhosted.org/packages/25/2d/b7df6ddb0c2a33afdb358f8af6ea3b8c4d1196ca45497dd37a56f0c122be/PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543", size = 204624, upload-time = "2022-01-07T22:06:00.085Z" }, - { url = "https://files.pythonhosted.org/packages/5e/22/d3db169895faaf3e2eda892f005f433a62db2decbcfbc2f61e6517adfa87/PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", size = 212141, upload-time = "2022-01-07T22:06:01.861Z" }, -] - -[[package]] -name = "pynvml" -version = "12.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-ml-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/6f/6b5880ed0239e85b9a39aed103b65b2ef81425beef9f45e5c035bf008330/pynvml-12.0.0.tar.gz", hash = "sha256:299ce2451a6a17e6822d6faee750103e25b415f06f59abb8db65d30f794166f5", size = 33636, upload-time = "2024-12-02T15:04:36.631Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/df/f7cf07a65a96dd11d71f346f9c2863accdd4784da83af7181b067d556cbc/pynvml-12.0.0-py3-none-any.whl", hash = "sha256:fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e", size = 26560, upload-time = "2024-12-02T15:04:35.047Z" }, -] - -[[package]] -name = "pyparsing" -version = "3.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bb/22/f1129e69d94ffff626bdb5c835506b3a5b4f3d070f17ea295e12c2c6f60f/pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be", size = 1088608, upload-time = "2025-03-25T05:01:28.114Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" }, -] - -[[package]] -name = "pypdf2" -version = "3.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/bb/18dc3062d37db6c491392007dfd1a7f524bb95886eb956569ac38a23a784/PyPDF2-3.0.1.tar.gz", hash = "sha256:a74408f69ba6271f71b9352ef4ed03dc53a31aa404d29b5d31f53bfecfee1440", size = 227419, upload-time = "2022-12-31T10:36:13.13Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/5e/c86a5643653825d3c913719e788e41386bee415c2b87b4f955432f2de6b2/pypdf2-3.0.1-py3-none-any.whl", hash = "sha256:d16e4205cfee272fbdc0568b68d82be796540b1537508cef59388f839c191928", size = 232572, upload-time = "2022-12-31T10:36:10.327Z" }, -] - -[[package]] -name = "pysocks" -version = "1.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, -] - -[[package]] -name = "pytest" -version = "8.3.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload-time = "2025-03-02T12:54:54.503Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634, upload-time = "2025-03-02T12:54:52.069Z" }, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "python-docx" -version = "1.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lxml" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/35/e4/386c514c53684772885009c12b67a7edd526c15157778ac1b138bc75063e/python_docx-1.1.2.tar.gz", hash = "sha256:0cf1f22e95b9002addca7948e16f2cd7acdfd498047f1941ca5d293db7762efd", size = 5656581, upload-time = "2024-05-01T19:41:57.772Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/3d/330d9efbdb816d3f60bf2ad92f05e1708e4a1b9abe80461ac3444c83f749/python_docx-1.1.2-py3-none-any.whl", hash = "sha256:08c20d6058916fb19853fcf080f7f42b6270d89eac9fa5f8c15f691c0017fabe", size = 244315, upload-time = "2024-05-01T19:41:47.006Z" }, -] - -[[package]] -name = "python-dotenv" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/88/2c/7bb1416c5620485aa793f2de31d3df393d3686aa8a8506d11e10e13c5baf/python_dotenv-1.1.0.tar.gz", hash = "sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5", size = 39920, upload-time = "2025-03-25T10:14:56.835Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/18/98a99ad95133c6a6e2005fe89faedf294a748bd5dc803008059409ac9b1e/python_dotenv-1.1.0-py3-none-any.whl", hash = "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d", size = 20256, upload-time = "2025-03-25T10:14:55.034Z" }, -] - -[[package]] -name = "python-engineio" -version = "4.11.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "simple-websocket" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/52/e0/a9e0fe427ce7f1b7dbf9531fa00ffe4b557c4a7bc8e71891c115af123170/python_engineio-4.11.2.tar.gz", hash = "sha256:145bb0daceb904b4bb2d3eb2d93f7dbb7bb87a6a0c4f20a94cc8654dec977129", size = 91381, upload-time = "2024-12-29T19:18:05.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/8f/978a0b913e3f8ad33a9a2fe204d32efe3d1ee34ecb1f2829c1cfbdd92082/python_engineio-4.11.2-py3-none-any.whl", hash = "sha256:f0971ac4c65accc489154fe12efd88f53ca8caf04754c46a66e85f5102ef22ad", size = 59239, upload-time = "2024-12-29T19:18:02.345Z" }, -] - -[[package]] -name = "python-frontmatter" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/96/de/910fa208120314a12f9a88ea63e03707261692af782c99283f1a2c8a5e6f/python-frontmatter-1.1.0.tar.gz", hash = "sha256:7118d2bd56af9149625745c58c9b51fb67e8d1294a0c76796dafdc72c36e5f6d", size = 16256, upload-time = "2024-01-16T18:50:04.052Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/87/3c8da047b3ec5f99511d1b4d7a5bc72d4b98751c7e78492d14dc736319c5/python_frontmatter-1.1.0-py3-none-any.whl", hash = "sha256:335465556358d9d0e6c98bbeb69b1c969f2a4a21360587b9873bfc3b213407c1", size = 9834, upload-time = "2024-01-16T18:50:00.911Z" }, -] - -[[package]] -name = "python-json-logger" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/de/d3144a0bceede957f961e975f3752760fbe390d57fbe194baf709d8f1f7b/python_json_logger-3.3.0.tar.gz", hash = "sha256:12b7e74b17775e7d565129296105bbe3910842d9d0eb083fc83a6a617aa8df84", size = 16642, upload-time = "2025-03-07T07:08:27.301Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/20/0f2523b9e50a8052bc6a8b732dfc8568abbdc42010aef03a2d750bdab3b2/python_json_logger-3.3.0-py3-none-any.whl", hash = "sha256:dd980fae8cffb24c13caf6e158d3d61c0d6d22342f932cb6e9deedab3d35eec7", size = 15163, upload-time = "2025-03-07T07:08:25.627Z" }, -] - -[[package]] -name = "python-multipart" -version = "0.0.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158, upload-time = "2024-12-16T19:45:46.972Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, -] - -[[package]] -name = "python-pptx" -version = "1.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lxml" }, - { name = "pillow" }, - { name = "typing-extensions" }, - { name = "xlsxwriter" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, -] - -[[package]] -name = "python-socketio" -version = "5.12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "bidict" }, - { name = "python-engineio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ce/d0/40ed38076e8aee94785d546d3e3a1cae393da5806a8530be877187e2875f/python_socketio-5.12.1.tar.gz", hash = "sha256:0299ff1f470b676c09c1bfab1dead25405077d227b2c13cf217a34dadc68ba9c", size = 119991, upload-time = "2024-12-29T20:11:51.506Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/a3/c69806f30dd81df5a99d592e7db4c930c3a9b098555aa97b0eb866b20b11/python_socketio-5.12.1-py3-none-any.whl", hash = "sha256:24a0ea7cfff0e021eb28c68edbf7914ee4111bdf030b95e4d250c4dc9af7a386", size = 76947, upload-time = "2024-12-29T20:11:48.876Z" }, -] - -[[package]] -name = "pytz" -version = "2025.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, -] - -[[package]] -name = "pywin32" -version = "310" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/ec/4fdbe47932f671d6e348474ea35ed94227fb5df56a7c30cbbb42cd396ed0/pywin32-310-cp312-cp312-win32.whl", hash = "sha256:8a75a5cc3893e83a108c05d82198880704c44bbaee4d06e442e471d3c9ea4f3d", size = 8796239, upload-time = "2025-03-17T00:55:58.807Z" }, - { url = "https://files.pythonhosted.org/packages/e3/e5/b0627f8bb84e06991bea89ad8153a9e50ace40b2e1195d68e9dff6b03d0f/pywin32-310-cp312-cp312-win_amd64.whl", hash = "sha256:bf5c397c9a9a19a6f62f3fb821fbf36cac08f03770056711f765ec1503972060", size = 9503839, upload-time = "2025-03-17T00:56:00.8Z" }, - { url = "https://files.pythonhosted.org/packages/1f/32/9ccf53748df72301a89713936645a664ec001abd35ecc8578beda593d37d/pywin32-310-cp312-cp312-win_arm64.whl", hash = "sha256:2349cc906eae872d0663d4d6290d13b90621eaf78964bb1578632ff20e152966", size = 8459470, upload-time = "2025-03-17T00:56:02.601Z" }, -] - -[[package]] -name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, -] - -[[package]] -name = "pyzmq" -version = "26.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/11/b9213d25230ac18a71b39b3723494e57adebe36e066397b961657b3b41c1/pyzmq-26.4.0.tar.gz", hash = "sha256:4bd13f85f80962f91a651a7356fe0472791a5f7a92f227822b5acf44795c626d", size = 278293, upload-time = "2025-04-04T12:05:44.049Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/44/a778555ebfdf6c7fc00816aad12d185d10a74d975800341b1bc36bad1187/pyzmq-26.4.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:5227cb8da4b6f68acfd48d20c588197fd67745c278827d5238c707daf579227b", size = 1341586, upload-time = "2025-04-04T12:03:41.954Z" }, - { url = "https://files.pythonhosted.org/packages/9c/4f/f3a58dc69ac757e5103be3bd41fb78721a5e17da7cc617ddb56d973a365c/pyzmq-26.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1c07a7fa7f7ba86554a2b1bef198c9fed570c08ee062fd2fd6a4dcacd45f905", size = 665880, upload-time = "2025-04-04T12:03:43.45Z" }, - { url = "https://files.pythonhosted.org/packages/fe/45/50230bcfb3ae5cb98bee683b6edeba1919f2565d7cc1851d3c38e2260795/pyzmq-26.4.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae775fa83f52f52de73183f7ef5395186f7105d5ed65b1ae65ba27cb1260de2b", size = 902216, upload-time = "2025-04-04T12:03:45.572Z" }, - { url = "https://files.pythonhosted.org/packages/41/59/56bbdc5689be5e13727491ad2ba5efd7cd564365750514f9bc8f212eef82/pyzmq-26.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66c760d0226ebd52f1e6b644a9e839b5db1e107a23f2fcd46ec0569a4fdd4e63", size = 859814, upload-time = "2025-04-04T12:03:47.188Z" }, - { url = "https://files.pythonhosted.org/packages/81/b1/57db58cfc8af592ce94f40649bd1804369c05b2190e4cbc0a2dad572baeb/pyzmq-26.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ef8c6ecc1d520debc147173eaa3765d53f06cd8dbe7bd377064cdbc53ab456f5", size = 855889, upload-time = "2025-04-04T12:03:49.223Z" }, - { url = "https://files.pythonhosted.org/packages/e8/92/47542e629cbac8f221c230a6d0f38dd3d9cff9f6f589ed45fdf572ffd726/pyzmq-26.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3150ef4084e163dec29ae667b10d96aad309b668fac6810c9e8c27cf543d6e0b", size = 1197153, upload-time = "2025-04-04T12:03:50.591Z" }, - { url = "https://files.pythonhosted.org/packages/07/e5/b10a979d1d565d54410afc87499b16c96b4a181af46e7645ab4831b1088c/pyzmq-26.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4448c9e55bf8329fa1dcedd32f661bf611214fa70c8e02fee4347bc589d39a84", size = 1507352, upload-time = "2025-04-04T12:03:52.473Z" }, - { url = "https://files.pythonhosted.org/packages/ab/58/5a23db84507ab9c01c04b1232a7a763be66e992aa2e66498521bbbc72a71/pyzmq-26.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e07dde3647afb084d985310d067a3efa6efad0621ee10826f2cb2f9a31b89d2f", size = 1406834, upload-time = "2025-04-04T12:03:54Z" }, - { url = "https://files.pythonhosted.org/packages/22/74/aaa837b331580c13b79ac39396601fb361454ee184ca85e8861914769b99/pyzmq-26.4.0-cp312-cp312-win32.whl", hash = "sha256:ba034a32ecf9af72adfa5ee383ad0fd4f4e38cdb62b13624278ef768fe5b5b44", size = 577992, upload-time = "2025-04-04T12:03:55.815Z" }, - { url = "https://files.pythonhosted.org/packages/30/0f/55f8c02c182856743b82dde46b2dc3e314edda7f1098c12a8227eeda0833/pyzmq-26.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:056a97aab4064f526ecb32f4343917a4022a5d9efb6b9df990ff72e1879e40be", size = 640466, upload-time = "2025-04-04T12:03:57.231Z" }, - { url = "https://files.pythonhosted.org/packages/e4/29/073779afc3ef6f830b8de95026ef20b2d1ec22d0324d767748d806e57379/pyzmq-26.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f23c750e485ce1eb639dbd576d27d168595908aa2d60b149e2d9e34c9df40e0", size = 556342, upload-time = "2025-04-04T12:03:59.218Z" }, -] - -[[package]] -name = "qtconsole" -version = "5.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ipykernel" }, - { name = "jupyter-client" }, - { name = "jupyter-core" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "qtpy" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f1/83/d2b11f2f737c276d8650a04ad3bf478d10fbcd55fe39f129cdb9e6843d31/qtconsole-5.6.1.tar.gz", hash = "sha256:5cad1c7e6c75d3ef8143857fd2ed28062b4b92b933c2cc328252d18a9cfd0be5", size = 435808, upload-time = "2024-10-28T23:59:26.449Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/8a/635610fb6131bc702229e2780d7b042416866ab78f8ed1ff24c4b23a2f4c/qtconsole-5.6.1-py3-none-any.whl", hash = "sha256:3d22490d9589bace566ad4f3455b61fa2209156f40e87e19e2c3cb64e9264950", size = 125035, upload-time = "2024-10-28T23:59:24.191Z" }, -] - -[[package]] -name = "qtpy" -version = "2.4.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/70/01/392eba83c8e47b946b929d7c46e0f04b35e9671f8bb6fc36b6f7945b4de8/qtpy-2.4.3.tar.gz", hash = "sha256:db744f7832e6d3da90568ba6ccbca3ee2b3b4a890c3d6fbbc63142f6e4cdf5bb", size = 66982, upload-time = "2025-02-11T15:09:25.759Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/76/37c0ccd5ab968a6a438f9c623aeecc84c202ab2fabc6a8fd927580c15b5a/QtPy-2.4.3-py3-none-any.whl", hash = "sha256:72095afe13673e017946cc258b8d5da43314197b741ed2890e563cf384b51aa1", size = 95045, upload-time = "2025-02-11T15:09:24.162Z" }, -] - -[[package]] -name = "ray" -version = "2.43.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiosignal" }, - { name = "click" }, - { name = "filelock" }, - { name = "frozenlist" }, - { name = "jsonschema" }, - { name = "msgpack" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "requests" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/f2/bafb73554f33bb9a84a3c02e7dee89c94bb06044a06db9d99e44cb2b2d2b/ray-2.43.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:5121fdf4bcbcb0fda3b9b71164dd6c8fcc79a2e258022a2a3957e401018913fb", size = 67858078, upload-time = "2025-02-27T19:08:39.168Z" }, - { url = "https://files.pythonhosted.org/packages/3a/02/9ba73313664a26072d92cf797cd1d17514b47444d6ffea32aa1d41707b54/ray-2.43.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f26f7b72da04c3c4422269c31b067abd15cb38424b7012d812ddfb2c77462ea", size = 65155290, upload-time = "2025-02-27T19:08:50.67Z" }, - { url = "https://files.pythonhosted.org/packages/84/f5/e567c655facc09df9a7cbe79eda8dc84465b83ca86b7092cd6c45461ce89/ray-2.43.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:73770d4c8a989730985ff2b4292129249e28c1e29e84589470c9ba1ae91ca832", size = 66871584, upload-time = "2025-02-27T19:09:01.454Z" }, - { url = "https://files.pythonhosted.org/packages/a1/52/99604614000eb5c12e46a3c423c3078494ce032451ac12d8a573cbffb710/ray-2.43.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:b45f478d29ce5df3fc19861df64fef9ed5c25f1e83fa10028d33fadefdeca095", size = 67783962, upload-time = "2025-02-27T19:09:15.226Z" }, - { url = "https://files.pythonhosted.org/packages/df/49/5663b8bf55dc85512783cb36fa8696bed552ed52e552c1ab8f410f5f1599/ray-2.43.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c3a9880112a8d561280a34e8ef9471070f81ca467e08b669e5e77a85e173c9c", size = 25565643, upload-time = "2025-02-27T19:09:24.144Z" }, -] - -[[package]] -name = "redis" -version = "5.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355, upload-time = "2024-12-06T09:50:41.956Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502, upload-time = "2024-12-06T09:50:39.656Z" }, -] - -[[package]] -name = "referencing" -version = "0.36.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" }, -] - -[[package]] -name = "regex" -version = "2024.11.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/5f/bd69653fbfb76cf8604468d3b4ec4c403197144c7bfe0e6a5fc9e02a07cb/regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519", size = 399494, upload-time = "2024-11-06T20:12:31.635Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/30/9a87ce8336b172cc232a0db89a3af97929d06c11ceaa19d97d84fa90a8f8/regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a", size = 483781, upload-time = "2024-11-06T20:10:07.07Z" }, - { url = "https://files.pythonhosted.org/packages/01/e8/00008ad4ff4be8b1844786ba6636035f7ef926db5686e4c0f98093612add/regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9", size = 288455, upload-time = "2024-11-06T20:10:09.117Z" }, - { url = "https://files.pythonhosted.org/packages/60/85/cebcc0aff603ea0a201667b203f13ba75d9fc8668fab917ac5b2de3967bc/regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2", size = 284759, upload-time = "2024-11-06T20:10:11.155Z" }, - { url = "https://files.pythonhosted.org/packages/94/2b/701a4b0585cb05472a4da28ee28fdfe155f3638f5e1ec92306d924e5faf0/regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4", size = 794976, upload-time = "2024-11-06T20:10:13.24Z" }, - { url = "https://files.pythonhosted.org/packages/4b/bf/fa87e563bf5fee75db8915f7352e1887b1249126a1be4813837f5dbec965/regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577", size = 833077, upload-time = "2024-11-06T20:10:15.37Z" }, - { url = "https://files.pythonhosted.org/packages/a1/56/7295e6bad94b047f4d0834e4779491b81216583c00c288252ef625c01d23/regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3", size = 823160, upload-time = "2024-11-06T20:10:19.027Z" }, - { url = "https://files.pythonhosted.org/packages/fb/13/e3b075031a738c9598c51cfbc4c7879e26729c53aa9cca59211c44235314/regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e", size = 796896, upload-time = "2024-11-06T20:10:21.85Z" }, - { url = "https://files.pythonhosted.org/packages/24/56/0b3f1b66d592be6efec23a795b37732682520b47c53da5a32c33ed7d84e3/regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe", size = 783997, upload-time = "2024-11-06T20:10:24.329Z" }, - { url = "https://files.pythonhosted.org/packages/f9/a1/eb378dada8b91c0e4c5f08ffb56f25fcae47bf52ad18f9b2f33b83e6d498/regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e", size = 781725, upload-time = "2024-11-06T20:10:28.067Z" }, - { url = "https://files.pythonhosted.org/packages/83/f2/033e7dec0cfd6dda93390089864732a3409246ffe8b042e9554afa9bff4e/regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29", size = 789481, upload-time = "2024-11-06T20:10:31.612Z" }, - { url = "https://files.pythonhosted.org/packages/83/23/15d4552ea28990a74e7696780c438aadd73a20318c47e527b47a4a5a596d/regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39", size = 852896, upload-time = "2024-11-06T20:10:34.054Z" }, - { url = "https://files.pythonhosted.org/packages/e3/39/ed4416bc90deedbfdada2568b2cb0bc1fdb98efe11f5378d9892b2a88f8f/regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51", size = 860138, upload-time = "2024-11-06T20:10:36.142Z" }, - { url = "https://files.pythonhosted.org/packages/93/2d/dd56bb76bd8e95bbce684326302f287455b56242a4f9c61f1bc76e28360e/regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad", size = 787692, upload-time = "2024-11-06T20:10:38.394Z" }, - { url = "https://files.pythonhosted.org/packages/0b/55/31877a249ab7a5156758246b9c59539abbeba22461b7d8adc9e8475ff73e/regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54", size = 262135, upload-time = "2024-11-06T20:10:40.367Z" }, - { url = "https://files.pythonhosted.org/packages/38/ec/ad2d7de49a600cdb8dd78434a1aeffe28b9d6fc42eb36afab4a27ad23384/regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b", size = 273567, upload-time = "2024-11-06T20:10:43.467Z" }, -] - -[[package]] -name = "requests" -version = "2.32.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218, upload-time = "2024-05-29T15:37:49.536Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload-time = "2024-05-29T15:37:47.027Z" }, -] - -[package.optional-dependencies] -socks = [ - { name = "pysocks" }, -] - -[[package]] -name = "requests-oauthlib" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "oauthlib" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, -] - -[[package]] -name = "retry" -version = "0.9.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "decorator" }, - { name = "py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/72/75d0b85443fbc8d9f38d08d2b1b67cc184ce35280e4a3813cda2f445f3a4/retry-0.9.2.tar.gz", hash = "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4", size = 6448, upload-time = "2016-05-11T13:58:51.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/0d/53aea75710af4528a25ed6837d71d117602b01946b307a3912cb3cfcbcba/retry-0.9.2-py2.py3-none-any.whl", hash = "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606", size = 7986, upload-time = "2016-05-11T13:58:39.925Z" }, -] - -[[package]] -name = "rich" -version = "14.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/53/830aa4c3066a8ab0ae9a9955976fb770fe9c6102117c8ec4ab3ea62d89e8/rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725", size = 224078, upload-time = "2025-03-30T14:15:14.23Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229, upload-time = "2025-03-30T14:15:12.283Z" }, -] - -[[package]] -name = "rich-toolkit" -version = "0.14.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "rich" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/69/e328fb8986814147562b2617f22b06723f60b0c85c85afc0408b9f324a97/rich_toolkit-0.14.3.tar.gz", hash = "sha256:b72a342e52253b912681b027e94226e2deea616494420eec0b09a7219a72a0a5", size = 104469, upload-time = "2025-04-23T14:54:52.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/09/e0c7b06657ca1d4317d9e37ea5657a88a20dc3507b2ee6939ace0ff9036e/rich_toolkit-0.14.3-py3-none-any.whl", hash = "sha256:2ec72dcdf1bbb09b6a9286a4eddcd4d43369da3b22fe3f28e5a92143618b8ac6", size = 24258, upload-time = "2025-04-23T14:54:51.443Z" }, -] - -[[package]] -name = "rpds-py" -version = "0.24.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/b3/52b213298a0ba7097c7ea96bee95e1947aa84cc816d48cebb539770cdf41/rpds_py-0.24.0.tar.gz", hash = "sha256:772cc1b2cd963e7e17e6cc55fe0371fb9c704d63e44cacec7b9b7f523b78919e", size = 26863, upload-time = "2025-03-26T14:56:01.518Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/e0/1c55f4a3be5f1ca1a4fd1f3ff1504a1478c1ed48d84de24574c4fa87e921/rpds_py-0.24.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:d8551e733626afec514b5d15befabea0dd70a343a9f23322860c4f16a9430205", size = 366945, upload-time = "2025-03-26T14:53:28.149Z" }, - { url = "https://files.pythonhosted.org/packages/39/1b/a3501574fbf29118164314dbc800d568b8c1c7b3258b505360e8abb3902c/rpds_py-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e374c0ce0ca82e5b67cd61fb964077d40ec177dd2c4eda67dba130de09085c7", size = 351935, upload-time = "2025-03-26T14:53:29.684Z" }, - { url = "https://files.pythonhosted.org/packages/dc/47/77d3d71c55f6a374edde29f1aca0b2e547325ed00a9da820cabbc9497d2b/rpds_py-0.24.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d69d003296df4840bd445a5d15fa5b6ff6ac40496f956a221c4d1f6f7b4bc4d9", size = 390817, upload-time = "2025-03-26T14:53:31.177Z" }, - { url = "https://files.pythonhosted.org/packages/4e/ec/1e336ee27484379e19c7f9cc170f4217c608aee406d3ae3a2e45336bff36/rpds_py-0.24.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8212ff58ac6dfde49946bea57474a386cca3f7706fc72c25b772b9ca4af6b79e", size = 401983, upload-time = "2025-03-26T14:53:33.163Z" }, - { url = "https://files.pythonhosted.org/packages/07/f8/39b65cbc272c635eaea6d393c2ad1ccc81c39eca2db6723a0ca4b2108fce/rpds_py-0.24.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:528927e63a70b4d5f3f5ccc1fa988a35456eb5d15f804d276709c33fc2f19bda", size = 451719, upload-time = "2025-03-26T14:53:34.721Z" }, - { url = "https://files.pythonhosted.org/packages/32/05/05c2b27dd9c30432f31738afed0300659cb9415db0ff7429b05dfb09bbde/rpds_py-0.24.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a824d2c7a703ba6daaca848f9c3d5cb93af0505be505de70e7e66829affd676e", size = 442546, upload-time = "2025-03-26T14:53:36.26Z" }, - { url = "https://files.pythonhosted.org/packages/7d/e0/19383c8b5d509bd741532a47821c3e96acf4543d0832beba41b4434bcc49/rpds_py-0.24.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44d51febb7a114293ffd56c6cf4736cb31cd68c0fddd6aa303ed09ea5a48e029", size = 393695, upload-time = "2025-03-26T14:53:37.728Z" }, - { url = "https://files.pythonhosted.org/packages/9d/15/39f14e96d94981d0275715ae8ea564772237f3fa89bc3c21e24de934f2c7/rpds_py-0.24.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3fab5f4a2c64a8fb64fc13b3d139848817a64d467dd6ed60dcdd6b479e7febc9", size = 427218, upload-time = "2025-03-26T14:53:39.326Z" }, - { url = "https://files.pythonhosted.org/packages/22/b9/12da7124905a680f690da7a9de6f11de770b5e359f5649972f7181c8bf51/rpds_py-0.24.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9be4f99bee42ac107870c61dfdb294d912bf81c3c6d45538aad7aecab468b6b7", size = 568062, upload-time = "2025-03-26T14:53:40.885Z" }, - { url = "https://files.pythonhosted.org/packages/88/17/75229017a2143d915f6f803721a6d721eca24f2659c5718a538afa276b4f/rpds_py-0.24.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:564c96b6076a98215af52f55efa90d8419cc2ef45d99e314fddefe816bc24f91", size = 596262, upload-time = "2025-03-26T14:53:42.544Z" }, - { url = "https://files.pythonhosted.org/packages/aa/64/8e8a1d8bd1b6b638d6acb6d41ab2cec7f2067a5b8b4c9175703875159a7c/rpds_py-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:75a810b7664c17f24bf2ffd7f92416c00ec84b49bb68e6a0d93e542406336b56", size = 564306, upload-time = "2025-03-26T14:53:44.2Z" }, - { url = "https://files.pythonhosted.org/packages/68/1c/a7eac8d8ed8cb234a9b1064647824c387753343c3fab6ed7c83481ed0be7/rpds_py-0.24.0-cp312-cp312-win32.whl", hash = "sha256:f6016bd950be4dcd047b7475fdf55fb1e1f59fc7403f387be0e8123e4a576d30", size = 224281, upload-time = "2025-03-26T14:53:45.769Z" }, - { url = "https://files.pythonhosted.org/packages/bb/46/b8b5424d1d21f2f2f3f2d468660085318d4f74a8df8289e3dd6ad224d488/rpds_py-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:998c01b8e71cf051c28f5d6f1187abbdf5cf45fc0efce5da6c06447cba997034", size = 239719, upload-time = "2025-03-26T14:53:47.187Z" }, -] - -[[package]] -name = "rsa" -version = "4.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/aa/65/7d973b89c4d2351d7fb232c2e452547ddfa243e93131e7cfa766da627b52/rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21", size = 29711, upload-time = "2022-07-20T10:28:36.115Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/97/fa78e3d2f65c02c8e1268b9aba606569fe97f6c8f7c2d74394553347c145/rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", size = 34315, upload-time = "2022-07-20T10:28:34.978Z" }, -] - -[[package]] -name = "ruff" -version = "0.11.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/45/71/5759b2a6b2279bb77fe15b1435b89473631c2cd6374d45ccdb6b785810be/ruff-0.11.5.tar.gz", hash = "sha256:cae2e2439cb88853e421901ec040a758960b576126dab520fa08e9de431d1bef", size = 3976488, upload-time = "2025-04-10T17:13:29.369Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/db/6efda6381778eec7f35875b5cbefd194904832a1153d68d36d6b269d81a8/ruff-0.11.5-py3-none-linux_armv6l.whl", hash = "sha256:2561294e108eb648e50f210671cc56aee590fb6167b594144401532138c66c7b", size = 10103150, upload-time = "2025-04-10T17:12:37.886Z" }, - { url = "https://files.pythonhosted.org/packages/44/f2/06cd9006077a8db61956768bc200a8e52515bf33a8f9b671ee527bb10d77/ruff-0.11.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ac12884b9e005c12d0bd121f56ccf8033e1614f736f766c118ad60780882a077", size = 10898637, upload-time = "2025-04-10T17:12:41.602Z" }, - { url = "https://files.pythonhosted.org/packages/18/f5/af390a013c56022fe6f72b95c86eb7b2585c89cc25d63882d3bfe411ecf1/ruff-0.11.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4bfd80a6ec559a5eeb96c33f832418bf0fb96752de0539905cf7b0cc1d31d779", size = 10236012, upload-time = "2025-04-10T17:12:44.584Z" }, - { url = "https://files.pythonhosted.org/packages/b8/ca/b9bf954cfed165e1a0c24b86305d5c8ea75def256707f2448439ac5e0d8b/ruff-0.11.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0947c0a1afa75dcb5db4b34b070ec2bccee869d40e6cc8ab25aca11a7d527794", size = 10415338, upload-time = "2025-04-10T17:12:47.172Z" }, - { url = "https://files.pythonhosted.org/packages/d9/4d/2522dde4e790f1b59885283f8786ab0046958dfd39959c81acc75d347467/ruff-0.11.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ad871ff74b5ec9caa66cb725b85d4ef89b53f8170f47c3406e32ef040400b038", size = 9965277, upload-time = "2025-04-10T17:12:50.628Z" }, - { url = "https://files.pythonhosted.org/packages/e5/7a/749f56f150eef71ce2f626a2f6988446c620af2f9ba2a7804295ca450397/ruff-0.11.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6cf918390cfe46d240732d4d72fa6e18e528ca1f60e318a10835cf2fa3dc19f", size = 11541614, upload-time = "2025-04-10T17:12:53.783Z" }, - { url = "https://files.pythonhosted.org/packages/89/b2/7d9b8435222485b6aac627d9c29793ba89be40b5de11584ca604b829e960/ruff-0.11.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:56145ee1478582f61c08f21076dc59153310d606ad663acc00ea3ab5b2125f82", size = 12198873, upload-time = "2025-04-10T17:12:56.956Z" }, - { url = "https://files.pythonhosted.org/packages/00/e0/a1a69ef5ffb5c5f9c31554b27e030a9c468fc6f57055886d27d316dfbabd/ruff-0.11.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5f66f8f1e8c9fc594cbd66fbc5f246a8d91f916cb9667e80208663ec3728304", size = 11670190, upload-time = "2025-04-10T17:13:00.194Z" }, - { url = "https://files.pythonhosted.org/packages/05/61/c1c16df6e92975072c07f8b20dad35cd858e8462b8865bc856fe5d6ccb63/ruff-0.11.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80b4df4d335a80315ab9afc81ed1cff62be112bd165e162b5eed8ac55bfc8470", size = 13902301, upload-time = "2025-04-10T17:13:03.246Z" }, - { url = "https://files.pythonhosted.org/packages/79/89/0af10c8af4363304fd8cb833bd407a2850c760b71edf742c18d5a87bb3ad/ruff-0.11.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3068befab73620b8a0cc2431bd46b3cd619bc17d6f7695a3e1bb166b652c382a", size = 11350132, upload-time = "2025-04-10T17:13:06.209Z" }, - { url = "https://files.pythonhosted.org/packages/b9/e1/ecb4c687cbf15164dd00e38cf62cbab238cad05dd8b6b0fc68b0c2785e15/ruff-0.11.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f5da2e710a9641828e09aa98b92c9ebbc60518fdf3921241326ca3e8f8e55b8b", size = 10312937, upload-time = "2025-04-10T17:13:08.855Z" }, - { url = "https://files.pythonhosted.org/packages/cf/4f/0e53fe5e500b65934500949361e3cd290c5ba60f0324ed59d15f46479c06/ruff-0.11.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ef39f19cb8ec98cbc762344921e216f3857a06c47412030374fffd413fb8fd3a", size = 9936683, upload-time = "2025-04-10T17:13:11.378Z" }, - { url = "https://files.pythonhosted.org/packages/04/a8/8183c4da6d35794ae7f76f96261ef5960853cd3f899c2671961f97a27d8e/ruff-0.11.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:b2a7cedf47244f431fd11aa5a7e2806dda2e0c365873bda7834e8f7d785ae159", size = 10950217, upload-time = "2025-04-10T17:13:14.565Z" }, - { url = "https://files.pythonhosted.org/packages/26/88/9b85a5a8af21e46a0639b107fcf9bfc31da4f1d263f2fc7fbe7199b47f0a/ruff-0.11.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:81be52e7519f3d1a0beadcf8e974715b2dfc808ae8ec729ecfc79bddf8dbb783", size = 11404521, upload-time = "2025-04-10T17:13:17.8Z" }, - { url = "https://files.pythonhosted.org/packages/fc/52/047f35d3b20fd1ae9ccfe28791ef0f3ca0ef0b3e6c1a58badd97d450131b/ruff-0.11.5-py3-none-win32.whl", hash = "sha256:e268da7b40f56e3eca571508a7e567e794f9bfcc0f412c4b607931d3af9c4afe", size = 10320697, upload-time = "2025-04-10T17:13:20.582Z" }, - { url = "https://files.pythonhosted.org/packages/b9/fe/00c78010e3332a6e92762424cf4c1919065707e962232797d0b57fd8267e/ruff-0.11.5-py3-none-win_amd64.whl", hash = "sha256:6c6dc38af3cfe2863213ea25b6dc616d679205732dc0fb673356c2d69608f800", size = 11378665, upload-time = "2025-04-10T17:13:23.349Z" }, - { url = "https://files.pythonhosted.org/packages/43/7c/c83fe5cbb70ff017612ff36654edfebec4b1ef79b558b8e5fd933bab836b/ruff-0.11.5-py3-none-win_arm64.whl", hash = "sha256:67e241b4314f4eacf14a601d586026a962f4002a475aa702c69980a38087aa4e", size = 10460287, upload-time = "2025-04-10T17:13:26.538Z" }, -] - -[[package]] -name = "runloop-api-client" -version = "0.24.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3e/17/fb66ec7207c92b72c81bff9a29f64fb98fc446d02151bfdf099d56acf331/runloop_api_client-0.24.0.tar.gz", hash = "sha256:1d47aee8996e18136aad104c5934979c03c56f9df589bd7458c909d0ad92c663", size = 173821, upload-time = "2025-02-19T01:07:22.469Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/f2/1d3dd51800d13078825ec13d4744a3ecca9169307b22d8b603db429c5670/runloop_api_client-0.24.0-py3-none-any.whl", hash = "sha256:9b7a061b0e110e1b6b66e13cba9fd17ed4cfae2e4846928b3003c6aded76835b", size = 209326, upload-time = "2025-02-19T01:07:20.951Z" }, -] - -[[package]] -name = "s3transfer" -version = "0.11.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0f/ec/aa1a215e5c126fe5decbee2e107468f51d9ce190b9763cb649f76bb45938/s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679", size = 148419, upload-time = "2025-03-04T20:29:15.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/62/8d3fc3ec6640161a5649b2cddbbf2b9fa39c92541225b33f117c37c5a2eb/s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d", size = 84412, upload-time = "2025-03-04T20:29:13.433Z" }, -] - -[[package]] -name = "safetensors" -version = "0.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/71/7e/2d5d6ee7b40c0682315367ec7475693d110f512922d582fef1bd4a63adc3/safetensors-0.5.3.tar.gz", hash = "sha256:b6b0d6ecacec39a4fdd99cc19f4576f5219ce858e6fd8dbe7609df0b8dc56965", size = 67210, upload-time = "2025-02-26T09:15:13.155Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/ae/88f6c49dbd0cc4da0e08610019a3c78a7d390879a919411a410a1876d03a/safetensors-0.5.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd20eb133db8ed15b40110b7c00c6df51655a2998132193de2f75f72d99c7073", size = 436917, upload-time = "2025-02-26T09:15:03.702Z" }, - { url = "https://files.pythonhosted.org/packages/b8/3b/11f1b4a2f5d2ab7da34ecc062b0bc301f2be024d110a6466726bec8c055c/safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:21d01c14ff6c415c485616b8b0bf961c46b3b343ca59110d38d744e577f9cce7", size = 418419, upload-time = "2025-02-26T09:15:01.765Z" }, - { url = "https://files.pythonhosted.org/packages/5d/9a/add3e6fef267658075c5a41573c26d42d80c935cdc992384dfae435feaef/safetensors-0.5.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11bce6164887cd491ca75c2326a113ba934be596e22b28b1742ce27b1d076467", size = 459493, upload-time = "2025-02-26T09:14:51.812Z" }, - { url = "https://files.pythonhosted.org/packages/df/5c/bf2cae92222513cc23b3ff85c4a1bb2811a2c3583ac0f8e8d502751de934/safetensors-0.5.3-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4a243be3590bc3301c821da7a18d87224ef35cbd3e5f5727e4e0728b8172411e", size = 472400, upload-time = "2025-02-26T09:14:53.549Z" }, - { url = "https://files.pythonhosted.org/packages/58/11/7456afb740bd45782d0f4c8e8e1bb9e572f1bf82899fb6ace58af47b4282/safetensors-0.5.3-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8bd84b12b1670a6f8e50f01e28156422a2bc07fb16fc4e98bded13039d688a0d", size = 522891, upload-time = "2025-02-26T09:14:55.717Z" }, - { url = "https://files.pythonhosted.org/packages/57/3d/fe73a9d2ace487e7285f6e157afee2383bd1ddb911b7cb44a55cf812eae3/safetensors-0.5.3-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:391ac8cab7c829452175f871fcaf414aa1e292b5448bd02620f675a7f3e7abb9", size = 537694, upload-time = "2025-02-26T09:14:57.036Z" }, - { url = "https://files.pythonhosted.org/packages/a6/f8/dae3421624fcc87a89d42e1898a798bc7ff72c61f38973a65d60df8f124c/safetensors-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cead1fa41fc54b1e61089fa57452e8834f798cb1dc7a09ba3524f1eb08e0317a", size = 471642, upload-time = "2025-02-26T09:15:00.544Z" }, - { url = "https://files.pythonhosted.org/packages/ce/20/1fbe16f9b815f6c5a672f5b760951e20e17e43f67f231428f871909a37f6/safetensors-0.5.3-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1077f3e94182d72618357b04b5ced540ceb71c8a813d3319f1aba448e68a770d", size = 502241, upload-time = "2025-02-26T09:14:58.303Z" }, - { url = "https://files.pythonhosted.org/packages/5f/18/8e108846b506487aa4629fe4116b27db65c3dde922de2c8e0cc1133f3f29/safetensors-0.5.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:799021e78287bac619c7b3f3606730a22da4cda27759ddf55d37c8db7511c74b", size = 638001, upload-time = "2025-02-26T09:15:05.79Z" }, - { url = "https://files.pythonhosted.org/packages/82/5a/c116111d8291af6c8c8a8b40628fe833b9db97d8141c2a82359d14d9e078/safetensors-0.5.3-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:df26da01aaac504334644e1b7642fa000bfec820e7cef83aeac4e355e03195ff", size = 734013, upload-time = "2025-02-26T09:15:07.892Z" }, - { url = "https://files.pythonhosted.org/packages/7d/ff/41fcc4d3b7de837963622e8610d998710705bbde9a8a17221d85e5d0baad/safetensors-0.5.3-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:32c3ef2d7af8b9f52ff685ed0bc43913cdcde135089ae322ee576de93eae5135", size = 670687, upload-time = "2025-02-26T09:15:09.979Z" }, - { url = "https://files.pythonhosted.org/packages/40/ad/2b113098e69c985a3d8fbda4b902778eae4a35b7d5188859b4a63d30c161/safetensors-0.5.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:37f1521be045e56fc2b54c606d4455573e717b2d887c579ee1dbba5f868ece04", size = 643147, upload-time = "2025-02-26T09:15:11.185Z" }, - { url = "https://files.pythonhosted.org/packages/0a/0c/95aeb51d4246bd9a3242d3d8349c1112b4ee7611a4b40f0c5c93b05f001d/safetensors-0.5.3-cp38-abi3-win32.whl", hash = "sha256:cfc0ec0846dcf6763b0ed3d1846ff36008c6e7290683b61616c4b040f6a54ace", size = 296677, upload-time = "2025-02-26T09:15:16.554Z" }, - { url = "https://files.pythonhosted.org/packages/69/e2/b011c38e5394c4c18fb5500778a55ec43ad6106126e74723ffaee246f56e/safetensors-0.5.3-cp38-abi3-win_amd64.whl", hash = "sha256:836cbbc320b47e80acd40e44c8682db0e8ad7123209f69b093def21ec7cafd11", size = 308878, upload-time = "2025-02-26T09:15:14.99Z" }, -] - -[[package]] -name = "scantree" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "pathspec" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/ce/828467ddfa0d2fe473673026442d2032d552a168e42cfbf25fd0e5264e0c/scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174", size = 20690, upload-time = "2024-08-03T20:08:58.137Z" }, -] - -[[package]] -name = "scikit-image" -version = "0.25.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "imageio" }, - { name = "lazy-loader" }, - { name = "networkx" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "scipy" }, - { name = "tifffile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c7/a8/3c0f256012b93dd2cb6fda9245e9f4bff7dc0486880b248005f15ea2255e/scikit_image-0.25.2.tar.gz", hash = "sha256:e5a37e6cd4d0c018a7a55b9d601357e3382826d3888c10d0213fc63bff977dde", size = 22693594, upload-time = "2025-02-18T18:05:24.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/8c/5df82881284459f6eec796a5ac2a0a304bb3384eec2e73f35cfdfcfbf20c/scikit_image-0.25.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8db8dd03663112783221bf01ccfc9512d1cc50ac9b5b0fe8f4023967564719fb", size = 13986000, upload-time = "2025-02-18T18:04:47.156Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e6/93bebe1abcdce9513ffec01d8af02528b4c41fb3c1e46336d70b9ed4ef0d/scikit_image-0.25.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:483bd8cc10c3d8a7a37fae36dfa5b21e239bd4ee121d91cad1f81bba10cfb0ed", size = 13235893, upload-time = "2025-02-18T18:04:51.049Z" }, - { url = "https://files.pythonhosted.org/packages/53/4b/eda616e33f67129e5979a9eb33c710013caa3aa8a921991e6cc0b22cea33/scikit_image-0.25.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d1e80107bcf2bf1291acfc0bf0425dceb8890abe9f38d8e94e23497cbf7ee0d", size = 14178389, upload-time = "2025-02-18T18:04:54.245Z" }, - { url = "https://files.pythonhosted.org/packages/6b/b5/b75527c0f9532dd8a93e8e7cd8e62e547b9f207d4c11e24f0006e8646b36/scikit_image-0.25.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a17e17eb8562660cc0d31bb55643a4da996a81944b82c54805c91b3fe66f4824", size = 15003435, upload-time = "2025-02-18T18:04:57.586Z" }, - { url = "https://files.pythonhosted.org/packages/34/e3/49beb08ebccda3c21e871b607c1cb2f258c3fa0d2f609fed0a5ba741b92d/scikit_image-0.25.2-cp312-cp312-win_amd64.whl", hash = "sha256:bdd2b8c1de0849964dbc54037f36b4e9420157e67e45a8709a80d727f52c7da2", size = 12899474, upload-time = "2025-02-18T18:05:01.166Z" }, -] - -[[package]] -name = "scipy" -version = "1.15.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/b9/31ba9cd990e626574baf93fbc1ac61cf9ed54faafd04c479117517661637/scipy-1.15.2.tar.gz", hash = "sha256:cd58a314d92838f7e6f755c8a2167ead4f27e1fd5c1251fd54289569ef3495ec", size = 59417316, upload-time = "2025-02-17T00:42:24.791Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/5d/3c78815cbab499610f26b5bae6aed33e227225a9fa5290008a733a64f6fc/scipy-1.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c4697a10da8f8765bb7c83e24a470da5797e37041edfd77fd95ba3811a47c4fd", size = 38756184, upload-time = "2025-02-17T00:31:50.623Z" }, - { url = "https://files.pythonhosted.org/packages/37/20/3d04eb066b471b6e171827548b9ddb3c21c6bbea72a4d84fc5989933910b/scipy-1.15.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:869269b767d5ee7ea6991ed7e22b3ca1f22de73ab9a49c44bad338b725603301", size = 30163558, upload-time = "2025-02-17T00:31:56.721Z" }, - { url = "https://files.pythonhosted.org/packages/a4/98/e5c964526c929ef1f795d4c343b2ff98634ad2051bd2bbadfef9e772e413/scipy-1.15.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:bad78d580270a4d32470563ea86c6590b465cb98f83d760ff5b0990cb5518a93", size = 22437211, upload-time = "2025-02-17T00:32:03.042Z" }, - { url = "https://files.pythonhosted.org/packages/1d/cd/1dc7371e29195ecbf5222f9afeedb210e0a75057d8afbd942aa6cf8c8eca/scipy-1.15.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b09ae80010f52efddb15551025f9016c910296cf70adbf03ce2a8704f3a5ad20", size = 25232260, upload-time = "2025-02-17T00:32:07.847Z" }, - { url = "https://files.pythonhosted.org/packages/f0/24/1a181a9e5050090e0b5138c5f496fee33293c342b788d02586bc410c6477/scipy-1.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a6fd6eac1ce74a9f77a7fc724080d507c5812d61e72bd5e4c489b042455865e", size = 35198095, upload-time = "2025-02-17T00:32:14.565Z" }, - { url = "https://files.pythonhosted.org/packages/c0/53/eaada1a414c026673eb983f8b4a55fe5eb172725d33d62c1b21f63ff6ca4/scipy-1.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b871df1fe1a3ba85d90e22742b93584f8d2b8e6124f8372ab15c71b73e428b8", size = 37297371, upload-time = "2025-02-17T00:32:21.411Z" }, - { url = "https://files.pythonhosted.org/packages/e9/06/0449b744892ed22b7e7b9a1994a866e64895363572677a316a9042af1fe5/scipy-1.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:03205d57a28e18dfd39f0377d5002725bf1f19a46f444108c29bdb246b6c8a11", size = 36872390, upload-time = "2025-02-17T00:32:29.421Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6f/a8ac3cfd9505ec695c1bc35edc034d13afbd2fc1882a7c6b473e280397bb/scipy-1.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:601881dfb761311045b03114c5fe718a12634e5608c3b403737ae463c9885d53", size = 39700276, upload-time = "2025-02-17T00:32:37.431Z" }, - { url = "https://files.pythonhosted.org/packages/f5/6f/e6e5aff77ea2a48dd96808bb51d7450875af154ee7cbe72188afb0b37929/scipy-1.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:e7c68b6a43259ba0aab737237876e5c2c549a031ddb7abc28c7b47f22e202ded", size = 40942317, upload-time = "2025-02-17T00:32:45.47Z" }, -] - -[[package]] -name = "seaborn" -version = "0.13.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "matplotlib" }, - { name = "numpy" }, - { name = "pandas" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, -] - -[[package]] -name = "sentencepiece" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/d2/b9c7ca067c26d8ff085d252c89b5f69609ca93fb85a00ede95f4857865d4/sentencepiece-0.2.0.tar.gz", hash = "sha256:a52c19171daaf2e697dc6cbe67684e0fa341b1248966f6aebb541de654d15843", size = 2632106, upload-time = "2024-02-19T17:06:47.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/5a/141b227ed54293360a9ffbb7bf8252b4e5efc0400cdeac5809340e5d2b21/sentencepiece-0.2.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ea5f536e32ea8ec96086ee00d7a4a131ce583a1b18d130711707c10e69601cb2", size = 2409370, upload-time = "2024-02-19T17:06:29.315Z" }, - { url = "https://files.pythonhosted.org/packages/2e/08/a4c135ad6fc2ce26798d14ab72790d66e813efc9589fd30a5316a88ca8d5/sentencepiece-0.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0cb51f53b6aae3c36bafe41e86167c71af8370a039f542c43b0cce5ef24a68c", size = 1239288, upload-time = "2024-02-19T17:06:31.674Z" }, - { url = "https://files.pythonhosted.org/packages/49/0a/2fe387f825ac5aad5a0bfe221904882106cac58e1b693ba7818785a882b6/sentencepiece-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3212121805afc58d8b00ab4e7dd1f8f76c203ddb9dc94aa4079618a31cf5da0f", size = 1181597, upload-time = "2024-02-19T17:06:33.763Z" }, - { url = "https://files.pythonhosted.org/packages/cc/38/e4698ee2293fe4835dc033c49796a39b3eebd8752098f6bd0aa53a14af1f/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a3149e3066c2a75e0d68a43eb632d7ae728c7925b517f4c05c40f6f7280ce08", size = 1259220, upload-time = "2024-02-19T17:06:35.85Z" }, - { url = "https://files.pythonhosted.org/packages/12/24/fd7ef967c9dad2f6e6e5386d0cadaf65cda8b7be6e3861a9ab3121035139/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:632f3594d3e7ac8b367bca204cb3fd05a01d5b21455acd097ea4c0e30e2f63d7", size = 1355962, upload-time = "2024-02-19T17:06:38.616Z" }, - { url = "https://files.pythonhosted.org/packages/4f/d2/18246f43ca730bb81918f87b7e886531eda32d835811ad9f4657c54eee35/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f295105c6bdbb05bd5e1b0cafbd78ff95036f5d3641e7949455a3f4e5e7c3109", size = 1301706, upload-time = "2024-02-19T17:06:40.712Z" }, - { url = "https://files.pythonhosted.org/packages/8a/47/ca237b562f420044ab56ddb4c278672f7e8c866e183730a20e413b38a989/sentencepiece-0.2.0-cp312-cp312-win32.whl", hash = "sha256:fb89f811e5efd18bab141afc3fea3de141c3f69f3fe9e898f710ae7fe3aab251", size = 936941, upload-time = "2024-02-19T17:06:42.802Z" }, - { url = "https://files.pythonhosted.org/packages/c6/97/d159c32642306ee2b70732077632895438867b3b6df282354bd550cf2a67/sentencepiece-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7a673a72aab81fef5ebe755c6e0cc60087d1f3a4700835d40537183c1703a45f", size = 991994, upload-time = "2024-02-19T17:06:45.01Z" }, -] - -[[package]] -name = "sentry-sdk" -version = "2.25.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/2f/a0f732270cc7c1834f5ec45539aec87c360d5483a8bd788217a9102ccfbd/sentry_sdk-2.25.1.tar.gz", hash = "sha256:f9041b7054a7cf12d41eadabe6458ce7c6d6eea7a97cfe1b760b6692e9562cf0", size = 322190, upload-time = "2025-04-02T15:13:10.225Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/b6/84049ab0967affbc7cc7590d86ae0170c1b494edb69df8786707100420e5/sentry_sdk-2.25.1-py2.py3-none-any.whl", hash = "sha256:60b016d0772789454dc55a284a6a44212044d4a16d9f8448725effee97aaf7f6", size = 339851, upload-time = "2025-04-02T15:13:08.616Z" }, -] - -[[package]] -name = "setproctitle" -version = "1.3.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c4/4d/6a840c8d2baa07b57329490e7094f90aac177a1d5226bc919046f1106860/setproctitle-1.3.5.tar.gz", hash = "sha256:1e6eaeaf8a734d428a95d8c104643b39af7d247d604f40a7bebcf3960a853c5e", size = 26737, upload-time = "2025-02-22T21:52:43.276Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/19/04755958495de57e4891de50f03e77b3fe9ca6716a86de00faa00ad0ee5a/setproctitle-1.3.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:523424b9be4dea97d95b8a584b183f35c7bab2d0a3d995b01febf5b8a8de90e4", size = 17250, upload-time = "2025-02-22T21:51:09.785Z" }, - { url = "https://files.pythonhosted.org/packages/b9/3d/2ca9df5aa49b975296411dcbbe272cdb1c5e514c43b8be7d61751bb71a46/setproctitle-1.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b6ec1d86c1b4d7b5f2bdceadf213310cf24696b82480a2a702194b8a0bfbcb47", size = 11878, upload-time = "2025-02-22T21:51:11.679Z" }, - { url = "https://files.pythonhosted.org/packages/36/d6/e90e23b4627e016a4f862d4f892be92c9765dd6bf1e27a48e52cd166d4a3/setproctitle-1.3.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea6c505264275a43e9b2acd2acfc11ac33caf52bc3167c9fced4418a810f6b1c", size = 31940, upload-time = "2025-02-22T21:51:12.977Z" }, - { url = "https://files.pythonhosted.org/packages/15/13/167cdd55e00a8e10b36aad79646c3bf3c23fba0c08a9b8db9b74622c1b13/setproctitle-1.3.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b91e68e6685998e6353f296100ecabc313a6cb3e413d66a03d74b988b61f5ff", size = 33370, upload-time = "2025-02-22T21:51:15.115Z" }, - { url = "https://files.pythonhosted.org/packages/9b/22/574a110527df133409a75053b7d6ff740993ccf30b8713d042f26840d351/setproctitle-1.3.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc1fda208ae3a2285ad27aeab44c41daf2328abe58fa3270157a739866779199", size = 30628, upload-time = "2025-02-22T21:51:16.324Z" }, - { url = "https://files.pythonhosted.org/packages/52/79/78b05c7d792c9167b917acdab1773b1ff73b016560f45d8155be2baa1a82/setproctitle-1.3.5-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:828727d220e46f048b82289018300a64547b46aaed96bf8810c05fe105426b41", size = 31672, upload-time = "2025-02-22T21:51:17.791Z" }, - { url = "https://files.pythonhosted.org/packages/b0/62/4509735be062129694751ac55d5e1fbb6d86fa46a8689b7d5e2c23dae5b0/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:83b016221cf80028b2947be20630faa14e3e72a403e35f0ba29550b4e856767b", size = 31378, upload-time = "2025-02-22T21:51:19.404Z" }, - { url = "https://files.pythonhosted.org/packages/72/e7/b394c55934b89f00c2ef7d5e6f18cca5d8dfa26ef628700c4de0c85e3f3d/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6d8a411e752e794d052434139ca4234ffeceeb8d8d8ddc390a9051d7942b2726", size = 30370, upload-time = "2025-02-22T21:51:21.218Z" }, - { url = "https://files.pythonhosted.org/packages/13/ee/e1f27bf52d2bec7060bb6311ab0ccede8de98ed5394e3a59e7a14a453fb5/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:50cfbf86b9c63a2c2903f1231f0a58edeb775e651ae1af84eec8430b0571f29b", size = 32875, upload-time = "2025-02-22T21:51:22.505Z" }, - { url = "https://files.pythonhosted.org/packages/6e/08/13b561085d2de53b9becfa5578545d99114e9ff2aa3dc151bcaadf80b17e/setproctitle-1.3.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f3b5e2eacd572444770026c9dd3ddc7543ce427cdf452d40a408d1e95beefb30", size = 30903, upload-time = "2025-02-22T21:51:23.732Z" }, - { url = "https://files.pythonhosted.org/packages/65/f0/6cd06fffff2553be7b0571447d0c0ef8b727ef44cc2d6a33452677a311c8/setproctitle-1.3.5-cp312-cp312-win32.whl", hash = "sha256:cf4e3ded98027de2596c6cc5bbd3302adfb3ca315c848f56516bb0b7e88de1e9", size = 11468, upload-time = "2025-02-22T21:51:25.45Z" }, - { url = "https://files.pythonhosted.org/packages/c1/8c/e8a7cb568c4552618838941b332203bfc77ab0f2d67c1cb8f24dee0370ec/setproctitle-1.3.5-cp312-cp312-win_amd64.whl", hash = "sha256:f7a8c01ffd013dda2bed6e7d5cb59fbb609e72f805abf3ee98360f38f7758d9b", size = 12190, upload-time = "2025-02-22T21:51:26.78Z" }, -] - -[[package]] -name = "setuptools" -version = "78.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/5a/0db4da3bc908df06e5efae42b44e75c81dd52716e10192ff36d0c1c8e379/setuptools-78.1.0.tar.gz", hash = "sha256:18fd474d4a82a5f83dac888df697af65afa82dec7323d09c3e37d1f14288da54", size = 1367827, upload-time = "2025-03-25T22:49:35.332Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/21/f43f0a1fa8b06b32812e0975981f4677d28e0f3271601dc88ac5a5b83220/setuptools-78.1.0-py3-none-any.whl", hash = "sha256:3e386e96793c8702ae83d17b853fb93d3e09ef82ec62722e61da5cd22376dcd8", size = 1256108, upload-time = "2025-03-25T22:49:33.13Z" }, -] - -[[package]] -name = "sgl-kernel" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/c3/60807976ba737d075324ecd756f90ae1da14512e0fee6ced5835fdc7b761/sgl_kernel-0.1.0-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:d3bdbd2b15eaae701540bf4110fb0bc88c059c5895bbeed2e92e5696f9609e96", size = 102264966, upload-time = "2025-04-23T21:45:49.024Z" }, -] - -[[package]] -name = "sglang" -version = "0.4.6.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "ipython" }, - { name = "numpy" }, - { name = "requests" }, - { name = "setproctitle" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/58/45f2d7ca294b4ccab9d10c26141e1fbcbf2ea5d3d382057792ba368ef0cb/sglang-0.4.6.post1.tar.gz", hash = "sha256:2ff518d7103efd9bd345d8e434b96bb152920738989ab007bd9771e758726a93", size = 873627, upload-time = "2025-04-28T19:57:34.353Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/38/d7ae5880f974d7a5af2269ec0ca350f22234b33ca9dbd38aaf578d931902/sglang-0.4.6.post1-py3-none-any.whl", hash = "sha256:ae62fff1bbdd2c8d4c0b208e79ab8f8ba5c1969775e0fe2ce19679983e997bc5", size = 1244043, upload-time = "2025-04-28T19:57:32.786Z" }, -] - -[package.optional-dependencies] -all = [ - { name = "anthropic" }, - { name = "compressed-tensors" }, - { name = "cuda-python" }, - { name = "datasets" }, - { name = "decord" }, - { name = "einops" }, - { name = "fastapi" }, - { name = "flashinfer-python" }, - { name = "hf-transfer" }, - { name = "huggingface-hub" }, - { name = "interegular" }, - { name = "litellm" }, - { name = "llguidance" }, - { name = "modelscope" }, - { name = "ninja" }, - { name = "openai" }, - { name = "orjson" }, - { name = "outlines" }, - { name = "packaging" }, - { name = "partial-json-parser" }, - { name = "pillow" }, - { name = "prometheus-client" }, - { name = "psutil" }, - { name = "pydantic" }, - { name = "pynvml" }, - { name = "python-multipart" }, - { name = "pyzmq" }, - { name = "sgl-kernel" }, - { name = "soundfile" }, - { name = "tiktoken" }, - { name = "torch" }, - { name = "torchao" }, - { name = "torchvision" }, - { name = "transformers" }, - { name = "uvicorn" }, - { name = "uvloop" }, - { name = "xgrammar" }, -] - -[[package]] -name = "shapely" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/fe/3b0d2f828ffaceadcdcb51b75b9c62d98e62dd95ce575278de35f24a1c20/shapely-2.1.0.tar.gz", hash = "sha256:2cbe90e86fa8fc3ca8af6ffb00a77b246b918c7cf28677b7c21489b678f6b02e", size = 313617, upload-time = "2025-04-03T09:15:05.725Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/d1/6a9371ec39d3ef08e13225594e6c55b045209629afd9e6d403204507c2a8/shapely-2.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:53e7ee8bd8609cf12ee6dce01ea5affe676976cf7049315751d53d8db6d2b4b2", size = 1830732, upload-time = "2025-04-03T09:14:25.047Z" }, - { url = "https://files.pythonhosted.org/packages/32/87/799e3e48be7ce848c08509b94d2180f4ddb02e846e3c62d0af33da4d78d3/shapely-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cab20b665d26dbec0b380e15749bea720885a481fa7b1eedc88195d4a98cfa4", size = 1638404, upload-time = "2025-04-03T09:14:26.456Z" }, - { url = "https://files.pythonhosted.org/packages/85/00/6665d77f9dd09478ab0993b8bc31668aec4fd3e5f1ddd1b28dd5830e47be/shapely-2.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4a38b39a09340273c3c92b3b9a374272a12cc7e468aeeea22c1c46217a03e5c", size = 2945316, upload-time = "2025-04-03T09:14:28.266Z" }, - { url = "https://files.pythonhosted.org/packages/34/49/738e07d10bbc67cae0dcfe5a484c6e518a517f4f90550dda2adf3a78b9f2/shapely-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:edaec656bdd9b71278b98e6f77c464b1c3b2daa9eace78012ff0f0b4b5b15b04", size = 3063099, upload-time = "2025-04-03T09:14:30.067Z" }, - { url = "https://files.pythonhosted.org/packages/88/b8/138098674559362ab29f152bff3b6630de423378fbb0324812742433a4ef/shapely-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c8a732ddd9b25e7a54aa748e7df8fd704e23e5d5d35b7d376d80bffbfc376d04", size = 3887873, upload-time = "2025-04-03T09:14:31.912Z" }, - { url = "https://files.pythonhosted.org/packages/67/a8/fdae7c2db009244991d86f4d2ca09d2f5ccc9d41c312c3b1ee1404dc55da/shapely-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9c93693ad8adfdc9138a5a2d42da02da94f728dd2e82d2f0f442f10e25027f5f", size = 4067004, upload-time = "2025-04-03T09:14:33.976Z" }, - { url = "https://files.pythonhosted.org/packages/ed/78/17e17d91b489019379df3ee1afc4bd39787b232aaa1d540f7d376f0280b7/shapely-2.1.0-cp312-cp312-win32.whl", hash = "sha256:d8ac6604eefe807e71a908524de23a37920133a1729fe3a4dfe0ed82c044cbf4", size = 1527366, upload-time = "2025-04-03T09:14:35.348Z" }, - { url = "https://files.pythonhosted.org/packages/b8/bd/9249bd6dda948441e25e4fb14cbbb5205146b0fff12c66b19331f1ff2141/shapely-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:f4f47e631aa4f9ec5576eac546eb3f38802e2f82aeb0552f9612cb9a14ece1db", size = 1708265, upload-time = "2025-04-03T09:14:36.878Z" }, -] - -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, -] - -[[package]] -name = "sigtools" -version = "4.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5f/db/669ca14166814da187b3087b908ca924cf83f5b504fe23b3859a3ef67d4f/sigtools-4.0.1.tar.gz", hash = "sha256:4b8e135a9cd4d2ea00da670c093372d74e672ba3abb87f4c98d8e73dea54445c", size = 71910, upload-time = "2022-10-13T07:03:54.149Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/91/853dbf6ec096197dba9cd5fd0c836c5fc19142038b7db60ebe6332b1bab1/sigtools-4.0.1-py2.py3-none-any.whl", hash = "sha256:d216b4cf920bbab0fce636ddc429ed8463a5b533d9e1492acb45a2a1bc36ac6c", size = 76419, upload-time = "2022-10-13T07:03:52.658Z" }, -] - -[[package]] -name = "simple-websocket" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wsproto" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "smmap" -version = "5.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, -] - -[[package]] -name = "sniffio" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, -] - -[[package]] -name = "soundfile" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/41/9b873a8c055582859b239be17902a85339bec6a30ad162f98c9b0288a2cc/soundfile-0.13.1.tar.gz", hash = "sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b", size = 46156, upload-time = "2025-01-25T09:17:04.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/28/e2a36573ccbcf3d57c00626a21fe51989380636e821b341d36ccca0c1c3a/soundfile-0.13.1-py2.py3-none-any.whl", hash = "sha256:a23c717560da2cf4c7b5ae1142514e0fd82d6bbd9dfc93a50423447142f2c445", size = 25751, upload-time = "2025-01-25T09:16:44.235Z" }, - { url = "https://files.pythonhosted.org/packages/ea/ab/73e97a5b3cc46bba7ff8650a1504348fa1863a6f9d57d7001c6b67c5f20e/soundfile-0.13.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:82dc664d19831933fe59adad199bf3945ad06d84bc111a5b4c0d3089a5b9ec33", size = 1142250, upload-time = "2025-01-25T09:16:47.583Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e5/58fd1a8d7b26fc113af244f966ee3aecf03cb9293cb935daaddc1e455e18/soundfile-0.13.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:743f12c12c4054921e15736c6be09ac26b3b3d603aef6fd69f9dde68748f2593", size = 1101406, upload-time = "2025-01-25T09:16:49.662Z" }, - { url = "https://files.pythonhosted.org/packages/58/ae/c0e4a53d77cf6e9a04179535766b3321b0b9ced5f70522e4caf9329f0046/soundfile-0.13.1-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9c9e855f5a4d06ce4213f31918653ab7de0c5a8d8107cd2427e44b42df547deb", size = 1235729, upload-time = "2025-01-25T09:16:53.018Z" }, - { url = "https://files.pythonhosted.org/packages/57/5e/70bdd9579b35003a489fc850b5047beeda26328053ebadc1fb60f320f7db/soundfile-0.13.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:03267c4e493315294834a0870f31dbb3b28a95561b80b134f0bd3cf2d5f0e618", size = 1313646, upload-time = "2025-01-25T09:16:54.872Z" }, - { url = "https://files.pythonhosted.org/packages/fe/df/8c11dc4dfceda14e3003bb81a0d0edcaaf0796dd7b4f826ea3e532146bba/soundfile-0.13.1-py2.py3-none-win32.whl", hash = "sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5", size = 899881, upload-time = "2025-01-25T09:16:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/14/e9/6b761de83277f2f02ded7e7ea6f07828ec78e4b229b80e4ca55dd205b9dc/soundfile-0.13.1-py2.py3-none-win_amd64.whl", hash = "sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9", size = 1019162, upload-time = "2025-01-25T09:16:59.573Z" }, -] - -[[package]] -name = "soupsieve" -version = "2.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/ce/fbaeed4f9fb8b2daa961f90591662df6a86c1abf25c548329a86920aedfb/soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb", size = 101569, upload-time = "2024-08-13T13:39:12.166Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/c2/fe97d779f3ef3b15f05c94a2f1e3d21732574ed441687474db9d342a7315/soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9", size = 36186, upload-time = "2024-08-13T13:39:10.986Z" }, -] - -[[package]] -name = "sse-starlette" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/a4/80d2a11af59fe75b48230846989e93979c892d3a20016b42bb44edb9e398/sse_starlette-2.2.1.tar.gz", hash = "sha256:54470d5f19274aeed6b2d473430b08b4b379ea851d953b11d7f1c4a2c118b419", size = 17376, upload-time = "2024-12-25T09:09:30.616Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/e0/5b8bd393f27f4a62461c5cf2479c75a2cc2ffa330976f9f00f5f6e4f50eb/sse_starlette-2.2.1-py3-none-any.whl", hash = "sha256:6410a3d3ba0c89e7675d4c273a301d64649c03a5ef1ca101f10b47f895fd0e99", size = 10120, upload-time = "2024-12-25T09:09:26.761Z" }, -] - -[[package]] -name = "stack-data" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asttokens" }, - { name = "executing" }, - { name = "pure-eval" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, -] - -[[package]] -name = "starlette" -version = "0.46.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/1b/52b27f2e13ceedc79a908e29eac426a63465a1a01248e5f24aa36a62aeb3/starlette-0.46.1.tar.gz", hash = "sha256:3c88d58ee4bd1bb807c0d1acb381838afc7752f9ddaec81bbe4383611d833230", size = 2580102, upload-time = "2025-03-08T10:55:34.504Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/4b/528ccf7a982216885a1ff4908e886b8fb5f19862d1962f56a3fce2435a70/starlette-0.46.1-py3-none-any.whl", hash = "sha256:77c74ed9d2720138b25875133f3a2dae6d854af2ec37dceb56aef370c1d8a227", size = 71995, upload-time = "2025-03-08T10:55:32.662Z" }, -] - -[[package]] -name = "streamlit" -version = "1.44.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "altair" }, - { name = "blinker" }, - { name = "cachetools" }, - { name = "click" }, - { name = "gitpython" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "pillow" }, - { name = "protobuf" }, - { name = "pyarrow" }, - { name = "pydeck" }, - { name = "requests" }, - { name = "tenacity" }, - { name = "toml" }, - { name = "tornado" }, - { name = "typing-extensions" }, - { name = "watchdog", marker = "sys_platform != 'darwin'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3e/c0/7286284567e5045f0c587c426d0c41aee5d10c0a2e360e627a83037e9f0c/streamlit-1.44.1.tar.gz", hash = "sha256:c6914ed6d5b76870b461510476806db370f36425ae0e6654d227c988288198d3", size = 9423685, upload-time = "2025-04-01T20:36:19.91Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/17/fc425e1d4d86e31b2aaf0812a2ef2163763a0670d671720c7c36e8679323/streamlit-1.44.1-py3-none-any.whl", hash = "sha256:9fe355f58b11f4eb71e74f115ce1f38c4c9eaff2733e6bcffb510ac1298a5990", size = 9812242, upload-time = "2025-04-01T20:36:16.785Z" }, -] - -[[package]] -name = "strenum" -version = "0.4.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/ad/430fb60d90e1d112a62ff57bdd1f286ec73a2a0331272febfddd21f330e1/StrEnum-0.4.15.tar.gz", hash = "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff", size = 23384, upload-time = "2023-06-29T22:02:58.399Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, -] - -[[package]] -name = "stripe" -version = "11.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/a4/7e16668d7adfb45229f4b67c04c79f53fe78335a6d0625303b32024d831c/stripe-11.6.0.tar.gz", hash = "sha256:0ced7cce23a6cb1a393c86a1f7f9435c9d83ae7cbd556362868caf62cb44a92c", size = 1390405, upload-time = "2025-02-24T22:39:46.962Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/67/e320a11da2049dfd50fd1726952093b173332563f32d1a94cc3758932147/stripe-11.6.0-py2.py3-none-any.whl", hash = "sha256:6e6cf09ebb6d5fc2d708401cb8868fd7bff987a6d09a0433caaa92c62f97dbc5", size = 1636766, upload-time = "2025-02-24T22:39:44.629Z" }, -] - -[[package]] -name = "swebench" -version = "2.0.13" -source = { git = "https://github.com/SWE-Gym/SWE-Bench-Fork.git#242429c188fcfd06aad13fce9a54d450470bf0ac" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "chardet" }, - { name = "datasets" }, - { name = "docker" }, - { name = "ghapi" }, - { name = "gitpython" }, - { name = "pre-commit" }, - { name = "python-dotenv" }, - { name = "requests" }, - { name = "rich" }, - { name = "tqdm" }, - { name = "unidiff" }, -] - -[[package]] -name = "swegym" -version = "2.0.13" -source = { git = "https://github.com/SWE-Gym/SWE-Bench-Package.git#16dd480cce9b27bf111a362d280881c6def5d2a7" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "chardet" }, - { name = "datasets" }, - { name = "docker" }, - { name = "ghapi" }, - { name = "gitpython" }, - { name = "pre-commit" }, - { name = "python-dotenv" }, - { name = "requests" }, - { name = "rich" }, - { name = "tqdm" }, - { name = "unidiff" }, -] - -[[package]] -name = "sympy" -version = "1.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ca/99/5a5b6f19ff9f083671ddf7b9632028436167cd3d33e11015754e41b249a4/sympy-1.13.1.tar.gz", hash = "sha256:9cebf7e04ff162015ce31c9c6c9144daa34a93bd082f54fd8f12deca4f47515f", size = 7533040, upload-time = "2024-07-19T09:26:51.238Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/fe/81695a1aa331a842b582453b605175f419fe8540355886031328089d840a/sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8", size = 6189177, upload-time = "2024-07-19T09:26:48.863Z" }, -] - -[[package]] -name = "synchronicity" -version = "0.9.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sigtools" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/52/f34a9ab6d514e0808d0f572affb360411d596b3439107318c00889277dd6/synchronicity-0.9.11.tar.gz", hash = "sha256:cb5dbbcb43d637e516ae50db05a776da51a705d1e1a9c0e301f6049afc3c2cae", size = 50323, upload-time = "2025-02-10T09:40:56.327Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/d5/7675cd9b8e18f05b9ea261acad5d197fcb8027d2a65b1a750427ec084593/synchronicity-0.9.11-py3-none-any.whl", hash = "sha256:231129654d2f56b1aa148e85ebd8545231be135771f6d2196d414175b1594ef6", size = 36827, upload-time = "2025-02-10T09:40:54.072Z" }, -] - -[[package]] -name = "tabulate" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090, upload-time = "2022-10-06T17:21:48.54Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, -] - -[[package]] -name = "tenacity" -version = "9.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, -] - -[[package]] -name = "tensordict" -version = "0.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cloudpickle" }, - { name = "numpy" }, - { name = "orjson" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/17/a2bdb51d5d1b2f06debdf8f60c159cf328c778b6cc8103940f2079e2e767/tensordict-0.6.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:529515ffcb0c6ab5d1c0c85798defe8972e5390412c356c79b87f4073e48b537", size = 668892, upload-time = "2024-11-14T15:02:37.036Z" }, - { url = "https://files.pythonhosted.org/packages/64/aa/2b562740c8be079aba4bb9eb37cb94ed042d65e48f739577eb70eee617c2/tensordict-0.6.2-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:6ce86d8f9ed53d1c893e0064729a799eb06371902e2d5a2f3c83c6af010cf3c2", size = 360647, upload-time = "2024-11-14T15:02:38.475Z" }, -] - -[[package]] -name = "termcolor" -version = "3.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/b6/8e2aaa8aeb570b5cc955cd913b083d96c5447bbe27eaf330dfd7cc8e3329/termcolor-3.0.1.tar.gz", hash = "sha256:a6abd5c6e1284cea2934443ba806e70e5ec8fd2449021be55c280f8a3731b611", size = 12935, upload-time = "2025-04-02T10:02:25.243Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/7e/a574ccd49ad07e8b117407bac361f1e096b01f1b620365daf60ff702c936/termcolor-3.0.1-py3-none-any.whl", hash = "sha256:da1ed4ec8a5dc5b2e17476d859febdb3cccb612be1c36e64511a6f2485c10c69", size = 7157, upload-time = "2025-04-02T10:02:24.088Z" }, -] - -[[package]] -name = "text-generation" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "huggingface-hub" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/53/1b2dc20686079464ae381f230a9fc412984a4255cea73c21afb6a46bc21f/text_generation-0.7.0.tar.gz", hash = "sha256:689200cd1f0d4141562af2515393c2c21cdbd9fac21c8398bf3043cdcc14184e", size = 10373, upload-time = "2024-03-22T16:09:22.167Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/79/8fc351fd919a41287243c998a47692c7eb0fa5acded13db0080f2c6f1852/text_generation-0.7.0-py3-none-any.whl", hash = "sha256:02ab337a0ee0e7c70e04a607b311c261caae74bde46a7d837c6fdd150108f4d8", size = 12718, upload-time = "2024-03-22T16:09:20.874Z" }, -] - -[[package]] -name = "tifffile" -version = "2025.3.30" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3c/54/d5ebe66a9de349b833e570e87bdbd9eec76ec54bd505c24b0591a15783ad/tifffile-2025.3.30.tar.gz", hash = "sha256:3cdee47fe06cd75367c16bc3ff34523713156dae6cd498e3a392e5b39a51b789", size = 366039, upload-time = "2025-03-30T04:45:30.503Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/be/10d23cfd4078fbec6aba768a357eff9e70c0b6d2a07398425985c524ad2a/tifffile-2025.3.30-py3-none-any.whl", hash = "sha256:0ed6eee7b66771db2d1bfc42262a51b01887505d35539daef118f4ff8c0f629c", size = 226837, upload-time = "2025-03-30T04:45:29Z" }, -] - -[[package]] -name = "tiktoken" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "regex" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ea/cf/756fedf6981e82897f2d570dd25fa597eb3f4459068ae0572d7e888cfd6f/tiktoken-0.9.0.tar.gz", hash = "sha256:d02a5ca6a938e0490e1ff957bc48c8b078c88cb83977be1625b1fd8aac792c5d", size = 35991, upload-time = "2025-02-14T06:03:01.003Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/e5/21ff33ecfa2101c1bb0f9b6df750553bd873b7fb532ce2cb276ff40b197f/tiktoken-0.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e88f121c1c22b726649ce67c089b90ddda8b9662545a8aeb03cfef15967ddd03", size = 1065073, upload-time = "2025-02-14T06:02:24.768Z" }, - { url = "https://files.pythonhosted.org/packages/8e/03/a95e7b4863ee9ceec1c55983e4cc9558bcfd8f4f80e19c4f8a99642f697d/tiktoken-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6600660f2f72369acb13a57fb3e212434ed38b045fd8cc6cdd74947b4b5d210", size = 1008075, upload-time = "2025-02-14T06:02:26.92Z" }, - { url = "https://files.pythonhosted.org/packages/40/10/1305bb02a561595088235a513ec73e50b32e74364fef4de519da69bc8010/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95e811743b5dfa74f4b227927ed86cbc57cad4df859cb3b643be797914e41794", size = 1140754, upload-time = "2025-02-14T06:02:28.124Z" }, - { url = "https://files.pythonhosted.org/packages/1b/40/da42522018ca496432ffd02793c3a72a739ac04c3794a4914570c9bb2925/tiktoken-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99376e1370d59bcf6935c933cb9ba64adc29033b7e73f5f7569f3aad86552b22", size = 1196678, upload-time = "2025-02-14T06:02:29.845Z" }, - { url = "https://files.pythonhosted.org/packages/5c/41/1e59dddaae270ba20187ceb8aa52c75b24ffc09f547233991d5fd822838b/tiktoken-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:badb947c32739fb6ddde173e14885fb3de4d32ab9d8c591cbd013c22b4c31dd2", size = 1259283, upload-time = "2025-02-14T06:02:33.838Z" }, - { url = "https://files.pythonhosted.org/packages/5b/64/b16003419a1d7728d0d8c0d56a4c24325e7b10a21a9dd1fc0f7115c02f0a/tiktoken-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:5a62d7a25225bafed786a524c1b9f0910a1128f4232615bf3f8257a73aaa3b16", size = 894897, upload-time = "2025-02-14T06:02:36.265Z" }, -] - -[[package]] -name = "tokenizers" -version = "0.21.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/92/76/5ac0c97f1117b91b7eb7323dcd61af80d72f790b4df71249a7850c195f30/tokenizers-0.21.1.tar.gz", hash = "sha256:a1bb04dc5b448985f86ecd4b05407f5a8d97cb2c0532199b2a302a604a0165ab", size = 343256, upload-time = "2025-03-13T10:51:18.189Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/1f/328aee25f9115bf04262e8b4e5a2050b7b7cf44b59c74e982db7270c7f30/tokenizers-0.21.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e78e413e9e668ad790a29456e677d9d3aa50a9ad311a40905d6861ba7692cf41", size = 2780767, upload-time = "2025-03-13T10:51:09.459Z" }, - { url = "https://files.pythonhosted.org/packages/ae/1a/4526797f3719b0287853f12c5ad563a9be09d446c44ac784cdd7c50f76ab/tokenizers-0.21.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:cd51cd0a91ecc801633829fcd1fda9cf8682ed3477c6243b9a095539de4aecf3", size = 2650555, upload-time = "2025-03-13T10:51:07.692Z" }, - { url = "https://files.pythonhosted.org/packages/4d/7a/a209b29f971a9fdc1da86f917fe4524564924db50d13f0724feed37b2a4d/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28da6b72d4fb14ee200a1bd386ff74ade8992d7f725f2bde2c495a9a98cf4d9f", size = 2937541, upload-time = "2025-03-13T10:50:56.679Z" }, - { url = "https://files.pythonhosted.org/packages/3c/1e/b788b50ffc6191e0b1fc2b0d49df8cff16fe415302e5ceb89f619d12c5bc/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:34d8cfde551c9916cb92014e040806122295a6800914bab5865deb85623931cf", size = 2819058, upload-time = "2025-03-13T10:50:59.525Z" }, - { url = "https://files.pythonhosted.org/packages/36/aa/3626dfa09a0ecc5b57a8c58eeaeb7dd7ca9a37ad9dd681edab5acd55764c/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aaa852d23e125b73d283c98f007e06d4595732104b65402f46e8ef24b588d9f8", size = 3133278, upload-time = "2025-03-13T10:51:04.678Z" }, - { url = "https://files.pythonhosted.org/packages/a4/4d/8fbc203838b3d26269f944a89459d94c858f5b3f9a9b6ee9728cdcf69161/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a21a15d5c8e603331b8a59548bbe113564136dc0f5ad8306dd5033459a226da0", size = 3144253, upload-time = "2025-03-13T10:51:01.261Z" }, - { url = "https://files.pythonhosted.org/packages/d8/1b/2bd062adeb7c7511b847b32e356024980c0ffcf35f28947792c2d8ad2288/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2fdbd4c067c60a0ac7eca14b6bd18a5bebace54eb757c706b47ea93204f7a37c", size = 3398225, upload-time = "2025-03-13T10:51:03.243Z" }, - { url = "https://files.pythonhosted.org/packages/8a/63/38be071b0c8e06840bc6046991636bcb30c27f6bb1e670f4f4bc87cf49cc/tokenizers-0.21.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dd9a0061e403546f7377df940e866c3e678d7d4e9643d0461ea442b4f89e61a", size = 3038874, upload-time = "2025-03-13T10:51:06.235Z" }, - { url = "https://files.pythonhosted.org/packages/ec/83/afa94193c09246417c23a3c75a8a0a96bf44ab5630a3015538d0c316dd4b/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:db9484aeb2e200c43b915a1a0150ea885e35f357a5a8fabf7373af333dcc8dbf", size = 9014448, upload-time = "2025-03-13T10:51:10.927Z" }, - { url = "https://files.pythonhosted.org/packages/ae/b3/0e1a37d4f84c0f014d43701c11eb8072704f6efe8d8fc2dcdb79c47d76de/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:ed248ab5279e601a30a4d67bdb897ecbe955a50f1e7bb62bd99f07dd11c2f5b6", size = 8937877, upload-time = "2025-03-13T10:51:12.688Z" }, - { url = "https://files.pythonhosted.org/packages/ac/33/ff08f50e6d615eb180a4a328c65907feb6ded0b8f990ec923969759dc379/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:9ac78b12e541d4ce67b4dfd970e44c060a2147b9b2a21f509566d556a509c67d", size = 9186645, upload-time = "2025-03-13T10:51:14.723Z" }, - { url = "https://files.pythonhosted.org/packages/5f/aa/8ae85f69a9f6012c6f8011c6f4aa1c96154c816e9eea2e1b758601157833/tokenizers-0.21.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e5a69c1a4496b81a5ee5d2c1f3f7fbdf95e90a0196101b0ee89ed9956b8a168f", size = 9384380, upload-time = "2025-03-13T10:51:16.526Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5b/a5d98c89f747455e8b7a9504910c865d5e51da55e825a7ae641fb5ff0a58/tokenizers-0.21.1-cp39-abi3-win32.whl", hash = "sha256:1039a3a5734944e09de1d48761ade94e00d0fa760c0e0551151d4dd851ba63e3", size = 2239506, upload-time = "2025-03-13T10:51:20.643Z" }, - { url = "https://files.pythonhosted.org/packages/e6/b6/072a8e053ae600dcc2ac0da81a23548e3b523301a442a6ca900e92ac35be/tokenizers-0.21.1-cp39-abi3-win_amd64.whl", hash = "sha256:0f0dcbcc9f6e13e675a66d7a5f2f225a736745ce484c1a4e07476a89ccdad382", size = 2435481, upload-time = "2025-03-13T10:51:19.243Z" }, -] - -[[package]] -name = "toml" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, -] - -[[package]] -name = "torch" -version = "2.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/35/0c52d708144c2deb595cd22819a609f78fdd699b95ff6f0ebcd456e3c7c1/torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:2bb8987f3bb1ef2675897034402373ddfc8f5ef0e156e2d8cfc47cacafdda4a9", size = 766624563, upload-time = "2025-01-29T16:23:19.084Z" }, - { url = "https://files.pythonhosted.org/packages/01/d6/455ab3fbb2c61c71c8842753b566012e1ed111e7a4c82e0e1c20d0c76b62/torch-2.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b789069020c5588c70d5c2158ac0aa23fd24a028f34a8b4fcb8fcb4d7efcf5fb", size = 95607867, upload-time = "2025-01-29T16:25:55.649Z" }, - { url = "https://files.pythonhosted.org/packages/18/cf/ae99bd066571656185be0d88ee70abc58467b76f2f7c8bfeb48735a71fe6/torch-2.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:7e1448426d0ba3620408218b50aa6ada88aeae34f7a239ba5431f6c8774b1239", size = 204120469, upload-time = "2025-01-29T16:24:01.821Z" }, - { url = "https://files.pythonhosted.org/packages/81/b4/605ae4173aa37fb5aa14605d100ff31f4f5d49f617928c9f486bb3aaec08/torch-2.6.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:9a610afe216a85a8b9bc9f8365ed561535c93e804c2a317ef7fabcc5deda0989", size = 66532538, upload-time = "2025-01-29T16:24:18.976Z" }, -] - -[[package]] -name = "torch-memory-saver" -version = "0.0.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/6c/3c49e9698c993dde605d32131fca0ca18ad69e7be06d314caff3aca67fe2/torch_memory_saver-0.0.5.tar.gz", hash = "sha256:6b0238bb49e47f44ef67f15e5873a6c06c4c0decb07e99a7db5442eca207cdcb", size = 5433, upload-time = "2025-03-26T08:59:25.157Z" } - -[[package]] -name = "torchao" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/30/ffceb0340ea16324a175cff4a3903b2946e7fdcf8a72a7a1ab94244ca749/torchao-0.10.0-cp39-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d984f39246817ac43b46549759af6fa09b3bf70e00f0cff3bd48978777a10e6", size = 5450290, upload-time = "2025-04-07T18:41:17.162Z" }, - { url = "https://files.pythonhosted.org/packages/da/3a/afb970951d5512d22d0288016ef737afb4ee5af159823c5886dbb3ede1d4/torchao-0.10.0-py3-none-any.whl", hash = "sha256:9489b71259a7e29ee5ad391c333046adeb47888b6d84cfd0111adcf439f829a9", size = 746628, upload-time = "2025-04-07T18:41:18.97Z" }, -] - -[[package]] -name = "torchaudio" -version = "2.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/4a/d71b932bda4171970bdf4997541b5c778daa0e2967ed5009d207fca86ded/torchaudio-2.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d0e4b08c42325bf4b887de9a25c44ed882997001740e1bd7d901f65581cf1ab", size = 1812899, upload-time = "2025-01-29T16:29:41.021Z" }, - { url = "https://files.pythonhosted.org/packages/ed/aa/9082e715a673dd8e22b6a60cec7f301e897406023672b2090f8bcd8a5959/torchaudio-2.6.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:715aa21f6bdbd085454c313ae3a2c7cc07bf2e8cf05752f819afb5b4c57f4e6f", size = 3379510, upload-time = "2025-01-29T16:29:14.127Z" }, - { url = "https://files.pythonhosted.org/packages/f2/e7/0bcb2e33f4bdec69477344eccfe25c515b90496888095e99f837ea422089/torchaudio-2.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6291d9507dc1d6b4ffe8843fbfb201e6c8270dd8c42ad70bb76226c0ebdcad56", size = 1653523, upload-time = "2025-01-29T16:29:32.803Z" }, - { url = "https://files.pythonhosted.org/packages/80/95/29e917905328337c7b104ce81f3bb5e2ad8dc70af2edf1d43f67eb621513/torchaudio-2.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:86d6239792bf94741a41acd6fe3d549faaf0d50e7275d17d076a190bd007e2f9", size = 2449191, upload-time = "2025-01-29T16:29:06.485Z" }, -] - -[[package]] -name = "torchdata" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "torch" }, - { name = "urllib3" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/d4/af694ef718aedbe95a72760ab9ff7a6a7a44ace2d7f70c27bfeb67c5c503/torchdata-0.11.0-py3-none-any.whl", hash = "sha256:52b940fbbe0e00fb21cabddf528449d1bec5bfb0d0823b7487b15f951658ee33", size = 61968, upload-time = "2025-02-20T22:26:30.666Z" }, -] - -[[package]] -name = "torchvision" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pillow" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/5b/76ca113a853b19c7b1da761f8a72cb6429b3bd0bf932537d8df4657f47c3/torchvision-0.21.0-1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ffa2a16499508fe6798323e455f312c7c55f2a88901c9a7c0fb1efa86cf7e327", size = 2329878, upload-time = "2025-03-18T17:25:50.039Z" }, - { url = "https://files.pythonhosted.org/packages/6e/1b/28f527b22d5e8800184d0bc847f801ae92c7573a8c15979d92b7091c0751/torchvision-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:97a5814a93c793aaf0179cfc7f916024f4b63218929aee977b645633d074a49f", size = 1784140, upload-time = "2025-01-29T16:28:44.694Z" }, - { url = "https://files.pythonhosted.org/packages/36/63/0722e153fd27d64d5b0af45b5c8cb0e80b35a68cf0130303bc9a8bb095c7/torchvision-0.21.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:b578bcad8a4083b40d34f689b19ca9f7c63e511758d806510ea03c29ac568f7b", size = 7238673, upload-time = "2025-01-29T16:28:27.631Z" }, - { url = "https://files.pythonhosted.org/packages/bb/ea/03541ed901cdc30b934f897060d09bbf7a98466a08ad1680320f9ce0cbe0/torchvision-0.21.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5083a5b1fec2351bf5ea9900a741d54086db75baec4b1d21e39451e00977f1b1", size = 14701186, upload-time = "2025-01-29T16:28:16.491Z" }, - { url = "https://files.pythonhosted.org/packages/4c/6a/c7752603060d076dfed95135b78b047dc71792630cbcb022e3693d6f32ef/torchvision-0.21.0-cp312-cp312-win_amd64.whl", hash = "sha256:6eb75d41e3bbfc2f7642d0abba9383cc9ae6c5a4ca8d6b00628c225e1eaa63b3", size = 1560520, upload-time = "2025-01-29T16:28:42.122Z" }, -] - -[[package]] -name = "tornado" -version = "6.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/59/45/a0daf161f7d6f36c3ea5fc0c2de619746cc3dd4c76402e9db545bd920f63/tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b", size = 501135, upload-time = "2024-11-22T03:06:38.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/7e/71f604d8cea1b58f82ba3590290b66da1e72d840aeb37e0d5f7291bd30db/tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1", size = 436299, upload-time = "2024-11-22T03:06:20.162Z" }, - { url = "https://files.pythonhosted.org/packages/96/44/87543a3b99016d0bf54fdaab30d24bf0af2e848f1d13d34a3a5380aabe16/tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803", size = 434253, upload-time = "2024-11-22T03:06:22.39Z" }, - { url = "https://files.pythonhosted.org/packages/cb/fb/fdf679b4ce51bcb7210801ef4f11fdac96e9885daa402861751353beea6e/tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec", size = 437602, upload-time = "2024-11-22T03:06:24.214Z" }, - { url = "https://files.pythonhosted.org/packages/4f/3b/e31aeffffc22b475a64dbeb273026a21b5b566f74dee48742817626c47dc/tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946", size = 436972, upload-time = "2024-11-22T03:06:25.559Z" }, - { url = "https://files.pythonhosted.org/packages/22/55/b78a464de78051a30599ceb6983b01d8f732e6f69bf37b4ed07f642ac0fc/tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf", size = 437173, upload-time = "2024-11-22T03:06:27.584Z" }, - { url = "https://files.pythonhosted.org/packages/79/5e/be4fb0d1684eb822c9a62fb18a3e44a06188f78aa466b2ad991d2ee31104/tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634", size = 437892, upload-time = "2024-11-22T03:06:28.933Z" }, - { url = "https://files.pythonhosted.org/packages/f5/33/4f91fdd94ea36e1d796147003b490fe60a0215ac5737b6f9c65e160d4fe0/tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73", size = 437334, upload-time = "2024-11-22T03:06:30.428Z" }, - { url = "https://files.pythonhosted.org/packages/2b/ae/c1b22d4524b0e10da2f29a176fb2890386f7bd1f63aacf186444873a88a0/tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c", size = 437261, upload-time = "2024-11-22T03:06:32.458Z" }, - { url = "https://files.pythonhosted.org/packages/b5/25/36dbd49ab6d179bcfc4c6c093a51795a4f3bed380543a8242ac3517a1751/tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482", size = 438463, upload-time = "2024-11-22T03:06:34.71Z" }, - { url = "https://files.pythonhosted.org/packages/61/cc/58b1adeb1bb46228442081e746fcdbc4540905c87e8add7c277540934edb/tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38", size = 438907, upload-time = "2024-11-22T03:06:36.71Z" }, -] - -[[package]] -name = "tqdm" -version = "4.67.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, -] - -[[package]] -name = "traitlets" -version = "5.14.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/eb/79/72064e6a701c2183016abbbfedaba506d81e30e232a68c9f0d6f6fcd1574/traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", size = 161621, upload-time = "2024-04-19T11:11:49.746Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, -] - -[[package]] -name = "transformers" -version = "4.51.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock" }, - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "regex" }, - { name = "requests" }, - { name = "safetensors" }, - { name = "tokenizers" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/85/9182a14e0cceb6ababd37c9832d99f1c939e89854c80fa676ee0dbb8615b/transformers-4.51.1.tar.gz", hash = "sha256:206ea0b75dfde142ed7495b911da76579dce6ea249cc3695fdd29a544a9e007b", size = 8935230, upload-time = "2025-04-08T09:23:41.883Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/e0/f1fce1b2f31da4acfb858732d909209fdb10f65adda1e7af0c7ac74927a5/transformers-4.51.1-py3-none-any.whl", hash = "sha256:c7038e216afb2a3e9b00dd12d87ad5e3af4c30895f70b28e92f65459eded0161", size = 10365727, upload-time = "2025-04-08T09:23:38.138Z" }, -] - -[[package]] -name = "tree-sitter" -version = "0.24.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/a2/698b9d31d08ad5558f8bfbfe3a0781bd4b1f284e89bde3ad18e05101a892/tree-sitter-0.24.0.tar.gz", hash = "sha256:abd95af65ca2f4f7eca356343391ed669e764f37748b5352946f00f7fc78e734", size = 168304, upload-time = "2025-01-17T05:06:38.115Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/57/3a590f287b5aa60c07d5545953912be3d252481bf5e178f750db75572bff/tree_sitter-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14beeff5f11e223c37be7d5d119819880601a80d0399abe8c738ae2288804afc", size = 140788, upload-time = "2025-01-17T05:06:08.492Z" }, - { url = "https://files.pythonhosted.org/packages/61/0b/fc289e0cba7dbe77c6655a4dd949cd23c663fd62a8b4d8f02f97e28d7fe5/tree_sitter-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26a5b130f70d5925d67b47db314da209063664585a2fd36fa69e0717738efaf4", size = 133945, upload-time = "2025-01-17T05:06:12.39Z" }, - { url = "https://files.pythonhosted.org/packages/86/d7/80767238308a137e0b5b5c947aa243e3c1e3e430e6d0d5ae94b9a9ffd1a2/tree_sitter-0.24.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fc5c3c26d83c9d0ecb4fc4304fba35f034b7761d35286b936c1db1217558b4e", size = 564819, upload-time = "2025-01-17T05:06:13.549Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b3/6c5574f4b937b836601f5fb556b24804b0a6341f2eb42f40c0e6464339f4/tree_sitter-0.24.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:772e1bd8c0931c866b848d0369b32218ac97c24b04790ec4b0e409901945dd8e", size = 579303, upload-time = "2025-01-17T05:06:16.685Z" }, - { url = "https://files.pythonhosted.org/packages/0a/f4/bd0ddf9abe242ea67cca18a64810f8af230fc1ea74b28bb702e838ccd874/tree_sitter-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:24a8dd03b0d6b8812425f3b84d2f4763322684e38baf74e5bb766128b5633dc7", size = 581054, upload-time = "2025-01-17T05:06:19.439Z" }, - { url = "https://files.pythonhosted.org/packages/8c/1c/ff23fa4931b6ef1bbeac461b904ca7e49eaec7e7e5398584e3eef836ec96/tree_sitter-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9e8b1605ab60ed43803100f067eed71b0b0e6c1fb9860a262727dbfbbb74751", size = 120221, upload-time = "2025-01-17T05:06:20.654Z" }, - { url = "https://files.pythonhosted.org/packages/b2/2a/9979c626f303177b7612a802237d0533155bf1e425ff6f73cc40f25453e2/tree_sitter-0.24.0-cp312-cp312-win_arm64.whl", hash = "sha256:f733a83d8355fc95561582b66bbea92ffd365c5d7a665bc9ebd25e049c2b2abb", size = 108234, upload-time = "2025-01-17T05:06:21.713Z" }, -] - -[[package]] -name = "tree-sitter-javascript" -version = "0.23.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/dc/1c55c33cc6bbe754359b330534cf9f261c1b9b2c26ddf23aef3c5fa67759/tree_sitter_javascript-0.23.1.tar.gz", hash = "sha256:b2059ce8b150162cda05a457ca3920450adbf915119c04b8c67b5241cd7fcfed", size = 110058, upload-time = "2024-11-10T05:40:42.357Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/d3/c67d7d49967344b51208ad19f105233be1afdf07d3dcb35b471900265227/tree_sitter_javascript-0.23.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6ca583dad4bd79d3053c310b9f7208cd597fd85f9947e4ab2294658bb5c11e35", size = 59333, upload-time = "2024-11-10T05:40:31.988Z" }, - { url = "https://files.pythonhosted.org/packages/a5/db/ea0ee1547679d1750e80a0c4bc60b3520b166eeaf048764cfdd1ba3fd5e5/tree_sitter_javascript-0.23.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:94100e491a6a247aa4d14caf61230c171b6376c863039b6d9cd71255c2d815ec", size = 61071, upload-time = "2024-11-10T05:40:33.458Z" }, - { url = "https://files.pythonhosted.org/packages/67/6e/07c4857e08be37bfb55bfb269863df8ec908b2f6a3f1893cd852b893ecab/tree_sitter_javascript-0.23.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a6bc1055b061c5055ec58f39ee9b2e9efb8e6e0ae970838af74da0afb811f0a", size = 96999, upload-time = "2024-11-10T05:40:34.869Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f5/4de730afe8b9422845bc2064020a8a8f49ebd1695c04261c38d1b3e3edec/tree_sitter_javascript-0.23.1-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:056dc04fb6b24293f8c5fec43c14e7e16ba2075b3009c643abf8c85edc4c7c3c", size = 94020, upload-time = "2024-11-10T05:40:35.735Z" }, - { url = "https://files.pythonhosted.org/packages/77/0a/f980520da86c4eff8392867840a945578ef43372c9d4a37922baa6b121fe/tree_sitter_javascript-0.23.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a11ca1c0f736da42967586b568dff8a465ee148a986c15ebdc9382806e0ce871", size = 92927, upload-time = "2024-11-10T05:40:37.92Z" }, - { url = "https://files.pythonhosted.org/packages/ff/5c/36a98d512aa1d1082409d6b7eda5d26b820bd4477a54100ad9f62212bc55/tree_sitter_javascript-0.23.1-cp39-abi3-win_amd64.whl", hash = "sha256:041fa22b34250ea6eb313d33104d5303f79504cb259d374d691e38bbdc49145b", size = 58824, upload-time = "2024-11-10T05:40:39.903Z" }, - { url = "https://files.pythonhosted.org/packages/dc/79/ceb21988e6de615355a63eebcf806cd2a0fe875bec27b429d58b63e7fb5f/tree_sitter_javascript-0.23.1-cp39-abi3-win_arm64.whl", hash = "sha256:eb28130cd2fb30d702d614cbf61ef44d1c7f6869e7d864a9cc17111e370be8f7", size = 57027, upload-time = "2024-11-10T05:40:40.841Z" }, -] - -[[package]] -name = "tree-sitter-languages" -version = "1.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tree-sitter" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/bf/a9bd2d6ecbd053de0a5a50c150105b69c90eb49089f9e1d4fc4937e86adc/tree_sitter_languages-1.10.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c0f4c8b2734c45859edc7fcaaeaab97a074114111b5ba51ab4ec7ed52104763c", size = 8884771, upload-time = "2024-02-04T10:28:39.655Z" }, - { url = "https://files.pythonhosted.org/packages/14/fb/1f6fe5903aeb7435cc66d4b56621e9a30a4de64420555b999de65b31fcae/tree_sitter_languages-1.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eecd3c1244ac3425b7a82ba9125b4ddb45d953bbe61de114c0334fd89b7fe782", size = 9724562, upload-time = "2024-02-04T10:28:42.275Z" }, - { url = "https://files.pythonhosted.org/packages/20/6c/1855a65c9d6b50600f7a68e0182153db7cb12ff81fdebd93e87851dfdd8f/tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15db3c8510bc39a80147ee7421bf4782c15c09581c1dc2237ea89cefbd95b846", size = 8678682, upload-time = "2024-02-04T10:28:44.642Z" }, - { url = "https://files.pythonhosted.org/packages/d0/75/eff180f187ce4dc3e5177b3f8508e0061ea786ac44f409cf69cf24bf31a6/tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92c6487a6feea683154d3e06e6db68c30e0ae749a7ce4ce90b9e4e46b78c85c7", size = 8595099, upload-time = "2024-02-04T10:28:47.767Z" }, - { url = "https://files.pythonhosted.org/packages/f2/e6/eddc76ad899d77adcb5fca6cdf651eb1d33b4a799456bf303540f6cf8204/tree_sitter_languages-1.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2f1cd1d1bdd65332f9c2b67d49dcf148cf1ded752851d159ac3e5ee4f4d260", size = 8433569, upload-time = "2024-02-04T10:28:50.404Z" }, - { url = "https://files.pythonhosted.org/packages/06/95/a13da048c33a876d0475974484bf66b1fae07226e8654b1365ab549309cd/tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:976c8039165b8e12f17a01ddee9f4e23ec6e352b165ad29b44d2bf04e2fbe77e", size = 9196003, upload-time = "2024-02-04T10:28:52.466Z" }, - { url = "https://files.pythonhosted.org/packages/ec/13/9e5cb03914d60dd51047ecbfab5400309fbab14bb25014af388f492da044/tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:dafbbdf16bf668a580902e1620f4baa1913e79438abcce721a50647564c687b9", size = 9175560, upload-time = "2024-02-04T10:28:55.064Z" }, - { url = "https://files.pythonhosted.org/packages/19/76/25bb32a9be1c476e388835d5c8de5af2920af055e295770003683896cfe2/tree_sitter_languages-1.10.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1aeabd3d60d6d276b73cd8f3739d595b1299d123cc079a317f1a5b3c5461e2ca", size = 8956249, upload-time = "2024-02-04T10:28:57.094Z" }, - { url = "https://files.pythonhosted.org/packages/52/01/8e2f97a444d25dde1380ec20b338722f733b6cc290524357b1be3dd452ab/tree_sitter_languages-1.10.2-cp312-cp312-win32.whl", hash = "sha256:fab8ee641914098e8933b87ea3d657bea4dd00723c1ee7038b847b12eeeef4f5", size = 8363094, upload-time = "2024-02-04T10:28:59.156Z" }, - { url = "https://files.pythonhosted.org/packages/47/58/0262e875dd899447476a8ffde7829df3716ffa772990095c65d6de1f053c/tree_sitter_languages-1.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e606430d736367e5787fa5a7a0c5a1ec9b85eded0b3596bbc0d83532a40810b", size = 8268983, upload-time = "2024-02-04T10:29:00.987Z" }, -] - -[[package]] -name = "tree-sitter-python" -version = "0.23.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/30/6766433b31be476fda6569a3a374c2220e45ffee0bff75460038a57bf23b/tree_sitter_python-0.23.6.tar.gz", hash = "sha256:354bfa0a2f9217431764a631516f85173e9711af2c13dbd796a8815acfe505d9", size = 155868, upload-time = "2024-12-22T23:09:55.918Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/67/577a02acae5f776007c924ca86ef14c19c12e71de0aa9d2a036f3c248e7b/tree_sitter_python-0.23.6-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:28fbec8f74eeb2b30292d97715e60fac9ccf8a8091ce19b9d93e9b580ed280fb", size = 74361, upload-time = "2024-12-22T23:09:42.37Z" }, - { url = "https://files.pythonhosted.org/packages/d2/a6/194b3625a7245c532ad418130d63077ce6cd241152524152f533e4d6edb0/tree_sitter_python-0.23.6-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:680b710051b144fedf61c95197db0094f2245e82551bf7f0c501356333571f7a", size = 76436, upload-time = "2024-12-22T23:09:43.566Z" }, - { url = "https://files.pythonhosted.org/packages/d0/62/1da112689d6d282920e62c40e67ab39ea56463b0e7167bfc5e81818a770e/tree_sitter_python-0.23.6-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a9dcef55507b6567207e8ee0a6b053d0688019b47ff7f26edc1764b7f4dc0a4", size = 112060, upload-time = "2024-12-22T23:09:44.721Z" }, - { url = "https://files.pythonhosted.org/packages/5d/62/c9358584c96e38318d69b6704653684fd8467601f7b74e88aa44f4e6903f/tree_sitter_python-0.23.6-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29dacdc0cd2f64e55e61d96c6906533ebb2791972bec988450c46cce60092f5d", size = 112338, upload-time = "2024-12-22T23:09:48.323Z" }, - { url = "https://files.pythonhosted.org/packages/1a/58/c5e61add45e34fb8ecbf057c500bae9d96ed7c9ca36edb7985da8ae45526/tree_sitter_python-0.23.6-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7e048733c36f564b379831689006801feb267d8194f9e793fbb395ef1723335d", size = 109382, upload-time = "2024-12-22T23:09:49.49Z" }, - { url = "https://files.pythonhosted.org/packages/e9/f3/9b30893cae9b3811fe652dc6f90aaadfda12ae0b2757f5722fc7266f423c/tree_sitter_python-0.23.6-cp39-abi3-win_amd64.whl", hash = "sha256:a24027248399fb41594b696f929f9956828ae7cc85596d9f775e6c239cd0c2be", size = 75904, upload-time = "2024-12-22T23:09:51.597Z" }, - { url = "https://files.pythonhosted.org/packages/87/cb/ce35a65f83a47b510d8a2f1eddf3bdbb0d57aabc87351c8788caf3309f76/tree_sitter_python-0.23.6-cp39-abi3-win_arm64.whl", hash = "sha256:71334371bd73d5fe080aed39fbff49ed8efb9506edebe16795b0c7567ed6a272", size = 73649, upload-time = "2024-12-22T23:09:53.71Z" }, -] - -[[package]] -name = "tree-sitter-ruby" -version = "0.23.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/09/5b/6d24be4fde4743481bd8e3fd24b434870cb6612238c8544b71fe129ed850/tree_sitter_ruby-0.23.1.tar.gz", hash = "sha256:886ed200bfd1f3ca7628bf1c9fefd42421bbdba70c627363abda67f662caa21e", size = 489602, upload-time = "2024-11-11T04:51:30.328Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/2e/2717b9451c712b60f833827a696baf29d8e50a0f7dccbf22a8d7006cc19e/tree_sitter_ruby-0.23.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:39f391322d2210843f07081182dbf00f8f69cfbfa4687b9575cac6d324bae443", size = 177959, upload-time = "2024-11-11T04:51:19.958Z" }, - { url = "https://files.pythonhosted.org/packages/e7/38/c41ecf7692b8ecccd26861d3293a88150a4a52fc081abe60f837030d7315/tree_sitter_ruby-0.23.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:aa4ee7433bd42fac22e2dad4a3c0f332292ecf482e610316828c711a0bb7f794", size = 195069, upload-time = "2024-11-11T04:51:21.82Z" }, - { url = "https://files.pythonhosted.org/packages/d8/01/14ef2d5107e6f42b64a400c3bbc3dd3b8fd24c3cef5306004ae03668f231/tree_sitter_ruby-0.23.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62b36813a56006b7569db7868f6b762caa3f4e419bd0f8cf9ccbb4abb1b6254c", size = 226761, upload-time = "2024-11-11T04:51:23.021Z" }, - { url = "https://files.pythonhosted.org/packages/23/dd/1171b5dd25da10f768732a20fb62d2e3ae66e3b42329351f2ce5bf723abb/tree_sitter_ruby-0.23.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7bcd93972b4ca2803856d4fe0fbd04123ff29c4592bbb9f12a27528bd252341", size = 214427, upload-time = "2024-11-11T04:51:24.854Z" }, - { url = "https://files.pythonhosted.org/packages/60/bc/de76c877a90fd8a62cd60f496d7832efddc1b18a148593d9aa9b4a9ce5e0/tree_sitter_ruby-0.23.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66c65d6c2a629783ca4ab2bab539bd6f271ce6f77cacb62845831e11665b5bd3", size = 210409, upload-time = "2024-11-11T04:51:26.093Z" }, - { url = "https://files.pythonhosted.org/packages/dd/4a/f5bcca350b84cdf75a53e918b8efa06c46ed650d99d3ef22195e9d8020cc/tree_sitter_ruby-0.23.1-cp39-abi3-win_amd64.whl", hash = "sha256:02e2c19ebefe29226c14aa63e11e291d990f5b5c20a99940ab6e7eda44e744e5", size = 179843, upload-time = "2024-11-11T04:51:27.265Z" }, - { url = "https://files.pythonhosted.org/packages/71/5c/a2e068ad4b2c4ba9b774a88b24149168d3bcd94f58b964e49dcabfe5fd24/tree_sitter_ruby-0.23.1-cp39-abi3-win_arm64.whl", hash = "sha256:ed042007e89f2cceeb1cbdd8b0caa68af1e2ce54c7eb2053ace760f90657ac9f", size = 178025, upload-time = "2024-11-11T04:51:29.051Z" }, -] - -[[package]] -name = "tree-sitter-typescript" -version = "0.23.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1e/fc/bb52958f7e399250aee093751e9373a6311cadbe76b6e0d109b853757f35/tree_sitter_typescript-0.23.2.tar.gz", hash = "sha256:7b167b5827c882261cb7a50dfa0fb567975f9b315e87ed87ad0a0a3aedb3834d", size = 773053, upload-time = "2024-11-11T02:36:11.396Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/95/4c00680866280e008e81dd621fd4d3f54aa3dad1b76b857a19da1b2cc426/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3cd752d70d8e5371fdac6a9a4df9d8924b63b6998d268586f7d374c9fba2a478", size = 286677, upload-time = "2024-11-11T02:35:58.839Z" }, - { url = "https://files.pythonhosted.org/packages/8f/2f/1f36fda564518d84593f2740d5905ac127d590baf5c5753cef2a88a89c15/tree_sitter_typescript-0.23.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c7cc1b0ff5d91bac863b0e38b1578d5505e718156c9db577c8baea2557f66de8", size = 302008, upload-time = "2024-11-11T02:36:00.733Z" }, - { url = "https://files.pythonhosted.org/packages/96/2d/975c2dad292aa9994f982eb0b69cc6fda0223e4b6c4ea714550477d8ec3a/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b1eed5b0b3a8134e86126b00b743d667ec27c63fc9de1b7bb23168803879e31", size = 351987, upload-time = "2024-11-11T02:36:02.669Z" }, - { url = "https://files.pythonhosted.org/packages/49/d1/a71c36da6e2b8a4ed5e2970819b86ef13ba77ac40d9e333cb17df6a2c5db/tree_sitter_typescript-0.23.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e96d36b85bcacdeb8ff5c2618d75593ef12ebaf1b4eace3477e2bdb2abb1752c", size = 344960, upload-time = "2024-11-11T02:36:04.443Z" }, - { url = "https://files.pythonhosted.org/packages/7f/cb/f57b149d7beed1a85b8266d0c60ebe4c46e79c9ba56bc17b898e17daf88e/tree_sitter_typescript-0.23.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8d4f0f9bcb61ad7b7509d49a1565ff2cc363863644a234e1e0fe10960e55aea0", size = 340245, upload-time = "2024-11-11T02:36:06.473Z" }, - { url = "https://files.pythonhosted.org/packages/8b/ab/dd84f0e2337296a5f09749f7b5483215d75c8fa9e33738522e5ed81f7254/tree_sitter_typescript-0.23.2-cp39-abi3-win_amd64.whl", hash = "sha256:3f730b66396bc3e11811e4465c41ee45d9e9edd6de355a58bbbc49fa770da8f9", size = 278015, upload-time = "2024-11-11T02:36:07.631Z" }, - { url = "https://files.pythonhosted.org/packages/9f/e4/81f9a935789233cf412a0ed5fe04c883841d2c8fb0b7e075958a35c65032/tree_sitter_typescript-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7", size = 274052, upload-time = "2024-11-11T02:36:09.514Z" }, -] - -[[package]] -name = "triton" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "platform_machine == 'aarch64' and sys_platform == 'linux'", - "platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux'", - "sys_platform == 'darwin'", - "sys_platform != 'darwin' and sys_platform != 'linux'", -] -dependencies = [ - { name = "filelock", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/eb/65f5ba83c2a123f6498a3097746607e5b2f16add29e36765305e4ac7fdd8/triton-3.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8182f42fd8080a7d39d666814fa36c5e30cc00ea7eeeb1a2983dbb4c99a0fdc", size = 209551444, upload-time = "2024-10-14T16:05:53.433Z" }, -] - -[[package]] -name = "triton" -version = "3.2.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "platform_machine == 'x86_64' and sys_platform == 'linux'", -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/00/59500052cb1cf8cf5316be93598946bc451f14072c6ff256904428eaf03c/triton-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d9b215efc1c26fa7eefb9a157915c92d52e000d2bf83e5f69704047e63f125c", size = 253159365, upload-time = "2025-01-22T19:13:24.648Z" }, -] - -[[package]] -name = "typer" -version = "0.15.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8b/6f/3991f0f1c7fcb2df31aef28e0594d8d54b05393a0e4e34c65e475c2a5d41/typer-0.15.2.tar.gz", hash = "sha256:ab2fab47533a813c49fe1f16b1a370fd5819099c00b119e0633df65f22144ba5", size = 100711, upload-time = "2025-02-27T19:17:34.807Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/fc/5b29fea8cee020515ca82cc68e3b8e1e34bb19a3535ad854cac9257b414c/typer-0.15.2-py3-none-any.whl", hash = "sha256:46a499c6107d645a9c13f7ee46c5d5096cae6f5fc57dd11eccbbb9ae3e44ddfc", size = 45061, upload-time = "2025-02-27T19:17:32.111Z" }, -] - -[[package]] -name = "types-certifi" -version = "2021.10.8.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/68/943c3aeaf14624712a0357c4a67814dba5cea36d194f5c764dad7959a00c/types-certifi-2021.10.8.3.tar.gz", hash = "sha256:72cf7798d165bc0b76e1c10dd1ea3097c7063c42c21d664523b928e88b554a4f", size = 2095, upload-time = "2022-06-09T15:19:05.244Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/63/2463d89481e811f007b0e1cd0a91e52e141b47f9de724d20db7b861dcfec/types_certifi-2021.10.8.3-py3-none-any.whl", hash = "sha256:b2d1e325e69f71f7c78e5943d410e650b4707bb0ef32e4ddf3da37f54176e88a", size = 2136, upload-time = "2022-06-09T15:19:03.127Z" }, -] - -[[package]] -name = "types-requests" -version = "2.32.0.20250328" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/00/7d/eb174f74e3f5634eaacb38031bbe467dfe2e545bc255e5c90096ec46bc46/types_requests-2.32.0.20250328.tar.gz", hash = "sha256:c9e67228ea103bd811c96984fac36ed2ae8da87a36a633964a21f199d60baf32", size = 22995, upload-time = "2025-03-28T02:55:13.271Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/15/3700282a9d4ea3b37044264d3e4d1b1f0095a4ebf860a99914fd544e3be3/types_requests-2.32.0.20250328-py3-none-any.whl", hash = "sha256:72ff80f84b15eb3aa7a8e2625fffb6a93f2ad5a0c20215fc1dcfa61117bcb2a2", size = 20663, upload-time = "2025-03-28T02:55:11.946Z" }, -] - -[[package]] -name = "types-toml" -version = "0.10.8.20240310" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/47/3e4c75042792bff8e90d7991aa5c51812cc668828cc6cce711e97f63a607/types-toml-0.10.8.20240310.tar.gz", hash = "sha256:3d41501302972436a6b8b239c850b26689657e25281b48ff0ec06345b8830331", size = 4392, upload-time = "2024-03-10T02:18:37.518Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/a2/d32ab58c0b216912638b140ab2170ee4b8644067c293b170e19fba340ccc/types_toml-0.10.8.20240310-py3-none-any.whl", hash = "sha256:627b47775d25fa29977d9c70dc0cbab3f314f32c8d8d0c012f2ef5de7aaec05d", size = 4777, upload-time = "2024-03-10T02:18:36.568Z" }, -] - -[[package]] -name = "types-tqdm" -version = "4.67.0.20250404" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "types-requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/a1/75e0d6f96f8c34a1bad6c232566182f2ae53ffdf7ab9c75afb61b2a07354/types_tqdm-4.67.0.20250404.tar.gz", hash = "sha256:e9997c655ffbba3ab78f4418b5511c05a54e76824d073d212166dc73aa56c768", size = 17159, upload-time = "2025-04-04T02:54:51.397Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/85/2c09e94d2554e8314cbc74f897c1b3012930b34a22f6905bb0b9fb79f40e/types_tqdm-4.67.0.20250404-py3-none-any.whl", hash = "sha256:4a9b897eb4036f757240f4cb4a794f296265c04de46fdd058e453891f0186eed", size = 24053, upload-time = "2025-04-04T02:54:50.2Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.13.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, -] - -[[package]] -name = "typing-inspect" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mypy-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/82/5c/e6082df02e215b846b4b8c0b887a64d7d08ffaba30605502639d44c06b82/typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122", size = 76222, upload-time = "2025-02-25T17:27:59.638Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/08/aa4fdfb71f7de5176385bd9e90852eaf6b5d622735020ad600f2bab54385/typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f", size = 14125, upload-time = "2025-02-25T17:27:57.754Z" }, -] - -[[package]] -name = "tzdata" -version = "2025.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, -] - -[[package]] -name = "unidiff" -version = "0.7.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/48/81be0ac96e423a877754153699731ef439fd7b80b4c8b5425c94ed079ebd/unidiff-0.7.5.tar.gz", hash = "sha256:2e5f0162052248946b9f0970a40e9e124236bf86c82b70821143a6fc1dea2574", size = 20931, upload-time = "2023-03-10T01:05:39.185Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/54/57c411a6e8f7bd7848c8b66e4dcaffa586bf4c02e63f2280db0327a4e6eb/unidiff-0.7.5-py2.py3-none-any.whl", hash = "sha256:c93bf2265cc1ba2a520e415ab05da587370bc2a3ae9e0414329f54f0c2fc09e8", size = 14386, upload-time = "2023-03-10T01:05:36.594Z" }, -] - -[[package]] -name = "uritemplate" -version = "4.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d2/5a/4742fdba39cd02a56226815abfa72fe0aa81c33bed16ed045647d6000eba/uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", size = 273898, upload-time = "2021-10-13T11:15:14.84Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c0/7461b49cd25aeece13766f02ee576d1db528f1c37ce69aee300e075b485b/uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e", size = 10356, upload-time = "2021-10-13T11:15:12.316Z" }, -] - -[[package]] -name = "urllib3" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/78/16493d9c386d8e60e442a35feac5e00f0913c0f4b7c217c11e8ec2ff53e0/urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", size = 390672, upload-time = "2025-04-10T15:23:39.232Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813", size = 128680, upload-time = "2025-04-10T15:23:37.377Z" }, -] - -[[package]] -name = "uvicorn" -version = "0.34.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4b/4d/938bd85e5bf2edeec766267a5015ad969730bb91e31b44021dfe8b22df6c/uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9", size = 76568, upload-time = "2024-12-15T13:33:30.42Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/14/33a3a1352cfa71812a3a21e8c9bfb83f60b0011f5e36f2b1399d51928209/uvicorn-0.34.0-py3-none-any.whl", hash = "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4", size = 62315, upload-time = "2024-12-15T13:33:27.467Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "httptools" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, - { name = "watchfiles" }, - { name = "websockets" }, -] - -[[package]] -name = "uvloop" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/c0/854216d09d33c543f12a44b393c402e89a920b1a0a7dc634c42de91b9cf6/uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3", size = 2492741, upload-time = "2024-10-14T23:38:35.489Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/4c/03f93178830dc7ce8b4cdee1d36770d2f5ebb6f3d37d354e061eefc73545/uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c", size = 1471284, upload-time = "2024-10-14T23:37:47.833Z" }, - { url = "https://files.pythonhosted.org/packages/43/3e/92c03f4d05e50f09251bd8b2b2b584a2a7f8fe600008bcc4523337abe676/uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2", size = 821349, upload-time = "2024-10-14T23:37:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/a6/ef/a02ec5da49909dbbfb1fd205a9a1ac4e88ea92dcae885e7c961847cd51e2/uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d", size = 4580089, upload-time = "2024-10-14T23:37:51.703Z" }, - { url = "https://files.pythonhosted.org/packages/06/a7/b4e6a19925c900be9f98bec0a75e6e8f79bb53bdeb891916609ab3958967/uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc", size = 4693770, upload-time = "2024-10-14T23:37:54.122Z" }, - { url = "https://files.pythonhosted.org/packages/ce/0c/f07435a18a4b94ce6bd0677d8319cd3de61f3a9eeb1e5f8ab4e8b5edfcb3/uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb", size = 4451321, upload-time = "2024-10-14T23:37:55.766Z" }, - { url = "https://files.pythonhosted.org/packages/8f/eb/f7032be105877bcf924709c97b1bf3b90255b4ec251f9340cef912559f28/uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f", size = 4659022, upload-time = "2024-10-14T23:37:58.195Z" }, -] - -[[package]] -name = "verl" -source = { editable = "." } -dependencies = [ - { name = "accelerate" }, - { name = "browsergym" }, - { name = "browsergym-miniwob" }, - { name = "browsergym-visualwebarena" }, - { name = "browsergym-webarena" }, - { name = "codetiming" }, - { name = "commit0" }, - { name = "datasets" }, - { name = "dill" }, - { name = "evaluate" }, - { name = "flash-attn" }, - { name = "func-timeout" }, - { name = "gdown" }, - { name = "hf-transfer" }, - { name = "hydra-core" }, - { name = "matplotlib" }, - { name = "numpy" }, - { name = "openhands-ai" }, - { name = "pandas" }, - { name = "peft" }, - { name = "pyarrow" }, - { name = "pybind11" }, - { name = "pylatexenc" }, - { name = "ray" }, - { name = "retry" }, - { name = "seaborn" }, - { name = "sglang", extra = ["all"] }, - { name = "streamlit" }, - { name = "swebench" }, - { name = "swegym" }, - { name = "sympy" }, - { name = "tabulate" }, - { name = "tensordict" }, - { name = "torch-memory-saver" }, - { name = "torchdata" }, - { name = "transformers" }, - { name = "vllm" }, - { name = "wandb" }, - { name = "whatthepatch" }, -] - -[package.optional-dependencies] -geo = [ - { name = "mathruler" }, -] -gpu = [ - { name = "flash-attn" }, - { name = "liger-kernel" }, -] -math = [ - { name = "math-verify" }, -] -test = [ - { name = "py-spy" }, - { name = "pytest" }, - { name = "yapf" }, -] -vllm = [ - { name = "tensordict" }, - { name = "vllm" }, -] - -[package.metadata] -requires-dist = [ - { name = "accelerate" }, - { name = "browsergym", specifier = "==0.10.2" }, - { name = "browsergym-miniwob", specifier = "==0.10.2" }, - { name = "browsergym-visualwebarena", specifier = "==0.10.2" }, - { name = "browsergym-webarena", specifier = "==0.10.2" }, - { name = "codetiming" }, - { name = "commit0" }, - { name = "datasets" }, - { name = "dill" }, - { name = "evaluate" }, - { name = "flash-attn", url = "https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl" }, - { name = "flash-attn", marker = "extra == 'gpu'" }, - { name = "func-timeout" }, - { name = "gdown" }, - { name = "hf-transfer" }, - { name = "hydra-core" }, - { name = "liger-kernel", marker = "extra == 'gpu'" }, - { name = "math-verify", marker = "extra == 'math'" }, - { name = "mathruler", marker = "extra == 'geo'" }, - { name = "matplotlib" }, - { name = "numpy" }, - { name = "openhands-ai", editable = "SkyRL-OpenHands" }, - { name = "pandas" }, - { name = "peft" }, - { name = "py-spy", marker = "extra == 'test'" }, - { name = "pyarrow", specifier = ">=15.0.0" }, - { name = "pybind11" }, - { name = "pylatexenc" }, - { name = "pytest", marker = "extra == 'test'" }, - { name = "ray", extras = ["default"], specifier = ">=2.10" }, - { name = "retry" }, - { name = "seaborn" }, - { name = "sglang", extras = ["all"], specifier = ">=0.4.6.post1" }, - { name = "streamlit" }, - { name = "swebench", git = "https://github.com/SWE-Gym/SWE-Bench-Fork.git" }, - { name = "swegym", git = "https://github.com/SWE-Gym/SWE-Bench-Package.git" }, - { name = "sympy" }, - { name = "tabulate" }, - { name = "tensordict", specifier = "<=0.6.2" }, - { name = "tensordict", marker = "extra == 'vllm'", specifier = "<=0.6.2" }, - { name = "torch-memory-saver", specifier = ">=0.0.5" }, - { name = "torchdata" }, - { name = "transformers" }, - { name = "vllm", specifier = ">=0.7.3" }, - { name = "vllm", marker = "extra == 'vllm'", specifier = "<=0.8.2" }, - { name = "wandb" }, - { name = "whatthepatch" }, - { name = "yapf", marker = "extra == 'test'" }, -] -provides-extras = ["test", "geo", "gpu", "math", "vllm"] - -[[package]] -name = "virtualenv" -version = "20.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "distlib" }, - { name = "filelock" }, - { name = "platformdirs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/e0/633e369b91bbc664df47dcb5454b6c7cf441e8f5b9d0c250ce9f0546401e/virtualenv-20.30.0.tar.gz", hash = "sha256:800863162bcaa5450a6e4d721049730e7f2dae07720e0902b0e4040bd6f9ada8", size = 4346945, upload-time = "2025-03-31T16:33:29.185Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/ed/3cfeb48175f0671ec430ede81f628f9fb2b1084c9064ca67ebe8c0ed6a05/virtualenv-20.30.0-py3-none-any.whl", hash = "sha256:e34302959180fca3af42d1800df014b35019490b119eba981af27f2fa486e5d6", size = 4329461, upload-time = "2025-03-31T16:33:26.758Z" }, -] - -[[package]] -name = "vllm" -version = "0.8.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "blake3" }, - { name = "cachetools" }, - { name = "cloudpickle" }, - { name = "compressed-tensors" }, - { name = "depyf" }, - { name = "einops" }, - { name = "fastapi", extra = ["standard"] }, - { name = "filelock" }, - { name = "gguf" }, - { name = "importlib-metadata" }, - { name = "lark" }, - { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "lm-format-enforcer" }, - { name = "mistral-common", extra = ["opencv"] }, - { name = "msgspec" }, - { name = "ninja" }, - { name = "numba" }, - { name = "numpy" }, - { name = "openai" }, - { name = "outlines" }, - { name = "partial-json-parser" }, - { name = "pillow" }, - { name = "prometheus-client" }, - { name = "prometheus-fastapi-instrumentator" }, - { name = "protobuf" }, - { name = "psutil" }, - { name = "py-cpuinfo" }, - { name = "pydantic" }, - { name = "python-json-logger" }, - { name = "pyyaml" }, - { name = "pyzmq" }, - { name = "ray" }, - { name = "requests" }, - { name = "scipy" }, - { name = "sentencepiece" }, - { name = "setuptools" }, - { name = "six" }, - { name = "tiktoken" }, - { name = "tokenizers" }, - { name = "torch" }, - { name = "torchaudio" }, - { name = "torchvision" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "typing-extensions" }, - { name = "watchfiles" }, - { name = "xformers", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "xgrammar" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/4d/6b27cc14d0c35e578a743a767953500a801ba296694b7e44cca709738b41/vllm-0.8.2.tar.gz", hash = "sha256:9b337b1c4072ccb94b1bf2b716593fadbe2dcb8d091f9bcbd6b5c6d37f9842ac", size = 6450146, upload-time = "2025-03-25T18:06:58.882Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/49/207364110b96d76139a4e80617e5831d46884abe824941b15c8a748ca5e0/vllm-0.8.2-cp38-abi3-manylinux1_x86_64.whl", hash = "sha256:32442b686c5dad8e6ddcf5a8b0cf3f741359fed6a9e9e940009f1daf80ae15de", size = 293643693, upload-time = "2025-03-25T18:06:50.924Z" }, -] - -[[package]] -name = "wandb" -version = "0.19.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "docker-pycreds" }, - { name = "gitpython" }, - { name = "platformdirs" }, - { name = "protobuf" }, - { name = "psutil" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "sentry-sdk" }, - { name = "setproctitle" }, - { name = "setuptools" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5f/07/c4f4ace4445cfafcd6a35dff94e03b3b875f2e472ae524c1d32fec8a5c33/wandb-0.19.9.tar.gz", hash = "sha256:a38881aa7770e6ea0f792268250e5b888a245c2e4829e5830ce08f5fa95a2ad1", size = 39264462, upload-time = "2025-04-01T21:22:05.475Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/32/39de0bcf45f0026aabbc9dcd41c287dc21f62b5cb6fe054bbb3b988ff01c/wandb-0.19.9-py3-none-any.whl", hash = "sha256:191b2c794b401bd04cc7c48b16339ca2832510af8499348744e7461a1609b6a9", size = 6335646, upload-time = "2025-04-01T21:21:39.799Z" }, - { url = "https://files.pythonhosted.org/packages/c6/61/f37280987c4ef624e238be4cbad4a0ffe71987fbb75c029fc89c6bd849b7/wandb-0.19.9-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:de5702416f29d1d61f9e85f4090346f23db1ee7f23cab37a4dd5d394fcf5f7da", size = 20468027, upload-time = "2025-04-01T21:21:42.422Z" }, - { url = "https://files.pythonhosted.org/packages/49/35/7af15563f7cc40c7ac95f2d2e5ca10fa144d1be20a62da8f475cc7ceebd3/wandb-0.19.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c598041f06be3cbed6bf6888cb67b033352ce156dc4e7882c4448e2750453a77", size = 19931553, upload-time = "2025-04-01T21:21:45.041Z" }, - { url = "https://files.pythonhosted.org/packages/f9/b2/aa02811d0eb66530e67c44cc1e7d001507064d54b036f2a801bc9c0a731f/wandb-0.19.9-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:7b99253c1f603d9331fa0626df61f47f0b544403321286c4fff50492572738f5", size = 20466642, upload-time = "2025-04-01T21:21:47.568Z" }, - { url = "https://files.pythonhosted.org/packages/56/07/47ab3b4f0f4a32d9269ecb60aa71da3e426faa2abe51c4f000778e2696c3/wandb-0.19.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a074ad070c4e8cbb03b2149a98abbe2d7562220f095a21c736e1abbca399eef", size = 19534405, upload-time = "2025-04-01T21:21:50.313Z" }, - { url = "https://files.pythonhosted.org/packages/89/d0/737d26d709bd7bc3f6b2250f41fda3d0787239cfdbd6eb13057c64c81ace/wandb-0.19.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5dc6c7180a5bf1eb5bd9cab8a1886fd980c76d54253c967082fe19d197443a2d", size = 20863142, upload-time = "2025-04-01T21:21:52.462Z" }, - { url = "https://files.pythonhosted.org/packages/28/a5/420515e3f724d730ef830e2362aaa53ce49f366d919fbfa9ed4c3a148095/wandb-0.19.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1e8cea987f326b053e79b7c40a0502c61a23792c9869b92ab6f2ac76086f66c2", size = 19545396, upload-time = "2025-04-01T21:21:54.707Z" }, - { url = "https://files.pythonhosted.org/packages/56/05/bab99791ede9eb78d03ddd27c54cbafcd89db1f279d4f4634156ebfcc24f/wandb-0.19.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:969a700cd625f56b2ac6c9e136ae828a0c3b2ebc4dc289c829986e9e7aded6bb", size = 20944223, upload-time = "2025-04-01T21:21:57.552Z" }, - { url = "https://files.pythonhosted.org/packages/07/ab/e3f728ff75307ba387546d3aedba7f18b4f5467d92d0343df2dee75af27e/wandb-0.19.9-py3-none-win32.whl", hash = "sha256:d3a192fec8fdfd89993243a81cad332c6c0e7127a9497b6bf9c244adeab2d717", size = 20247779, upload-time = "2025-04-01T21:22:00.265Z" }, - { url = "https://files.pythonhosted.org/packages/78/89/fee9e1513cc7045e9a6235b14d8e6e5b625fe4fe3a281fda9e799c6269ed/wandb-0.19.9-py3-none-win_amd64.whl", hash = "sha256:2033dacee312988e5418d0f257fd9888d28b6785ec39ec43cb2f6b8fd2f13176", size = 20247782, upload-time = "2025-04-01T21:22:02.983Z" }, -] - -[[package]] -name = "watchdog" -version = "6.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, - { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, - { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, - { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, - { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, - { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, - { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, - { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, - { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, -] - -[[package]] -name = "watchfiles" -version = "1.0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/03/e2/8ed598c42057de7aa5d97c472254af4906ff0a59a66699d426fc9ef795d7/watchfiles-1.0.5.tar.gz", hash = "sha256:b7529b5dcc114679d43827d8c35a07c493ad6f083633d573d81c660abc5979e9", size = 94537, upload-time = "2025-04-08T10:36:26.722Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/8c/4f0b9bdb75a1bfbd9c78fad7d8854369283f74fe7cf03eb16be77054536d/watchfiles-1.0.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b5eb568c2aa6018e26da9e6c86f3ec3fd958cee7f0311b35c2630fa4217d17f2", size = 401511, upload-time = "2025-04-08T10:35:17.956Z" }, - { url = "https://files.pythonhosted.org/packages/dc/4e/7e15825def77f8bd359b6d3f379f0c9dac4eb09dd4ddd58fd7d14127179c/watchfiles-1.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a04059f4923ce4e856b4b4e5e783a70f49d9663d22a4c3b3298165996d1377f", size = 392715, upload-time = "2025-04-08T10:35:19.202Z" }, - { url = "https://files.pythonhosted.org/packages/58/65/b72fb817518728e08de5840d5d38571466c1b4a3f724d190cec909ee6f3f/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e380c89983ce6e6fe2dd1e1921b9952fb4e6da882931abd1824c092ed495dec", size = 454138, upload-time = "2025-04-08T10:35:20.586Z" }, - { url = "https://files.pythonhosted.org/packages/3e/a4/86833fd2ea2e50ae28989f5950b5c3f91022d67092bfec08f8300d8b347b/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fe43139b2c0fdc4a14d4f8d5b5d967f7a2777fd3d38ecf5b1ec669b0d7e43c21", size = 458592, upload-time = "2025-04-08T10:35:21.87Z" }, - { url = "https://files.pythonhosted.org/packages/38/7e/42cb8df8be9a37e50dd3a818816501cf7a20d635d76d6bd65aae3dbbff68/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee0822ce1b8a14fe5a066f93edd20aada932acfe348bede8aa2149f1a4489512", size = 487532, upload-time = "2025-04-08T10:35:23.143Z" }, - { url = "https://files.pythonhosted.org/packages/fc/fd/13d26721c85d7f3df6169d8b495fcac8ab0dc8f0945ebea8845de4681dab/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0dbcb1c2d8f2ab6e0a81c6699b236932bd264d4cef1ac475858d16c403de74d", size = 522865, upload-time = "2025-04-08T10:35:24.702Z" }, - { url = "https://files.pythonhosted.org/packages/a1/0d/7f9ae243c04e96c5455d111e21b09087d0eeaf9a1369e13a01c7d3d82478/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a2014a2b18ad3ca53b1f6c23f8cd94a18ce930c1837bd891262c182640eb40a6", size = 499887, upload-time = "2025-04-08T10:35:25.969Z" }, - { url = "https://files.pythonhosted.org/packages/8e/0f/a257766998e26aca4b3acf2ae97dff04b57071e991a510857d3799247c67/watchfiles-1.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f6ae86d5cb647bf58f9f655fcf577f713915a5d69057a0371bc257e2553234", size = 454498, upload-time = "2025-04-08T10:35:27.353Z" }, - { url = "https://files.pythonhosted.org/packages/81/79/8bf142575a03e0af9c3d5f8bcae911ee6683ae93a625d349d4ecf4c8f7df/watchfiles-1.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1a7bac2bde1d661fb31f4d4e8e539e178774b76db3c2c17c4bb3e960a5de07a2", size = 630663, upload-time = "2025-04-08T10:35:28.685Z" }, - { url = "https://files.pythonhosted.org/packages/f1/80/abe2e79f610e45c63a70d271caea90c49bbf93eb00fa947fa9b803a1d51f/watchfiles-1.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ab626da2fc1ac277bbf752446470b367f84b50295264d2d313e28dc4405d663", size = 625410, upload-time = "2025-04-08T10:35:30.42Z" }, - { url = "https://files.pythonhosted.org/packages/91/6f/bc7fbecb84a41a9069c2c6eb6319f7f7df113adf113e358c57fc1aff7ff5/watchfiles-1.0.5-cp312-cp312-win32.whl", hash = "sha256:9f4571a783914feda92018ef3901dab8caf5b029325b5fe4558c074582815249", size = 277965, upload-time = "2025-04-08T10:35:32.023Z" }, - { url = "https://files.pythonhosted.org/packages/99/a5/bf1c297ea6649ec59e935ab311f63d8af5faa8f0b86993e3282b984263e3/watchfiles-1.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:360a398c3a19672cf93527f7e8d8b60d8275119c5d900f2e184d32483117a705", size = 291693, upload-time = "2025-04-08T10:35:33.225Z" }, - { url = "https://files.pythonhosted.org/packages/7f/7b/fd01087cc21db5c47e5beae507b87965db341cce8a86f9eb12bf5219d4e0/watchfiles-1.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:1a2902ede862969077b97523987c38db28abbe09fb19866e711485d9fbf0d417", size = 283287, upload-time = "2025-04-08T10:35:34.568Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.2.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, -] - -[[package]] -name = "websockets" -version = "15.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, - { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, - { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, - { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, - { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, - { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, - { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, - { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, - { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, - { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, - { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, - { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, -] - -[[package]] -name = "werkzeug" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" }, -] - -[[package]] -name = "whatthepatch" -version = "1.0.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/28/55bc3e107a56fdcf7d5022cb32b8c21d98a9cc2df5cd9f3b93e10419099e/whatthepatch-1.0.7.tar.gz", hash = "sha256:9eefb4ebea5200408e02d413d2b4bc28daea6b78bb4b4d53431af7245f7d7edf", size = 34612, upload-time = "2024-11-16T17:21:22.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/93/af1d6ccb69ab6b5a00e03fa0cefa563f9862412667776ea15dd4eece3a90/whatthepatch-1.0.7-py3-none-any.whl", hash = "sha256:1b6f655fd31091c001c209529dfaabbabdbad438f5de14e3951266ea0fc6e7ed", size = 11964, upload-time = "2024-11-16T17:21:20.761Z" }, -] - -[[package]] -name = "widgetsnbextension" -version = "4.0.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/53/2e0253c5efd69c9656b1843892052a31c36d37ad42812b5da45c62191f7e/widgetsnbextension-4.0.14.tar.gz", hash = "sha256:a3629b04e3edb893212df862038c7232f62973373869db5084aed739b437b5af", size = 1097428, upload-time = "2025-04-10T13:01:25.628Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/51/5447876806d1088a0f8f71e16542bf350918128d0a69437df26047c8e46f/widgetsnbextension-4.0.14-py3-none-any.whl", hash = "sha256:4875a9eaf72fbf5079dc372a51a9f268fc38d46f767cbf85c43a36da5cb9b575", size = 2196503, upload-time = "2025-04-10T13:01:23.086Z" }, -] - -[[package]] -name = "wrapt" -version = "1.17.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/fc/e91cc220803d7bc4db93fb02facd8461c37364151b8494762cc88b0fbcef/wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3", size = 55531, upload-time = "2025-01-14T10:35:45.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/bd/ab55f849fd1f9a58ed7ea47f5559ff09741b25f00c191231f9f059c83949/wrapt-1.17.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d5e2439eecc762cd85e7bd37161d4714aa03a33c5ba884e26c81559817ca0925", size = 53799, upload-time = "2025-01-14T10:33:57.4Z" }, - { url = "https://files.pythonhosted.org/packages/53/18/75ddc64c3f63988f5a1d7e10fb204ffe5762bc663f8023f18ecaf31a332e/wrapt-1.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fc7cb4c1c744f8c05cd5f9438a3caa6ab94ce8344e952d7c45a8ed59dd88392", size = 38821, upload-time = "2025-01-14T10:33:59.334Z" }, - { url = "https://files.pythonhosted.org/packages/48/2a/97928387d6ed1c1ebbfd4efc4133a0633546bec8481a2dd5ec961313a1c7/wrapt-1.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8fdbdb757d5390f7c675e558fd3186d590973244fab0c5fe63d373ade3e99d40", size = 38919, upload-time = "2025-01-14T10:34:04.093Z" }, - { url = "https://files.pythonhosted.org/packages/73/54/3bfe5a1febbbccb7a2f77de47b989c0b85ed3a6a41614b104204a788c20e/wrapt-1.17.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bb1d0dbf99411f3d871deb6faa9aabb9d4e744d67dcaaa05399af89d847a91d", size = 88721, upload-time = "2025-01-14T10:34:07.163Z" }, - { url = "https://files.pythonhosted.org/packages/25/cb/7262bc1b0300b4b64af50c2720ef958c2c1917525238d661c3e9a2b71b7b/wrapt-1.17.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d18a4865f46b8579d44e4fe1e2bcbc6472ad83d98e22a26c963d46e4c125ef0b", size = 80899, upload-time = "2025-01-14T10:34:09.82Z" }, - { url = "https://files.pythonhosted.org/packages/2a/5a/04cde32b07a7431d4ed0553a76fdb7a61270e78c5fd5a603e190ac389f14/wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc570b5f14a79734437cb7b0500376b6b791153314986074486e0b0fa8d71d98", size = 89222, upload-time = "2025-01-14T10:34:11.258Z" }, - { url = "https://files.pythonhosted.org/packages/09/28/2e45a4f4771fcfb109e244d5dbe54259e970362a311b67a965555ba65026/wrapt-1.17.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6d9187b01bebc3875bac9b087948a2bccefe464a7d8f627cf6e48b1bbae30f82", size = 86707, upload-time = "2025-01-14T10:34:12.49Z" }, - { url = "https://files.pythonhosted.org/packages/c6/d2/dcb56bf5f32fcd4bd9aacc77b50a539abdd5b6536872413fd3f428b21bed/wrapt-1.17.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9e8659775f1adf02eb1e6f109751268e493c73716ca5761f8acb695e52a756ae", size = 79685, upload-time = "2025-01-14T10:34:15.043Z" }, - { url = "https://files.pythonhosted.org/packages/80/4e/eb8b353e36711347893f502ce91c770b0b0929f8f0bed2670a6856e667a9/wrapt-1.17.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8b2816ebef96d83657b56306152a93909a83f23994f4b30ad4573b00bd11bb9", size = 87567, upload-time = "2025-01-14T10:34:16.563Z" }, - { url = "https://files.pythonhosted.org/packages/17/27/4fe749a54e7fae6e7146f1c7d914d28ef599dacd4416566c055564080fe2/wrapt-1.17.2-cp312-cp312-win32.whl", hash = "sha256:468090021f391fe0056ad3e807e3d9034e0fd01adcd3bdfba977b6fdf4213ea9", size = 36672, upload-time = "2025-01-14T10:34:17.727Z" }, - { url = "https://files.pythonhosted.org/packages/15/06/1dbf478ea45c03e78a6a8c4be4fdc3c3bddea5c8de8a93bc971415e47f0f/wrapt-1.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:ec89ed91f2fa8e3f52ae53cd3cf640d6feff92ba90d62236a81e4e563ac0e991", size = 38865, upload-time = "2025-01-14T10:34:19.577Z" }, - { url = "https://files.pythonhosted.org/packages/2d/82/f56956041adef78f849db6b289b282e72b55ab8045a75abad81898c28d19/wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", size = 23594, upload-time = "2025-01-14T10:35:44.018Z" }, -] - -[[package]] -name = "wsproto" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c9/4a/44d3c295350d776427904d73c189e10aeae66d7f555bb2feee16d1e4ba5a/wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", size = 53425, upload-time = "2022-08-23T19:58:21.447Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/58/e860788190eba3bcce367f74d29c4675466ce8dddfba85f7827588416f01/wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736", size = 24226, upload-time = "2022-08-23T19:58:19.96Z" }, -] - -[[package]] -name = "xformers" -version = "0.0.29.post2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "torch", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/27/ed/04ec7ef97a7e1c836add41ef5a2aef8cbdd45c0190ca42cc08f3c21e2b7b/xformers-0.0.29.post2.tar.gz", hash = "sha256:6ca3d1a6db6f2abff25c1154adee96987f77f4dfd5141771805afa5fc13e9395", size = 8468494, upload-time = "2025-02-01T02:33:48.209Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/ac/1aca7e44c93876dbda00e80f79c0bda78bc65e236c68ceb2fc6b26f77df5/xformers-0.0.29.post2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0d0eb14db56cf08ec3fb9cb36ed5e98de1303411571539ca4dc080c5861e2744", size = 44289739, upload-time = "2025-02-01T02:32:54.559Z" }, -] - -[[package]] -name = "xgrammar" -version = "0.1.17" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nanobind" }, - { name = "ninja" }, - { name = "pydantic" }, - { name = "sentencepiece" }, - { name = "tiktoken" }, - { name = "torch" }, - { name = "transformers" }, - { name = "triton", version = "3.1.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and platform_system == 'linux' and sys_platform != 'linux'" }, - { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and platform_system == 'linux' and sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e6/f9/6d530ce703cf5aae65d594a5ab984b9c0c4956e6fdbcc3279e8b1eaa358e/xgrammar-0.1.17.tar.gz", hash = "sha256:8f6cd7b3436482ad8c94b6cc93892a7f36381315c443e8e7f256f8d71c3efdee", size = 1679977, upload-time = "2025-03-25T11:08:13.458Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/32/deaee8f04d24bc2ed38c14fb01d6faa2319fb361353bbbebac4bdf801ac6/xgrammar-0.1.17-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:cc8e1e4a3298aae9856416e1366ccd07d4c6b5556921ecd108c579b1184522d2", size = 371412, upload-time = "2025-03-25T11:07:54.23Z" }, - { url = "https://files.pythonhosted.org/packages/35/ed/59a89ef003235f746fa989bf82e8425e6b046d65349feacd1b57b4763141/xgrammar-0.1.17-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a7712942793727f0c490f6f2388d5995632cc0c8258a7aff33577ff0f47bc513", size = 340973, upload-time = "2025-03-25T11:07:55.76Z" }, - { url = "https://files.pythonhosted.org/packages/48/bc/f6f5f16d9cb57684f23a62d3f51deed410da6c9708bf3d5eb679dd867dc0/xgrammar-0.1.17-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b3e998ab30662b5f090978d04928f20467df973116c17624f868fa7717ff683", size = 4236280, upload-time = "2025-03-25T11:07:57.085Z" }, - { url = "https://files.pythonhosted.org/packages/8a/89/8d4b7a8bf5af80564081555f1734d668e5496e90171280de9153d0696065/xgrammar-0.1.17-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1687ce767c5ca0fe101f699c2691762a037a6b0159608f6c4a720bccdb57ee8a", size = 4310624, upload-time = "2025-03-25T11:07:58.554Z" }, - { url = "https://files.pythonhosted.org/packages/27/37/8e31a5a44b21e89755795103df04fadb390db395c9fe65179acc9bf067b4/xgrammar-0.1.17-cp312-cp312-win_amd64.whl", hash = "sha256:9572b4c571cf39f6ffd29915b73d3cc13303c72aa86043660f46f66746b5b947", size = 421404, upload-time = "2025-03-25T11:07:59.954Z" }, -] - -[[package]] -name = "xlsxwriter" -version = "3.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/08/26f69d1e9264e8107253018de9fc6b96f9219817d01c5f021e927384a8d1/xlsxwriter-3.2.2.tar.gz", hash = "sha256:befc7f92578a85fed261639fb6cde1fd51b79c5e854040847dde59d4317077dc", size = 205202, upload-time = "2025-01-28T20:23:14.387Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/07/df054f7413bdfff5e98f75056e4ed0977d0c8716424011fac2587864d1d3/XlsxWriter-3.2.2-py3-none-any.whl", hash = "sha256:272ce861e7fa5e82a4a6ebc24511f2cb952fde3461f6c6e1a1e81d3272db1471", size = 165121, upload-time = "2025-01-28T20:23:11.654Z" }, -] - -[[package]] -name = "xxhash" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/5e/d6e5258d69df8b4ed8c83b6664f2b47d30d2dec551a29ad72a6c69eafd31/xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f", size = 84241, upload-time = "2024-08-17T09:20:38.972Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/0e/1bfce2502c57d7e2e787600b31c83535af83746885aa1a5f153d8c8059d6/xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00", size = 31969, upload-time = "2024-08-17T09:18:24.025Z" }, - { url = "https://files.pythonhosted.org/packages/3f/d6/8ca450d6fe5b71ce521b4e5db69622383d039e2b253e9b2f24f93265b52c/xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9", size = 30787, upload-time = "2024-08-17T09:18:25.318Z" }, - { url = "https://files.pythonhosted.org/packages/5b/84/de7c89bc6ef63d750159086a6ada6416cc4349eab23f76ab870407178b93/xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84", size = 220959, upload-time = "2024-08-17T09:18:26.518Z" }, - { url = "https://files.pythonhosted.org/packages/fe/86/51258d3e8a8545ff26468c977101964c14d56a8a37f5835bc0082426c672/xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793", size = 200006, upload-time = "2024-08-17T09:18:27.905Z" }, - { url = "https://files.pythonhosted.org/packages/02/0a/96973bd325412feccf23cf3680fd2246aebf4b789122f938d5557c54a6b2/xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be", size = 428326, upload-time = "2024-08-17T09:18:29.335Z" }, - { url = "https://files.pythonhosted.org/packages/11/a7/81dba5010f7e733de88af9555725146fc133be97ce36533867f4c7e75066/xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6", size = 194380, upload-time = "2024-08-17T09:18:30.706Z" }, - { url = "https://files.pythonhosted.org/packages/fb/7d/f29006ab398a173f4501c0e4977ba288f1c621d878ec217b4ff516810c04/xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90", size = 207934, upload-time = "2024-08-17T09:18:32.133Z" }, - { url = "https://files.pythonhosted.org/packages/8a/6e/6e88b8f24612510e73d4d70d9b0c7dff62a2e78451b9f0d042a5462c8d03/xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27", size = 216301, upload-time = "2024-08-17T09:18:33.474Z" }, - { url = "https://files.pythonhosted.org/packages/af/51/7862f4fa4b75a25c3b4163c8a873f070532fe5f2d3f9b3fc869c8337a398/xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2", size = 203351, upload-time = "2024-08-17T09:18:34.889Z" }, - { url = "https://files.pythonhosted.org/packages/22/61/8d6a40f288f791cf79ed5bb113159abf0c81d6efb86e734334f698eb4c59/xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d", size = 210294, upload-time = "2024-08-17T09:18:36.355Z" }, - { url = "https://files.pythonhosted.org/packages/17/02/215c4698955762d45a8158117190261b2dbefe9ae7e5b906768c09d8bc74/xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab", size = 414674, upload-time = "2024-08-17T09:18:38.536Z" }, - { url = "https://files.pythonhosted.org/packages/31/5c/b7a8db8a3237cff3d535261325d95de509f6a8ae439a5a7a4ffcff478189/xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e", size = 192022, upload-time = "2024-08-17T09:18:40.138Z" }, - { url = "https://files.pythonhosted.org/packages/78/e3/dd76659b2811b3fd06892a8beb850e1996b63e9235af5a86ea348f053e9e/xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8", size = 30170, upload-time = "2024-08-17T09:18:42.163Z" }, - { url = "https://files.pythonhosted.org/packages/d9/6b/1c443fe6cfeb4ad1dcf231cdec96eb94fb43d6498b4469ed8b51f8b59a37/xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e", size = 30040, upload-time = "2024-08-17T09:18:43.699Z" }, - { url = "https://files.pythonhosted.org/packages/0f/eb/04405305f290173acc0350eba6d2f1a794b57925df0398861a20fbafa415/xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2", size = 26796, upload-time = "2024-08-17T09:18:45.29Z" }, -] - -[[package]] -name = "yapf" -version = "0.43.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "platformdirs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907, upload-time = "2024-11-14T00:11:41.584Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/81/6acd6601f61e31cfb8729d3da6d5df966f80f374b78eff83760714487338/yapf-0.43.0-py3-none-any.whl", hash = "sha256:224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca", size = 256158, upload-time = "2024-11-14T00:11:39.37Z" }, -] - -[[package]] -name = "yarl" -version = "1.19.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "multidict" }, - { name = "propcache" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/4d/8a8f57caccce49573e567744926f88c6ab3ca0b47a257806d1cf88584c5f/yarl-1.19.0.tar.gz", hash = "sha256:01e02bb80ae0dbed44273c304095295106e1d9470460e773268a27d11e594892", size = 184396, upload-time = "2025-04-06T02:36:26.161Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/70/44ef8f69d61cb5123167a4dda87f6c739a833fbdb2ed52960b4e8409d65c/yarl-1.19.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7b687c334da3ff8eab848c9620c47a253d005e78335e9ce0d6868ed7e8fd170b", size = 146855, upload-time = "2025-04-06T02:34:40.177Z" }, - { url = "https://files.pythonhosted.org/packages/c3/94/38c14d6c8217cc818647689f2dd647b976ced8fea08d0ac84e3c8168252b/yarl-1.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b0fe766febcf523a2930b819c87bb92407ae1368662c1bc267234e79b20ff894", size = 97523, upload-time = "2025-04-06T02:34:41.817Z" }, - { url = "https://files.pythonhosted.org/packages/35/a5/43a613586a6255105c4655a911c307ef3420e49e540d6ae2c5829863fb25/yarl-1.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:742ceffd3c7beeb2b20d47cdb92c513eef83c9ef88c46829f88d5b06be6734ee", size = 95540, upload-time = "2025-04-06T02:34:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/d4/60/ed26049f4a8b06ebfa6d5f3cb6a51b152fd57081aa818b6497474f65a631/yarl-1.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2af682a1e97437382ee0791eacbf540318bd487a942e068e7e0a6c571fadbbd3", size = 344386, upload-time = "2025-04-06T02:34:45.275Z" }, - { url = "https://files.pythonhosted.org/packages/49/a6/b84899cab411f49af5986cfb44b514040788d81c8084f5811e6a7c0f1ce6/yarl-1.19.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:63702f1a098d0eaaea755e9c9d63172be1acb9e2d4aeb28b187092bcc9ca2d17", size = 338889, upload-time = "2025-04-06T02:34:47.359Z" }, - { url = "https://files.pythonhosted.org/packages/cc/ce/0704f7166a781b1f81bdd45c4f49eadbae0230ebd35b9ec7cd7769d3a6ff/yarl-1.19.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3560dcba3c71ae7382975dc1e912ee76e50b4cd7c34b454ed620d55464f11876", size = 353107, upload-time = "2025-04-06T02:34:49.32Z" }, - { url = "https://files.pythonhosted.org/packages/75/e5/0ecd6f2a9cc4264c16d8dfb0d3d71ba8d03cb58f3bcd42b1df4358331189/yarl-1.19.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68972df6a0cc47c8abaf77525a76ee5c5f6ea9bbdb79b9565b3234ded3c5e675", size = 353128, upload-time = "2025-04-06T02:34:51.312Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c7/cd0fd1de581f1c2e8f996e704c9fd979e00106f18eebd91b0173cf1a13c6/yarl-1.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5684e7ff93ea74e47542232bd132f608df4d449f8968fde6b05aaf9e08a140f9", size = 349107, upload-time = "2025-04-06T02:34:53.022Z" }, - { url = "https://files.pythonhosted.org/packages/e6/34/ba3e5a20bd1d6a09034fc7985aaf1309976f2a7a5aefd093c9e56f6e1e0c/yarl-1.19.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8182ad422bfacdebd4759ce3adc6055c0c79d4740aea1104e05652a81cd868c6", size = 335144, upload-time = "2025-04-06T02:34:54.767Z" }, - { url = "https://files.pythonhosted.org/packages/1e/98/d9b7beb932fade015906efe0980aa7d522b8f93cf5ebf1082e74faa314b7/yarl-1.19.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aee5b90a5a9b71ac57400a7bdd0feaa27c51e8f961decc8d412e720a004a1791", size = 360795, upload-time = "2025-04-06T02:34:57.08Z" }, - { url = "https://files.pythonhosted.org/packages/9a/11/70b8770039cc54af5948970591517a1e1d093df3f04f328c655c9a0fefb7/yarl-1.19.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:8c0b2371858d5a814b08542d5d548adb03ff2d7ab32f23160e54e92250961a72", size = 360140, upload-time = "2025-04-06T02:34:59.341Z" }, - { url = "https://files.pythonhosted.org/packages/d4/67/708e3e36fafc4d9d96b4eecc6c8b9f37c8ad50df8a16c7a1d5ba9df53050/yarl-1.19.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cd430c2b7df4ae92498da09e9b12cad5bdbb140d22d138f9e507de1aa3edfea3", size = 364431, upload-time = "2025-04-06T02:35:01.126Z" }, - { url = "https://files.pythonhosted.org/packages/c3/8b/937fbbcc895553a7e16fcd86ae4e0724c6ac9468237ad8e7c29cc3b1c9d9/yarl-1.19.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a93208282c0ccdf73065fd76c6c129bd428dba5ff65d338ae7d2ab27169861a0", size = 373832, upload-time = "2025-04-06T02:35:03.244Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ca/288ddc2230c9b6647fe907504f1119adb41252ac533eb564d3fc73511215/yarl-1.19.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:b8179280cdeb4c36eb18d6534a328f9d40da60d2b96ac4a295c5f93e2799e9d9", size = 378122, upload-time = "2025-04-06T02:35:05.365Z" }, - { url = "https://files.pythonhosted.org/packages/4f/5a/79e1ef31d14968fbfc0ecec70a6683b574890d9c7550c376dd6d40de7754/yarl-1.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eda3c2b42dc0c389b7cfda2c4df81c12eeb552019e0de28bde8f913fc3d1fcf3", size = 375178, upload-time = "2025-04-06T02:35:07.106Z" }, - { url = "https://files.pythonhosted.org/packages/95/38/9b0e56bf14026c3f550ad6425679f6d1a2f4821d70767f39d6f4c56a0820/yarl-1.19.0-cp312-cp312-win32.whl", hash = "sha256:57f3fed859af367b9ca316ecc05ce79ce327d6466342734305aa5cc380e4d8be", size = 86172, upload-time = "2025-04-06T02:35:09.224Z" }, - { url = "https://files.pythonhosted.org/packages/b3/96/5c2f3987c4bb4e5cdebea3caf99a45946b13a9516f849c02222203d99860/yarl-1.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:5507c1f7dd3d41251b67eecba331c8b2157cfd324849879bebf74676ce76aff7", size = 92617, upload-time = "2025-04-06T02:35:11.283Z" }, - { url = "https://files.pythonhosted.org/packages/a4/06/ae25a353e8f032322df6f30d6bb1fc329773ee48e1a80a2196ccb8d1206b/yarl-1.19.0-py3-none-any.whl", hash = "sha256:a727101eb27f66727576630d02985d8a065d09cd0b5fcbe38a5793f71b2a97ef", size = 45990, upload-time = "2025-04-06T02:36:24.343Z" }, -] - -[[package]] -name = "zipp" -version = "3.21.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e9468ebd0bcd6505de3b275e06f202c2cb016e3ff56f/zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4", size = 24545, upload-time = "2024-11-10T15:05:20.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, -] - -[[package]] -name = "zope-event" -version = "5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/c2/427f1867bb96555d1d34342f1dd97f8c420966ab564d58d18469a1db8736/zope.event-5.0.tar.gz", hash = "sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd", size = 17350, upload-time = "2023-06-23T06:28:35.709Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/42/f8dbc2b9ad59e927940325a22d6d3931d630c3644dae7e2369ef5d9ba230/zope.event-5.0-py3-none-any.whl", hash = "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", size = 6824, upload-time = "2023-06-23T06:28:32.652Z" }, -] - -[[package]] -name = "zope-interface" -version = "7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/0b/c7516bc3bad144c2496f355e35bd699443b82e9437aa02d9867653203b4a/zope.interface-7.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:086ee2f51eaef1e4a52bd7d3111a0404081dadae87f84c0ad4ce2649d4f708b7", size = 208959, upload-time = "2024-11-28T08:47:47.788Z" }, - { url = "https://files.pythonhosted.org/packages/a2/e9/1463036df1f78ff8c45a02642a7bf6931ae4a38a4acd6a8e07c128e387a7/zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21328fcc9d5b80768bf051faa35ab98fb979080c18e6f84ab3f27ce703bce465", size = 209357, upload-time = "2024-11-28T08:47:50.897Z" }, - { url = "https://files.pythonhosted.org/packages/07/a8/106ca4c2add440728e382f1b16c7d886563602487bdd90004788d45eb310/zope.interface-7.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6dd02ec01f4468da0f234da9d9c8545c5412fef80bc590cc51d8dd084138a89", size = 264235, upload-time = "2024-11-28T09:18:15.56Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ca/57286866285f4b8a4634c12ca1957c24bdac06eae28fd4a3a578e30cf906/zope.interface-7.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e7da17f53e25d1a3bde5da4601e026adc9e8071f9f6f936d0fe3fe84ace6d54", size = 259253, upload-time = "2024-11-28T08:48:29.025Z" }, - { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702, upload-time = "2024-11-28T08:48:37.363Z" }, - { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466, upload-time = "2024-11-28T08:49:14.397Z" }, -] diff --git a/verl.egg-info/PKG-INFO b/verl.egg-info/PKG-INFO deleted file mode 100644 index dd85dd72e4..0000000000 --- a/verl.egg-info/PKG-INFO +++ /dev/null @@ -1,484 +0,0 @@ -Metadata-Version: 2.4 -Name: verl -Version: 0.2.0.dev0 -Summary: verl: Volcano Engine Reinforcement Learning for LLM -Home-page: https://github.com/volcengine/verl -Author: Bytedance - Seed - MLSys -Author-email: zhangchi.usc1992@bytedance.com, gmsheng@connect.hku.hk -License: - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -Requires-Python: ==3.12.* -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: accelerate -Requires-Dist: codetiming -Requires-Dist: datasets -Requires-Dist: dill -Requires-Dist: hydra-core -Requires-Dist: numpy -Requires-Dist: pandas -Requires-Dist: datasets -Requires-Dist: peft -Requires-Dist: pyarrow>=15.0.0 -Requires-Dist: pybind11 -Requires-Dist: pylatexenc -Requires-Dist: ray[default]>=2.10 -Requires-Dist: tensordict<=0.6.2 -Requires-Dist: torchdata -Requires-Dist: transformers -Requires-Dist: wandb -Requires-Dist: hf_transfer -Requires-Dist: torchdata -Requires-Dist: openhands-ai -Requires-Dist: sglang[all]>=0.4.6.post1 -Requires-Dist: flash-attn@ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl -Requires-Dist: streamlit -Requires-Dist: whatthepatch -Requires-Dist: retry -Requires-Dist: evaluate -Requires-Dist: swebench@ https://github.com/SWE-Gym/SWE-Bench-Fork.git -Requires-Dist: swegym@ https://github.com/SWE-Gym/SWE-Bench-Package.git -Requires-Dist: commit0 -Requires-Dist: func_timeout -Requires-Dist: sympy -Requires-Dist: gdown -Requires-Dist: matplotlib -Requires-Dist: seaborn -Requires-Dist: tabulate -Requires-Dist: browsergym==0.10.2 -Requires-Dist: browsergym-webarena==0.10.2 -Requires-Dist: browsergym-miniwob==0.10.2 -Requires-Dist: browsergym-visualwebarena==0.10.2 -Requires-Dist: tensordict<=0.6.2 -Requires-Dist: torch-memory-saver>=0.0.5 -Requires-Dist: vllm>=0.7.3 -Provides-Extra: test -Requires-Dist: pytest; extra == "test" -Requires-Dist: yapf; extra == "test" -Requires-Dist: py-spy; extra == "test" -Provides-Extra: geo -Requires-Dist: mathruler; extra == "geo" -Provides-Extra: gpu -Requires-Dist: liger-kernel; extra == "gpu" -Requires-Dist: flash-attn; extra == "gpu" -Provides-Extra: math -Requires-Dist: math-verify; extra == "math" -Provides-Extra: vllm -Requires-Dist: tensordict<=0.6.2; extra == "vllm" -Requires-Dist: vllm<=0.8.2; extra == "vllm" -Dynamic: author -Dynamic: author-email -Dynamic: home-page -Dynamic: license-file -Dynamic: provides-extra - -# SkyRL-v0: Training Code - -This repository contains training code for the `SkyRL-v0`release. Our implementation is a fork of [VERL](https://github.com/volcengine/verl). - -## Installation - -The only pre-requisite is having `uv` [installed](docs.astral.sh/uv/getting-started/installation) on your system. We use the `uv` + `ray` integration to easily manage dependencies in multi-node training. - -### Clone SkyRL-OpenHands - -We use [SkyRL-OpenHands](https://github.com/NovaSky-AI/SkyRL-OpenHands) to be able to connect to our remote runtime server. Clone the repository and place it in the git root: - -```bash -git clone https://github.com/NovaSky-AI/SkyRL-OpenHands -``` - -### Installation dry run - -You can dry run your installation with the following command: - -```bash -uv run --isolated --frozen pip show torch -``` - -NOTE: With a CPU head node, you might encounter installation issues with `torch-memory-saver`. To fix this, you need to install CUDA and make sure your CUDA libraries are linked in `/usr/lib`. For example, - -```bash -sudo ln -s /usr/local/cuda-12.4/compat/libcuda.so /usr/lib/libcuda.so -sudo ln -s /usr/local/cuda-12.4/compat/libcuda.so.1 /usr/lib/libcuda.so.1 -``` - - -## Scripts for reproduction - -For reproducing our results for SkyRL-Agent-14B-v0 you can refer to to [examples/sky](./examples/sky/) - - -## Original README - -We reproduce the original README from VERL below: - -

verl: Volcano Engine Reinforcement Learning for LLM

- -[![GitHub Repo stars](https://img.shields.io/github/stars/volcengine/verl)](https://github.com/volcengine/verl/stargazers) -![GitHub forks](https://img.shields.io/github/forks/volcengine/verl) -[![Twitter](https://img.shields.io/twitter/follow/verl_project)](https://twitter.com/verl_project) - - -![GitHub contributors](https://img.shields.io/github/contributors/volcengine/verl) -[![Documentation](https://img.shields.io/badge/documentation-blue)](https://verl.readthedocs.io/en/latest/) - - - -verl is a flexible, efficient and production-ready RL training library for large language models (LLMs). - -verl is the open-source version of **[HybridFlow: A Flexible and Efficient RLHF Framework](https://arxiv.org/abs/2409.19256v2)** paper. - -verl is flexible and easy to use with: - -- **Easy extension of diverse RL algorithms**: The hybrid-controller programming model enables flexible representation and efficient execution of complex Post-Training dataflows. Build RL dataflows such as GRPO, PPO in a few lines of code. - -- **Seamless integration of existing LLM infra with modular APIs**: Decouples computation and data dependencies, enabling seamless integration with existing LLM frameworks, such as FSDP, Megatron-LM, vLLM, SGLang, etc - -- **Flexible device mapping**: Supports various placement of models onto different sets of GPUs for efficient resource utilization and scalability across different cluster sizes. - -- Ready integration with popular HuggingFace models - - -verl is fast with: - -- **State-of-the-art throughput**: SOTA LLM training and inference engine integrations and SOTA RL throughput. - -- **Efficient actor model resharding with 3D-HybridEngine**: Eliminates memory redundancy and significantly reduces communication overhead during transitions between training and generation phases. - -

- -## News -- [2025/03] [DAPO](https://dapo-sia.github.io/) is the open-sourced SOTA RL algorithm that achieves 50 points on AIME 2024 based on the Qwen2.5-32B pre-trained model, surpassing the previous SOTA achieved by DeepSeek's GRPO (DeepSeek-R1-Zero-Qwen-32B). DAPO's training is fully powered by verl and the reproduction code is [publicly available](https://github.com/volcengine/verl/tree/gm-tyx/puffin/main/recipe/dapo) now. -- [2025/03] We will present verl(HybridFlow) at EuroSys 2025. See you in Rotterdam! -- [2025/03] We introduced the programming model of verl at the [vLLM Beijing Meetup](https://mp.weixin.qq.com/s/n77GibL2corAtQHtVEAzfg) and [verl intro and updates](https://github.com/eric-haibin-lin/verl-community/blob/main/slides/verl-lmsys-meetup.pdf) at the [LMSys Meetup](https://lu.ma/ntjrr7ig) in Sunnyvale mid March. -- [2025/02] verl v0.2.0.post2 is released! See [release note](https://github.com/volcengine/verl/releases/) for details. -- [2025/01] [Doubao-1.5-pro](https://team.doubao.com/zh/special/doubao_1_5_pro) is released with SOTA-level performance on LLM & VLM. The RL scaling preview model is trained using verl, reaching OpenAI O1-level performance on math benchmarks (70.0 pass@1 on AIME). -
more... - -
- -## Key Features - -- **FSDP** and **Megatron-LM** for training. -- **vLLM**, **SGLang**(experimental) and **HF Transformers** for rollout generation. -- Compatible with Hugging Face Transformers and Modelscope Hub: Qwen-2.5, Llama3.1, Gemma2, DeepSeek-LLM, etc -- Supervised fine-tuning. -- Reinforcement learning with [PPO](examples/ppo_trainer/), [GRPO](examples/grpo_trainer/), [ReMax](examples/remax_trainer/), [REINFORCE++](https://verl.readthedocs.io/en/latest/examples/config.html#algorithm), [RLOO](examples/rloo_trainer/), [PRIME](recipe/prime/), etc. - - Support model-based reward and function-based reward (verifiable reward) - - Support vision-language models (VLMs) and [multi-modal RL](examples/grpo_trainer/run_qwen2_5_vl-7b.sh) -- Flash attention 2, [sequence packing](examples/ppo_trainer/run_qwen2-7b_seq_balance.sh), [sequence parallelism](examples/ppo_trainer/run_deepseek7b_llm_sp2.sh) support via DeepSpeed Ulysses, [LoRA](examples/sft/gsm8k/run_qwen_05_peft.sh), [Liger-kernel](examples/sft/gsm8k/run_qwen_05_sp2_liger.sh). -- Scales up to 70B models and hundreds of GPUs. -- Experiment tracking with wandb, swanlab, mlflow and tensorboard. - -## Upcoming Features -- DeepSeek 671b optimizations with Megatron v0.11 -- Multi-turn rollout optimizations - -## Getting Started - -Documentation - -**Quickstart:** -- [Installation](https://verl.readthedocs.io/en/latest/start/install.html) -- [Quickstart](https://verl.readthedocs.io/en/latest/start/quickstart.html) -- [Programming Guide](https://verl.readthedocs.io/en/latest/hybrid_flow.html) - -**Running a PPO example step-by-step:** -- Data and Reward Preparation - - [Prepare Data for Post-Training](https://verl.readthedocs.io/en/latest/preparation/prepare_data.html) - - [Implement Reward Function for Dataset](https://verl.readthedocs.io/en/latest/preparation/reward_function.html) -- Understanding the PPO Example - - [PPO Example Architecture](https://verl.readthedocs.io/en/latest/examples/ppo_code_architecture.html) - - [Config Explanation](https://verl.readthedocs.io/en/latest/examples/config.html) - - [Run GSM8K Example](https://verl.readthedocs.io/en/latest/examples/gsm8k_example.html) - -**Reproducible algorithm baselines:** -- [PPO, GRPO, ReMax](https://verl.readthedocs.io/en/latest/experiment/ppo.html) - -**For code explanation and advance usage (extension):** -- PPO Trainer and Workers - - [PPO Ray Trainer](https://verl.readthedocs.io/en/latest/workers/ray_trainer.html) - - [PyTorch FSDP Backend](https://verl.readthedocs.io/en/latest/workers/fsdp_workers.html) - - [Megatron-LM Backend](https://verl.readthedocs.io/en/latest/index.html) -- Advance Usage and Extension - - [Ray API design tutorial](https://verl.readthedocs.io/en/latest/advance/placement.html) - - [Extend to Other RL(HF) algorithms](https://verl.readthedocs.io/en/latest/advance/dpo_extension.html) - - [Add Models with the FSDP Backend](https://verl.readthedocs.io/en/latest/advance/fsdp_extension.html) - - [Add Models with the Megatron-LM Backend](https://verl.readthedocs.io/en/latest/advance/megatron_extension.html) - - [Deployment using Separate GPU Resources](https://github.com/volcengine/verl/tree/main/examples/split_placement) - -**Blogs from the community** -- [使用verl进行GRPO分布式强化学习训练最佳实践](https://www.volcengine.com/docs/6459/1463942) -- [HybridFlow veRL 原文浅析](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/readme.md) -- [最高提升20倍吞吐量！豆包大模型团队发布全新 RLHF 框架，现已开源！](https://team.doubao.com/en/blog/%E6%9C%80%E9%AB%98%E6%8F%90%E5%8D%8720%E5%80%8D%E5%90%9E%E5%90%90%E9%87%8F-%E8%B1%86%E5%8C%85%E5%A4%A7%E6%A8%A1%E5%9E%8B%E5%9B%A2%E9%98%9F%E5%8F%91%E5%B8%83%E5%85%A8%E6%96%B0-rlhf-%E6%A1%86%E6%9E%B6-%E7%8E%B0%E5%B7%B2%E5%BC%80%E6%BA%90) - - -## Performance Tuning Guide -The performance is essential for on-policy RL algorithm. We have written a detailed [performance tuning guide](https://verl.readthedocs.io/en/latest/perf/perf_tuning.html) to help you optimize performance. - -## Use vLLM v0.8 -veRL now supports vLLM>=0.8.0 when using FSDP as the training backend. Please refer to [this document](https://github.com/volcengine/verl/blob/main/docs/README_vllm0.8.md) for installation guide and more information. - -## Citation and acknowledgement - -If you find the project helpful, please cite: -- [HybridFlow: A Flexible and Efficient RLHF Framework](https://arxiv.org/abs/2409.19256v2) -- [A Framework for Training Large Language Models for Code Generation via Proximal Policy Optimization](https://i.cs.hku.hk/~cwu/papers/gmsheng-NL2Code24.pdf) - -```bibtex -@article{sheng2024hybridflow, - title = {HybridFlow: A Flexible and Efficient RLHF Framework}, - author = {Guangming Sheng and Chi Zhang and Zilingfeng Ye and Xibin Wu and Wang Zhang and Ru Zhang and Yanghua Peng and Haibin Lin and Chuan Wu}, - year = {2024}, - journal = {arXiv preprint arXiv: 2409.19256} -} -``` - -verl is inspired by the design of Nemo-Aligner, Deepspeed-chat and OpenRLHF. The project is adopted and supported by Anyscale, Bytedance, LMSys.org, Shanghai AI Lab, Tsinghua University, UC Berkeley, UCLA, UIUC, University of Hong Kong, and many more. - -## Awesome work using verl -- [TinyZero](https://github.com/Jiayi-Pan/TinyZero): a reproduction of **DeepSeek R1 Zero** recipe for reasoning tasks ![GitHub Repo stars](https://img.shields.io/github/stars/Jiayi-Pan/TinyZero) -- [DAPO](https://dapo-sia.github.io/): the fully open source SOTA RL algorithm that beats DeepSeek-R1-zero-32B ![GitHub Repo stars](https://img.shields.io/github/stars/volcengine/verl) -- [SkyThought](https://github.com/NovaSky-AI/SkyThought): RL training for Sky-T1-7B by NovaSky AI team. ![GitHub Repo stars](https://img.shields.io/github/stars/NovaSky-AI/SkyThought) -- [simpleRL-reason](https://github.com/hkust-nlp/simpleRL-reason): SimpleRL-Zoo: Investigating and Taming Zero Reinforcement Learning for Open Base Models in the Wild ![GitHub Repo stars](https://img.shields.io/github/stars/hkust-nlp/simpleRL-reason) -- [Easy-R1](https://github.com/hiyouga/EasyR1): **Multi-modal** RL training framework ![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/EasyR1) -- [OpenManus-RL](https://github.com/OpenManus/OpenManus-RL): LLM Agents RL tunning framework for multiple agent environments. ![GitHub Repo stars](https://img.shields.io/github/stars/OpenManus/OpenManus-RL) -- [deepscaler](https://github.com/agentica-project/deepscaler): iterative context scaling with GRPO ![GitHub Repo stars](https://img.shields.io/github/stars/agentica-project/deepscaler) -- [PRIME](https://github.com/PRIME-RL/PRIME): Process reinforcement through implicit rewards ![GitHub Repo stars](https://img.shields.io/github/stars/PRIME-RL/PRIME) -- [RAGEN](https://github.com/ZihanWang314/ragen): a general-purpose reasoning **agent** training framework ![GitHub Repo stars](https://img.shields.io/github/stars/ZihanWang314/ragen) -- [Logic-RL](https://github.com/Unakar/Logic-RL): a reproduction of DeepSeek R1 Zero on 2K Tiny Logic Puzzle Dataset. ![GitHub Repo stars](https://img.shields.io/github/stars/Unakar/Logic-RL) -- [Search-R1](https://github.com/PeterGriffinJin/Search-R1): RL with reasoning and **searching (tool-call)** interleaved LLMs ![GitHub Repo stars](https://img.shields.io/github/stars/PeterGriffinJin/Search-R1) -- [ReSearch](https://github.com/Agent-RL/ReSearch): Learning to **Re**ason with **Search** for LLMs via Reinforcement Learning ![GitHub Repo stars](https://img.shields.io/github/stars/Agent-RL/ReSearch) -- [DeepRetrieval](https://github.com/pat-jj/DeepRetrieval): Hacking **Real Search Engines** and **retrievers** with LLMs via RL for **information retrieval** ![GitHub Repo stars](https://img.shields.io/github/stars/pat-jj/DeepRetrieval) -- [cognitive-behaviors](https://github.com/kanishkg/cognitive-behaviors): Cognitive Behaviors that Enable Self-Improving Reasoners, or, Four Habits of Highly Effective STaRs ![GitHub Repo stars](https://img.shields.io/github/stars/kanishkg/cognitive-behaviors) -- [PURE](https://github.com/CJReinforce/PURE): **Credit assignment** is the key to successful reinforcement fine-tuning using **process reward model** ![GitHub Repo stars](https://img.shields.io/github/stars/CJReinforce/PURE) -- [MetaSpatial](https://github.com/PzySeere/MetaSpatial): Reinforcing **3D Spatial Reasoning** in **VLMs** for the **Metaverse** ![GitHub Repo stars](https://img.shields.io/github/stars/PzySeere/MetaSpatial) -- [DeepEnlighten](https://github.com/DolbyUUU/DeepEnlighten): Reproduce R1 with **social reasoning** tasks and analyze key findings ![GitHub Repo stars](https://img.shields.io/github/stars/DolbyUUU/DeepEnlighten) -- [Code-R1](https://github.com/ganler/code-r1): Reproducing R1 for **Code** with Reliable Rewards ![GitHub Repo stars](https://img.shields.io/github/stars/ganler/code-r1) -- [self-rewarding-reasoning-LLM](https://arxiv.org/pdf/2502.19613): self-rewarding and correction with **generative reward models** ![GitHub Repo stars](https://img.shields.io/github/stars/RLHFlow/Self-rewarding-reasoning-LLM) -- [critic-rl](https://github.com/HKUNLP/critic-rl): LLM critics for code generation ![GitHub Repo stars](https://img.shields.io/github/stars/HKUNLP/critic-rl) -- [DQO](https://arxiv.org/abs/2410.09302): Enhancing multi-Step reasoning abilities of language models through direct Q-function optimization -- [FIRE](https://arxiv.org/abs/2410.21236): Flaming-hot initiation with regular execution sampling for large language models -- [Rec-R1](https://arxiv.org/pdf/2503.24289): Bridging Generative Large Language Models and Recommendation Systems via Reinforcement Learning - -## Contribution Guide -Contributions from the community are welcome! Please check out our [project roadmap](https://github.com/volcengine/verl/issues/22) and [release plan](https://github.com/volcengine/verl/issues/354) to see where you can contribute. - -### Code formatting -We use yapf (Google style) to enforce strict code formatting when reviewing PRs. To reformat your code locally, make sure you have installed the **latest** version of `yapf` -```bash -pip3 install yapf --upgrade -``` -Then, make sure you are at top level of verl repo and run -```bash -bash scripts/format.sh -``` -We are HIRING! Send us an [email](mailto:haibin.lin@bytedance.com) if you are interested in internship/FTE opportunities in MLSys/LLM reasoning/multimodal alignment. diff --git a/verl.egg-info/SOURCES.txt b/verl.egg-info/SOURCES.txt deleted file mode 100644 index 926917ccc7..0000000000 --- a/verl.egg-info/SOURCES.txt +++ /dev/null @@ -1,477 +0,0 @@ -LICENSE -README.md -pyproject.toml -setup.py -./tests/__init__.py -./tests/test_reward.py -./tests/e2e/__init__.py -./tests/e2e/check_custom_rwd_fn.py -./tests/e2e/check_results.py -./tests/e2e/envs/__init__.py -./tests/e2e/envs/digit_completion/__init__.py -./tests/e2e/envs/digit_completion/task.py -./tests/e2e/envs/digit_completion/tokenizer.py -./verl/__init__.py -./verl/protocol.py -./verl/models/__init__.py -./verl/models/registry.py -./verl/models/weight_loader_registry.py -./verl/models/llama/__init__.py -./verl/models/llama/megatron/__init__.py -./verl/models/llama/megatron/modeling_llama_megatron.py -./verl/models/llama/megatron/checkpoint_utils/__init__.py -./verl/models/llama/megatron/checkpoint_utils/llama_loader.py -./verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py -./verl/models/llama/megatron/checkpoint_utils/llama_saver.py -./verl/models/llama/megatron/layers/__init__.py -./verl/models/llama/megatron/layers/parallel_attention.py -./verl/models/llama/megatron/layers/parallel_decoder.py -./verl/models/llama/megatron/layers/parallel_linear.py -./verl/models/llama/megatron/layers/parallel_mlp.py -./verl/models/llama/megatron/layers/parallel_rmsnorm.py -./verl/models/mcore/__init__.py -./verl/models/mcore/gpt_model.py -./verl/models/mcore/loader.py -./verl/models/mcore/saver.py -./verl/models/qwen2/__init__.py -./verl/models/qwen2/megatron/__init__.py -./verl/models/qwen2/megatron/modeling_qwen2_megatron.py -./verl/models/qwen2/megatron/checkpoint_utils/__init__.py -./verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py -./verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py -./verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py -./verl/models/qwen2/megatron/layers/__init__.py -./verl/models/qwen2/megatron/layers/parallel_attention.py -./verl/models/qwen2/megatron/layers/parallel_decoder.py -./verl/models/qwen2/megatron/layers/parallel_linear.py -./verl/models/qwen2/megatron/layers/parallel_mlp.py -./verl/models/qwen2/megatron/layers/parallel_rmsnorm.py -./verl/models/transformers/__init__.py -./verl/models/transformers/llama.py -./verl/models/transformers/monkey_patch.py -./verl/models/transformers/qwen2.py -./verl/models/transformers/qwen2_vl.py -./verl/single_controller/__init__.py -./verl/single_controller/base/__init__.py -./verl/single_controller/base/decorator.py -./verl/single_controller/base/worker.py -./verl/single_controller/base/worker_group.py -./verl/single_controller/base/megatron/__init__.py -./verl/single_controller/base/megatron/worker.py -./verl/single_controller/base/megatron/worker_group.py -./verl/single_controller/base/register_center/__init__.py -./verl/single_controller/base/register_center/ray.py -./verl/single_controller/ray/__init__.py -./verl/single_controller/ray/base.py -./verl/single_controller/ray/megatron.py -./verl/third_party/__init__.py -./verl/third_party/sglang/__init__.py -./verl/third_party/sglang/parallel_state.py -./verl/third_party/vllm/__init__.py -./verl/third_party/vllm/vllm_v_0_3_1/__init__.py -./verl/third_party/vllm/vllm_v_0_3_1/arg_utils.py -./verl/third_party/vllm/vllm_v_0_3_1/config.py -./verl/third_party/vllm/vllm_v_0_3_1/llm.py -./verl/third_party/vllm/vllm_v_0_3_1/llm_engine_sp.py -./verl/third_party/vllm/vllm_v_0_3_1/model_loader.py -./verl/third_party/vllm/vllm_v_0_3_1/model_runner.py -./verl/third_party/vllm/vllm_v_0_3_1/parallel_state.py -./verl/third_party/vllm/vllm_v_0_3_1/tokenizer.py -./verl/third_party/vllm/vllm_v_0_3_1/weight_loaders.py -./verl/third_party/vllm/vllm_v_0_3_1/worker.py -./verl/third_party/vllm/vllm_v_0_4_2/__init__.py -./verl/third_party/vllm/vllm_v_0_4_2/arg_utils.py -./verl/third_party/vllm/vllm_v_0_4_2/config.py -./verl/third_party/vllm/vllm_v_0_4_2/dtensor_weight_loaders.py -./verl/third_party/vllm/vllm_v_0_4_2/hf_weight_loader.py -./verl/third_party/vllm/vllm_v_0_4_2/llm.py -./verl/third_party/vllm/vllm_v_0_4_2/llm_engine_sp.py -./verl/third_party/vllm/vllm_v_0_4_2/megatron_weight_loaders.py -./verl/third_party/vllm/vllm_v_0_4_2/model_loader.py -./verl/third_party/vllm/vllm_v_0_4_2/model_runner.py -./verl/third_party/vllm/vllm_v_0_4_2/parallel_state.py -./verl/third_party/vllm/vllm_v_0_4_2/spmd_gpu_executor.py -./verl/third_party/vllm/vllm_v_0_4_2/tokenizer.py -./verl/third_party/vllm/vllm_v_0_4_2/worker.py -./verl/third_party/vllm/vllm_v_0_5_4/__init__.py -./verl/third_party/vllm/vllm_v_0_5_4/arg_utils.py -./verl/third_party/vllm/vllm_v_0_5_4/config.py -./verl/third_party/vllm/vllm_v_0_5_4/dtensor_weight_loaders.py -./verl/third_party/vllm/vllm_v_0_5_4/hf_weight_loader.py -./verl/third_party/vllm/vllm_v_0_5_4/llm.py -./verl/third_party/vllm/vllm_v_0_5_4/llm_engine_sp.py -./verl/third_party/vllm/vllm_v_0_5_4/megatron_weight_loaders.py -./verl/third_party/vllm/vllm_v_0_5_4/model_loader.py -./verl/third_party/vllm/vllm_v_0_5_4/model_runner.py -./verl/third_party/vllm/vllm_v_0_5_4/parallel_state.py -./verl/third_party/vllm/vllm_v_0_5_4/spmd_gpu_executor.py -./verl/third_party/vllm/vllm_v_0_5_4/tokenizer.py -./verl/third_party/vllm/vllm_v_0_5_4/worker.py -./verl/third_party/vllm/vllm_v_0_6_3/__init__.py -./verl/third_party/vllm/vllm_v_0_6_3/arg_utils.py -./verl/third_party/vllm/vllm_v_0_6_3/config.py -./verl/third_party/vllm/vllm_v_0_6_3/dtensor_weight_loaders.py -./verl/third_party/vllm/vllm_v_0_6_3/hf_weight_loader.py -./verl/third_party/vllm/vllm_v_0_6_3/llm.py -./verl/third_party/vllm/vllm_v_0_6_3/llm_engine_sp.py -./verl/third_party/vllm/vllm_v_0_6_3/megatron_weight_loaders.py -./verl/third_party/vllm/vllm_v_0_6_3/model_loader.py -./verl/third_party/vllm/vllm_v_0_6_3/model_runner.py -./verl/third_party/vllm/vllm_v_0_6_3/parallel_state.py -./verl/third_party/vllm/vllm_v_0_6_3/spmd_gpu_executor.py -./verl/third_party/vllm/vllm_v_0_6_3/tokenizer.py -./verl/third_party/vllm/vllm_v_0_6_3/worker.py -./verl/trainer/__init__.py -./verl/trainer/fsdp_sft_trainer.py -./verl/trainer/main_eval.py -./verl/trainer/main_generation.py -./verl/trainer/main_ppo.py -./verl/trainer/config/evaluation.yaml -./verl/trainer/config/generation.yaml -./verl/trainer/config/ppo_megatron_trainer.yaml -./verl/trainer/config/ppo_trainer.yaml -./verl/trainer/config/sft_trainer.yaml -./verl/trainer/ppo/__init__.py -./verl/trainer/ppo/core_algos.py -./verl/trainer/ppo/metric_utils.py -./verl/trainer/ppo/ray_trainer.py -./verl/utils/__init__.py -./verl/utils/config.py -./verl/utils/distributed.py -./verl/utils/flops_counter.py -./verl/utils/fs.py -./verl/utils/fsdp_utils.py -./verl/utils/hdfs_io.py -./verl/utils/import_utils.py -./verl/utils/logging_utils.py -./verl/utils/megatron_utils.py -./verl/utils/memory_buffer.py -./verl/utils/model.py -./verl/utils/py_functional.py -./verl/utils/ray_utils.py -./verl/utils/seqlen_balancing.py -./verl/utils/swedev_utils.py -./verl/utils/tokenizer.py -./verl/utils/torch_dtypes.py -./verl/utils/torch_functional.py -./verl/utils/tracking.py -./verl/utils/ulysses.py -./verl/utils/checkpoint/__init__.py -./verl/utils/checkpoint/checkpoint_manager.py -./verl/utils/checkpoint/fsdp_checkpoint_manager.py -./verl/utils/checkpoint/megatron_checkpoint_manager.py -./verl/utils/checkpoint/upload_utils.py -./verl/utils/dataset/__init__.py -./verl/utils/dataset/rl_dataset.py -./verl/utils/dataset/rm_dataset.py -./verl/utils/dataset/sft_dataset.py -./verl/utils/debug/__init__.py -./verl/utils/debug/performance.py -./verl/utils/debug/trajectory_tracker.py -./verl/utils/logger/__init__.py -./verl/utils/logger/aggregate_logger.py -./verl/utils/megatron/__init__.py -./verl/utils/megatron/memory.py -./verl/utils/megatron/optimizer.py -./verl/utils/megatron/pipeline_parallel.py -./verl/utils/megatron/sequence_parallel.py -./verl/utils/megatron/tensor_parallel.py -./verl/utils/rendezvous/__init__.py -./verl/utils/rendezvous/ray_backend.py -./verl/utils/reward_score/__init__.py -./verl/utils/reward_score/geo3k.py -./verl/utils/reward_score/gsm8k.py -./verl/utils/reward_score/math.py -./verl/utils/reward_score/math_dapo.py -./verl/utils/reward_score/math_verify.py -./verl/utils/reward_score/openhands_swebench/__init__.py -./verl/utils/reward_score/prime_code/__init__.py -./verl/utils/reward_score/prime_code/testing_util.py -./verl/utils/reward_score/prime_code/utils.py -./verl/utils/reward_score/prime_math/__init__.py -./verl/utils/reward_score/prime_math/grader.py -./verl/utils/reward_score/prime_math/math_normalize.py -./verl/version/version -./verl/workers/__init__.py -./verl/workers/fsdp_workers.py -./verl/workers/megatron_workers.py -./verl/workers/actor/__init__.py -./verl/workers/actor/base.py -./verl/workers/actor/dp_actor.py -./verl/workers/actor/megatron_actor.py -./verl/workers/agentic/__init__.py -./verl/workers/agentic/async_rollout.py -./verl/workers/agentic/codeact.py -./verl/workers/agentic/fsdp_sgl.py -./verl/workers/agentic/utils.py -./verl/workers/critic/__init__.py -./verl/workers/critic/base.py -./verl/workers/critic/dp_critic.py -./verl/workers/critic/megatron_critic.py -./verl/workers/reward_manager/__init__.py -./verl/workers/reward_manager/dapo.py -./verl/workers/reward_manager/naive.py -./verl/workers/reward_manager/prime.py -./verl/workers/reward_manager/swebench.py -./verl/workers/reward_manager/with_ray.py -./verl/workers/reward_model/__init__.py -./verl/workers/reward_model/base.py -./verl/workers/reward_model/megatron/__init__.py -./verl/workers/reward_model/megatron/reward_model.py -./verl/workers/rollout/__init__.py -./verl/workers/rollout/base.py -./verl/workers/rollout/hf_rollout.py -./verl/workers/rollout/tokenizer.py -./verl/workers/rollout/naive/__init__.py -./verl/workers/rollout/naive/naive_rollout.py -./verl/workers/rollout/sglang_rollout/__init__.py -./verl/workers/rollout/sglang_rollout/sglang_rollout.py -./verl/workers/rollout/vllm_rollout/__init__.py -./verl/workers/rollout/vllm_rollout/fire_vllm_rollout.py -./verl/workers/rollout/vllm_rollout/vllm_rollout.py -./verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py -./verl/workers/sharding_manager/__init__.py -./verl/workers/sharding_manager/base.py -./verl/workers/sharding_manager/fsdp_sglang.py -./verl/workers/sharding_manager/fsdp_ulysses.py -./verl/workers/sharding_manager/fsdp_vllm.py -./verl/workers/sharding_manager/megatron_vllm.py -tests/__init__.py -tests/test_reward.py -tests/e2e/__init__.py -tests/e2e/check_custom_rwd_fn.py -tests/e2e/check_results.py -tests/e2e/envs/__init__.py -tests/e2e/envs/digit_completion/__init__.py -tests/e2e/envs/digit_completion/task.py -tests/e2e/envs/digit_completion/tokenizer.py -verl/__init__.py -verl/protocol.py -verl.egg-info/PKG-INFO -verl.egg-info/SOURCES.txt -verl.egg-info/dependency_links.txt -verl.egg-info/requires.txt -verl.egg-info/top_level.txt -verl/models/__init__.py -verl/models/registry.py -verl/models/weight_loader_registry.py -verl/models/llama/__init__.py -verl/models/llama/megatron/__init__.py -verl/models/llama/megatron/modeling_llama_megatron.py -verl/models/llama/megatron/checkpoint_utils/__init__.py -verl/models/llama/megatron/checkpoint_utils/llama_loader.py -verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py -verl/models/llama/megatron/checkpoint_utils/llama_saver.py -verl/models/llama/megatron/layers/__init__.py -verl/models/llama/megatron/layers/parallel_attention.py -verl/models/llama/megatron/layers/parallel_decoder.py -verl/models/llama/megatron/layers/parallel_linear.py -verl/models/llama/megatron/layers/parallel_mlp.py -verl/models/llama/megatron/layers/parallel_rmsnorm.py -verl/models/mcore/__init__.py -verl/models/mcore/gpt_model.py -verl/models/mcore/loader.py -verl/models/mcore/saver.py -verl/models/qwen2/__init__.py -verl/models/qwen2/megatron/__init__.py -verl/models/qwen2/megatron/modeling_qwen2_megatron.py -verl/models/qwen2/megatron/checkpoint_utils/__init__.py -verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py -verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py -verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py -verl/models/qwen2/megatron/layers/__init__.py -verl/models/qwen2/megatron/layers/parallel_attention.py -verl/models/qwen2/megatron/layers/parallel_decoder.py -verl/models/qwen2/megatron/layers/parallel_linear.py -verl/models/qwen2/megatron/layers/parallel_mlp.py -verl/models/qwen2/megatron/layers/parallel_rmsnorm.py -verl/models/transformers/__init__.py -verl/models/transformers/llama.py -verl/models/transformers/monkey_patch.py -verl/models/transformers/qwen2.py -verl/models/transformers/qwen2_vl.py -verl/single_controller/__init__.py -verl/single_controller/base/__init__.py -verl/single_controller/base/decorator.py -verl/single_controller/base/worker.py -verl/single_controller/base/worker_group.py -verl/single_controller/base/megatron/__init__.py -verl/single_controller/base/megatron/worker.py -verl/single_controller/base/megatron/worker_group.py -verl/single_controller/base/register_center/__init__.py -verl/single_controller/base/register_center/ray.py -verl/single_controller/ray/__init__.py -verl/single_controller/ray/base.py -verl/single_controller/ray/megatron.py -verl/third_party/__init__.py -verl/third_party/sglang/__init__.py -verl/third_party/sglang/parallel_state.py -verl/third_party/vllm/__init__.py -verl/third_party/vllm/vllm_v_0_3_1/__init__.py -verl/third_party/vllm/vllm_v_0_3_1/arg_utils.py -verl/third_party/vllm/vllm_v_0_3_1/config.py -verl/third_party/vllm/vllm_v_0_3_1/llm.py -verl/third_party/vllm/vllm_v_0_3_1/llm_engine_sp.py -verl/third_party/vllm/vllm_v_0_3_1/model_loader.py -verl/third_party/vllm/vllm_v_0_3_1/model_runner.py -verl/third_party/vllm/vllm_v_0_3_1/parallel_state.py -verl/third_party/vllm/vllm_v_0_3_1/tokenizer.py -verl/third_party/vllm/vllm_v_0_3_1/weight_loaders.py -verl/third_party/vllm/vllm_v_0_3_1/worker.py -verl/third_party/vllm/vllm_v_0_4_2/__init__.py -verl/third_party/vllm/vllm_v_0_4_2/arg_utils.py -verl/third_party/vllm/vllm_v_0_4_2/config.py -verl/third_party/vllm/vllm_v_0_4_2/dtensor_weight_loaders.py -verl/third_party/vllm/vllm_v_0_4_2/hf_weight_loader.py -verl/third_party/vllm/vllm_v_0_4_2/llm.py -verl/third_party/vllm/vllm_v_0_4_2/llm_engine_sp.py -verl/third_party/vllm/vllm_v_0_4_2/megatron_weight_loaders.py -verl/third_party/vllm/vllm_v_0_4_2/model_loader.py -verl/third_party/vllm/vllm_v_0_4_2/model_runner.py -verl/third_party/vllm/vllm_v_0_4_2/parallel_state.py -verl/third_party/vllm/vllm_v_0_4_2/spmd_gpu_executor.py -verl/third_party/vllm/vllm_v_0_4_2/tokenizer.py -verl/third_party/vllm/vllm_v_0_4_2/worker.py -verl/third_party/vllm/vllm_v_0_5_4/__init__.py -verl/third_party/vllm/vllm_v_0_5_4/arg_utils.py -verl/third_party/vllm/vllm_v_0_5_4/config.py -verl/third_party/vllm/vllm_v_0_5_4/dtensor_weight_loaders.py -verl/third_party/vllm/vllm_v_0_5_4/hf_weight_loader.py -verl/third_party/vllm/vllm_v_0_5_4/llm.py -verl/third_party/vllm/vllm_v_0_5_4/llm_engine_sp.py -verl/third_party/vllm/vllm_v_0_5_4/megatron_weight_loaders.py -verl/third_party/vllm/vllm_v_0_5_4/model_loader.py -verl/third_party/vllm/vllm_v_0_5_4/model_runner.py -verl/third_party/vllm/vllm_v_0_5_4/parallel_state.py -verl/third_party/vllm/vllm_v_0_5_4/spmd_gpu_executor.py -verl/third_party/vllm/vllm_v_0_5_4/tokenizer.py -verl/third_party/vllm/vllm_v_0_5_4/worker.py -verl/third_party/vllm/vllm_v_0_6_3/__init__.py -verl/third_party/vllm/vllm_v_0_6_3/arg_utils.py -verl/third_party/vllm/vllm_v_0_6_3/config.py -verl/third_party/vllm/vllm_v_0_6_3/dtensor_weight_loaders.py -verl/third_party/vllm/vllm_v_0_6_3/hf_weight_loader.py -verl/third_party/vllm/vllm_v_0_6_3/llm.py -verl/third_party/vllm/vllm_v_0_6_3/llm_engine_sp.py -verl/third_party/vllm/vllm_v_0_6_3/megatron_weight_loaders.py -verl/third_party/vllm/vllm_v_0_6_3/model_loader.py -verl/third_party/vllm/vllm_v_0_6_3/model_runner.py -verl/third_party/vllm/vllm_v_0_6_3/parallel_state.py -verl/third_party/vllm/vllm_v_0_6_3/spmd_gpu_executor.py -verl/third_party/vllm/vllm_v_0_6_3/tokenizer.py -verl/third_party/vllm/vllm_v_0_6_3/worker.py -verl/trainer/__init__.py -verl/trainer/fsdp_sft_trainer.py -verl/trainer/main_eval.py -verl/trainer/main_generation.py -verl/trainer/main_ppo.py -verl/trainer/config/evaluation.yaml -verl/trainer/config/generation.yaml -verl/trainer/config/ppo_megatron_trainer.yaml -verl/trainer/config/ppo_trainer.yaml -verl/trainer/config/sft_trainer.yaml -verl/trainer/ppo/__init__.py -verl/trainer/ppo/core_algos.py -verl/trainer/ppo/metric_utils.py -verl/trainer/ppo/ray_trainer.py -verl/utils/__init__.py -verl/utils/config.py -verl/utils/distributed.py -verl/utils/flops_counter.py -verl/utils/fs.py -verl/utils/fsdp_utils.py -verl/utils/hdfs_io.py -verl/utils/import_utils.py -verl/utils/logging_utils.py -verl/utils/megatron_utils.py -verl/utils/memory_buffer.py -verl/utils/model.py -verl/utils/py_functional.py -verl/utils/ray_utils.py -verl/utils/seqlen_balancing.py -verl/utils/swedev_utils.py -verl/utils/tokenizer.py -verl/utils/torch_dtypes.py -verl/utils/torch_functional.py -verl/utils/tracking.py -verl/utils/ulysses.py -verl/utils/checkpoint/__init__.py -verl/utils/checkpoint/checkpoint_manager.py -verl/utils/checkpoint/fsdp_checkpoint_manager.py -verl/utils/checkpoint/megatron_checkpoint_manager.py -verl/utils/checkpoint/upload_utils.py -verl/utils/dataset/__init__.py -verl/utils/dataset/rl_dataset.py -verl/utils/dataset/rm_dataset.py -verl/utils/dataset/sft_dataset.py -verl/utils/debug/__init__.py -verl/utils/debug/performance.py -verl/utils/debug/trajectory_tracker.py -verl/utils/logger/__init__.py -verl/utils/logger/aggregate_logger.py -verl/utils/megatron/__init__.py -verl/utils/megatron/memory.py -verl/utils/megatron/optimizer.py -verl/utils/megatron/pipeline_parallel.py -verl/utils/megatron/sequence_parallel.py -verl/utils/megatron/tensor_parallel.py -verl/utils/rendezvous/__init__.py -verl/utils/rendezvous/ray_backend.py -verl/utils/reward_score/__init__.py -verl/utils/reward_score/geo3k.py -verl/utils/reward_score/gsm8k.py -verl/utils/reward_score/math.py -verl/utils/reward_score/math_dapo.py -verl/utils/reward_score/math_verify.py -verl/utils/reward_score/openhands_swebench/__init__.py -verl/utils/reward_score/prime_code/__init__.py -verl/utils/reward_score/prime_code/testing_util.py -verl/utils/reward_score/prime_code/utils.py -verl/utils/reward_score/prime_math/__init__.py -verl/utils/reward_score/prime_math/grader.py -verl/utils/reward_score/prime_math/math_normalize.py -verl/version/version -verl/workers/__init__.py -verl/workers/fsdp_workers.py -verl/workers/megatron_workers.py -verl/workers/actor/__init__.py -verl/workers/actor/base.py -verl/workers/actor/dp_actor.py -verl/workers/actor/megatron_actor.py -verl/workers/agentic/__init__.py -verl/workers/agentic/async_rollout.py -verl/workers/agentic/codeact.py -verl/workers/agentic/fsdp_sgl.py -verl/workers/agentic/utils.py -verl/workers/critic/__init__.py -verl/workers/critic/base.py -verl/workers/critic/dp_critic.py -verl/workers/critic/megatron_critic.py -verl/workers/reward_manager/__init__.py -verl/workers/reward_manager/dapo.py -verl/workers/reward_manager/naive.py -verl/workers/reward_manager/prime.py -verl/workers/reward_manager/swebench.py -verl/workers/reward_manager/with_ray.py -verl/workers/reward_model/__init__.py -verl/workers/reward_model/base.py -verl/workers/reward_model/megatron/__init__.py -verl/workers/reward_model/megatron/reward_model.py -verl/workers/rollout/__init__.py -verl/workers/rollout/base.py -verl/workers/rollout/hf_rollout.py -verl/workers/rollout/tokenizer.py -verl/workers/rollout/naive/__init__.py -verl/workers/rollout/naive/naive_rollout.py -verl/workers/rollout/sglang_rollout/__init__.py -verl/workers/rollout/sglang_rollout/sglang_rollout.py -verl/workers/rollout/vllm_rollout/__init__.py -verl/workers/rollout/vllm_rollout/fire_vllm_rollout.py -verl/workers/rollout/vllm_rollout/vllm_rollout.py -verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py -verl/workers/sharding_manager/__init__.py -verl/workers/sharding_manager/base.py -verl/workers/sharding_manager/fsdp_sglang.py -verl/workers/sharding_manager/fsdp_ulysses.py -verl/workers/sharding_manager/fsdp_vllm.py -verl/workers/sharding_manager/megatron_vllm.py \ No newline at end of file diff --git a/verl.egg-info/dependency_links.txt b/verl.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789179..0000000000 --- a/verl.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/verl.egg-info/requires.txt b/verl.egg-info/requires.txt deleted file mode 100644 index 2424fa9661..0000000000 --- a/verl.egg-info/requires.txt +++ /dev/null @@ -1,61 +0,0 @@ -accelerate -codetiming -datasets -dill -hydra-core -numpy -pandas -datasets -peft -pyarrow>=15.0.0 -pybind11 -pylatexenc -ray[default]>=2.10 -tensordict<=0.6.2 -torchdata -transformers -wandb -hf_transfer -torchdata -openhands-ai -sglang[all]>=0.4.6.post1 -flash-attn@ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abifalse-cp312-cp312-linux_x86_64.whl -streamlit -whatthepatch -retry -evaluate -swebench@ https://github.com/SWE-Gym/SWE-Bench-Fork.git -swegym@ https://github.com/SWE-Gym/SWE-Bench-Package.git -commit0 -func_timeout -sympy -gdown -matplotlib -seaborn -tabulate -browsergym==0.10.2 -browsergym-webarena==0.10.2 -browsergym-miniwob==0.10.2 -browsergym-visualwebarena==0.10.2 -tensordict<=0.6.2 -torch-memory-saver>=0.0.5 -vllm>=0.7.3 - -[geo] -mathruler - -[gpu] -liger-kernel -flash-attn - -[math] -math-verify - -[test] -pytest -yapf -py-spy - -[vllm] -tensordict<=0.6.2 -vllm<=0.8.2 diff --git a/verl.egg-info/top_level.txt b/verl.egg-info/top_level.txt deleted file mode 100644 index 79460bbcbf..0000000000 --- a/verl.egg-info/top_level.txt +++ /dev/null @@ -1,2 +0,0 @@ -tests -verl diff --git a/verl/__init__.py b/verl/__init__.py deleted file mode 100644 index 9b174f43df..0000000000 --- a/verl/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__))) - -with open(os.path.join(version_folder, 'version/version')) as f: - __version__ = f.read().strip() - -from .protocol import DataProto - -from .utils.logging_utils import set_basic_config -import logging - -set_basic_config(level=logging.WARNING) - -from . import single_controller - -__all__ = ['DataProto', "__version__"] - -if os.getenv('VERL_USE_MODELSCOPE', 'False').lower() == 'true': - import importlib - if importlib.util.find_spec("modelscope") is None: - raise ImportError(f'You are using the modelscope hub, please install modelscope by `pip install modelscope -U`') - # Patch hub to download models from modelscope to speed up. - from modelscope.utils.hf_util import patch_hub - patch_hub() diff --git a/verl/models/README.md b/verl/models/README.md deleted file mode 100644 index 677b92f387..0000000000 --- a/verl/models/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Models -Common modelzoo such as huggingface/transformers stuggles when using Pytorch native model parallelism. Following the design principle of vLLM, we keep a simple, parallelizable, highly-optimized with packed inputs in verl. -## Adding a New Huggingface Model -### Step 1: Copy the model file from HF to verl -- Add a new file under verl/models/hf -- Copy ONLY the model file from huggingface/transformers/models to verl/models/hf - -### Step 2: Modify the model file to use packed inputs -- Remove all the code related to inference (kv cache) -- Modify the inputs to include only - - input_ids (total_nnz,) - - cu_seqlens (total_nnz + 1,) - - max_seqlen_in_batch: int -- Note that this requires using flash attention with causal mask. - -### Step 2.5: Add tests -- Add a test to compare this version and the huggingface version -- Following the infrastructure and add tests to tests/models/hf - -### Step 3: Add a function to apply tensor parallelism -- Please follow - - https://pytorch.org/docs/stable/distributed.tensor.parallel.html - - https://pytorch.org/tutorials/intermediate/TP_tutorial.html -- General comments - - Tensor Parallelism in native Pytorch is NOT auto-parallelism. The way it works is to specify how model parameters and input/output reshards using configs. These configs are then registered as hooks to perform input/output resharding before/after model forward. - -### Step 4: Add a function to apply data parallelism -- Please use FSDP2 APIs -- See demo here https://github.com/pytorch/torchtitan/blob/main/torchtitan/parallelisms/parallelize_llama.py#L413 - -### Step 5: Add a function to apply pipeline parallelism -- Comes in Pytorch 2.4 -- Currently only in alpha in nightly version -- Check torchtitan for more details - diff --git a/verl/models/__init__.py b/verl/models/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/models/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/models/llama/__init__.py b/verl/models/llama/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/models/llama/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/models/llama/megatron/__init__.py b/verl/models/llama/megatron/__init__.py deleted file mode 100644 index b188b3ee62..0000000000 --- a/verl/models/llama/megatron/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .modeling_llama_megatron import ( - # original model with megatron - ParallelLlamaModel, - ParallelLlamaForCausalLM, - # rmpad with megatron - ParallelLlamaForCausalLMRmPad, - ParallelLlamaForValueRmPad, - # rmpad with megatron and pipeline parallelism - ParallelLlamaForCausalLMRmPadPP, - ParallelLlamaForValueRmPadPP) diff --git a/verl/models/llama/megatron/checkpoint_utils/__init__.py b/verl/models/llama/megatron/checkpoint_utils/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/models/llama/megatron/checkpoint_utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/models/llama/megatron/checkpoint_utils/llama_loader.py b/verl/models/llama/megatron/checkpoint_utils/llama_loader.py deleted file mode 100644 index 4f602b5f46..0000000000 --- a/verl/models/llama/megatron/checkpoint_utils/llama_loader.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -from packaging.version import Version -import torch -import time -from typing import Dict, Any, Callable, Optional -import torch.distributed as dist - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - from megatron.core import mpu - - print(f'get megatron data parallel size: {mpu.get_data_parallel_world_size()}') - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def load_state_dict_to_megatron_llama(state_dict, - wrapped_models, - config, - params_dtype, - is_value_model=False, - tie_word_embeddings=False): - """Load merged state_dict to sharded Megatron module in training. - """ - from megatron.core import mpu - from verl.utils.megatron_utils import print_rank_0, unwrap_model - from megatron.core.transformer.module import Float16Module - from megatron.core import DistributedDataParallel as LocalDDP - from torch.nn.parallel import DistributedDataParallel as torchDDP - - start_time = time.time() - - def _get_gpt_model(model): - return model - - def fetch_params(module): - for param in module.parameters(): - torch.distributed.fetch(param.data, - src=mpu.get_data_parallel_src_rank(), - group=mpu.get_data_parallel_group()) - - dp_rank = mpu.get_data_parallel_rank() - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if torch.distributed.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, f'num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size {virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}' - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - gpt_model_module = _get_gpt_model(models[i]) - assert len(gpt_model_module.model.layers) == num_layers_per_model - - def _fetch_tensor(tensor, name) -> torch.Tensor: - """fetch tensor""" - nonlocal state_dict - if tensor is not None: - tensor.data.copy_(state_dict[name]) - - def _fetch_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """fetch tensor in tp shards""" - nonlocal state_dict - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - if name in state_dict: - full_weight = state_dict[name] - - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - if tensor is not None: - tensor.data.copy_(tensor_chunk[tp_rank]) - else: - print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - - def _fetch_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """fetch tensor in tp shards""" - nonlocal state_dict - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - if name in state_dict: - full_weight = state_dict[name] - - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - if tensor is not None: - tensor.data.copy_(tensor_chunk[tp_rank]) - else: - print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - - def _fetch_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: - """fetch gate_up tensor in tp shards""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - if gate_name in state_dict and up_name in state_dict: - gate_weight = state_dict[gate_name] - up_weight = state_dict[up_name] - new_gate_up_weight = torch.empty(config.intermediate_size * 2, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_tp = gate_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - up_weight_tp = up_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - new_gate_up_weight[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)].copy_( - torch.cat([gate_weight_tp, up_weight_tp], dim=0)) - - tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) - if tensor is not None: - tensor.data.copy_(tensor_chunk[tp_rank]) - else: - print(f"tp_shard tensor:[{gate_name}, {up_name}] not in state_dict, skip loading") - - def _fetch_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name) -> torch.Tensor: - """fetch tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - assert (q_name in state_dict and k_name in state_dict and v_name in state_dict) - full_weight_q = state_dict[q_name] - full_weight_k = state_dict[k_name] - full_weight_v = state_dict[v_name] - - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - k_part = full_weight_k[i * kv_size_tp:(i + 1) * kv_size_tp] - v_part = full_weight_v[i * kv_size_tp:(i + 1) * kv_size_tp] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) - - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head - end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head - k_part = full_weight_k[start_idx:end_idx] - v_part = full_weight_v[start_idx:end_idx] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) - - tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) - if tensor is not None: - tensor.data.copy_(tensor_chunk[tp_rank]) - - # Embeddings - # ------------------- - print_rank_0("loading embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - embed_tokens_weight = None - if pp_rank == 0: - embed_tokens_weight = gpt_model_module.model.embed_tokens.weight - _fetch_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - num_layer_per_pp = config.num_hidden_layers // pp_size - vpp_size = mpu.get_virtual_pipeline_model_parallel_world_size() - - layer_list = [] - if vpp_size is not None: - for vpp_rank in range(vpp_size): - num_layer_vpp_chunk = num_layer_per_pp // vpp_size - num_layer_this_model = num_layer_vpp_chunk - offset = vpp_rank * ( - config.num_hidden_layers // mpu.get_virtual_pipeline_model_parallel_world_size()) + \ - (mpu.get_pipeline_model_parallel_rank() * num_layer_vpp_chunk) - layer_list.extend(list(range(offset, offset + num_layer_this_model))) - else: - num_layer_this_model = num_layer_per_pp - offset = pp_rank * num_layer_per_pp - layer_list.extend(list(range(offset, offset + num_layer_this_model))) - - for layer in layer_list: - print_rank_0(f"loading layer #{layer}...") - layer_name = f"model.layers.{layer}" - dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] - - gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) - sync_layer = gpt_model_module.model.layers[dst_layer_idx] - - _fetch_tensor( - sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.input_layernorm.weight", - ) - - _fetch_tp_shard_tensor_qkv( - sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - ) - - _fetch_tp_shard_tensor( - sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.o_proj.weight", - chunk_dim=1, - ) - - _fetch_tensor( - sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.post_attention_layernorm.weight", - ) - - _fetch_tp_shard_tensor_gate_up(sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.gate_proj.weight", f"{layer_name}.mlp.up_proj.weight") - - _fetch_tp_shard_tensor( - sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.down_proj.weight", - chunk_dim=1, - ) - # Final Layernorm - # ------------------- - print_rank_0("loading final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _fetch_tensor( - getattr(gpt_model_module.model.norm, "weight", None), - "model.norm.weight", - ) - - print_rank_0("loading lm_head...") - if pp_rank + 1 == pp_size: - lm_head_weight = gpt_model_module.lm_head.weight - - if is_value_model: - if 'lm_head.weight' in state_dict and state_dict['lm_head.weight'].shape[0] == 1: - _fetch_tensor(lm_head_weight, "lm_head.weight") - print_rank_0('load lm_head weight') - elif 'reward_head.weight' in state_dict and state_dict['reward_head.weight'].shape[0] == 1: - _fetch_tensor(lm_head_weight, "reward_head.weight") - print_rank_0('load lm_head from value_head weight') - else: - _fetch_tensor(None, "lm_head.weight") - print_rank_0('fail to match lm_head in value_model') - else: - _fetch_tp_shard_tensor(lm_head_weight, "lm_head.weight") - - dist.barrier() - torch.cuda.empty_cache() - print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py b/verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py deleted file mode 100644 index 7f1b6a4a02..0000000000 --- a/verl/models/llama/megatron/checkpoint_utils/llama_loader_depracated.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -from packaging.version import Version -import torch -import time -from typing import Dict, Any, Callable, Optional -import torch.distributed as dist - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - from megatron.core import mpu - - print(f'get megatron data parallel size: {mpu.get_data_parallel_world_size()}') - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def load_state_dict_to_megatron_llama(state_dict, - wrapped_models, - config, - params_dtype, - is_value_model=False, - tie_word_embeddings=False): - """Load merged state_dict to sharded Megatron module in training. - """ - from megatron.core import mpu - from verl.utils.megatron_utils import print_rank_0, unwrap_model - from megatron.core.transformer.module import Float16Module - from megatron.core import DistributedDataParallel as LocalDDP - from torch.nn.parallel import DistributedDataParallel as torchDDP - - start_time = time.time() - - def _get_gpt_model(model): - return model - - def broadcast_params(module): - for param in module.parameters(): - torch.distributed.broadcast(param.data, - src=mpu.get_data_parallel_src_rank(), - group=mpu.get_data_parallel_group()) - - dp_rank = mpu.get_data_parallel_rank() - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if torch.distributed.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, f'num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size {virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}' - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - gpt_model_module = _get_gpt_model(models[i]) - assert len(gpt_model_module.model.layers) == num_layers_per_model - - def _broadcast_tensor(tensor, name) -> torch.Tensor: - """broadcast tensor from rank0 across mp_group""" - nonlocal state_dict - nonlocal mp_group - if torch.distributed.get_rank() == 0: - if name in state_dict: - weight = state_dict[name] - tensor_shape = weight.shape - else: - tensor_shape = None - else: - weight = None - tensor_shape = None - - obj_list = [tensor_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - tensor_shape = obj_list[0] - - if tensor_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tensor:[{name}] not in state_dict, skip load") - return - - if tensor is None: - tensor = torch.empty( - tensor_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - if torch.distributed.get_rank() == 0: - tensor.data.copy_(weight) - dist.broadcast(tensor, src=0, group=mp_group) - - def _broadcast_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - if name in state_dict: - full_weight = state_dict[name] - - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - if name in state_dict: - full_weight = state_dict[name] - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - gate_weight = state_dict[gate_name] - up_weight = state_dict[up_name] - new_gate_up_weight = torch.empty(config.intermediate_size * 2, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_tp = gate_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - up_weight_tp = up_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - new_gate_up_weight[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)].copy_( - torch.cat([gate_weight_tp, up_weight_tp], dim=0)) - - tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert ( - tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank() == 0:} tensor {gate_name, up_name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - assert (q_name in state_dict and k_name in state_dict and v_name in state_dict) - full_weight_q = state_dict[q_name] - full_weight_k = state_dict[k_name] - full_weight_v = state_dict[v_name] - - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - k_part = full_weight_k[i * kv_size_tp:(i + 1) * kv_size_tp] - v_part = full_weight_v[i * kv_size_tp:(i + 1) * kv_size_tp] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], - dim=0)) - - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head - end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head - k_part = full_weight_k[start_idx:end_idx] - v_part = full_weight_v[start_idx:end_idx] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], - dim=0)) - - tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{q_name, k_name, v_name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {q_name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - if dp_rank == 0: - # Embeddings - # ------------------- - print_rank_0("loading embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - embed_tokens_weight = None - if pp_rank == 0: - embed_tokens_weight = gpt_model_module.model.embed_tokens.weight - _broadcast_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - - for layer in range(config.num_hidden_layers): - print_rank_0(f"loading layer #{layer}...") - layer_name = f"model.layers.{layer}" - dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] - - gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) - sync_layer = gpt_model_module.model.layers[dst_layer_idx] - - _broadcast_tensor( - sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.input_layernorm.weight", - ) - - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - ) - - _broadcast_tp_shard_tensor( - sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.o_proj.weight", - chunk_dim=1, - ) - - _broadcast_tensor( - sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.post_attention_layernorm.weight", - ) - - _broadcast_tp_shard_tensor_gate_up(sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.gate_proj.weight", f"{layer_name}.mlp.up_proj.weight") - - _broadcast_tp_shard_tensor( - sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.down_proj.weight", - chunk_dim=1, - ) - # Final Layernorm - # ------------------- - print_rank_0("loading final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _broadcast_tensor( - getattr(gpt_model_module.model.norm, "weight", None), - "model.norm.weight", - ) - - print_rank_0("loading lm_head...") - lm_head_weight = None - if pp_rank + 1 == pp_size: - lm_head_weight = gpt_model_module.lm_head.weight - - if is_value_model: - if 'lm_head.weight' in state_dict and state_dict['lm_head.weight'].shape[0] == 1: - _broadcast_tensor(lm_head_weight, "lm_head.weight") - print_rank_0('load lm_head weight') - elif 'reward_head.weight' in state_dict and state_dict['reward_head.weight'].shape[0] == 1: - _broadcast_tensor(lm_head_weight, "reward_head.weight") - print_rank_0('load lm_head from value_head weight') - else: - _broadcast_tensor(None, "lm_head.weight") - print_rank_0('fail to match lm_head in value_model') - else: - _broadcast_tp_shard_tensor(lm_head_weight, "lm_head.weight") - dist.barrier() - # Broadcast weights inside data parallel groups - for wrapped_model in wrapped_models: - broadcast_params(wrapped_model) - - torch.cuda.empty_cache() - print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") \ No newline at end of file diff --git a/verl/models/llama/megatron/checkpoint_utils/llama_saver.py b/verl/models/llama/megatron/checkpoint_utils/llama_saver.py deleted file mode 100644 index 5ee15a9498..0000000000 --- a/verl/models/llama/megatron/checkpoint_utils/llama_saver.py +++ /dev/null @@ -1,441 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -import torch -import torch.distributed as dist -from megatron.core import mpu -from megatron.core.distributed import DistributedDataParallel as LocalDDP -from megatron.core.transformer.module import Float16Module -from torch.nn.parallel import DistributedDataParallel as torchDDP - -from verl.utils.megatron_utils import print_rank_0, unwrap_model - - -def _megatron_calc_global_rank(tp_rank: int = 0, dp_rank: int = 0, pp_rank: int = 0): - """given TP,DP,PP rank to get the global rank.""" - - tp_size = mpu.get_tensor_model_parallel_world_size() - dp_size = mpu.get_data_parallel_world_size() - pp_size = mpu.get_pipeline_model_parallel_world_size() - assert (tp_size * dp_size * pp_size == torch.distributed.get_world_size() - ), f"{tp_size} x {dp_size} x {pp_size} != {torch.distributed.get_world_size()}" - # We only support TP-DP-PP grouping, for correctness when resharding - return (pp_rank * dp_size + dp_rank) * tp_size + tp_rank - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - from megatron.core import mpu - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def merge_megatron_ckpt_llama(wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False): - """Merge sharded parameters of a Megatron module into a merged checkpoint. - - Args: - wrapped_models (list of megatron.core.distributed.DistributedDataParallel): - The local DDP wrapped megatron modules. - config (str or None): - HF config for model - dtype: model params type - is_value_model: if model is value model - tie_word_embeddings: tie_word_embeddings, not used in llama, only to keep same interface with qwen2 - Returns: - state_dict (dict): - The merged state_dict in rank 0, and an empty dictionary in other ranks. - """ - start_time = time.time() - - def _get_gpt_model(model): - return model - - dp_rank = mpu.get_data_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - pp_rank = mpu.get_pipeline_model_parallel_rank() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if dist.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - assert len(models[i].model.layers - ) == num_layers_per_model, 'len model layers {} not equal to num_layers_per_model {}'.format( - len(models[i].model.layers), num_layers_per_model) - - state_dict = dict() - - def _get_cpu_tensor(tensor: torch.Tensor): - if tensor is None: - return None - if tensor.device == torch.device("cpu"): - return tensor.detach().clone() - return tensor.detach().cpu() - - def _broadcast_tensor(tensor, name, src_pp_rank) -> torch.Tensor: - """broadcast tensor across mp_group""" - nonlocal state_dict - nonlocal mp_group - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - if tensor is None: - weight = None - tensor_shape = None - else: - weight = tensor - tensor_shape = weight.shape - else: - weight = None - tensor_shape = None - - obj_list = [tensor_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - tensor_shape = obj_list[0] - - if tensor_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tensor:[{name}] not exist, skip collect") - return - - if weight is None: - weight = torch.empty( - tensor_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - dist.broadcast(weight, src=src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - state_dict[name] = _get_cpu_tensor(weight) - - def _broadcast_tp_shard_tensor(tensor, name, src_pp_rank, concat_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=concat_dim) - if mutate_func is not None: - full_tensor = mutate_func(full_tensor) - state_dict[name] = full_tensor - - def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name, src_pp_rank) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=0) - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_list = [] - up_weight_list = [] - for i in range(tp_size): - gate_up_weight_tp = full_tensor[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)] - gate_weight_tp = gate_up_weight_tp[:intermediate_size_tp] - up_weight_tp = gate_up_weight_tp[intermediate_size_tp:] - gate_weight_list.append(gate_weight_tp) - up_weight_list.append(up_weight_tp) - - state_dict[gate_name] = torch.cat(gate_weight_list, dim=0) - state_dict[up_name] = torch.cat(up_weight_list, dim=0) - - def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, src_pp_rank): - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{q_name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=0) - q_weight_list = [] - k_weight_list = [] - v_weight_list = [] - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - for i in range(tp_size): - qkv_part = full_tensor[i * total_size:(i + 1) * total_size] - q_part = qkv_part[:q_size_tp] - k_part = qkv_part[q_size_tp:q_size_tp + kv_size_tp] - v_part = qkv_part[q_size_tp + kv_size_tp:total_size] - q_weight_list.append(q_part) - k_weight_list.append(k_part) - v_weight_list.append(v_part) - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - for i in range(tp_size): - qkv_part = full_tensor[i * total_size:(i + 1) * total_size] - q_part = qkv_part[:q_size_tp] - k_part = qkv_part[q_size_tp:q_size_tp + kv_size_tp] - v_part = qkv_part[q_size_tp + kv_size_tp:total_size] - q_weight_list.append(q_part) - if i * config.num_key_value_heads % tp_size == 0: - k_weight_list.append(k_part) - v_weight_list.append(v_part) - - state_dict[q_name] = torch.cat(q_weight_list, dim=0) - state_dict[k_name] = torch.cat(k_weight_list, dim=0) - state_dict[v_name] = torch.cat(v_weight_list, dim=0) - - # empty cache before collecting weights - torch.cuda.empty_cache() - # Embeddings - # ------------------- - if dp_rank == 0: - # Embeddings - # ------------------- - print_rank_0("collecting embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - _broadcast_tp_shard_tensor( - gpt_model_module.model.embed_tokens.weight if pp_rank == 0 else None, - "model.embed_tokens.weight", - src_pp_rank=0, - ) - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - for layer in range(config.num_hidden_layers): - print_rank_0(f"collecting layer #{layer}...") - layer_name = f"model.layers.{layer}" - src_pp_rank, src_virtual_pp_rank, src_layer_idx = layer_map[layer] - - gpt_model_module = _get_gpt_model(models[src_virtual_pp_rank]) - sync_layer = gpt_model_module.model.layers[src_layer_idx] - - _broadcast_tensor( - sync_layer.input_layernorm.weight, - f"{layer_name}.input_layernorm.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attn.qkv_proj.weight, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor( - sync_layer.self_attn.o_proj.weight, - f"{layer_name}.self_attn.o_proj.weight", - concat_dim=1, - src_pp_rank=src_pp_rank, - ) - - _broadcast_tensor( - sync_layer.post_attention_layernorm.weight, - f"{layer_name}.post_attention_layernorm.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor_gate_up(sync_layer.mlp.gate_up_proj.weight, - f"{layer_name}.mlp.gate_proj.weight", - f"{layer_name}.mlp.up_proj.weight", - src_pp_rank=src_pp_rank) - - _broadcast_tp_shard_tensor( - sync_layer.mlp.down_proj.weight, - f"{layer_name}.mlp.down_proj.weight", - concat_dim=1, - src_pp_rank=src_pp_rank, - ) - - # Final Layernorm - # ------------------- - print_rank_0("collecting final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _broadcast_tensor( - getattr(gpt_model_module.model.norm, "weight", None), - "model.norm.weight", - src_pp_rank=pp_size - 1, - ) - - print_rank_0("collecting lm_head...") - - if is_value_model: - if pp_rank == pp_size - 1: - print(f'gpt_model_module.lm_head.weight: {gpt_model_module.lm_head.weight.shape}') - _broadcast_tensor(gpt_model_module.lm_head.weight if pp_rank == pp_size - 1 else None, - "lm_head.weight", - src_pp_rank=pp_size - 1) - _broadcast_tensor(gpt_model_module.reward_head.weight if pp_rank == pp_size - 1 and - getattr(gpt_model_module, "reward_weight", None) is not None else None, - "reward_head.weight", - src_pp_rank=pp_size - 1) - - else: - _broadcast_tp_shard_tensor( - getattr(gpt_model_module.lm_head, "weight", None) if pp_rank == pp_size - 1 else None, - "lm_head.weight", - src_pp_rank=pp_size - 1, - ) - - dist.barrier() - - torch.cuda.empty_cache() - if torch.distributed.get_rank() == 0: - if dtype not in [torch.float16, torch.bfloat16, torch.float32]: - print(f'Unknown/unsupported dtype to save: {dtype}"') - exit(1) - for k, v in state_dict.items(): - if dtype != v.dtype: - state_dict[k] = v.to(dtype) - - print_rank_0(f"merge megatron ckpt done, time elapsed {time.time() - start_time}s") - return state_dict diff --git a/verl/models/llama/megatron/layers/__init__.py b/verl/models/llama/megatron/layers/__init__.py deleted file mode 100644 index 3761bae7db..0000000000 --- a/verl/models/llama/megatron/layers/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .parallel_attention import ParallelLlamaAttention -from .parallel_decoder import ParallelLlamaDecoderLayer, ParallelLlamaDecoderLayerRmPad -from .parallel_mlp import ParallelLlamaMLP -from .parallel_rmsnorm import ParallelLlamaRMSNorm diff --git a/verl/models/llama/megatron/layers/parallel_attention.py b/verl/models/llama/megatron/layers/parallel_attention.py deleted file mode 100644 index 3d3ecd7906..0000000000 --- a/verl/models/llama/megatron/layers/parallel_attention.py +++ /dev/null @@ -1,458 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import math -from typing import Optional, Tuple - -import torch -from megatron.core import parallel_state as mpu -from megatron.core import tensor_parallel -from megatron.core import ModelParallelConfig -from torch import nn -from transformers import LlamaConfig -from verl.models.llama.megatron.layers.parallel_linear import QKVParallelLinear - -from verl.utils.megatron import tensor_parallel as tp_utils - - -class LlamaRotaryEmbedding(nn.Module): - - def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): - super().__init__() - - self.dim = dim - self.max_position_embeddings = max_position_embeddings - self.base = base - inv_freq = 1.0 / (self.base**(torch.arange(0, self.dim, 2).float().to(device) / self.dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - # Build here to make `torch.jit.trace` work. - self._set_cos_sin_cache(seq_len=max_position_embeddings, - device=self.inv_freq.device, - dtype=torch.get_default_dtype()) - - def _set_cos_sin_cache(self, seq_len, device, dtype): - self.max_seq_len_cached = seq_len - t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - - freqs = torch.einsum("i,j->ij", t, self.inv_freq) - # Different from paper, but it uses a different permutation in order to obtain the same calculation - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - - def forward(self, x, seq_len=None): - # x: [bs, num_attention_heads, seq_len, head_size] - if seq_len > self.max_seq_len_cached: - self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) - - return ( - self.cos_cached[:seq_len].to(dtype=x.dtype), - self.sin_cached[:seq_len].to(dtype=x.dtype), - ) - - -class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding): - """LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev""" - - def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): - self.scaling_factor = scaling_factor - super().__init__(dim, max_position_embeddings, base, device) - - def _set_cos_sin_cache(self, seq_len, device, dtype): - self.max_seq_len_cached = seq_len - t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - t = t / self.scaling_factor - - freqs = torch.einsum("i,j->ij", t, self.inv_freq) - # Different from paper, but it uses a different permutation in order to obtain the same calculation - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - - -class LlamaDynamicNTKScalingRotaryEmbedding(LlamaRotaryEmbedding): - """LlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla""" - - def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): - self.scaling_factor = scaling_factor - super().__init__(dim, max_position_embeddings, base, device) - - def _set_cos_sin_cache(self, seq_len, device, dtype): - self.max_seq_len_cached = seq_len - - if seq_len > self.max_position_embeddings: - base = self.base * ((self.scaling_factor * seq_len / self.max_position_embeddings) - - (self.scaling_factor - 1))**(self.dim / (self.dim - 2)) - inv_freq = 1.0 / (base**(torch.arange(0, self.dim, 2).float().to(device) / self.dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - - freqs = torch.einsum("i,j->ij", t, self.inv_freq) - # Different from paper, but it uses a different permutation in order to obtain the same calculation - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - - -class LlamaLlama3ScalingRotaryEmbedding(LlamaRotaryEmbedding): - - def __init__(self, dim, config, max_position_embeddings=2048, base=10000, device=None): - super().__init__(dim, max_position_embeddings, base, device) - - self.factor = config.rope_scaling["factor"] # `8` in the original implementation - self.high_freq_factor = config.rope_scaling["high_freq_factor"] # `1` in the original implementation - self.low_freq_factor = config.rope_scaling["low_freq_factor"] # `4` in the original implementation - self.old_context_len = config.rope_scaling[ - "original_max_position_embeddings"] # `8192` in the original implementation - - low_freq_wavelen = self.old_context_len / self.low_freq_factor - high_freq_wavelen = self.old_context_len / self.high_freq_factor - - wavelen = 2 * math.pi / self.inv_freq - # wavelen < high_freq_wavelen: do nothing; wavelen > low_freq_wavelen: divide by factor - inv_freq_llama = torch.where(wavelen > low_freq_wavelen, self.inv_freq / self.factor, self.inv_freq) - # otherwise: interpolate between the two, using a smooth factor - smooth_factor = (self.old_context_len / wavelen - self.low_freq_factor) / (self.high_freq_factor - - self.low_freq_factor) - smoothed_inv_freq = (1 - smooth_factor) * inv_freq_llama / self.factor + smooth_factor * inv_freq_llama - is_medium_freq = ~(wavelen < high_freq_wavelen) * ~(wavelen > low_freq_wavelen) - inv_freq = torch.where(is_medium_freq, smoothed_inv_freq, inv_freq_llama) - - self.register_buffer("inv_freq", inv_freq, persistent=False) - - # Build here to make `torch.jit.trace` work. - self._set_cos_sin_cache(seq_len=max_position_embeddings, - device=self.inv_freq.device, - dtype=torch.get_default_dtype()) - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., :x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2:] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb(q, k, cos, sin, position_ids): - cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] - sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - - -class ParallelLlamaAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): - super().__init__() - self.config = config - self.megatron_config = megatron_config - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - - # assign values after tp - tp_size = mpu.get_tensor_model_parallel_world_size() - assert self.num_heads % tp_size == 0, f'num_head must be divisible by tp_size. Got num_head={self.num_heads}, tp_size={tp_size}' - assert self.num_key_value_heads % tp_size == 0, \ - f'num_key_value_heads must be divisible by tp_size. Got num_key_value_heads={self.num_key_value_heads}, tp_size={tp_size}' - - self.num_heads_per_tp = self.num_heads // tp_size - self.num_key_value_heads_per_tp = self.num_key_value_heads // tp_size - self.hidden_size_per_tp = self.hidden_size // tp_size - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError(f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads}).") - - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() - - if megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - assert row_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) - tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) - - # [self.q_size, self.k_size, self.v_size] - self.qkv_proj = QKVParallelLinear(input_size=self.hidden_size, - num_heads=self.num_heads, - num_key_value_heads=self.num_key_value_heads, - head_dim=self.head_dim, - bias=config.attention_bias, - gather_output=False, - skip_bias_add=False, - **column_kwargs) - - self.q_size = self.num_heads_per_tp * self.head_dim - self.k_size = self.num_key_value_heads_per_tp * self.head_dim - self.v_size = self.num_key_value_heads_per_tp * self.head_dim - - self.o_proj = tensor_parallel.RowParallelLinear(input_size=self.num_heads * self.head_dim, - output_size=self.hidden_size, - bias=config.attention_bias, - input_is_parallel=True, - skip_bias_add=False, - **row_kwargs) - - self._init_rope() - - def _init_rope(self): - if self.config.rope_scaling is None: - self.rotary_emb = LlamaRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - else: - rope_type_key = "type" if "type" in self.config.rope_scaling else "rope_type" - scaling_type = self.config.rope_scaling[rope_type_key] - scaling_factor = self.config.rope_scaling["factor"] - if scaling_type == "linear": - self.rotary_emb = LlamaLinearScalingRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - scaling_factor=scaling_factor, - base=self.rope_theta, - ) - elif scaling_type == "dynamic": - self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - scaling_factor=scaling_factor, - base=self.rope_theta, - ) - elif scaling_type == "llama3": - self.rotary_emb = LlamaLlama3ScalingRotaryEmbedding( - self.head_dim, - self.config, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - else: - raise ValueError(f"Unknown RoPE scaling type {scaling_type}") - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - bsz, q_len, _ = hidden_states.size() - qkv = self.qkv_proj(hidden_states)[0] - query_states, key_states, value_states = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) - - query_states = query_states.view(bsz, q_len, self.num_heads_per_tp, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attn_weights.size() != (bsz, self.num_heads_per_tp, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads_per_tp, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}") - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}") - attn_weights = attn_weights + attention_mask - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads_per_tp, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads_per_tp, q_len, self.head_dim)}, but is" - f" {attn_output.size()}") - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size_per_tp) - attn_output = self.o_proj(attn_output)[0] - return attn_output - - -""" -Remove padding Attention -- Using Flash-attn 2 -- Compatible with sequence parallel -""" - -from transformers.utils import is_flash_attn_2_available -import torch.nn.functional as F - -from einops import rearrange - -if is_flash_attn_2_available(): - from flash_attn import flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - - -def apply_rotary_pos_emb_rmpad(q, k, cos, sin, position_ids, indices, sequence_length): - batch_size = position_ids.shape[0] - - q = pad_input(q, indices, batch_size, sequence_length) # (batch_size, seqlen, num_head, head_dim) - k = pad_input(k, indices, batch_size, sequence_length) - cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] - sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - - q_embed = index_first_axis(rearrange(q_embed, "b s ... -> (b s) ..."), indices) - k_embed = index_first_axis(rearrange(k_embed, "b s ... -> (b s) ..."), indices) - - return q_embed, k_embed - - -from flash_attn.layers.rotary import apply_rotary_emb - - -# use flash-attn rotary embeddings with rmpad -# cos/sin shoudl be: (seq_length, rotary_dim / 2) -def apply_rotary_pos_emb_rmpad_flash(q, k, cos, sin, cu_seqlens, max_seqlen): - q_embed = apply_rotary_emb(q, - cos, - sin, - interleaved=False, - inplace=False, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen) - k_embed = apply_rotary_emb(k, - cos, - sin, - interleaved=False, - inplace=False, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen) - return q_embed, k_embed - - -class ParallelLlamaAttentionRmPad(ParallelLlamaAttention): - - def forward(self, - hidden_states: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: torch.Tensor = None, - max_seqlen_in_batch: int = None): - total_nnz, _, _ = hidden_states.size() # This is the total_nnz padded after sequence parallel - - if self.megatron_config.sequence_parallel: - total_nnz = total_nnz * mpu.get_tensor_model_parallel_world_size() - - qkv = self.qkv_proj(hidden_states)[0] - query_states, key_states, value_states = qkv.split([self.q_size, self.k_size, self.v_size], - dim=-1) # (total_nnz, 1, hidden_size) - - if self.megatron_config.sequence_parallel: - sequence_parallel_pad = total_nnz - cu_seqlens[-1] - total_nnz = cu_seqlens[-1] # total_nnz before sp padding - query_states = query_states[:total_nnz] - key_states = key_states[:total_nnz] - value_states = value_states[:total_nnz] - - # Flash attention requires the input to have the shape - # batch_size x seq_length x head_dime x hidden_dim - # therefore we just need to keep the original shape - query_states = query_states.view(total_nnz, self.num_heads_per_tp, self.head_dim) - key_states = key_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) - value_states = value_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) - - cos, sin = self.rotary_emb(value_states, seq_len=sequence_length) - cos, sin = cos[:, :cos.shape[1] // 2], sin[:, :sin.shape[1] // 2] # flash attn only needs half - query_states, key_states = apply_rotary_pos_emb_rmpad_flash(query_states, - key_states, - cos, - sin, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen_in_batch) - # query_states, key_states = apply_rotary_pos_emb_rmpad(query_states, key_states, cos, sin, position_ids, indices, - - # TODO: llama does not have dropout in the config?? - # It is recommended to use dropout with FA according to the docs - # when training. - dropout_rate = 0.0 # if not self.training else self.attn_dropout - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in float16 just to be sure everything works as expected. - # This might slowdown training & inference so it is recommended to not cast the LayerNorms - # in fp32. (LlamaRMSNorm handles it correctly) - input_dtype = query_states.dtype - if input_dtype == torch.float32: - query_states = query_states.to(torch.float16) - key_states = key_states.to(torch.float16) - value_states = value_states.to(torch.float16) - - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens, - cu_seqlens_k=cu_seqlens, - max_seqlen_q=max_seqlen_in_batch, - max_seqlen_k=max_seqlen_in_batch, - dropout_p=dropout_rate, - softmax_scale=None, - causal=True, - ) - - attn_output_unpad = attn_output_unpad.to(input_dtype) - attn_output_unpad = attn_output_unpad.reshape(total_nnz, 1, self.hidden_size_per_tp).contiguous() - - # sequence parallel reduce_scatter is performed inside RowColumnParallel if enabled - # Here we need to repad - if self.megatron_config.sequence_parallel: - attn_output_unpad = F.pad(attn_output_unpad, pad=(0, 0, 0, 0, 0, sequence_parallel_pad)) - - attn_output_unpad = self.o_proj(attn_output_unpad)[0] - return attn_output_unpad diff --git a/verl/models/llama/megatron/layers/parallel_decoder.py b/verl/models/llama/megatron/layers/parallel_decoder.py deleted file mode 100644 index e51632a33c..0000000000 --- a/verl/models/llama/megatron/layers/parallel_decoder.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Tuple - -import torch -from torch import nn -from transformers import LlamaConfig -from megatron.core import ModelParallelConfig - -from .parallel_attention import ParallelLlamaAttention, ParallelLlamaAttentionRmPad -from .parallel_mlp import ParallelLlamaMLP -from .parallel_rmsnorm import ParallelLlamaRMSNorm - -from verl.utils.megatron_utils import TransformerConfig, convert_config - - -class ParallelLlamaDecoderLayer(nn.Module): - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig, layer_idx: int): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.layer_idx = layer_idx - self.hidden_size = config.hidden_size - self.self_attn = ParallelLlamaAttention(config=config, megatron_config=megatron_config) - - self.mlp = ParallelLlamaMLP(config, megatron_config=megatron_config) - self.input_layernorm = ParallelLlamaRMSNorm(config, megatron_config) - self.post_attention_layernorm = ParallelLlamaRMSNorm(config, megatron_config) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: - """ - Args: - hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` - attention_mask (`torch.FloatTensor`, *optional*): attention mask of size - `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states - """ - - residual = hidden_states - - hidden_states = self.input_layernorm(hidden_states) - - # Note: sequence parallel is hidden inside ColumnParallelLinear - # reduce scatter is hidden inside RowParallelLinear - - # Self Attention - hidden_states = self.self_attn( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - ) - - # TODO: add sequence parallel operator reduce_scatter here - - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - - # TODO: add sequence parallel operator all_gather here - - hidden_states = self.mlp(hidden_states) - - # TODO: add sequence parallel operator reduce_scatter here - - hidden_states = residual + hidden_states - - outputs = hidden_states - - return outputs - - -class ParallelLlamaDecoderLayerRmPad(nn.Module): - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig, layer_idx: int): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.layer_idx = layer_idx - self.hidden_size = config.hidden_size - self.self_attn = ParallelLlamaAttentionRmPad(config=config, megatron_config=megatron_config) - - self.mlp = ParallelLlamaMLP(config, megatron_config=megatron_config) - self.input_layernorm = ParallelLlamaRMSNorm(config, megatron_config) - self.post_attention_layernorm = ParallelLlamaRMSNorm(config, megatron_config) - - def forward( - self, - hidden_states: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: int = None, - max_seqlen_in_batch: int = None - ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: - residual = hidden_states # (total_nnz // sp, 1, hidden_size) - - hidden_states = self.input_layernorm(hidden_states) - - # Self Attention - # (total_nnz // sp, 1, hidden_size) -> all-gather (total_nnz, 1, hidden_size) - # -> col + row -> reduce-scatter -> (total_nnz // sp, 1, hidden_size) - hidden_states = self.self_attn(hidden_states=hidden_states, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = residual + hidden_states - - # Fully Connected - # shape changes same as attn - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - - outputs = hidden_states - - return outputs diff --git a/verl/models/llama/megatron/layers/parallel_linear.py b/verl/models/llama/megatron/layers/parallel_linear.py deleted file mode 100644 index e3e4e43855..0000000000 --- a/verl/models/llama/megatron/layers/parallel_linear.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/linear.py - -from typing import Optional, Tuple - -from megatron.core import tensor_parallel - - -class QKVParallelLinear(tensor_parallel.ColumnParallelLinear): - - def __init__(self, - input_size, - num_heads, - num_key_value_heads, - head_dim, - *, - bias=True, - gather_output=True, - skip_bias_add=False, - **kwargs): - # Keep input parameters, and already restrict the head numbers - self.input_size = input_size - self.q_output_size = num_heads * head_dim - self.kv_output_size = num_key_value_heads * head_dim - self.head_dim = head_dim - self.gather_output = gather_output - self.skip_bias_add = skip_bias_add - - input_size = self.input_size - output_size = (num_heads + 2 * num_key_value_heads) * self.head_dim - - super().__init__(input_size=input_size, - output_size=output_size, - bias=bias, - gather_output=gather_output, - skip_bias_add=skip_bias_add, - **kwargs) - - -class MergedColumnParallelLinear(tensor_parallel.ColumnParallelLinear): - - def __init__(self, - input_size, - gate_ouput_size, - up_output_size, - *, - bias=True, - gather_output=True, - skip_bias_add=False, - **kwargs): - # Keep input parameters, and already restrict the head numbers - self.input_size = input_size - self.output_size = gate_ouput_size + up_output_size - self.gather_output = gather_output - self.skip_bias_add = skip_bias_add - - super().__init__(input_size=self.input_size, - output_size=self.output_size, - bias=bias, - gather_output=gather_output, - skip_bias_add=skip_bias_add, - **kwargs) - - -import torch - - -class LinearForLastLayer(torch.nn.Linear): - - def __init__( - self, - input_size, - output_size, - *, - config, - bias=True, - ): - super().__init__(in_features=input_size, out_features=output_size, bias=bias) - self.sequence_parallel = config.sequence_parallel - if self.sequence_parallel: - setattr(self.weight, 'sequence_parallel', True) - - def forward( - self, - input_, - weight=None, - runtime_gather_output=None, - ): - logits = super().forward(input_) - logits = logits.float() - if self.sequence_parallel: - logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) - return logits, None diff --git a/verl/models/llama/megatron/layers/parallel_mlp.py b/verl/models/llama/megatron/layers/parallel_mlp.py deleted file mode 100644 index 21ad9b16a6..0000000000 --- a/verl/models/llama/megatron/layers/parallel_mlp.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from megatron.core import parallel_state as mpu -from megatron.core import tensor_parallel -from megatron.core import ModelParallelConfig -from torch import nn -from transformers.activations import ACT2FN -from verl.models.llama.megatron.layers.parallel_linear import MergedColumnParallelLinear - -from verl.utils.megatron import tensor_parallel as tp_utils - - -class ParallelLlamaMLP(nn.Module): - - def __init__(self, config, megatron_config: ModelParallelConfig = None) -> None: - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size - # The weight is only [hidden_size, intermediate_size // model_parallel_world_size] - - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() - - if megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - assert row_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) - tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) - - tp_size = mpu.get_tensor_model_parallel_world_size() - - self.gate_up_proj = MergedColumnParallelLinear( - input_size=self.hidden_size, - gate_ouput_size=self.intermediate_size, - up_output_size=self.intermediate_size, - bias=False, - gather_output=False, - skip_bias_add=False, - **column_kwargs, - ) - self.gate_size = self.intermediate_size // tp_size - - self.down_proj = tensor_parallel.RowParallelLinear(input_size=self.intermediate_size, - output_size=self.hidden_size, - bias=False, - input_is_parallel=True, - skip_bias_add=False, - **row_kwargs) - - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, x): - gate_up = self.gate_up_proj(x)[0] - gate, up = gate_up.split(self.gate_size, dim=-1) - return self.down_proj(self.act_fn(gate) * up)[0] diff --git a/verl/models/llama/megatron/layers/parallel_rmsnorm.py b/verl/models/llama/megatron/layers/parallel_rmsnorm.py deleted file mode 100644 index 7027036bf4..0000000000 --- a/verl/models/llama/megatron/layers/parallel_rmsnorm.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numbers -import torch -from megatron.core import ModelParallelConfig -from torch import nn -from transformers import LlamaConfig - -from apex.normalization.fused_layer_norm import fused_rms_norm_affine -from verl.utils.megatron import sequence_parallel as sp_utils - - -class ParallelLlamaRMSNorm(nn.Module): - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): - """ - LlamaRMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - if isinstance(config.hidden_size, numbers.Integral): - normalized_shape = (config.hidden_size,) - self.normalized_shape = torch.Size(normalized_shape) - self.weight = nn.Parameter(torch.ones(self.normalized_shape)) - self.variance_epsilon = config.rms_norm_eps - - if megatron_config.sequence_parallel: - sp_utils.mark_parameter_as_sequence_parallel(self.weight) - - def forward(self, hidden_states): - return fused_rms_norm_affine(input=hidden_states, - weight=self.weight, - normalized_shape=self.normalized_shape, - eps=self.variance_epsilon, - memory_efficient=True) \ No newline at end of file diff --git a/verl/models/llama/megatron/modeling_llama_megatron.py b/verl/models/llama/megatron/modeling_llama_megatron.py deleted file mode 100644 index 83d0f0f2bb..0000000000 --- a/verl/models/llama/megatron/modeling_llama_megatron.py +++ /dev/null @@ -1,667 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""PyTorch LLaMA model with Megatron-style acceleration.""" - -from typing import Optional, Tuple, Union - -import torch -import torch.utils.checkpoint -from megatron.core import tensor_parallel -from megatron.core import ModelParallelConfig -from megatron.core import mpu - -from torch import nn -from transformers.modeling_outputs import BaseModelOutputWithPast -from transformers.models.llama.configuration_llama import LlamaConfig -from transformers.models.llama.modeling_llama import CausalLMOutputWithPast - -from verl.utils.megatron import sequence_parallel as sp_utils -from verl.utils.megatron import tensor_parallel as tp_utils -from verl.utils.megatron_utils import TransformerConfig, convert_config -from .layers import ParallelLlamaDecoderLayer, ParallelLlamaRMSNorm, ParallelLlamaDecoderLayerRmPad -""" -TODO: -1. Add weight initialization. Here we need to be careful on TP weight init. -2. Add sequence parallel -3. Load checkpoint from meta LLama pretrained checkpoint -""" - - -# Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device): - """ - Make causal mask used for bi-directional self-attention. - """ - bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) - mask_cond = torch.arange(mask.size(-1), device=device) - mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) - mask = mask.to(dtype) - return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len) - - -# Copied from transformers.models.bart.modeling_bart._expand_mask -def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): - """ - Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. - """ - bsz, src_len = mask.size() - tgt_len = tgt_len if tgt_len is not None else src_len - - expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) - - inverted_mask = 1.0 - expanded_mask - - return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) - - -class ParallelLlamaModel(nn.Module): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] - - Args: - config: LlamaConfig - """ - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() - if megatron_config is not None: - assert embedding_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) - self.embed_tokens = tensor_parallel.VocabParallelEmbedding(num_embeddings=config.vocab_size, - embedding_dim=config.hidden_size, - **embedding_kwargs) - - self.layers = nn.ModuleList( - [ParallelLlamaDecoderLayer(config, megatron_config) for _ in range(config.num_hidden_layers)]) - self.norm = ParallelLlamaRMSNorm(config, megatron_config) - - # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask - def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds): - # create causal mask - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - combined_attention_mask = None - if input_shape[-1] > 1: - combined_attention_mask = _make_causal_mask( - input_shape, - inputs_embeds.dtype, - device=inputs_embeds.device, - ) - - if attention_mask is not None: - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, - tgt_len=input_shape[-1]).to(inputs_embeds.device) - combined_attention_mask = (expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + - combined_attention_mask) - - return combined_attention_mask - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, BaseModelOutputWithPast]: - """ - - Args: - input_ids: input ids. shape (batch_size, seq_length) - attention_mask: attention_mask. shape (batch_size, seq_length) - position_ids: position ids. shape (batch_size, seq_length) - - Returns: - - """ - batch_size, seq_length = input_ids.shape - inputs_embeds = self.embed_tokens(input_ids) - # embed positions - - attention_mask = self._prepare_decoder_attention_mask(attention_mask, (batch_size, seq_length), inputs_embeds) - - hidden_states = inputs_embeds - - for idx, decoder_layer in enumerate(self.layers): - layer_outputs = decoder_layer( - hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - ) - - hidden_states = layer_outputs - - hidden_states = self.norm(hidden_states) - - return hidden_states - - -class ParallelLlamaForCausalLM(nn.Module): - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.model = ParallelLlamaModel(config, megatron_config=megatron_config) - self.vocab_size = config.vocab_size - - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - - self.lm_head = tensor_parallel.ColumnParallelLinear(input_size=config.hidden_size, - output_size=config.vocab_size, - bias=False, - gather_output=False, - skip_bias_add=False, - **column_kwargs) - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - ```""" - - # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - ) - - hidden_states = outputs - logits = self.lm_head(hidden_states)[0] - - logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) - - logits = logits.float() - return CausalLMOutputWithPast( - loss=None, - logits=logits, - past_key_values=None, - hidden_states=None, - attentions=None, - ) - - -from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - - -class ParallelLlamaModelRmPad(nn.Module): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] - - Args: - config: LlamaConfig - """ - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() - self.megatron_config = megatron_config - if megatron_config is not None: - assert embedding_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) - self.embed_tokens = tensor_parallel.VocabParallelEmbedding(num_embeddings=config.vocab_size, - embedding_dim=config.hidden_size, - **embedding_kwargs) - - self.layers = nn.ModuleList( - [ParallelLlamaDecoderLayerRmPad(config, megatron_config) for _ in range(config.num_hidden_layers)]) - self.norm = ParallelLlamaRMSNorm(config, megatron_config) - - def forward(self, - input_ids: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: int = None, - max_seqlen_in_batch: int = None) -> Union[Tuple, BaseModelOutputWithPast]: - """ - - Args: - input_ids: input ids. shape (1, totol_nnz) - position_ids: position ids. shape (batch_size, seq_length) - - Returns: - - """ - inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) - - # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) - inputs_embeds = inputs_embeds.transpose(0, 1) - if self.megatron_config.sequence_parallel: - inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) - - hidden_states = inputs_embeds - for idx, decoder_layer in enumerate(self.layers): - layer_outputs = decoder_layer(hidden_states, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = layer_outputs - - hidden_states = self.norm(hidden_states) - - return hidden_states - - -class ParallelLlamaForCausalLMRmPad(nn.Module): - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.megatron_config = megatron_config - self.model = ParallelLlamaModelRmPad(config, megatron_config=megatron_config) - self.vocab_size = config.vocab_size - self._init_head(config) - - def _init_head(self, config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = tensor_parallel.ColumnParallelLinear(input_size=config.hidden_size, - output_size=config.vocab_size, - bias=False, - gather_output=False, - skip_bias_add=False, - **column_kwargs) - - def _forward_head(self, hidden_states): - # all_gather from sequence parallel region is performed inside lm_head - logits = self.lm_head(hidden_states)[0] - logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) - logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) # (total_nnz_padded, 1, vocab_size) - return logits - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - ```""" - batch_size, sequence_length = input_ids.shape - - # remove padding here - input_ids, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input(input_ids.unsqueeze(dim=-1), - attention_mask) # (total_nnz, 1) - - # pad input_ids to multiple of tp for all tp ranks - # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap - if self.megatron_config.sequence_parallel: - input_ids = sp_utils.pad_to_sequence_parallel(input_ids) - - input_ids = input_ids.transpose(0, 1) # (1, total_nnz+pad) - - outputs = self.model(input_ids=input_ids, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = outputs - - logits = self._forward_head(hidden_states) - - # remove padding from sequence parallel - if self.megatron_config.sequence_parallel: - totol_nnz = cu_seqlens[-1] - logits = logits[:totol_nnz] # (total_nnz_padded) - - logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension - # add removed padding back - logits = pad_input(logits, indices, batch_size, - seqlen=sequence_length) # (batch_size, sequence_length, vocab_size) - - return CausalLMOutputWithPast( - loss=None, - logits=logits, - past_key_values=None, - hidden_states=None, - attentions=None, - ) - - -class ParallelLlamaForValueRmPad(ParallelLlamaForCausalLMRmPad): - - def _init_head(self, config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) - # lm_head is effectively the same as sequence parallel - sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) - - def _forward_head(self, hidden_states): - logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) - logits = logits.float() - if self.megatron_config.sequence_parallel: - logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) - return logits - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - output = super().forward(input_ids, attention_mask, position_ids) - output.logits = torch.squeeze(output.logits, dim=-1) - return output - - -""" -Support pipeline parallelism -""" - - -class ParallelLlamaModelRmPadPP(nn.Module): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] - This model definition supports pipeline parallelism. To support pp and vpp, - - This model only contains layer in this pp stage and vpp chunk - - When calling get_model in Megatron, this rank will instantiate all the vpp chunks in this pp. - Args: - config: LlamaConfig - """ - - def __init__(self, config: LlamaConfig, megatron_config: ModelParallelConfig, pre_process, post_process): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - self.pre_process = pre_process - self.post_process = post_process - self.megatron_config = megatron_config - embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() - if megatron_config is not None: - assert embedding_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) - if pre_process: - self.embed_tokens = tensor_parallel.VocabParallelEmbedding(num_embeddings=config.vocab_size, - embedding_dim=config.hidden_size, - **embedding_kwargs) - else: - self.embed_tokens = None - - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = megatron_config.pipeline_model_parallel_size - self.num_layer_per_pp = config.num_hidden_layers // pp_size - vpp_size = megatron_config.virtual_pipeline_model_parallel_size - vpp_rank = mpu.get_virtual_pipeline_model_parallel_rank() - - if vpp_size is not None: - self.layers = nn.ModuleList() - self.num_layer_vpp_chunk = self.num_layer_per_pp // vpp_size - self.num_layer_this_model = self.num_layer_vpp_chunk - offset = vpp_rank * ( - config.num_hidden_layers // vpp_size) + \ - (pp_rank * self.num_layer_vpp_chunk) - else: - self.num_layer_this_model = self.num_layer_per_pp - offset = pp_rank * self.num_layer_per_pp - - self.layers = nn.ModuleList() - for i in range(self.num_layer_this_model): - layer = ParallelLlamaDecoderLayerRmPad(config, megatron_config, layer_idx=offset + i) - self.layers.add_module(f'{i}', layer) - - if post_process: - self.norm = ParallelLlamaRMSNorm(config, megatron_config) - else: - self.norm = None - - def set_input_tensor(self, input_tensor): - """Set input tensor to be used instead of forward()'s input. - - When doing pipeline parallelism the input from the previous - stage comes from communication, not from the input, so the - model's forward_step_func won't have it. This function is thus - used by internal code to bypass the input provided by the - forward_step_func""" - self.input_tensor = input_tensor - - def forward(self, - input_ids: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: int = None, - max_seqlen_in_batch: int = None) -> Union[Tuple, BaseModelOutputWithPast]: - """ - - Args: - input_ids: input ids. shape (1, totol_nnz) - position_ids: position ids. shape (batch_size, seq_length) - - Returns: - - """ - if self.pre_process: - inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) - - # vocab parallel embedding will not do sequence parallel reduce-scatter in open source megatron - # so need to deal with it by handle here: - # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) - inputs_embeds = inputs_embeds.transpose(0, 1) - if self.megatron_config.sequence_parallel: - inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) - - hidden_states = inputs_embeds - else: - # self.hidden_states should be passed by Megatron - hidden_states = self.input_tensor - - for idx, decoder_layer in enumerate(self.layers): - layer_outputs = decoder_layer(hidden_states, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = layer_outputs - - if self.post_process: - hidden_states = self.norm(hidden_states) - - return hidden_states - - -class ParallelLlamaForCausalLMRmPadPP(nn.Module): - - def __init__(self, - config: LlamaConfig, - megatron_config: ModelParallelConfig, - pre_process, - post_process, - share_embeddings_and_output_weights=False): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.megatron_config = megatron_config - self.model = ParallelLlamaModelRmPadPP(config, - megatron_config=megatron_config, - pre_process=pre_process, - post_process=post_process) - assert share_embeddings_and_output_weights == False, f'Llama Model not supports sharing embedding and output weights' - self.share_embeddings_and_output_weights = share_embeddings_and_output_weights - self.vocab_size = config.vocab_size - self.pre_process = pre_process - self.post_process = post_process - if post_process: - self._init_head(config) - - def set_input_tensor(self, input_tensor): - """Set input tensor to be used instead of forward()'s input. - - When doing pipeline parallelism the input from the previous - stage comes from communication, not from the input, so the - model's forward_step_func won't have it. This function is thus - used by internal code to bypass the input provided by the - forward_step_func""" - assert len(input_tensor) == 1 - self.model.set_input_tensor(input_tensor[0]) - - def _init_head(self, config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = tensor_parallel.ColumnParallelLinear(input_size=config.hidden_size, - output_size=config.vocab_size, - bias=False, - gather_output=False, - skip_bias_add=False, - **column_kwargs) - - def _forward_head(self, hidden_states): - # all_gather from sequence parallel region is performed inside lm_head - # logits shape before forward_head hidden_states.shape: [4, 32, 4096] - logits = self.lm_head(hidden_states)[0] - # logits shape after forward_head logits.shape: [8, 32, 8] - logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) - return logits - - def forward( - self, - # original input - *, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - ```""" - - # Note that input_ids, attention_mask and position_ids should be passed to every pp layer. - # In the first pp, input_ids will be used, in other pp layers hidden_states will be used inside self.model - batch_size, sequence_length = input_ids.shape - # remove padding here - input_ids_rmpad, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input(input_ids.unsqueeze(dim=-1), - attention_mask) # (total_nnz, 1) - - # pad input_ids to multiple of tp for all tp ranks - # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap - if self.megatron_config.sequence_parallel: - input_ids_rmpad = sp_utils.pad_to_sequence_parallel(input_ids_rmpad) - - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz+pad) - - outputs = self.model(input_ids=input_ids_rmpad, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - if self.post_process: - hidden_states = outputs - # print(f'hidden_states.shape = {hidden_states.shape}') # torch.Size([4, 32, 4096]) - logits = self._forward_head(hidden_states) - logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension # torch.Size([8, 32, 16]) - - # remove padding from sequence parallel - if self.megatron_config.sequence_parallel: - totol_nnz = cu_seqlens[-1] - logits = logits[:totol_nnz] # (total_nnz_padded) - # add removed padding back. If input is already rmpad, we let the caller pad_input - logits = pad_input(logits, indices, batch_size, - seqlen=sequence_length) # (batch_size, sequence_length, vocab_size) - - return CausalLMOutputWithPast( - loss=None, - logits=logits, - past_key_values=None, - hidden_states=None, - attentions=None, - ) - else: - return outputs - - -class ParallelLlamaForValueRmPadPP(ParallelLlamaForCausalLMRmPadPP): - - def _init_head(self, config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) - # lm_head is effectively the same as sequence parallel - sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) - - def _forward_head(self, hidden_states): - logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) - logits = logits.float() - if self.megatron_config.sequence_parallel: - logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) - return logits - - def forward( - self, - *, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - output = super().forward(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids) - if self.post_process: - output.logits = torch.squeeze(output.logits, dim=-1) - return output - else: - return output diff --git a/verl/models/mcore/__init__.py b/verl/models/mcore/__init__.py deleted file mode 100644 index e4e2a3861a..0000000000 --- a/verl/models/mcore/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2025 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .gpt_model import gptmodel_forward \ No newline at end of file diff --git a/verl/models/mcore/gpt_model.py b/verl/models/mcore/gpt_model.py deleted file mode 100644 index 172857b797..0000000000 --- a/verl/models/mcore/gpt_model.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2025 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl.utils.megatron import sequence_parallel as sp_utils -from verl.utils.megatron import tensor_parallel as tp_utils -import torch -from megatron.core.packed_seq_params import PackedSeqParams -from megatron.core import parallel_state as mpu -from verl.utils.megatron_utils import unwrap_model - - -def gptmodel_forward(model, - input_ids, - attention_mask, - position_ids, - sequence_parallel, - value_model=False, - pack_seqs=True): - pre_process = unwrap_model(model).pre_process - post_process = unwrap_model(model).post_process - if pack_seqs: - batch_size, seq_len = attention_mask.shape[:2] - input_ids_rmpad, packed_seq_params = preprocess_packed_seqs(input_ids, attention_mask, pre_process=pre_process) - input_ids_rmpad = input_ids_rmpad.contiguous() - output = model(input_ids=input_ids_rmpad, - attention_mask=None, - position_ids=position_ids, - packed_seq_params=packed_seq_params) - output = postprocess_packed_seqs(output, - packed_seq_params, - attention_mask, - batch_size, - seq_len, - post_process=post_process) - else: - batch_size, sequence_length = attention_mask.shape - new_input_ids, new_attention_mask, new_position_ids = remove_left_padding(input_ids, - attention_mask, - position_ids, - sequence_parallel, - pre_process=pre_process) - output = model(input_ids=new_input_ids, attention_mask=new_attention_mask, position_ids=new_position_ids) - output = recover_left_padding(output, - new_attention_mask, - attention_mask, - sequence_length, - post_process=post_process) - if value_model and post_process: - output = output[..., 0] - return output - - -def preprocess_packed_seqs(input_ids: torch.Tensor, - attention_mask: torch.Tensor, - pre_process: bool = True) -> tuple[torch.Tensor, PackedSeqParams]: - """ - Preprocess packed sequences - """ - batch_size = input_ids.shape[0] - - seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) - tp_size = mpu.get_tensor_model_parallel_world_size() - pad_size = (tp_size - seqlens_in_batch % tp_size) % tp_size - seqlens_in_batch_padded = seqlens_in_batch + pad_size - cu_seqlens = torch.zeros(batch_size + 1, dtype=torch.int32, device=input_ids.device) - cu_seqlens[1:] = torch.cumsum(seqlens_in_batch, dim=0) - cu_seqlens_padded = torch.zeros(batch_size + 1, dtype=torch.int32, device=input_ids.device) - cu_seqlens_padded[1:] = torch.cumsum(seqlens_in_batch_padded, dim=0) - max_seqlen_in_batch = seqlens_in_batch_padded.max().item() - - shape = list(input_ids.shape[1:]) - shape[0] = seqlens_in_batch_padded.sum().item() - if pre_process: - input_ids_rmpad = torch.zeros(shape, dtype=input_ids.dtype, device=input_ids.device) - for i in range(batch_size): - seqlen = seqlens_in_batch[i] - input_ids_rmpad[cu_seqlens_padded[i]:cu_seqlens_padded[i] + seqlen] = input_ids[i, attention_mask[i]] - - packed_seq_params = PackedSeqParams(qkv_format='thd', - cu_seqlens_q=cu_seqlens_padded, - max_seqlen_q=max_seqlen_in_batch, - cu_seqlens_kv=cu_seqlens_padded, - max_seqlen_kv=max_seqlen_in_batch, - cu_seqlens_q_padded=cu_seqlens_padded, - cu_seqlens_kv_padded=cu_seqlens_padded) - if pre_process: - return input_ids_rmpad.unsqueeze(0), packed_seq_params - else: - return input_ids, packed_seq_params - - -def postprocess_packed_seqs(output: torch.Tensor, - packed_seq_params: PackedSeqParams, - attention_mask: torch.Tensor, - batch_size: int, - seq_len: int, - post_process: bool = True) -> torch.Tensor: - """ - Postprocess packed sequences - """ - if not post_process: - return output - shape = [batch_size, seq_len] + list(output.shape[2:]) # 1,packed, dim -> batch_size, seq_len, dim - output_new = torch.zeros(shape, dtype=output.dtype, device=output.device) - for i in range(batch_size): - s = attention_mask[i].sum().item() - output_new[i, - attention_mask[i]] = output[0][packed_seq_params. - cu_seqlens_q_padded[i]:packed_seq_params.cu_seqlens_q_padded[i] + s] - - return output_new - - -def remove_left_padding(input_ids: torch.Tensor, - attention_mask: torch.Tensor, - position_ids: torch.Tensor, - sequence_parallel: bool = False, - pre_process: bool = True): - """ - Remove left padding from input_ids, attention_mask and position_ids - return new_input_ids, new_attention_mask, new_position_ids - """ - assert attention_mask.ndim == 2 - assert position_ids.ndim == 2 - - batch_size = input_ids.shape[0] - shape = list(input_ids.shape) # batch_size, seq_len,... - seq_lens = attention_mask.sum(dim=1) - seq_len = seq_lens.max().item() - if sequence_parallel: - from megatron.core import parallel_state as mpu - sp_world_size = mpu.get_tensor_model_parallel_world_size() - pad_size = (sp_world_size - seq_len % sp_world_size) % sp_world_size - seq_len = seq_len + pad_size - shape[1] = seq_len - if pre_process: - new_input_ids = torch.zeros(dtype=input_ids.dtype, device=input_ids.device, size=shape) - new_attention_mask = torch.zeros(dtype=attention_mask.dtype, - device=attention_mask.device, - size=(batch_size, seq_len)) - new_position_ids = torch.zeros(dtype=position_ids.dtype, device=position_ids.device, size=(batch_size, seq_len)) - for i in range(batch_size): - if pre_process: - new_input_ids[i, :seq_lens[i]] = input_ids[i, attention_mask[i]] - new_attention_mask[i, :seq_lens[i]] = attention_mask[i, attention_mask[i]] - new_position_ids[i, :seq_lens[i]] = position_ids[i, attention_mask[i]] - if pre_process: - return new_input_ids, new_attention_mask, new_position_ids - else: - return input_ids, new_attention_mask, new_position_ids - - -def recover_left_padding(result, - attention_mask: torch.Tensor, - original_attention_mask: torch.Tensor, - origin_seqlen: int, - post_process: bool = True): - """ - Recover left padding from result - return result - """ - if not post_process: - return result - shape = list(result.shape) - batch_size = shape[0] - shape[1] = origin_seqlen - new_result = torch.zeros(dtype=result.dtype, device=result.device, size=shape) - for i in range(batch_size): - new_result[i, original_attention_mask[i]] = result[i, attention_mask[i]] - return new_result diff --git a/verl/models/mcore/loader.py b/verl/models/mcore/loader.py deleted file mode 100644 index 1a782aa18f..0000000000 --- a/verl/models/mcore/loader.py +++ /dev/null @@ -1,465 +0,0 @@ -# Copyright 2025 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import time -import torch.distributed as dist - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - import megatron - from megatron.core import mpu - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def load_state_dict_to_megatron_gptmodel(state_dict, wrapped_models, config, params_dtype, is_value_model=False): - """Load merged state_dict to sharded Megatron module in training. - """ - import megatron - from megatron.core import mpu - from verl.utils.megatron_utils import print_rank_0, unwrap_model - from megatron.core.transformer.module import Float16Module - from megatron.core import DistributedDataParallel as LocalDDP - from torch.nn.parallel import DistributedDataParallel as torchDDP - - start_time = time.time() - - def _get_gpt_model(model): - return model - - def broadcast_params(module): - for param in module.parameters(): - torch.distributed.broadcast(param.data, - src=mpu.get_data_parallel_src_rank(), - group=mpu.get_data_parallel_group()) - - dp_rank = mpu.get_data_parallel_rank() - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if torch.distributed.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - gpt_model_module = _get_gpt_model(models[i]) - assert len(gpt_model_module.decoder.layers) == num_layers_per_model - - def _broadcast_tensor(tensor, name) -> torch.Tensor: - """broadcast tensor from rank0 across mp_group""" - nonlocal state_dict - nonlocal mp_group - if torch.distributed.get_rank() == 0: - if name in state_dict: - weight = state_dict[name] - tensor_shape = weight.shape - else: - tensor_shape = None - else: - weight = None - tensor_shape = None - - obj_list = [tensor_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - tensor_shape = obj_list[0] - - if tensor_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tensor:[{name}] not in state_dict, skip load") - return - - if tensor is None: - tensor = torch.empty( - tensor_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - if torch.distributed.get_rank() == 0: - tensor.data.copy_(weight) - dist.broadcast(tensor, src=0, group=mp_group) - - def _broadcast_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - if name in state_dict: - full_weight = state_dict[name] - - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - if name in state_dict: - full_weight = state_dict[name] - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - gate_weight = state_dict[gate_name] - up_weight = state_dict[up_name] - new_gate_up_weight = torch.empty(config.intermediate_size * 2, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_tp = gate_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - up_weight_tp = up_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - new_gate_up_weight[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)].copy_( - torch.cat([gate_weight_tp, up_weight_tp], dim=0)) - - tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert ( - tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank() == 0:} tensor {gate_name, up_name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, bias=False) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - assert (q_name in state_dict and k_name in state_dict and v_name in state_dict) - full_weight_q = state_dict[q_name] - full_weight_k = state_dict[k_name] - full_weight_v = state_dict[v_name] - - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - sizes = [total_size * tp_size] - if not bias: - sizes.append(config.hidden_size) - new_weight_qkv = torch.empty(*sizes, dtype=params_dtype, device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - k_part = full_weight_k[i * kv_size_tp:(i + 1) * kv_size_tp] - v_part = full_weight_v[i * kv_size_tp:(i + 1) * kv_size_tp] - num_query_groups_per_partition = models[0].config.num_query_groups // tp_size - new_weight_qkv_this_tp = new_weight_qkv[i * total_size:(i + 1) * total_size] - q_part_per_head = torch.chunk(q_part, num_query_groups_per_partition, dim=0) - k_part_per_head = torch.chunk(k_part, num_query_groups_per_partition, dim=0) - v_part_per_head = torch.chunk(v_part, num_query_groups_per_partition, dim=0) - total_size_per_head = total_size // num_query_groups_per_partition - for j in range(num_query_groups_per_partition): - new_weight_qkv_this_tp[j * total_size_per_head:(j + 1) * total_size_per_head].copy_( - torch.cat([q_part_per_head[j], k_part_per_head[j], v_part_per_head[j]], dim=0)) - - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - sizes = [total_size * tp_size] - if not bias: - sizes.append(config.hidden_size) - new_weight_qkv = torch.empty(*sizes, dtype=params_dtype, device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head - end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head - k_part = full_weight_k[start_idx:end_idx] - v_part = full_weight_v[start_idx:end_idx] - new_weight_qkv_this_tp = new_weight_qkv[i * total_size:(i + 1) * total_size] - q_part_per_head = torch.chunk(q_part, config.num_attention_heads, dim=0) - k_part_per_head = torch.chunk(k_part, config.num_attention_heads, dim=0) - v_part_per_head = torch.chunk(v_part, config.num_attention_heads, dim=0) - total_size_per_head = total_size // config.num_attention_heads - for j in range(config.num_attention_heads): - new_weight_qkv_this_tp[j * total_size_per_head:(j + 1) * total_size_per_head].copy_( - torch.cat([q_part_per_head[j], k_part_per_head[j], v_part_per_head[j]], dim=0)) - - tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {q_name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - if dp_rank == 0: - # Embeddings - # ------------------- - print_rank_0("loading embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - embed_tokens_weight = None - if pp_rank == 0: - embed_tokens_weight = gpt_model_module.embedding.word_embeddings.weight - _broadcast_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - - for layer in range(config.num_hidden_layers): - print_rank_0(f"loading layer #{layer}...") - layer_name = f"model.layers.{layer}" - dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] - - gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) - sync_layer = gpt_model_module.decoder.layers[dst_layer_idx] - - _broadcast_tensor( - sync_layer.self_attention.linear_qkv.layer_norm_weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.input_layernorm.weight", - ) - - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attention.linear_qkv.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - ) - if f"{layer_name}.self_attn.q_proj.bias" in state_dict: - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attention.linear_qkv.bias if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.bias", - f"{layer_name}.self_attn.k_proj.bias", - f"{layer_name}.self_attn.v_proj.bias", - bias=True) - - _broadcast_tp_shard_tensor( - sync_layer.self_attention.linear_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.o_proj.weight", - chunk_dim=1, - ) - _broadcast_tensor( - sync_layer.mlp.linear_fc1.layer_norm_weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.post_attention_layernorm.weight", - ) - - _broadcast_tp_shard_tensor_gate_up(sync_layer.mlp.linear_fc1.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.gate_proj.weight", f"{layer_name}.mlp.up_proj.weight") - - _broadcast_tp_shard_tensor( - sync_layer.mlp.linear_fc2.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.down_proj.weight", - chunk_dim=1, - ) - # Final Layernorm - # ------------------- - print_rank_0("loading final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _broadcast_tensor( - getattr(gpt_model_module.decoder.final_layernorm, "weight", None), - "model.norm.weight", - ) - - print_rank_0("loading lm_head...") - lm_head_weight = None - if pp_rank + 1 == pp_size: - lm_head_weight = gpt_model_module.output_layer.weight - - if is_value_model: - # if torch.distributed.get_rank() == 0: - if 'lm_head.weight' in state_dict and state_dict['lm_head.weight'].shape[0] == 1: - _broadcast_tensor(lm_head_weight, "lm_head.weight") - elif 'reward_head.weight' in state_dict and state_dict['reward_head.weight'].shape[0] == 1: - _broadcast_tensor(lm_head_weight, "reward_head.weight") - print_rank_0('load lm_head from value_head weight') - else: - _broadcast_tensor(None, "lm_head.weight") - print_rank_0('fail to match lm_head in value_model') - # else: - - # _broadcast_tensor(lm_head_weight, "lm_head.weight") - - else: - _broadcast_tp_shard_tensor(lm_head_weight, "lm_head.weight") - dist.barrier() - # Broadcast weights inside data parallel groups - for wrapped_model in wrapped_models: - broadcast_params(wrapped_model) - pass - torch.cuda.empty_cache() - print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/verl/models/mcore/saver.py b/verl/models/mcore/saver.py deleted file mode 100644 index 446bc3120a..0000000000 --- a/verl/models/mcore/saver.py +++ /dev/null @@ -1,455 +0,0 @@ -# Copyright 2025 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl.utils.megatron_utils import print_rank_0, unwrap_model -from megatron.core import mpu -from megatron.core.transformer.module import Float16Module -from megatron.core.distributed import DistributedDataParallel as LocalDDP -from torch.nn.parallel import DistributedDataParallel as torchDDP -import torch -import time - -import torch -import torch.distributed as dist - - -def _megatron_calc_global_rank(tp_rank: int = 0, dp_rank: int = 0, pp_rank: int = 0): - """given TP,DP,PP rank to get the global rank.""" - - tp_size = mpu.get_tensor_model_parallel_world_size() - dp_size = mpu.get_data_parallel_world_size() - pp_size = mpu.get_pipeline_model_parallel_world_size() - assert (tp_size * dp_size * pp_size == torch.distributed.get_world_size() - ), f"{tp_size} x {dp_size} x {pp_size} != {torch.distributed.get_world_size()}" - # We only support TP-DP-PP grouping, for correctness when resharding - return (pp_rank * dp_size + dp_rank) * tp_size + tp_rank - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - from megatron.core import mpu - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def merge_megatron_ckpt_gptmodel(wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False): - """Merge sharded parameters of a Megatron module into a merged checkpoint. - - Args: - wrapped_models (list of megatron.core.distributed.DistributedDataParallel): - The local DDP wrapped megatron modules. - config (str or None): - HF config for model - dtype: model params type - is_value_model: if model is value model - tie_word_embeddings: tie_word_embeddings - Returns: - state_dict (dict): - The merged state_dict in rank 0, and an empty dictionary in other ranks. - """ - start_time = time.time() - - def _get_gpt_model(model): - return model - - dp_rank = mpu.get_data_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - pp_rank = mpu.get_pipeline_model_parallel_rank() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if dist.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - assert len(models[i].decoder.layers - ) == num_layers_per_model, 'len model layers {} not equal to num_layers_per_model {}'.format( - len(models[i].decoder.layers), num_layers_per_model) - - state_dict = dict() - - def _get_cpu_tensor(tensor: torch.Tensor): - if tensor is None: - return None - if tensor.device == torch.device("cpu"): - return tensor.detach().clone() - return tensor.detach().cpu() - - def _broadcast_tensor(tensor, name, src_pp_rank) -> torch.Tensor: - """broadcast tensor across mp_group""" - nonlocal state_dict - nonlocal mp_group - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - if tensor is None: - weight = None - tensor_shape = None - else: - weight = tensor - tensor_shape = weight.shape - else: - weight = None - tensor_shape = None - - obj_list = [tensor_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - tensor_shape = obj_list[0] - - if tensor_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tensor:[{name}] not exist, skip collect") - return - - if weight is None: - weight = torch.empty( - tensor_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - dist.broadcast(weight, src=src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - state_dict[name] = _get_cpu_tensor(weight) - - def _broadcast_tp_shard_tensor(tensor, name, src_pp_rank, concat_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=concat_dim) - if mutate_func is not None: - full_tensor = mutate_func(full_tensor) - state_dict[name] = full_tensor - - def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name, src_pp_rank) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=0) - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_list = [] - up_weight_list = [] - for i in range(tp_size): - gate_up_weight_tp = full_tensor[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)] - gate_weight_tp = gate_up_weight_tp[:intermediate_size_tp] - up_weight_tp = gate_up_weight_tp[intermediate_size_tp:] - gate_weight_list.append(gate_weight_tp) - up_weight_list.append(up_weight_tp) - - state_dict[gate_name] = torch.cat(gate_weight_list, dim=0) - state_dict[up_name] = torch.cat(up_weight_list, dim=0) - - def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, src_pp_rank): - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{q_name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=0) - q_weight_list = [] - k_weight_list = [] - v_weight_list = [] - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - for i in range(tp_size): - num_query_groups_per_partition = wrapped_models[0].config.num_query_groups // tp_size - qkv_part = full_tensor[i * total_size:(i + 1) * total_size] - q_size_chunk = q_size_tp // num_query_groups_per_partition - kv_size_chunk = kv_size_tp // num_query_groups_per_partition - for qkv_part_chunk in qkv_part.chunk(num_query_groups_per_partition): - q_part = qkv_part_chunk[:q_size_chunk] - k_part = qkv_part_chunk[q_size_chunk:q_size_chunk + kv_size_chunk] - v_part = qkv_part_chunk[q_size_chunk + kv_size_chunk:] - q_weight_list.append(q_part) - k_weight_list.append(k_part) - v_weight_list.append(v_part) - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - for i in range(tp_size): - num_query_groups_per_partition = wrapped_models[0].config.num_query_groups // tp_size - qkv_part = full_tensor[i * total_size:(i + 1) * total_size] - q_size_chunk = q_size_tp // num_query_groups_per_partition - kv_size_chunk = kv_size_tp // num_query_groups_per_partition - for qkv_part_chunk in qkv_part.chunk(num_query_groups_per_partition): - q_part = qkv_part_chunk[:q_size_chunk] - k_part = qkv_part_chunk[q_size_chunk:q_size_chunk + kv_size_chunk] - v_part = qkv_part_chunk[q_size_chunk + kv_size_chunk:] - q_weight_list.append(q_part) - if i * config.num_key_value_heads % tp_size == 0: - k_weight_list.append(k_part) - v_weight_list.append(v_part) - - state_dict[q_name] = torch.cat(q_weight_list, dim=0) - state_dict[k_name] = torch.cat(k_weight_list, dim=0) - state_dict[v_name] = torch.cat(v_weight_list, dim=0) - - # empty cache before collecting weights - torch.cuda.empty_cache() - # Embeddings - # ------------------- - if dp_rank == 0: - # Embeddings - # ------------------- - print_rank_0("collecting embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - _broadcast_tp_shard_tensor( - gpt_model_module.embedding.word_embeddings.weight if pp_rank == 0 else None, - "model.embed_tokens.weight", - src_pp_rank=0, - ) - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - for layer in range(config.num_hidden_layers): - print_rank_0(f"collecting layer #{layer}...") - layer_name = f"model.layers.{layer}" - src_pp_rank, src_virtual_pp_rank, src_layer_idx = layer_map[layer] - - gpt_model_module = _get_gpt_model(models[src_virtual_pp_rank]) - sync_layer = gpt_model_module.decoder.layers[src_layer_idx] - - _broadcast_tensor( - sync_layer.self_attention.linear_qkv.layer_norm_weight, - f"{layer_name}.input_layernorm.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attention.linear_qkv.weight, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - src_pp_rank=src_pp_rank, - ) - - if getattr(sync_layer.self_attention.linear_qkv, 'bias', - None) is not None and sync_layer.self_attention.linear_qkv.bias.numel() > 0: - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attention.linear_qkv.bias, - f"{layer_name}.self_attn.q_proj.bias", - f"{layer_name}.self_attn.k_proj.bias", - f"{layer_name}.self_attn.v_proj.bias", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor( - sync_layer.self_attention.linear_proj.weight, - f"{layer_name}.self_attn.o_proj.weight", - concat_dim=1, - src_pp_rank=src_pp_rank, - ) - - _broadcast_tensor( - sync_layer.mlp.linear_fc1.layer_norm_weight, - f"{layer_name}.post_attention_layernorm.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor_gate_up(sync_layer.mlp.linear_fc1.weight, - f"{layer_name}.mlp.gate_proj.weight", - f"{layer_name}.mlp.up_proj.weight", - src_pp_rank=src_pp_rank) - - _broadcast_tp_shard_tensor( - sync_layer.mlp.linear_fc2.weight, - f"{layer_name}.mlp.down_proj.weight", - concat_dim=1, - src_pp_rank=src_pp_rank, - ) - - # Final Layernorm - # ------------------- - print_rank_0("collecting final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _broadcast_tensor( - getattr(gpt_model_module.decoder.final_layernorm, "weight", None), - "model.norm.weight", - src_pp_rank=pp_size - 1, - ) - - if tie_word_embeddings: - print_rank_0(f"tie word embedding skip load lm_head...") - else: - print_rank_0("collecting lm_head...") - - if is_value_model: - lm_head_weight = None - if pp_rank == pp_size - 1: - lm_head_weight = getattr(gpt_model_module.output_layer, "weight", None) - _broadcast_tensor(lm_head_weight, "lm_head.weight", src_pp_rank=pp_size - 1) - - else: - _broadcast_tp_shard_tensor( - getattr(gpt_model_module.output_layer, "weight", None) if pp_rank == pp_size - 1 else None, - "lm_head.weight", - src_pp_rank=pp_size - 1, - ) - - dist.barrier() - torch.cuda.empty_cache() - if torch.distributed.get_rank() == 0: - - for k, v in state_dict.items(): - if dtype != v.dtype: - state_dict[k] = v.to(dtype) - - print_rank_0(f"merge megatron ckpt done, time elapsed {time.time() - start_time}s") - return state_dict diff --git a/verl/models/qwen2/__init__.py b/verl/models/qwen2/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/models/qwen2/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/models/qwen2/megatron/__init__.py b/verl/models/qwen2/megatron/__init__.py deleted file mode 100644 index 26ff013785..0000000000 --- a/verl/models/qwen2/megatron/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .modeling_qwen2_megatron import ( - # original model with megatron - ParallelQwen2Model, - ParallelQwen2ForCausalLM, - # rmpad with megatron - ParallelQwen2ForCausalLMRmPad, - ParallelQwen2ForValueRmPad, - # rmpad with megatron and pipeline parallelism - ParallelQwen2ForCausalLMRmPadPP, - ParallelQwen2ForValueRmPadPP) diff --git a/verl/models/qwen2/megatron/checkpoint_utils/__init__.py b/verl/models/qwen2/megatron/checkpoint_utils/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/models/qwen2/megatron/checkpoint_utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py b/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py deleted file mode 100644 index 87e1561a40..0000000000 --- a/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import time -from typing import Dict, Any, Callable, Optional -import torch.distributed as dist - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - from megatron.core import mpu - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def load_state_dict_to_megatron_qwen2(state_dict, - wrapped_models, - config, - params_dtype, - is_value_model=False, - tie_word_embeddings=False): - """Load merged state_dict to sharded Megatron module in training. - """ - from verl.utils.megatron_utils import print_rank_0, unwrap_model - from megatron.core import mpu - from megatron.core.transformer.module import Float16Module - from megatron.core import DistributedDataParallel as LocalDDP - from torch.nn.parallel import DistributedDataParallel as torchDDP - - start_time = time.time() - - def _get_gpt_model(model): - return model - - def fetch_params(module): - for param in module.parameters(): - torch.distributed.fetch(param.data, - src=mpu.get_data_parallel_src_rank(), - group=mpu.get_data_parallel_group()) - - dp_rank = mpu.get_data_parallel_rank() - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if torch.distributed.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, f'num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size: {virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}' - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - gpt_model_module = _get_gpt_model(models[i]) - assert len(gpt_model_module.model.layers) == num_layers_per_model - - def _fetch_tensor(tensor, name) -> torch.Tensor: - """fetch tensor""" - nonlocal state_dict - if tensor is not None: - tensor = tensor.data.copy_(state_dict[name], non_blocking=True) - - def _fetch_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """fetch tensor in tp shards""" - nonlocal state_dict - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - if name in state_dict: - full_weight = state_dict[name] - - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - if tensor is not None: - tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) - else: - print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - - def _fetch_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """fetch tensor in tp shards""" - nonlocal state_dict - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - if name in state_dict: - full_weight = state_dict[name] - - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - if tensor is not None: - tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) - else: - print(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - - def _fetch_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: - """fetch gate_up tensor in tp shards""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - if gate_name in state_dict and up_name in state_dict: - gate_weight = state_dict[gate_name] - up_weight = state_dict[up_name] - new_gate_up_weight = torch.empty(config.intermediate_size * 2, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_tp = gate_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - up_weight_tp = up_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - new_gate_up_weight[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)].copy_( - torch.cat([gate_weight_tp, up_weight_tp], dim=0)) - - tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) - if tensor is not None: - tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) - else: - print(f"tp_shard tensor:[{gate_name}, {up_name}] not in state_dict, skip loading") - - def _fetch_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, bias=False) -> torch.Tensor: - """fetch tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - assert (q_name in state_dict and k_name in state_dict and v_name in state_dict) - full_weight_q = state_dict[q_name] - full_weight_k = state_dict[k_name] - full_weight_v = state_dict[v_name] - - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - if not bias: - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - else: - new_weight_qkv = torch.empty(total_size * tp_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - k_part = full_weight_k[i * kv_size_tp:(i + 1) * kv_size_tp] - v_part = full_weight_v[i * kv_size_tp:(i + 1) * kv_size_tp] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) - - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - if not bias: - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - else: - new_weight_qkv = torch.empty(total_size * tp_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head - end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head - k_part = full_weight_k[start_idx:end_idx] - v_part = full_weight_v[start_idx:end_idx] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], dim=0)) - - tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) - if tensor is not None: - tensor = tensor.data.copy_(tensor_chunk[tp_rank], non_blocking=True) - - # Embeddings - # ------------------- - print_rank_0("loading embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - if pp_rank == 0: - embed_tokens_weight = gpt_model_module.model.embed_tokens.weight - _fetch_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - num_layer_per_pp = config.num_hidden_layers // pp_size - vpp_size = mpu.get_virtual_pipeline_model_parallel_world_size() - - layer_list = [] - if vpp_size is not None: - for vpp_rank in range(vpp_size): - num_layer_vpp_chunk = num_layer_per_pp // vpp_size - num_layer_this_model = num_layer_vpp_chunk - offset = vpp_rank * ( - config.num_hidden_layers // mpu.get_virtual_pipeline_model_parallel_world_size()) + \ - (mpu.get_pipeline_model_parallel_rank() * num_layer_vpp_chunk) - layer_list.extend(list(range(offset, offset + num_layer_this_model))) - else: - num_layer_this_model = num_layer_per_pp - offset = pp_rank * num_layer_per_pp - layer_list.extend(list(range(offset, offset + num_layer_this_model))) - - for layer in layer_list: - print(f"{torch.distributed.get_rank()} loading layer #{layer}...") - layer_name = f"model.layers.{layer}" - dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] - - print( - f'{torch.distributed.get_rank()} offset: {offset}, num_layer_this_model: {num_layer_this_model}, layer_name: {layer_name}, layer_map[layer]: {layer_map[layer]}' - ) - - gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) - sync_layer = gpt_model_module.model.layers[dst_layer_idx] - - _fetch_tensor( - sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.input_layernorm.weight", - ) - - _fetch_tp_shard_tensor_qkv( - sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - ) - - _fetch_tp_shard_tensor_qkv(sync_layer.self_attn.qkv_proj.bias if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.bias", - f"{layer_name}.self_attn.k_proj.bias", - f"{layer_name}.self_attn.v_proj.bias", - bias=True) - - _fetch_tp_shard_tensor( - sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.o_proj.weight", - chunk_dim=1, - ) - - _fetch_tensor( - sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.post_attention_layernorm.weight", - ) - - _fetch_tp_shard_tensor_gate_up(sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.gate_proj.weight", f"{layer_name}.mlp.up_proj.weight") - - _fetch_tp_shard_tensor( - sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.down_proj.weight", - chunk_dim=1, - ) - # Final Layernorm - # ------------------- - print_rank_0("loading final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _fetch_tensor( - getattr(gpt_model_module.model.norm, "weight", None), - "model.norm.weight", - ) - - if tie_word_embeddings: - print_rank_0("tie_word_embeddings skip load lm_head") - else: - print_rank_0("loading lm_head...") - if pp_rank + 1 == pp_size: - lm_head_weight = gpt_model_module.lm_head.weight - - if is_value_model: - if 'lm_head.weight' in state_dict and state_dict['lm_head.weight'].shape[0] == 1: - _fetch_tensor(lm_head_weight, "lm_head.weight") - print_rank_0('load lm_head from value_head weight') - elif 'reward_head.weight' in state_dict and state_dict['reward_head.weight'].shape[0] == 1: - _fetch_tensor(lm_head_weight, "reward_head.weight") - print_rank_0('load lm_head from value_head weight') - else: - _fetch_tensor(None, "lm_head.weight") - print_rank_0('fail to match lm_head in value_model') - - else: - _fetch_tp_shard_tensor(lm_head_weight, "lm_head.weight") - - dist.barrier() - torch.cuda.empty_cache() - print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") diff --git a/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py b/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py deleted file mode 100644 index 3d14887cdc..0000000000 --- a/verl/models/qwen2/megatron/checkpoint_utils/qwen2_loader_depracated.py +++ /dev/null @@ -1,466 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import time -from typing import Dict, Any, Callable, Optional -import torch.distributed as dist - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - from megatron.core import mpu - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def load_state_dict_to_megatron_qwen2(state_dict, - wrapped_models, - config, - params_dtype, - is_value_model=False, - tie_word_embeddings=False): - """Load merged state_dict to sharded Megatron module in training. - """ - from verl.utils.megatron_utils import print_rank_0, unwrap_model - from megatron.core import mpu - from megatron.core.transformer.module import Float16Module - from megatron.core import DistributedDataParallel as LocalDDP - from torch.nn.parallel import DistributedDataParallel as torchDDP - - start_time = time.time() - - def _get_gpt_model(model): - return model - - def broadcast_params(module): - for param in module.parameters(): - torch.distributed.broadcast(param.data, - src=mpu.get_data_parallel_src_rank(), - group=mpu.get_data_parallel_group()) - - dp_rank = mpu.get_data_parallel_rank() - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if torch.distributed.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers, f'num_layers_per_model: {num_layers_per_model} * pp_size: {pp_size} * virtual_pp_size: {virtual_pp_size} != config.num_hidden_layers: {config.num_hidden_layers}' - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - gpt_model_module = _get_gpt_model(models[i]) - assert len(gpt_model_module.model.layers) == num_layers_per_model - - def _broadcast_tensor(tensor, name) -> torch.Tensor: - """broadcast tensor from rank0 across mp_group""" - nonlocal state_dict - nonlocal mp_group - if torch.distributed.get_rank() == 0: - if name in state_dict: - weight = state_dict[name] - tensor_shape = weight.shape - else: - tensor_shape = None - else: - weight = None - tensor_shape = None - - obj_list = [tensor_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - tensor_shape = obj_list[0] - - if tensor_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tensor:[{name}] not in state_dict, skip load") - return - - if tensor is None: - tensor = torch.empty( - tensor_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - if torch.distributed.get_rank() == 0: - tensor.data.copy_(weight) - dist.broadcast(tensor, src=0, group=mp_group) - - def _broadcast_tp_shard_tensor_vocab(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - if name in state_dict: - full_weight = state_dict[name] - - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor(tensor, name, chunk_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - if name in state_dict: - full_weight = state_dict[name] - if mutate_func is not None: - full_weight = mutate_func(full_weight) - tensor_chunk = torch.chunk(full_weight, tp_size, dim=chunk_dim) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - gate_weight = state_dict[gate_name] - up_weight = state_dict[up_name] - new_gate_up_weight = torch.empty(config.intermediate_size * 2, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_tp = gate_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - up_weight_tp = up_weight[i * intermediate_size_tp:(i + 1) * intermediate_size_tp] - new_gate_up_weight[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)].copy_( - torch.cat([gate_weight_tp, up_weight_tp], dim=0)) - - tensor_chunk = torch.chunk(new_gate_up_weight, tp_size, dim=0) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert ( - tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank() == 0:} tensor {gate_name, up_name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, bias=False) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - - if torch.distributed.get_rank() == 0: - assert (q_name in state_dict and k_name in state_dict and v_name in state_dict) - full_weight_q = state_dict[q_name] - full_weight_k = state_dict[k_name] - full_weight_v = state_dict[v_name] - - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - if not bias: - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - else: - new_weight_qkv = torch.empty(total_size * tp_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - k_part = full_weight_k[i * kv_size_tp:(i + 1) * kv_size_tp] - v_part = full_weight_v[i * kv_size_tp:(i + 1) * kv_size_tp] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], - dim=0)) - - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - if not bias: - new_weight_qkv = torch.empty(total_size * tp_size, - config.hidden_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - else: - new_weight_qkv = torch.empty(total_size * tp_size, - dtype=params_dtype, - device=torch.cuda.current_device()) - for i in range(tp_size): - q_part = full_weight_q[i * q_size_tp:(i + 1) * q_size_tp] - start_idx = i * config.num_key_value_heads // tp_size * hidden_size_per_head - end_idx = (i * config.num_key_value_heads // tp_size + 1) * hidden_size_per_head - k_part = full_weight_k[start_idx:end_idx] - v_part = full_weight_v[start_idx:end_idx] - new_weight_qkv[i * total_size:(i + 1) * total_size].copy_(torch.cat([q_part, k_part, v_part], - dim=0)) - - tensor_chunk = torch.chunk(new_weight_qkv, tp_size, dim=0) - chunk_shape = tensor_chunk[0].shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=0, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{q_name, k_name, v_name}] not in state_dict, skip loading") - return - - if tensor is None: - sync_tensor = torch.empty( - chunk_shape, - dtype=params_dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - assert (tensor.shape == chunk_shape - ), f"rank #{torch.distributed.get_rank()} tensor {q_name} shape {tensor.shape} != {chunk_shape}" - sync_tensor = torch.empty_like(tensor, device=torch.cuda.current_device(), requires_grad=False) - - for i in range(tp_size): - if torch.distributed.get_rank() == 0: - sync_tensor.data.copy_(tensor_chunk[i]) - dist.broadcast(sync_tensor, src=0, group=mp_group) - if (i == tp_rank) and (tensor is not None): - tensor.data.copy_(sync_tensor) - - if dp_rank == 0: - # Embeddings - # ------------------- - print_rank_0("loading embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - embed_tokens_weight = None - if pp_rank == 0: - embed_tokens_weight = gpt_model_module.model.embed_tokens.weight - _broadcast_tp_shard_tensor_vocab(embed_tokens_weight, "model.embed_tokens.weight") - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - - for layer in range(config.num_hidden_layers): - print_rank_0(f"loading layer #{layer}...") - layer_name = f"model.layers.{layer}" - dst_pp_rank, dst_virtual_pp_rank, dst_layer_idx = layer_map[layer] - - gpt_model_module = _get_gpt_model(models[dst_virtual_pp_rank]) - sync_layer = gpt_model_module.model.layers[dst_layer_idx] - - _broadcast_tensor( - sync_layer.input_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.input_layernorm.weight", - ) - - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attn.qkv_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - ) - - _broadcast_tp_shard_tensor_qkv(sync_layer.self_attn.qkv_proj.bias if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.q_proj.bias", - f"{layer_name}.self_attn.k_proj.bias", - f"{layer_name}.self_attn.v_proj.bias", - bias=True) - - _broadcast_tp_shard_tensor( - sync_layer.self_attn.o_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.self_attn.o_proj.weight", - chunk_dim=1, - ) - - _broadcast_tensor( - sync_layer.post_attention_layernorm.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.post_attention_layernorm.weight", - ) - - _broadcast_tp_shard_tensor_gate_up(sync_layer.mlp.gate_up_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.gate_proj.weight", f"{layer_name}.mlp.up_proj.weight") - - _broadcast_tp_shard_tensor( - sync_layer.mlp.down_proj.weight if dst_pp_rank == pp_rank else None, - f"{layer_name}.mlp.down_proj.weight", - chunk_dim=1, - ) - # Final Layernorm - # ------------------- - print_rank_0("loading final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _broadcast_tensor( - getattr(gpt_model_module.model.norm, "weight", None), - "model.norm.weight", - ) - - if tie_word_embeddings: - print_rank_0("tie_word_embeddings skip load lm_head") - else: - print_rank_0("loading lm_head...") - lm_head_weight = None - if pp_rank + 1 == pp_size: - lm_head_weight = gpt_model_module.lm_head.weight - - if is_value_model: - if 'lm_head.weight' in state_dict and state_dict['lm_head.weight'].shape[0] == 1: - _broadcast_tensor(lm_head_weight, "lm_head.weight") - print_rank_0('load lm_head from value_head weight') - elif 'reward_head.weight' in state_dict and state_dict['reward_head.weight'].shape[0] == 1: - _broadcast_tensor(lm_head_weight, "reward_head.weight") - print_rank_0('load lm_head from value_head weight') - else: - _broadcast_tensor(None, "lm_head.weight") - print_rank_0('fail to match lm_head in value_model') - - else: - _broadcast_tp_shard_tensor(lm_head_weight, "lm_head.weight") - - dist.barrier() - # Broadcast weights inside data parallel groups - for wrapped_model in wrapped_models: - broadcast_params(wrapped_model) - - torch.cuda.empty_cache() - print_rank_0(f"loading megatron ckpt done, time elapsed {time.time() - start_time}s") \ No newline at end of file diff --git a/verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py b/verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py deleted file mode 100644 index 547a097b1c..0000000000 --- a/verl/models/qwen2/megatron/checkpoint_utils/qwen2_saver.py +++ /dev/null @@ -1,448 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl.utils.megatron_utils import print_rank_0, unwrap_model -from megatron.core import mpu -from megatron.core.transformer.module import Float16Module -from megatron.core.distributed import DistributedDataParallel as LocalDDP -from torch.nn.parallel import DistributedDataParallel as torchDDP -import torch -import time - -import torch -import torch.distributed as dist - - -def _megatron_calc_global_rank(tp_rank: int = 0, dp_rank: int = 0, pp_rank: int = 0): - """given TP,DP,PP rank to get the global rank.""" - - tp_size = mpu.get_tensor_model_parallel_world_size() - dp_size = mpu.get_data_parallel_world_size() - pp_size = mpu.get_pipeline_model_parallel_world_size() - assert (tp_size * dp_size * pp_size == torch.distributed.get_world_size() - ), f"{tp_size} x {dp_size} x {pp_size} != {torch.distributed.get_world_size()}" - # We only support TP-DP-PP grouping, for correctness when resharding - return (pp_rank * dp_size + dp_rank) * tp_size + tp_rank - - -def _megatron_calc_layer_map(config): - """Calculate the mapping of global layer_idx to local layer_idx - Returns: - layer_map (Dict: int -> tuple(int, int, int)): - mapping from the global layer index to - a tuple of (pp_rank, virtual_pp_rank, layer_idx inside model) - """ - from megatron.core import mpu - - pp_size = mpu.get_pipeline_model_parallel_world_size() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - - layer_map = dict() - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - for pp_rank_idx in range(pp_size): - for virtual_pp_rank_idx in range(virtual_pp_size): - layer_offset = (virtual_pp_rank_idx * (config.num_hidden_layers // virtual_pp_size) + - pp_rank_idx * num_layers_per_model) - for layer_idx in range(num_layers_per_model): - layer_map[layer_offset + layer_idx] = ( - pp_rank_idx, - virtual_pp_rank_idx, - layer_idx, - ) - return layer_map - - -def merge_megatron_ckpt_qwen2(wrapped_models, config, dtype, is_value_model=False, tie_word_embeddings=False): - """Merge sharded parameters of a Megatron module into a merged checkpoint. - - Args: - wrapped_models (list of megatron.core.distributed.DistributedDataParallel): - The local DDP wrapped megatron modules. - config (str or None): - HF config for model - dtype: model params type - is_value_model: if model is value model - tie_word_embeddings: tie_word_embeddings - Returns: - state_dict (dict): - The merged state_dict in rank 0, and an empty dictionary in other ranks. - """ - start_time = time.time() - - def _get_gpt_model(model): - return model - - dp_rank = mpu.get_data_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - pp_rank = mpu.get_pipeline_model_parallel_rank() - virtual_pp_size = mpu.get_virtual_pipeline_model_parallel_world_size() or 1 - mp_group = mpu.get_model_parallel_group() - - if dist.get_rank() == 0: - assert mp_group.rank() == 0, f"mp_rank:[{mp_group.rank}] != 0 on rank #0" - assert pp_rank == 0, f"pp_rank:[{pp_rank}] != 0 on rank #0" - assert dp_rank == 0, f"dp_rank:[{dp_rank}] != 0 on rank #0" - - if not isinstance(wrapped_models, (list, tuple)): - wrapped_models = list(wrapped_models) - - assert len(wrapped_models) == virtual_pp_size - num_layers_per_model = config.num_hidden_layers // pp_size // virtual_pp_size - assert num_layers_per_model * pp_size * virtual_pp_size == config.num_hidden_layers - - models = [None] * len(wrapped_models) - - for i, wrapped_model in enumerate(wrapped_models): - models[i] = unwrap_model(wrapped_model, (torchDDP, LocalDDP, Float16Module)) - assert len(models[i].model.layers - ) == num_layers_per_model, 'len model layers {} not equal to num_layers_per_model {}'.format( - len(models[i].model.layers), num_layers_per_model) - - state_dict = dict() - - def _get_cpu_tensor(tensor: torch.Tensor): - if tensor is None: - return None - if tensor.device == torch.device("cpu"): - return tensor.detach().clone() - return tensor.detach().cpu() - - def _broadcast_tensor(tensor, name, src_pp_rank) -> torch.Tensor: - """broadcast tensor across mp_group""" - nonlocal state_dict - nonlocal mp_group - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - if tensor is None: - weight = None - tensor_shape = None - else: - weight = tensor - tensor_shape = weight.shape - else: - weight = None - tensor_shape = None - - obj_list = [tensor_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - tensor_shape = obj_list[0] - - if tensor_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tensor:[{name}] not exist, skip collect") - return - - if weight is None: - weight = torch.empty( - tensor_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - dist.broadcast(weight, src=src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - state_dict[name] = _get_cpu_tensor(weight) - - def _broadcast_tp_shard_tensor(tensor, name, src_pp_rank, concat_dim=0, mutate_func=None) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=concat_dim) - if mutate_func is not None: - full_tensor = mutate_func(full_tensor) - state_dict[name] = full_tensor - - def _broadcast_tp_shard_tensor_gate_up(tensor, gate_name, up_name, src_pp_rank) -> torch.Tensor: - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{gate_name, up_name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=0) - intermediate_size_tp = config.intermediate_size // tp_size - gate_weight_list = [] - up_weight_list = [] - for i in range(tp_size): - gate_up_weight_tp = full_tensor[intermediate_size_tp * 2 * i:intermediate_size_tp * 2 * (i + 1)] - gate_weight_tp = gate_up_weight_tp[:intermediate_size_tp] - up_weight_tp = gate_up_weight_tp[intermediate_size_tp:] - gate_weight_list.append(gate_weight_tp) - up_weight_list.append(up_weight_tp) - - state_dict[gate_name] = torch.cat(gate_weight_list, dim=0) - state_dict[up_name] = torch.cat(up_weight_list, dim=0) - - def _broadcast_tp_shard_tensor_qkv(tensor, q_name, k_name, v_name, src_pp_rank): - """broadcast tensor in tp shards across mp_group""" - nonlocal state_dict - nonlocal mp_group - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - src_rank = _megatron_calc_global_rank(tp_rank=0, dp_rank=0, pp_rank=src_pp_rank) - - if torch.distributed.get_rank() == src_rank: - chunk_shape = tensor.shape - else: - chunk_shape = None - - obj_list = [chunk_shape] - dist.broadcast_object_list(obj_list, src=src_rank, group=mp_group) - chunk_shape = obj_list[0] - if chunk_shape is None: - # all or none ranks in the mp_group should reach here - print_rank_0(f"tp_shard tensor:[{q_name}] not exist, skip collecting") - return - - buffer_tensor = torch.empty( - chunk_shape, - dtype=dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - - chunk_tensors = [None] * tp_size - - for i in range(tp_size): - cur_src_rank = _megatron_calc_global_rank(tp_rank=i, dp_rank=0, pp_rank=src_pp_rank) - sync_tensor = tensor if torch.distributed.get_rank() == cur_src_rank else buffer_tensor - dist.broadcast(sync_tensor, src=cur_src_rank, group=mp_group) - - if torch.distributed.get_rank() == 0: - chunk_tensors[i] = _get_cpu_tensor(sync_tensor) - - if torch.distributed.get_rank() == 0: - full_tensor = torch.concat(chunk_tensors, dim=0) - q_weight_list = [] - k_weight_list = [] - v_weight_list = [] - hidden_size_per_head = config.hidden_size // config.num_attention_heads - - if config.num_key_value_heads >= tp_size: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head * config.num_key_value_heads // tp_size - total_size = q_size_tp + 2 * kv_size_tp - for i in range(tp_size): - qkv_part = full_tensor[i * total_size:(i + 1) * total_size] - q_part = qkv_part[:q_size_tp] - k_part = qkv_part[q_size_tp:q_size_tp + kv_size_tp] - v_part = qkv_part[q_size_tp + kv_size_tp:total_size] - q_weight_list.append(q_part) - k_weight_list.append(k_part) - v_weight_list.append(v_part) - else: - q_size_tp = config.hidden_size // tp_size - kv_size_tp = hidden_size_per_head - total_size = q_size_tp + 2 * kv_size_tp - for i in range(tp_size): - qkv_part = full_tensor[i * total_size:(i + 1) * total_size] - q_part = qkv_part[:q_size_tp] - k_part = qkv_part[q_size_tp:q_size_tp + kv_size_tp] - v_part = qkv_part[q_size_tp + kv_size_tp:total_size] - q_weight_list.append(q_part) - if i * config.num_key_value_heads % tp_size == 0: - k_weight_list.append(k_part) - v_weight_list.append(v_part) - - state_dict[q_name] = torch.cat(q_weight_list, dim=0) - state_dict[k_name] = torch.cat(k_weight_list, dim=0) - state_dict[v_name] = torch.cat(v_weight_list, dim=0) - - # empty cache before collecting weights - torch.cuda.empty_cache() - # Embeddings - # ------------------- - if dp_rank == 0: - # Embeddings - # ------------------- - print_rank_0("collecting embeddings...") - gpt_model_module = _get_gpt_model(models[0]) - _broadcast_tp_shard_tensor( - gpt_model_module.model.embed_tokens.weight if pp_rank == 0 else None, - "model.embed_tokens.weight", - src_pp_rank=0, - ) - - # Transformer layers - # ------------------- - layer_map = _megatron_calc_layer_map(config) - for layer in range(config.num_hidden_layers): - print_rank_0(f"collecting layer #{layer}...") - layer_name = f"model.layers.{layer}" - src_pp_rank, src_virtual_pp_rank, src_layer_idx = layer_map[layer] - - gpt_model_module = _get_gpt_model(models[src_virtual_pp_rank]) - sync_layer = gpt_model_module.model.layers[src_layer_idx] - - _broadcast_tensor( - sync_layer.input_layernorm.weight, - f"{layer_name}.input_layernorm.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attn.qkv_proj.weight, - f"{layer_name}.self_attn.q_proj.weight", - f"{layer_name}.self_attn.k_proj.weight", - f"{layer_name}.self_attn.v_proj.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor_qkv( - sync_layer.self_attn.qkv_proj.bias, - f"{layer_name}.self_attn.q_proj.bias", - f"{layer_name}.self_attn.k_proj.bias", - f"{layer_name}.self_attn.v_proj.bias", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor( - sync_layer.self_attn.o_proj.weight, - f"{layer_name}.self_attn.o_proj.weight", - concat_dim=1, - src_pp_rank=src_pp_rank, - ) - - _broadcast_tensor( - sync_layer.post_attention_layernorm.weight, - f"{layer_name}.post_attention_layernorm.weight", - src_pp_rank=src_pp_rank, - ) - - _broadcast_tp_shard_tensor_gate_up(sync_layer.mlp.gate_up_proj.weight, - f"{layer_name}.mlp.gate_proj.weight", - f"{layer_name}.mlp.up_proj.weight", - src_pp_rank=src_pp_rank) - - _broadcast_tp_shard_tensor( - sync_layer.mlp.down_proj.weight, - f"{layer_name}.mlp.down_proj.weight", - concat_dim=1, - src_pp_rank=src_pp_rank, - ) - - # Final Layernorm - # ------------------- - print_rank_0("collecting final layernorm...") - gpt_model_module = _get_gpt_model(models[-1]) - _broadcast_tensor( - getattr(gpt_model_module.model.norm, "weight", None), - "model.norm.weight", - src_pp_rank=pp_size - 1, - ) - - if tie_word_embeddings: - print_rank_0(f"tie word embedding skip load lm_head...") - else: - print_rank_0("collecting lm_head...") - - if is_value_model: - _broadcast_tensor(gpt_model_module.lm_head.weight if pp_rank == pp_size - 1 else None, - "lm_head.weight", - src_pp_rank=pp_size - 1) - _broadcast_tensor(gpt_model_module.reward_head.weight if pp_rank == pp_size - 1 and - getattr(gpt_model_module, "reward_weight", None) is not None else None, - "reward_head.weight", - src_pp_rank=pp_size - 1) - - else: - _broadcast_tp_shard_tensor( - getattr(gpt_model_module.lm_head, "weight", None) if pp_rank == pp_size - 1 else None, - "lm_head.weight", - src_pp_rank=pp_size - 1, - ) - - dist.barrier() - - torch.cuda.empty_cache() - if torch.distributed.get_rank() == 0: - - for k, v in state_dict.items(): - if dtype != v.dtype: - state_dict[k] = v.to(dtype) - - print_rank_0(f"merge megatron ckpt done, time elapsed {time.time() - start_time}s") - return state_dict diff --git a/verl/models/qwen2/megatron/layers/__init__.py b/verl/models/qwen2/megatron/layers/__init__.py deleted file mode 100644 index 2d19bef216..0000000000 --- a/verl/models/qwen2/megatron/layers/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .parallel_attention import ParallelQwen2Attention -from .parallel_decoder import ParallelQwen2DecoderLayer, ParallelQwen2DecoderLayerRmPad -from .parallel_mlp import ParallelQwen2MLP -from .parallel_rmsnorm import ParallelQwen2RMSNorm diff --git a/verl/models/qwen2/megatron/layers/parallel_attention.py b/verl/models/qwen2/megatron/layers/parallel_attention.py deleted file mode 100644 index a4f65e892a..0000000000 --- a/verl/models/qwen2/megatron/layers/parallel_attention.py +++ /dev/null @@ -1,401 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import math -from typing import Optional, Tuple - -import torch -from megatron.core import parallel_state as mpu -from megatron.core import tensor_parallel -from megatron.core import ModelParallelConfig -from torch import nn -from transformers import Qwen2Config -from verl.models.qwen2.megatron.layers.parallel_linear import QKVParallelLinear - -from verl.utils.megatron import tensor_parallel as tp_utils - - -class Qwen2RotaryEmbedding(nn.Module): - - def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): - super().__init__() - - self.dim = dim - self.max_position_embeddings = max_position_embeddings - self.base = base - inv_freq = 1.0 / (self.base**(torch.arange(0, self.dim, 2).float().to(device) / self.dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - # Build here to make `torch.jit.trace` work. - self._set_cos_sin_cache(seq_len=max_position_embeddings, - device=self.inv_freq.device, - dtype=torch.get_default_dtype()) - - def _set_cos_sin_cache(self, seq_len, device, dtype): - self.max_seq_len_cached = seq_len - t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - - freqs = torch.einsum("i,j->ij", t, self.inv_freq) - # Different from paper, but it uses a different permutation in order to obtain the same calculation - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - - def forward(self, x, seq_len=None): - # x: [bs, num_attention_heads, seq_len, head_size] - if seq_len > self.max_seq_len_cached: - self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) - - return ( - self.cos_cached[:seq_len].to(dtype=x.dtype), - self.sin_cached[:seq_len].to(dtype=x.dtype), - ) - - -class Qwen2LinearScalingRotaryEmbedding(Qwen2RotaryEmbedding): - """Qwen2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev""" - - def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): - self.scaling_factor = scaling_factor - super().__init__(dim, max_position_embeddings, base, device) - - def _set_cos_sin_cache(self, seq_len, device, dtype): - self.max_seq_len_cached = seq_len - t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - t = t / self.scaling_factor - - freqs = torch.einsum("i,j->ij", t, self.inv_freq) - # Different from paper, but it uses a different permutation in order to obtain the same calculation - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - - -class Qwen2DynamicNTKScalingRotaryEmbedding(Qwen2RotaryEmbedding): - """Qwen2RotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla""" - - def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0): - self.scaling_factor = scaling_factor - super().__init__(dim, max_position_embeddings, base, device) - - def _set_cos_sin_cache(self, seq_len, device, dtype): - self.max_seq_len_cached = seq_len - - if seq_len > self.max_position_embeddings: - base = self.base * ((self.scaling_factor * seq_len / self.max_position_embeddings) - - (self.scaling_factor - 1))**(self.dim / (self.dim - 2)) - inv_freq = 1.0 / (base**(torch.arange(0, self.dim, 2).float().to(device) / self.dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype) - - freqs = torch.einsum("i,j->ij", t, self.inv_freq) - # Different from paper, but it uses a different permutation in order to obtain the same calculation - emb = torch.cat((freqs, freqs), dim=-1) - self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) - self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., :x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2:] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb(q, k, cos, sin, position_ids): - cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] - sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim] - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - - -class ParallelQwen2Attention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): - super().__init__() - self.config = config - self.megatron_config = megatron_config - self.hidden_size = config.hidden_size - self.num_heads = config.num_attention_heads - self.head_dim = self.hidden_size // self.num_heads - self.num_key_value_heads = config.num_key_value_heads - self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.max_position_embeddings = config.max_position_embeddings - self.rope_theta = config.rope_theta - - # assign values after tp - tp_size = mpu.get_tensor_model_parallel_world_size() - assert self.num_heads % tp_size == 0, f'num_head must be divisible by tp_size. Got num_head={self.num_heads}, tp_size={tp_size}' - assert self.num_key_value_heads % tp_size == 0, \ - f'num_key_value_heads must be divisible by tp_size. Got num_key_value_heads={self.num_key_value_heads}, tp_size={tp_size}' - - self.num_heads_per_tp = self.num_heads // tp_size - self.num_key_value_heads_per_tp = self.num_key_value_heads // tp_size - self.hidden_size_per_tp = self.hidden_size // tp_size - - if (self.head_dim * self.num_heads) != self.hidden_size: - raise ValueError(f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {self.num_heads}).") - - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() - - if megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - assert row_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) - tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) - - # [self.q_size, self.k_size, self.v_size] - self.qkv_proj = QKVParallelLinear( - input_size=self.hidden_size, - num_heads=self.num_heads, - num_key_value_heads=self.num_key_value_heads, - head_dim=self.head_dim, - # bias=config.attention_bias, - bias=True, - gather_output=False, - skip_bias_add=False, - **column_kwargs) - - self.q_size = self.num_heads_per_tp * self.head_dim - self.k_size = self.num_key_value_heads_per_tp * self.head_dim - self.v_size = self.num_key_value_heads_per_tp * self.head_dim - - self.o_proj = tensor_parallel.RowParallelLinear( - input_size=self.num_heads * self.head_dim, - output_size=self.hidden_size, - # bias=config.attention_bias, - bias=False, - input_is_parallel=True, - skip_bias_add=False, - **row_kwargs) - - self._init_rope() - - def _init_rope(self): - self.rotary_emb = Qwen2RotaryEmbedding( - self.head_dim, - max_position_embeddings=self.max_position_embeddings, - base=self.rope_theta, - ) - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - bsz, q_len, _ = hidden_states.size() - qkv = self.qkv_proj(hidden_states)[0] - query_states, key_states, value_states = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) - - query_states = query_states.view(bsz, q_len, self.num_heads_per_tp, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads_per_tp, self.head_dim).transpose(1, 2) - - kv_seq_len = key_states.shape[-2] - cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) - - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - - attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) - - if attn_weights.size() != (bsz, self.num_heads_per_tp, q_len, kv_seq_len): - raise ValueError( - f"Attention weights should be of size {(bsz, self.num_heads_per_tp, q_len, kv_seq_len)}, but is" - f" {attn_weights.size()}") - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}") - attn_weights = attn_weights + attention_mask - - # upcast attention to fp32 - attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) - attn_output = torch.matmul(attn_weights, value_states) - - if attn_output.size() != (bsz, self.num_heads_per_tp, q_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads_per_tp, q_len, self.head_dim)}, but is" - f" {attn_output.size()}") - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size_per_tp) - attn_output = self.o_proj(attn_output)[0] - return attn_output - - -""" -Remove padding Attention -- Using Flash-attn 2 -- Compatible with sequence parallel -""" - -from transformers.utils import is_flash_attn_2_available -import torch.nn.functional as F - -from einops import rearrange - -if is_flash_attn_2_available(): - from flash_attn import flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - - -def apply_rotary_pos_emb_rmpad(q, k, cos, sin, position_ids, indices, sequence_length): - batch_size = position_ids.shape[0] - - q = pad_input(q, indices, batch_size, sequence_length) # (batch_size, seqlen, num_head, head_dim) - k = pad_input(k, indices, batch_size, sequence_length) - cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] - sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - - q_embed = index_first_axis(rearrange(q_embed, "b s ... -> (b s) ..."), indices) - k_embed = index_first_axis(rearrange(k_embed, "b s ... -> (b s) ..."), indices) - - return q_embed, k_embed - - -from flash_attn.layers.rotary import apply_rotary_emb - - -# use flash-attn rotary embeddings with rmpad -# cos/sin shoudl be: (seq_length, rotary_dim / 2) -def apply_rotary_pos_emb_rmpad_flash(q, k, cos, sin, cu_seqlens, max_seqlen): - q_embed = apply_rotary_emb(q, - cos, - sin, - interleaved=False, - inplace=False, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen) - k_embed = apply_rotary_emb(k, - cos, - sin, - interleaved=False, - inplace=False, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen) - return q_embed, k_embed - - -class ParallelQwen2AttentionRmPad(ParallelQwen2Attention): - - def forward(self, - hidden_states: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: torch.Tensor = None, - max_seqlen_in_batch: int = None): - total_nnz, _, _ = hidden_states.size() # This is the total_nnz padded after sequence parallel - - if self.megatron_config.sequence_parallel: - total_nnz = total_nnz * mpu.get_tensor_model_parallel_world_size() - - qkv = self.qkv_proj(hidden_states)[0] - query_states, key_states, value_states = qkv.split([self.q_size, self.k_size, self.v_size], - dim=-1) # (total_nnz, 1, hidden_size) - - if self.megatron_config.sequence_parallel: - sequence_parallel_pad = total_nnz - cu_seqlens[-1] - total_nnz = cu_seqlens[-1] # total_nnz before sp padding - query_states = query_states[:total_nnz] - key_states = key_states[:total_nnz] - value_states = value_states[:total_nnz] - - # Flash attention requires the input to have the shape - # batch_size x seq_length x head_dime x hidden_dim - # therefore we just need to keep the original shape - query_states = query_states.view(total_nnz, self.num_heads_per_tp, self.head_dim) - key_states = key_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) - value_states = value_states.view(total_nnz, self.num_key_value_heads_per_tp, self.head_dim) - - cos, sin = self.rotary_emb(value_states, seq_len=sequence_length) - cos, sin = cos[:, :cos.shape[1] // 2], sin[:, :sin.shape[1] // 2] # flash attn only needs half - query_states, key_states = apply_rotary_pos_emb_rmpad_flash(query_states, - key_states, - cos, - sin, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen_in_batch) - # query_states, key_states = apply_rotary_pos_emb_rmpad(query_states, key_states, cos, sin, position_ids, indices, - - # It is recommended to use dropout with FA according to the docs - # when training. - dropout_rate = 0.0 # if not self.training else self.attn_dropout - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in float16 just to be sure everything works as expected. - # This might slowdown training & inference so it is recommended to not cast the LayerNorms - # in fp32. (Qwen2RMSNorm handles it correctly) - input_dtype = query_states.dtype - if input_dtype == torch.float32: - query_states = query_states.to(torch.float16) - key_states = key_states.to(torch.float16) - value_states = value_states.to(torch.float16) - - attn_output_unpad = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens, - cu_seqlens_k=cu_seqlens, - max_seqlen_q=max_seqlen_in_batch, - max_seqlen_k=max_seqlen_in_batch, - dropout_p=dropout_rate, - softmax_scale=None, - causal=True, - ) - - attn_output_unpad = attn_output_unpad.to(input_dtype) - attn_output_unpad = attn_output_unpad.reshape(total_nnz, 1, self.hidden_size_per_tp).contiguous() - - # sequence parallel reduce_scatter is performed inside RowColumnParallel if enabled - # Here we need to repad - if self.megatron_config.sequence_parallel: - attn_output_unpad = F.pad(attn_output_unpad, pad=(0, 0, 0, 0, 0, sequence_parallel_pad)) - - attn_output_unpad = self.o_proj(attn_output_unpad)[0] - return attn_output_unpad diff --git a/verl/models/qwen2/megatron/layers/parallel_decoder.py b/verl/models/qwen2/megatron/layers/parallel_decoder.py deleted file mode 100644 index 84562b3bca..0000000000 --- a/verl/models/qwen2/megatron/layers/parallel_decoder.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Tuple - -import torch -from torch import nn -from transformers import Qwen2Config -from megatron.core import ModelParallelConfig - -from .parallel_attention import ParallelQwen2Attention, ParallelQwen2AttentionRmPad -from .parallel_mlp import ParallelQwen2MLP -from .parallel_rmsnorm import ParallelQwen2RMSNorm - -from verl.utils.megatron_utils import TransformerConfig, convert_config - - -class ParallelQwen2DecoderLayer(nn.Module): - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig, layer_idx: int): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.layer_idx = layer_idx - self.hidden_size = config.hidden_size - self.self_attn = ParallelQwen2Attention(config=config, megatron_config=megatron_config) - - self.mlp = ParallelQwen2MLP(config, megatron_config=megatron_config) - self.input_layernorm = ParallelQwen2RMSNorm(config, megatron_config) - self.post_attention_layernorm = ParallelQwen2RMSNorm(config, megatron_config) - - def forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: - """ - Args: - hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` - attention_mask (`torch.FloatTensor`, *optional*): attention mask of size - `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states - """ - - residual = hidden_states - - hidden_states = self.input_layernorm(hidden_states) - - # Note: sequence parallel is hidden inside ColumnParallelLinear - # reduce scatter is hidden inside RowParallelLinear - - # Self Attention - hidden_states = self.self_attn( - hidden_states=hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - ) - - # TODO: add sequence parallel operator reduce_scatter here - - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - - # TODO: add sequence parallel operator all_gather here - - hidden_states = self.mlp(hidden_states) - - # TODO: add sequence parallel operator reduce_scatter here - - hidden_states = residual + hidden_states - - outputs = hidden_states - - return outputs - - -class ParallelQwen2DecoderLayerRmPad(nn.Module): - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig, layer_idx: int): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.hidden_size = config.hidden_size - self.layer_idx = layer_idx - self.self_attn = ParallelQwen2AttentionRmPad(config=config, megatron_config=megatron_config) - - self.mlp = ParallelQwen2MLP(config, megatron_config=megatron_config) - self.input_layernorm = ParallelQwen2RMSNorm(config, megatron_config) - self.post_attention_layernorm = ParallelQwen2RMSNorm(config, megatron_config) - - def forward( - self, - hidden_states: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: int = None, - max_seqlen_in_batch: int = None - ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: - residual = hidden_states # (total_nnz // sp, 1, hidden_size) - - hidden_states = self.input_layernorm(hidden_states) - - # Self Attention - # (total_nnz // sp, 1, hidden_size) -> all-gather (total_nnz, 1, hidden_size) - # -> col + row -> reduce-scatter -> (total_nnz // sp, 1, hidden_size) - hidden_states = self.self_attn(hidden_states=hidden_states, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = residual + hidden_states - - # Fully Connected - # shape changes same as attn - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states - - outputs = hidden_states - - return outputs diff --git a/verl/models/qwen2/megatron/layers/parallel_linear.py b/verl/models/qwen2/megatron/layers/parallel_linear.py deleted file mode 100644 index bfe5cf4e65..0000000000 --- a/verl/models/qwen2/megatron/layers/parallel_linear.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/linear.py - -from typing import Optional, Tuple - -from megatron.core import tensor_parallel - - -class QKVParallelLinear(tensor_parallel.ColumnParallelLinear): - - def __init__(self, - input_size, - num_heads, - num_key_value_heads, - head_dim, - *, - bias=True, - gather_output=True, - skip_bias_add=False, - **kwargs): - # Keep input parameters, and already restrict the head numbers - self.input_size = input_size - self.q_output_size = num_heads * head_dim - self.kv_output_size = num_key_value_heads * head_dim - self.head_dim = head_dim - self.gather_output = gather_output - self.skip_bias_add = skip_bias_add - - input_size = self.input_size - output_size = (num_heads + 2 * num_key_value_heads) * self.head_dim - - super().__init__(input_size=input_size, - output_size=output_size, - bias=bias, - gather_output=gather_output, - skip_bias_add=skip_bias_add, - **kwargs) - - -class MergedColumnParallelLinear(tensor_parallel.ColumnParallelLinear): - - def __init__(self, - input_size, - gate_ouput_size, - up_output_size, - *, - bias=True, - gather_output=True, - skip_bias_add=False, - **kwargs): - # Keep input parameters, and already restrict the head numbers - self.input_size = input_size - self.output_size = gate_ouput_size + up_output_size - self.gather_output = gather_output - self.skip_bias_add = skip_bias_add - - super().__init__(input_size=self.input_size, - output_size=self.output_size, - bias=bias, - gather_output=gather_output, - skip_bias_add=skip_bias_add, - **kwargs) diff --git a/verl/models/qwen2/megatron/layers/parallel_mlp.py b/verl/models/qwen2/megatron/layers/parallel_mlp.py deleted file mode 100644 index 48b9771192..0000000000 --- a/verl/models/qwen2/megatron/layers/parallel_mlp.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from megatron.core import parallel_state as mpu -from megatron.core import tensor_parallel -from megatron.core import ModelParallelConfig -from torch import nn -from transformers.activations import ACT2FN -from verl.models.qwen2.megatron.layers.parallel_linear import MergedColumnParallelLinear - -from verl.utils.megatron import tensor_parallel as tp_utils - - -class ParallelQwen2MLP(nn.Module): - - def __init__(self, config, megatron_config: ModelParallelConfig = None) -> None: - super().__init__() - self.config = config - self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size - # The weight is only [hidden_size, intermediate_size // model_parallel_world_size] - - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - row_kwargs = tp_utils.get_default_kwargs_for_row_parallel_linear() - - if megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - assert row_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(row_kwargs, megatron_config) - tp_utils.update_kwargs_with_config(column_kwargs, megatron_config) - - tp_size = mpu.get_tensor_model_parallel_world_size() - - self.gate_up_proj = MergedColumnParallelLinear( - input_size=self.hidden_size, - gate_ouput_size=self.intermediate_size, - up_output_size=self.intermediate_size, - bias=False, - gather_output=False, - skip_bias_add=False, - **column_kwargs, - ) - self.gate_size = self.intermediate_size // tp_size - - self.down_proj = tensor_parallel.RowParallelLinear(input_size=self.intermediate_size, - output_size=self.hidden_size, - bias=False, - input_is_parallel=True, - skip_bias_add=False, - **row_kwargs) - - self.act_fn = ACT2FN[config.hidden_act] - - def forward(self, x): - gate_up = self.gate_up_proj(x)[0] - gate, up = gate_up.split(self.gate_size, dim=-1) - return self.down_proj(self.act_fn(gate) * up)[0] diff --git a/verl/models/qwen2/megatron/layers/parallel_rmsnorm.py b/verl/models/qwen2/megatron/layers/parallel_rmsnorm.py deleted file mode 100644 index 726eb7f893..0000000000 --- a/verl/models/qwen2/megatron/layers/parallel_rmsnorm.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numbers -import torch -from megatron.core import ModelParallelConfig -from torch import nn -from transformers import Qwen2Config - -from apex.normalization.fused_layer_norm import fused_rms_norm_affine -from verl.utils.megatron import sequence_parallel as sp_utils - - -class ParallelQwen2RMSNorm(nn.Module): - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): - """ - Qwen2RMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - if isinstance(config.hidden_size, numbers.Integral): - normalized_shape = (config.hidden_size,) - self.normalized_shape = torch.Size(normalized_shape) - self.weight = nn.Parameter(torch.ones(self.normalized_shape)) - self.variance_epsilon = config.rms_norm_eps - - if megatron_config.sequence_parallel: - sp_utils.mark_parameter_as_sequence_parallel(self.weight) - - def forward(self, hidden_states): - return fused_rms_norm_affine(input=hidden_states, - weight=self.weight, - normalized_shape=self.normalized_shape, - eps=self.variance_epsilon, - memory_efficient=True) \ No newline at end of file diff --git a/verl/models/qwen2/megatron/modeling_qwen2_megatron.py b/verl/models/qwen2/megatron/modeling_qwen2_megatron.py deleted file mode 100644 index c15111bd6b..0000000000 --- a/verl/models/qwen2/megatron/modeling_qwen2_megatron.py +++ /dev/null @@ -1,714 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" PyTorch Qwen2 model.""" - -from typing import Optional, Tuple, Union - -import torch -import torch.utils.checkpoint -from megatron.core import tensor_parallel, parallel_state -from megatron.core import ModelParallelConfig -from megatron.core import mpu - -from torch import nn -from transformers.modeling_outputs import BaseModelOutputWithPast -from transformers.models.qwen2.configuration_qwen2 import Qwen2Config -from transformers.models.qwen2.modeling_qwen2 import CausalLMOutputWithPast - -from verl.utils.megatron import sequence_parallel as sp_utils -from verl.utils.megatron import tensor_parallel as tp_utils -from verl.utils.megatron_utils import TransformerConfig, convert_config -from .layers import ParallelQwen2DecoderLayer, ParallelQwen2RMSNorm, ParallelQwen2DecoderLayerRmPad -""" -TODO: -1. Add weight initialization. Here we need to be careful on TP weight init. -2. Add sequence parallel -3. Load checkpoint from Qwen2 pretrained checkpoint -""" - - -# Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device): - """ - Make causal mask used for bi-directional self-attention. - """ - bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) - mask_cond = torch.arange(mask.size(-1), device=device) - mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) - mask = mask.to(dtype) - return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len) - - -# Copied from transformers.models.bart.modeling_bart._expand_mask -def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): - """ - Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. - """ - bsz, src_len = mask.size() - tgt_len = tgt_len if tgt_len is not None else src_len - - expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) - - inverted_mask = 1.0 - expanded_mask - - return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) - - -class ParallelQwen2Model(nn.Module): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`] - - Args: - config: Qwen2Config - """ - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() - if megatron_config is not None: - assert embedding_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(embedding_kwargs, megatron_config) - self.embed_tokens = tensor_parallel.VocabParallelEmbedding(num_embeddings=config.vocab_size, - embedding_dim=config.hidden_size, - **embedding_kwargs) - - self.layers = nn.ModuleList( - [ParallelQwen2DecoderLayer(config, megatron_config) for _ in range(config.num_hidden_layers)]) - self.norm = ParallelQwen2RMSNorm(config, megatron_config) - - # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask - def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds): - # create causal mask - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - combined_attention_mask = None - if input_shape[-1] > 1: - combined_attention_mask = _make_causal_mask( - input_shape, - inputs_embeds.dtype, - device=inputs_embeds.device, - ) - - if attention_mask is not None: - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, - tgt_len=input_shape[-1]).to(inputs_embeds.device) - combined_attention_mask = (expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + - combined_attention_mask) - - return combined_attention_mask - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, BaseModelOutputWithPast]: - """ - - Args: - input_ids: input ids. shape (batch_size, seq_length) - attention_mask: attention_mask. shape (batch_size, seq_length) - position_ids: position ids. shape (batch_size, seq_length) - - Returns: - - """ - batch_size, seq_length = input_ids.shape - inputs_embeds = self.embed_tokens(input_ids) - # embed positions - - attention_mask = self._prepare_decoder_attention_mask(attention_mask, (batch_size, seq_length), inputs_embeds) - - hidden_states = inputs_embeds - - for idx, decoder_layer in enumerate(self.layers): - layer_outputs = decoder_layer( - hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - ) - - hidden_states = layer_outputs - - hidden_states = self.norm(hidden_states) - - return hidden_states - - -class ParallelQwen2ForCausalLM(nn.Module): - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.model = ParallelQwen2Model(config, megatron_config=megatron_config) - self.vocab_size = config.vocab_size - - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - - self.lm_head = tensor_parallel.ColumnParallelLinear(input_size=config.hidden_size, - output_size=config.vocab_size, - bias=False, - gather_output=False, - skip_bias_add=False, - **column_kwargs) - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - ```""" - - # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - ) - - hidden_states = outputs - logits = self.lm_head(hidden_states)[0] - - logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) - - logits = logits.float() - return CausalLMOutputWithPast( - loss=None, - logits=logits, - past_key_values=None, - hidden_states=None, - attentions=None, - ) - - -from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - - -class ParallelQwen2ModelRmPad(nn.Module): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`] - - Args: - config: Qwen2Config - """ - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() - self.megatron_config = megatron_config - if megatron_config is not None: - assert embedding_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) - self.embed_tokens = tensor_parallel.VocabParallelEmbedding(num_embeddings=config.vocab_size, - embedding_dim=config.hidden_size, - **embedding_kwargs) - - self.layers = nn.ModuleList( - [ParallelQwen2DecoderLayerRmPad(config, megatron_config) for _ in range(config.num_hidden_layers)]) - self.norm = ParallelQwen2RMSNorm(config, megatron_config) - - def forward(self, - input_ids: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: int = None, - max_seqlen_in_batch: int = None) -> Union[Tuple, BaseModelOutputWithPast]: - """ - - Args: - input_ids: input ids. shape (1, totol_nnz) - position_ids: position ids. shape (batch_size, seq_length) - - Returns: - - """ - inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) - - # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) - inputs_embeds = inputs_embeds.transpose(0, 1) - if self.megatron_config.sequence_parallel: - inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) - - hidden_states = inputs_embeds - for idx, decoder_layer in enumerate(self.layers): - layer_outputs = decoder_layer(hidden_states, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = layer_outputs - - hidden_states = self.norm(hidden_states) - - return hidden_states - - -class ParallelQwen2ForCausalLMRmPad(nn.Module): - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.megatron_config = megatron_config - self.model = ParallelQwen2ModelRmPad(config, megatron_config=megatron_config) - self.vocab_size = config.vocab_size - self._init_head(config) - - def _init_head(self, config: Qwen2Config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = tensor_parallel.ColumnParallelLinear(input_size=config.hidden_size, - output_size=config.vocab_size, - bias=False, - gather_output=False, - skip_bias_add=False, - **column_kwargs) - - def _forward_head(self, hidden_states): - # all_gather from sequence parallel region is performed inside lm_head - logits = self.lm_head(hidden_states)[0] - logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) - logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) # (total_nnz_padded, 1, vocab_size) - return logits - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - ```""" - batch_size, sequence_length = input_ids.shape - - # remove padding here - input_ids, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input(input_ids.unsqueeze(dim=-1), - attention_mask) # (total_nnz, 1) - - # pad input_ids to multiple of tp for all tp ranks - # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap - if self.megatron_config.sequence_parallel: - input_ids = sp_utils.pad_to_sequence_parallel(input_ids) - - input_ids = input_ids.transpose(0, 1) # (1, total_nnz+pad) - - outputs = self.model(input_ids=input_ids, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = outputs - - logits = self._forward_head(hidden_states) - - # remove padding from sequence parallel - if self.megatron_config.sequence_parallel: - totol_nnz = cu_seqlens[-1] - logits = logits[:totol_nnz] # (total_nnz_padded) - - logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension - # add removed padding back - logits = pad_input(logits, indices, batch_size, - seqlen=sequence_length) # (batch_size, sequence_length, vocab_size) - - return CausalLMOutputWithPast( - loss=None, - logits=logits, - past_key_values=None, - hidden_states=None, - attentions=None, - ) - - -class ParallelQwen2ForValueRmPad(ParallelQwen2ForCausalLMRmPad): - - def _init_head(self, config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) - # lm_head is effectively the same as sequence parallel - sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) - - def _forward_head(self, hidden_states): - logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) - logits = logits.float() - if self.megatron_config.sequence_parallel: - logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) - return logits - - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - output = super().forward(input_ids, attention_mask, position_ids) - output.logits = torch.squeeze(output.logits, dim=-1) - return output - - -""" -Support pipeline parallelism -""" - - -class ParallelQwen2ModelRmPadPP(nn.Module): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`] - This model definition supports pipeline parallelism. To support pp and vpp, - - This model only contains layer in this pp stage and vpp chunk - - When calling get_model in Megatron, this rank will instantiate all the vpp chunks in this pp. - Args: - config: Qwen2Config - """ - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig, pre_process, post_process): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.padding_idx = config.pad_token_id - self.vocab_size = config.vocab_size - self.pre_process = pre_process - self.post_process = post_process - self.megatron_config = megatron_config - embedding_kwargs = tp_utils.get_default_kwargs_for_parallel_embedding() - if megatron_config is not None: - assert embedding_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(embedding_kwargs, self.megatron_config) - if pre_process: - self.embed_tokens = tensor_parallel.VocabParallelEmbedding(num_embeddings=config.vocab_size, - embedding_dim=config.hidden_size, - **embedding_kwargs) - else: - self.embed_tokens = None - - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = megatron_config.pipeline_model_parallel_size - self.num_layer_per_pp = config.num_hidden_layers // pp_size - vpp_size = megatron_config.virtual_pipeline_model_parallel_size - vpp_rank = mpu.get_virtual_pipeline_model_parallel_rank() - - if vpp_size is not None: - self.num_layer_vpp_chunk = self.num_layer_per_pp // vpp_size - self.num_layer_this_model = self.num_layer_vpp_chunk - offset = vpp_rank * ( - config.num_hidden_layers // vpp_size) + \ - (pp_rank * self.num_layer_vpp_chunk) - else: - self.num_layer_this_model = self.num_layer_per_pp - offset = pp_rank * self.num_layer_per_pp - - self.layers = nn.ModuleList() - for i in range(self.num_layer_this_model): - layer = ParallelQwen2DecoderLayerRmPad(config, megatron_config, layer_idx=i + offset) - self.layers.add_module(f'{i}', layer) - - if post_process: - self.norm = ParallelQwen2RMSNorm(config, megatron_config) - else: - self.norm = None - - def set_input_tensor(self, input_tensor): - """Set input tensor to be used instead of forward()'s input. - - When doing pipeline parallelism the input from the previous - stage comes from communication, not from the input, so the - model's forward_step_func won't have it. This function is thus - used by internal code to bypass the input provided by the - forward_step_func""" - self.input_tensor = input_tensor - - def forward(self, - input_ids: torch.Tensor, - position_ids: Optional[torch.LongTensor] = None, - sequence_length: int = None, - indices: torch.Tensor = None, - cu_seqlens: int = None, - max_seqlen_in_batch: int = None) -> Union[Tuple, BaseModelOutputWithPast]: - """ - - Args: - input_ids: input ids. shape (1, totol_nnz) - position_ids: position ids. shape (batch_size, seq_length) - - Returns: - - """ - if self.pre_process: - inputs_embeds = self.embed_tokens(input_ids) # (1, total_nnz) -> (1, total_nnz, hidden_size) - - # vocab parallel embedding will not do sequence parallel reduce-scatter in open source megatron - # so need to deal with it by handle here: - # (1, total_nnz, hidden_size) -> (total_nnz, 1, hidden_size) -> (total_nnz // sp, 1, hidden_size) - inputs_embeds = inputs_embeds.transpose(0, 1) - if self.megatron_config.sequence_parallel: - inputs_embeds = tensor_parallel.scatter_to_sequence_parallel_region(inputs_embeds) - - hidden_states = inputs_embeds - else: - # self.hidden_states should be passed by Megatron - hidden_states = self.input_tensor - - for idx, decoder_layer in enumerate(self.layers): - layer_outputs = decoder_layer(hidden_states, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - hidden_states = layer_outputs - - if self.post_process: - hidden_states = self.norm(hidden_states) - - return hidden_states - - -class ParallelQwen2ForCausalLMRmPadPP(nn.Module): - - def __init__(self, config: Qwen2Config, megatron_config: ModelParallelConfig, pre_process, post_process, - share_embeddings_and_output_weights): - super().__init__() - self.config: TransformerConfig = convert_config(config, megatron_config) - self.megatron_config = megatron_config - self.model = ParallelQwen2ModelRmPadPP(config, - megatron_config=megatron_config, - pre_process=pre_process, - post_process=post_process) - self.share_embeddings_and_output_weights = share_embeddings_and_output_weights - self.vocab_size = config.vocab_size - self.pre_process = pre_process - self.post_process = post_process - if post_process: - self._init_head(config) - if pre_process or post_process: - self.setup_embeddings_and_output_layer() - - def set_input_tensor(self, input_tensor): - """Set input tensor to be used instead of forward()'s input. - - When doing pipeline parallelism the input from the previous - stage comes from communication, not from the input, so the - model's forward_step_func won't have it. This function is thus - used by internal code to bypass the input provided by the - forward_step_func""" - assert len(input_tensor) == 1 - self.model.set_input_tensor(input_tensor[0]) - - def _init_head(self, config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = tensor_parallel.ColumnParallelLinear(input_size=config.hidden_size, - output_size=config.vocab_size, - bias=False, - gather_output=False, - skip_bias_add=False, - skip_weight_param_allocation=self.pre_process and - self.share_embeddings_and_output_weights, - **column_kwargs) - - def setup_embeddings_and_output_layer(self) -> None: - """Sets up embedding layer in first stage and output layer in last stage. - - This function initalizes word embeddings in the final stage when we are - using pipeline parallelism and sharing word embeddings, and sets up param - attributes on the embedding and output layers. - """ - # Set `is_embedding_or_output_parameter` attribute. - if self.pre_process: - self.model.embed_tokens.weight.is_embedding_or_output_parameter = True - if self.post_process and self.lm_head.weight is not None: - self.lm_head.weight.is_embedding_or_output_parameter = True - - if not self.share_embeddings_and_output_weights: - return - - if parallel_state.get_pipeline_model_parallel_world_size() == 1: - # Zero out wgrad if sharing embeddings between two layers on same - # pipeline stage to make sure grad accumulation into main_grad is - # correct and does not include garbage values (e.g., from torch.empty). - self.shared_embedding_or_output_weight().zero_out_wgrad = True - return - - if parallel_state.is_pipeline_first_stage() and self.pre_process and not self.post_process: - self.shared_embedding_or_output_weight().shared_embedding = True - - if self.post_process and not self.pre_process: - assert not parallel_state.is_pipeline_first_stage() - # set word_embeddings weights to 0 here, then copy first - # stage's weights using all_reduce below. - self.lm_head.weight.data.fill_(0) - self.lm_head.weight.shared = True - self.lm_head.weight.shared_embedding = True - - if torch.distributed.is_initialized(): - if parallel_state.is_rank_in_embedding_group(): - weight = self.shared_embedding_or_output_weight() - weight.data = weight.data.cuda() - torch.distributed.all_reduce(weight.data, group=parallel_state.get_embedding_group()) - - def shared_embedding_or_output_weight(self) -> torch.Tensor: - if self.pre_process: - return self.model.embed_tokens.weight - elif self.post_process: - return self.lm_head.weight - return None - - def _forward_head(self, hidden_states): - # all_gather from sequence parallel region is performed inside lm_head - # print(f'logits shape before forward_head: {hidden_states.shape}, vocab_size = {self.config.vocab_size}') # [4, 32, 4096] - output_weight = None - if self.share_embeddings_and_output_weights: - output_weight = self.shared_embedding_or_output_weight() - logits = self.lm_head(hidden_states, weight=output_weight)[0] - # print(f'logits shape after forward_head: {logits.shape}') # [8, 32, 8] - logits = logits.float() # (total_nnz_padded, 1, vocab_size // tp) - return logits - - def forward( - self, - # original input - *, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - ```""" - - # Note that input_ids, attention_mask and position_ids should be passed to every pp layer. - # In the first pp, input_ids will be used, in other pp layers hidden_states will be used inside self.model - batch_size, sequence_length = input_ids.shape - # remove padding here - input_ids_rmpad, indices, cu_seqlens, max_seqlen_in_batch, *_ = unpad_input(input_ids.unsqueeze(dim=-1), - attention_mask) # (total_nnz, 1) - - # pad input_ids to multiple of tp for all tp ranks - # TODO: for better performance, the sp padding should be removed at each layer. Not sure the performance gap - if self.megatron_config.sequence_parallel: - input_ids_rmpad = sp_utils.pad_to_sequence_parallel(input_ids_rmpad) - - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz+pad) - - outputs = self.model(input_ids=input_ids_rmpad, - position_ids=position_ids, - sequence_length=sequence_length, - indices=indices, - cu_seqlens=cu_seqlens, - max_seqlen_in_batch=max_seqlen_in_batch) - - if self.post_process: - hidden_states = outputs - logits = self._forward_head(hidden_states) - logits = torch.squeeze(logits, dim=1) # remove the artificial batch dimension # torch.Size([8, 32, 16]) - - # remove padding from sequence parallel - if self.megatron_config.sequence_parallel: - totol_nnz = cu_seqlens[-1] - logits = logits[:totol_nnz] # (total_nnz_padded) - # add removed padding back. If input is already rmpad, we let the caller pad_input - logits = pad_input(logits, indices, batch_size, - seqlen=sequence_length) # (batch_size, sequence_length, vocab_size) - - return CausalLMOutputWithPast( - loss=None, - logits=logits, - past_key_values=None, - hidden_states=None, - attentions=None, - ) - else: - return outputs - - -class ParallelQwen2ForValueRmPadPP(ParallelQwen2ForCausalLMRmPadPP): - - def _init_head(self, config): - column_kwargs = tp_utils.get_default_kwargs_for_column_parallel_linear() - if self.megatron_config is not None: - assert column_kwargs.get('config', False), 'must have ModelParallelConfig' - tp_utils.update_kwargs_with_config(column_kwargs, self.megatron_config) - self.lm_head = nn.Linear(in_features=config.hidden_size, out_features=1, bias=False) - # lm_head is effectively the same as sequence parallel - sp_utils.mark_parameter_as_sequence_parallel(self.lm_head.weight) - - def _forward_head(self, hidden_states): - logits = self.lm_head(hidden_states) # (total_nnz_padded // tp, 1, 1) - logits = logits.float() - if self.megatron_config.sequence_parallel: - logits = tensor_parallel.gather_from_sequence_parallel_region(logits, tensor_parallel_output_grad=False) - return logits - - def forward( - self, - *, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - ) -> Union[Tuple, CausalLMOutputWithPast]: - output = super().forward(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids) - if self.post_process: - output.logits = torch.squeeze(output.logits, dim=-1) - return output - else: - return output diff --git a/verl/models/registry.py b/verl/models/registry.py deleted file mode 100644 index 7cdf9cd99d..0000000000 --- a/verl/models/registry.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -from typing import List, Optional, Type - -import torch.nn as nn - -# Supported models using HF Rmpad -# TODO(sgm): HF may supported more than listed here, we should add more after testing -_MODELS_SUPPORT_RMPAD = {'llama', 'mistral', 'gemma', 'qwen2', 'qwen2_vl', 'qwen2_5_vl'} - -# by lurui: try a dummy Chatglm configuration, but it failed, so use_remove_padding=False is needed for glm -# _REOVEPAD_MODELS['chatglm'] = LlamaConfig - - -def check_model_support_rmpad(model_type: str): - assert isinstance(model_type, str) - if not model_type in _MODELS_SUPPORT_RMPAD: - raise ValueError(f"Model architecture {model_type} is not supported for now. " - f"RMPad supported architectures: {_MODELS_SUPPORT_RMPAD}." - f"Please set `use_remove_padding=False` in the model config.") - - if model_type in ("qwen2_vl", "qwen2_5_vl"): # patch remove padding for qwen2vl mrope - from verl.models.transformers.qwen2_vl import ulysses_flash_attn_forward - from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLFlashAttention2 - from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLFlashAttention2 - - Qwen2VLFlashAttention2.forward = ulysses_flash_attn_forward - Qwen2_5_VLFlashAttention2.forward = ulysses_flash_attn_forward - print("Qwen2vl patch applied!") - - -# Supported models in Megatron-LM -# Architecture -> (module, class). -_MODELS = { - "LlamaForCausalLM": - ("llama", ("ParallelLlamaForCausalLMRmPadPP", "ParallelLlamaForValueRmPadPP", "ParallelLlamaForCausalLMRmPad")), - "Qwen2ForCausalLM": - ("qwen2", ("ParallelQwen2ForCausalLMRmPadPP", "ParallelQwen2ForValueRmPadPP", "ParallelQwen2ForCausalLMRmPad")), - "MistralForCausalLM": ("mistral", ("ParallelMistralForCausalLMRmPadPP", "ParallelMistralForValueRmPadPP", - "ParallelMistralForCausalLMRmPad")) -} - - -# return model class -class ModelRegistry: - - @staticmethod - def load_model_cls(model_arch: str, value=False) -> Optional[Type[nn.Module]]: - if model_arch not in _MODELS: - return None - - megatron = "megatron" - - module_name, model_cls_name = _MODELS[model_arch] - if not value: # actor/ref - model_cls_name = model_cls_name[0] - elif value: # critic/rm - model_cls_name = model_cls_name[1] - - module = importlib.import_module(f"verl.models.{module_name}.{megatron}.modeling_{module_name}_megatron") - return getattr(module, model_cls_name, None) - - @staticmethod - def get_supported_archs() -> List[str]: - return list(_MODELS.keys()) diff --git a/verl/models/transformers/__init__.py b/verl/models/transformers/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/models/transformers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/models/transformers/llama.py b/verl/models/transformers/llama.py deleted file mode 100644 index 886ccb67d6..0000000000 --- a/verl/models/transformers/llama.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -from typing import Optional, Tuple, Callable -import sys -if sys.version_info >= (3, 11): - from typing import Unpack -else: - from typing_extensions import Unpack - -from transformers.models.llama.modeling_llama import apply_rotary_pos_emb -from transformers.cache_utils import Cache -from transformers.utils import logging -from transformers.modeling_flash_attention_utils import _flash_attention_forward -from verl.utils.ulysses import gather_heads_scatter_seq, gather_seq_scatter_heads, \ - get_ulysses_sequence_parallel_world_size, validate_ulysses_config - -logger = logging.get_logger(__name__) - -def llama_flash_attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.LongTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 - **kwargs, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """ - Adapted from transformers 4.47.1 to support Ulysses sequence parallelism. - - NOTE: This function is used for transformers versions in the range [4.45.0, 4.47.1]. - """ - output_attentions = False - - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - # Flash attention requires the input to have the shape - # batch_size x seq_length x head_dim x hidden_dim - # therefore we just need to keep the original shape - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - # trade off: repeat first and then all to all - # key_states = repeat_kv(key_states, self.num_key_value_groups) - # value_states = repeat_kv(value_states, self.num_key_value_groups) - - ########## AlltoAll for Ulysses ########## - ulysses_sp_size = get_ulysses_sequence_parallel_world_size() - - if ulysses_sp_size > 1: - validate_ulysses_config(self.num_heads, ulysses_sp_size) - - # (bsz, n_head, seq_len/n, head_dim) -> (bsz, n_head/n, seq_len, head_dim) - query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) - key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) - value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) - - full_q_len = query_states.size(2) # full seq length - - if position_embeddings is None: - logger.warning_once( - "The attention layers in this model are transitioning from computing the RoPE embeddings internally " - "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " - "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " - "removed and `position_embeddings` will be mandatory.") - cos, sin = self.rotary_emb(value_states, position_ids) - else: - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_value is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache - # to be able to avoid many of these transpose/reshape/view. - query_states = query_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - - dropout_rate = self.attention_dropout if self.training else 0.0 - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in the correct dtype just to be sure everything works as expected. - # This might slowdown training & inference so it is recommended to not cast the LayerNorms - # in fp32. (LlamaRMSNorm handles it correctly) - - input_dtype = query_states.dtype - if input_dtype == torch.float32: - if torch.is_autocast_enabled(): - target_dtype = torch.get_autocast_gpu_dtype() - # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype - else: - target_dtype = self.q_proj.weight.dtype - - logger.warning_once( - f"The input hidden states seems to be silently casted in float32, this might be related to" - f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" - f" {target_dtype}.") - - query_states = query_states.to(target_dtype) - key_states = key_states.to(target_dtype) - value_states = value_states.to(target_dtype) - - attn_output = _flash_attention_forward( - query_states, - key_states, - value_states, - attention_mask, - full_q_len, - position_ids=position_ids, - dropout=dropout_rate, - sliding_window=getattr(self, "sliding_window", None), - use_top_left_mask=self._flash_attn_uses_top_left_mask, - is_causal=self.is_causal, - **kwargs, - ) - - attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() - ########## AlltoAll for Ulysses ########## - if ulysses_sp_size > 1: - attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) - attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - -def llama_attn_forward( - self, - hidden_states: torch.Tensor, - position_embeddings: Tuple[torch.Tensor, torch.Tensor], - attention_mask: Optional[torch.Tensor], - past_key_value: Optional[Cache] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """ - Adapted from transformers 4.49.0 to support Ulysses sequence parallelism for transformers >= 4.48.0. - - NOTE: This function has been tested only on transformers versions between 4.48.0 and 4.50.0. - """ - from transformers.models.llama.modeling_llama import eager_attention_forward - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - bsz, q_len, _ = hidden_states.shape - - query_states = self.q_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - key_states = self.k_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - value_states = self.v_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - - ########## AlltoAll for Ulysses ########## - ulysses_sp_size = get_ulysses_sequence_parallel_world_size() - - if ulysses_sp_size > 1: - validate_ulysses_config(self.config.num_attention_heads, ulysses_sp_size) - - query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) - key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) - value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) - - full_q_len = query_states.size(2) - - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_value is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False): - logger.warning_once( - "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to " - 'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' - ) - else: - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - **kwargs, - ) - - attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() - ########## AlltoAll for Ulysses ########## - if ulysses_sp_size > 1: - attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) - attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights diff --git a/verl/models/transformers/monkey_patch.py b/verl/models/transformers/monkey_patch.py deleted file mode 100644 index 6a6f47e8cd..0000000000 --- a/verl/models/transformers/monkey_patch.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Apply monkey-patch function to models -""" -import sys -from typing import Optional - -import torch -from transformers.modeling_utils import PreTrainedModel -from transformers.modeling_flash_attention_utils import _flash_attention_forward - -from verl.utils.ulysses import ( - gather_heads_scatter_seq, - gather_seq_scatter_heads, - get_ulysses_sequence_parallel_world_size, - get_ulysses_sequence_parallel_group, -) - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=2, repeats=n_rep). The hidden states go from (batch, - seqlen, num_key_value_heads, head_dim) to (batch, seqlen, num_attention_heads, head_dim) - """ - batch, slen, num_key_value_heads, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, :, None, :].expand(batch, slen, num_key_value_heads, n_rep, head_dim) - return hidden_states.reshape(batch, slen, num_key_value_heads * n_rep, head_dim) - - -def _ulysses_flash_attention_forward( - query_states: torch.Tensor, - key_states: torch.Tensor, - value_states: torch.Tensor, - *args, - position_ids: Optional[torch.Tensor] = None, - **kwargs, -): - """Insert all-to-all before and after flash attention. - DeepSpeed-Ulysses: https://arxiv.org/pdf/2309.14509 - - Args: - query_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads, head_dim) - key_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads_k, head_dim) - value_states (torch.Tensor): (batch_size, seqlen/sp_size, nheads_k, head_dim) - position_ids (torch.Tensor, optional): (batch_size, seqlen/sp_size) - - Returns: - torch.Tensor: (batch_size, seqlen/sp_size, nheads, head_dim) - """ - ulysses_sp_size = get_ulysses_sequence_parallel_world_size() - - ########## AlltoAll for Ulysses ########## - if ulysses_sp_size > 1: - assert position_ids is not None, "position_ids is required for Ulysses sequence parallelism" - - # NOTE: repeat kv heads to be divided by sequence parallel. Instead of repeating nheads_q//nheads_k, - # we choose to repeat sp_size//nheads_k, since flash_attention supports MQA/GQA. - # For example: - # - nheads_k=4, sp=8, repeats=2 - # - nheads_k=8, sp=8, repeats=1 - # - nheads_k=16, sp=8, repeats=1 - repeats = max(ulysses_sp_size // key_states.size(2), 1) - key_states = repeat_kv(key_states, repeats) - value_states = repeat_kv(value_states, repeats) - - # (bsz, seq_len/n, n_head, head_dim) -> (bsz, seq_len, n_head/n, head_dim) - query_states = gather_seq_scatter_heads(query_states, seq_dim=1, head_dim=2) - key_states = gather_seq_scatter_heads(key_states, seq_dim=1, head_dim=2) - value_states = gather_seq_scatter_heads(value_states, seq_dim=1, head_dim=2) - - # TODO: all_gather position_ids because `prepare_fa2_from_position_ids` needs it, we can eliminate - # this all_gather by passing cu_seq_lens_q, cu_seq_lens_k, max_length_k, max_length_q explicitly. - # https://github.com/huggingface/transformers/pull/33932 - - # (bsz, seq_len/n) -> (bsz, seq_len) - position_ids_list = [torch.empty_like(position_ids) for _ in range(ulysses_sp_size)] - torch.distributed.all_gather(position_ids_list, position_ids, group=get_ulysses_sequence_parallel_group()) - position_ids = torch.concat(position_ids_list, dim=-1) - - # (bsz, seq_len, n_head/n, head_dim) - attn_output = _flash_attention_forward(query_states, - key_states, - value_states, - *args, - position_ids=position_ids, - **kwargs) - - ########## AlltoAll for Ulysses ########## - if ulysses_sp_size > 1: - # (bsz, seq_len, n_head/n, head_dim) -> (bsz, seq_len/n, n_head, head_dim) - attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) - - return attn_output - - -def apply_monkey_patch(model: PreTrainedModel, ulysses_sp_size: int): - """Replace _flash_attention_forward to _ulysses_flash_attention_forward""" - module = sys.modules[model.__module__] - - num_attention_heads, num_key_value_heads = model.config.num_attention_heads, model.config.num_key_value_heads - assert num_attention_heads % ulysses_sp_size == 0, \ - f"num_attention_heads {num_attention_heads} must be divisible by ulysses_sp_size {ulysses_sp_size}" - assert num_key_value_heads % ulysses_sp_size == 0 or ulysses_sp_size % num_key_value_heads == 0, ( - f"num_key_value_heads {num_key_value_heads} must be divisible by ulysses_sp_size {ulysses_sp_size}" - f"or vise versa. Upon ulysses_sp_size % num_key_value_heads == 0," - f"kv heads are repeated to ensure correctness.") - # TODO: VLM models only, unify monkey patch to LLM models. - if model.config.model_type in ("qwen2_vl", "qwen2_5_vl"): # patch remove padding for qwen2vl mrope - from verl.models.transformers.qwen2_vl import ulysses_flash_attn_forward - from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLFlashAttention2 - from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLFlashAttention2 - - Qwen2VLFlashAttention2.forward = ulysses_flash_attn_forward - Qwen2_5_VLFlashAttention2.forward = ulysses_flash_attn_forward - print("Monkey patch FlashAttention2.forward in Qwen2VL") - return - - # transformers<=4.47.1 - if hasattr(module, "_flash_attention_forward"): - module._flash_attention_forward = _ulysses_flash_attention_forward - print(f"Monkey patch _flash_attention_forward in {model.__module__}") - else: - # transformers>=4.48.0 - from transformers.integrations import flash_attention - flash_attention._flash_attention_forward = _ulysses_flash_attention_forward - print(f"Monkey patch _flash_attention_forward in {flash_attention.__name__}") - - -from functools import lru_cache -from packaging import version -import importlib.metadata - - -@lru_cache() -def is_transformers_version_in_range(min_version: str, max_version: str) -> bool: - try: - # Get the installed version of the transformers library - transformers_version = importlib.metadata.version("transformers") - except importlib.metadata.PackageNotFoundError: - raise ModuleNotFoundError("The `transformers` package is not installed.") - - # Check if the version is within the specified range - return version.parse(min_version) <= version.parse(transformers_version) <= version.parse(max_version) diff --git a/verl/models/transformers/qwen2.py b/verl/models/transformers/qwen2.py deleted file mode 100644 index 63d9ae98b5..0000000000 --- a/verl/models/transformers/qwen2.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -from typing import Optional, Tuple, Callable - -from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv -from transformers.cache_utils import Cache -from transformers.utils import logging -from transformers.modeling_flash_attention_utils import _flash_attention_forward -from transformers.processing_utils import Unpack -from verl.utils.ulysses import gather_heads_scatter_seq, gather_seq_scatter_heads, \ - get_ulysses_sequence_parallel_world_size, validate_ulysses_config - -logger = logging.get_logger(__name__) - - -def qwen2_flash_attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional[Cache] = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 -): - """ - Adapted from transformers 4.47.1 to support Ulysses sequence parallelism. - - NOTE: This function is only tested on transformers versions between 4.45.0 and 4.47.1. - """ - bsz, q_len, _ = hidden_states.size() - - query_states = self.q_proj(hidden_states) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) - - ########## AlltoAll for Ulysses ########## - ulysses_sp_size = get_ulysses_sequence_parallel_world_size() - - if ulysses_sp_size > 1: - validate_ulysses_config(self.num_heads, ulysses_sp_size) - - # (bsz, n_head, seq_len/n, head_dim) -> (bsz, n_head/n, seq_len, head_dim) - query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) - key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) - value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) - - full_q_len = query_states.size(2) # full seq length - - if position_embeddings is None: - logger.warning_once( - "The attention layers in this model are transitioning from computing the RoPE embeddings internally " - "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " - "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " - "removed and `position_embeddings` will be mandatory.") - cos, sin = self.rotary_emb(value_states, position_ids) - else: - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_value is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - # repeat k/v heads if n_kv_heads < n_heads - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - dropout_rate = 0.0 if not self.training else self.attention_dropout - - # In PEFT, usually we cast the layer norms in float32 for training stability reasons - # therefore the input hidden states gets silently casted in float32. Hence, we need - # cast them back in float16 just to be sure everything works as expected. - input_dtype = query_states.dtype - if input_dtype == torch.float32: - if torch.is_autocast_enabled(): - target_dtype = torch.get_autocast_gpu_dtype() - # Handle the case where the model is quantized - elif hasattr(self.config, "_pre_quantization_dtype"): - target_dtype = self.config._pre_quantization_dtype - else: - target_dtype = self.q_proj.weight.dtype - - logger.warning_once( - f"The input hidden states seems to be silently casted in float32, this might be related to" - f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" - f" {target_dtype}.") - - query_states = query_states.to(target_dtype) - key_states = key_states.to(target_dtype) - value_states = value_states.to(target_dtype) - - # Reashape to the expected shape for Flash Attention - query_states = query_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - - if (self.config.use_sliding_window and getattr(self.config, "sliding_window", None) is not None and - self.layer_idx >= self.config.max_window_layers): - sliding_window = self.config.sliding_window - else: - sliding_window = None - - attn_output = _flash_attention_forward( - query_states, - key_states, - value_states, - attention_mask, - full_q_len, - position_ids=position_ids, - dropout=dropout_rate, - sliding_window=sliding_window, - is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, - ) - - # use full_q_len to reshape - attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() - ########## AlltoAll for Ulysses ########## - if ulysses_sp_size > 1: - attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) - attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() - attn_output = self.o_proj(attn_output) - - if not output_attentions: - attn_weights = None - - return attn_output, attn_weights, past_key_value - - -def qwen2_attn_forward( - self, - hidden_states: torch.Tensor, - position_embeddings: Tuple[torch.Tensor, torch.Tensor], - attention_mask: Optional[torch.Tensor], - past_key_value: Optional[Cache] = None, - cache_position: Optional[torch.LongTensor] = None, - **kwargs, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """ - Adapted from transformers 4.49.0 to support Ulysses sequence parallelism for transformers >= 4.48.0. - - NOTE: This function has been tested only on transformers versions between 4.48.0 and 4.50.0. - """ - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - bsz, q_len, _ = hidden_states.shape - hidden_shape = (bsz, q_len, -1, self.head_dim) - - query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) - key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) - value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) - - ########## AlltoAll for Ulysses ########## - ulysses_sp_size = get_ulysses_sequence_parallel_world_size() - - if ulysses_sp_size > 1: - validate_ulysses_config(self.config.num_attention_heads, ulysses_sp_size) - - # (bsz, n_head, seq_len/n, head_dim) -> (bsz, n_head/n, seq_len, head_dim) - query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) - key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) - value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) - - full_q_len = query_states.size(2) - - cos, sin = position_embeddings - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) - - if past_key_value is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - sliding_window = None - if (self.config.use_sliding_window and getattr(self.config, "sliding_window", None) is not None and - self.layer_idx >= self.config.max_window_layers): - sliding_window = self.config.sliding_window - - from transformers.models.qwen2.modeling_qwen2 import eager_attention_forward - attention_interface: Callable = eager_attention_forward - if self.config._attn_implementation != "eager": - if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False): - logger.warning_once( - "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to " - 'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' - ) - else: - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - attention_mask, - dropout=0.0 if not self.training else self.attention_dropout, - scaling=self.scaling, - sliding_window=sliding_window, # main diff with Llama - **kwargs, - ) - - attn_output = attn_output.reshape(bsz, full_q_len, -1, self.head_dim).contiguous() - ########## AlltoAll for Ulysses ########## - if ulysses_sp_size > 1: - # (bsz, seq_len, n_head/n, head_dim) -> (bsz, seq_len/n, n_head, head_dim) - attn_output = gather_heads_scatter_seq(attn_output, seq_dim=1, head_dim=2) - attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, attn_weights diff --git a/verl/models/transformers/qwen2_vl.py b/verl/models/transformers/qwen2_vl.py deleted file mode 100644 index 718b9ca6f5..0000000000 --- a/verl/models/transformers/qwen2_vl.py +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Tuple -import inspect -import torch -import os -from transformers.utils import is_flash_attn_greater_or_equal -from transformers.modeling_flash_attention_utils import _flash_attention_forward -from verl.utils.ulysses import gather_heads_scatter_seq, gather_seq_scatter_heads, \ - get_ulysses_sequence_parallel_world_size, validate_ulysses_config - -try: - from flash_attn import flash_attn_func, flash_attn_varlen_func - - _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) -except ImportError: - flash_attn_varlen_func = None - - -def get_rope_index( - processor, - input_ids: torch.Tensor, - image_grid_thw: Optional[torch.Tensor] = None, - video_grid_thw: Optional[torch.Tensor] = None, - second_per_grid_ts: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """ - Gets the position ids for Qwen2-VL, it should be generated before sharding the sequence. - The batch dim has been removed and the input_ids should be a 1D tensor representing a single example. - https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py#L1546 - """ - spatial_merge_size = processor.image_processor.merge_size - tokens_per_second = 2 - image_token_id = processor.tokenizer.convert_tokens_to_ids("<|image_pad|>") - video_token_id = processor.tokenizer.convert_tokens_to_ids("<|video_pad|>") - vision_start_token_id = processor.tokenizer.convert_tokens_to_ids("<|vision_start|>") - if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): - if attention_mask is None: - attention_mask = torch.ones_like(input_ids) - - position_ids = torch.ones(3, input_ids.size(0), dtype=input_ids.dtype, device=input_ids.device) # (3, seqlen) - image_index, video_index = 0, 0 - input_ids = input_ids[attention_mask == 1] - image_nums, video_nums = 0, 0 - vision_start_indices = torch.argwhere(input_ids == vision_start_token_id) - vision_tokens = input_ids[vision_start_indices + 1] - image_nums = (vision_tokens == image_token_id).sum() - video_nums = (vision_tokens == video_token_id).sum() - input_tokens = input_ids.tolist() - llm_pos_ids_list: list = [] - st = 0 - remain_images, remain_videos = image_nums, video_nums - for _ in range(image_nums + video_nums): - if image_token_id in input_tokens and remain_images > 0: - ed_image = input_tokens.index(image_token_id, st) - else: - ed_image = len(input_tokens) + 1 - if video_token_id in input_tokens and remain_videos > 0: - ed_video = input_tokens.index(video_token_id, st) - else: - ed_video = len(input_tokens) + 1 - if ed_image < ed_video: - t, h, w = ( - image_grid_thw[image_index][0], - image_grid_thw[image_index][1], - image_grid_thw[image_index][2], - ) - second_per_grid_t = 0 - image_index += 1 - remain_images -= 1 - ed = ed_image - else: - t, h, w = ( - video_grid_thw[video_index][0], - video_grid_thw[video_index][1], - video_grid_thw[video_index][2], - ) - if second_per_grid_ts is not None: - second_per_grid_t = second_per_grid_ts[video_index] - else: - second_per_grid_t = 1.0 - - video_index += 1 - remain_videos -= 1 - ed = ed_video - - llm_grid_t, llm_grid_h, llm_grid_w = ( - t.item(), - h.item() // spatial_merge_size, - w.item() // spatial_merge_size, - ) - text_len = ed - st - - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w) - t_index = (t_index * second_per_grid_t * tokens_per_second).long().flatten() - h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() - w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() - llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) - st = ed + llm_grid_t * llm_grid_h * llm_grid_w - - if st < len(input_tokens): - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - text_len = len(input_tokens) - st - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) - position_ids[..., attention_mask == 1] = llm_positions.to(position_ids.device) - else: - if attention_mask is not None: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - position_ids = position_ids.unsqueeze(0).expand(3, -1).to(input_ids.device) - else: - position_ids = torch.arange(input_ids.shape[1], device=input_ids.device).view(1, -1).expand(3, -1) - - return position_ids - - -def prepare_fa2_from_position_ids(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - position_ids: torch.Tensor): - query = query.view(-1, query.size(-2), query.size(-1)) - key = key.view(-1, key.size(-2), key.size(-1)) - value = value.view(-1, value.size(-2), value.size(-1)) - position_ids = position_ids.flatten() - indices_q = torch.arange(position_ids.size(0), device=position_ids.device, dtype=torch.int32) - cu_seqlens = torch.cat(( - indices_q[position_ids == 0], - torch.tensor(position_ids.size(), device=position_ids.device, dtype=torch.int32), - )) - max_length = cu_seqlens.diff().max() # use cu_seqlens to infer max_length for qwen2vl mrope - return (query, key, value, indices_q, (cu_seqlens, cu_seqlens), (max_length, max_length)) - - -def flash_attention_forward( - query_states: torch.Tensor, - key_states: torch.Tensor, - value_states: torch.Tensor, - attention_mask: torch.Tensor, - query_length: int, - is_causal: bool = True, - position_ids: Optional[torch.Tensor] = None, - sliding_window: Optional[int] = None, - use_top_left_mask: bool = False, - deterministic: Optional[bool] = None, - **kwargs, -): - """ - Patches flash attention forward to handle 3D position ids in mrope. (3, batch_size, seq_length) - """ - if not use_top_left_mask: - causal = is_causal - else: - causal = is_causal and query_length != 1 - - # Assuming 4D tensors, key_states.shape[1] is the key/value sequence length (source length). - use_sliding_windows = (_flash_supports_window_size and sliding_window is not None and - key_states.shape[1] > sliding_window) - flash_kwargs = {"window_size": (sliding_window, sliding_window)} if use_sliding_windows else {} - - if is_flash_attn_greater_or_equal("2.4.1"): - if deterministic is None: - deterministic = os.environ.get("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" - flash_kwargs["deterministic"] = deterministic - - if position_ids is not None and query_length != 1 and not (torch.diff(position_ids[0], dim=-1) >= 0).all(): - batch_size = query_states.size(0) - query_states, key_states, value_states, _, cu_seq_lens, max_seq_lens = prepare_fa2_from_position_ids( - query_states, key_states, value_states, position_ids[0]) # remove channel dimension - cu_seqlens_q, cu_seqlens_k = cu_seq_lens - max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens - attn_output = flash_attn_varlen_func( - query_states, - key_states, - value_states, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_in_batch_q, - max_seqlen_k=max_seqlen_in_batch_k, - dropout_p=kwargs.pop("dropout", 0.0), - softmax_scale=kwargs.pop("softmax_scale", None), - causal=causal, - **flash_kwargs, - ) - attn_output = attn_output.view(batch_size, -1, attn_output.size(-2), attn_output.size(-1)) - else: - attn_output = _flash_attention_forward( - query_states, - key_states, - value_states, - attention_mask, - query_length, - is_causal=is_causal, - sliding_window=sliding_window, - use_top_left_mask=use_top_left_mask, - deterministic=deterministic, - **kwargs, - ) # do not pass position_ids to old flash_attention_forward - - return attn_output - - -def ulysses_flash_attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 - **kwargs, -) -> Tuple[torch.Tensor, None, None]: - from transformers.models.qwen2_vl.modeling_qwen2_vl import repeat_kv, apply_multimodal_rotary_pos_emb - - bsz, q_len, _ = hidden_states.size() # q_len = seq_length / sp_size - query_states = self.q_proj(hidden_states) # (batch_size, seq_length / sp_size, num_heads * head_size) - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) - - ulysses_sp_size = get_ulysses_sequence_parallel_world_size() - - if ulysses_sp_size > 1: - validate_ulysses_config(self.num_heads, ulysses_sp_size) - - key_states = repeat_kv(key_states, self.num_key_value_groups) - value_states = repeat_kv(value_states, self.num_key_value_groups) - query_states = gather_seq_scatter_heads(query_states, seq_dim=2, head_dim=1) - key_states = gather_seq_scatter_heads(key_states, seq_dim=2, head_dim=1) - value_states = gather_seq_scatter_heads(value_states, seq_dim=2, head_dim=1) - # (batch_size, num_head / sp_size, seq_length, head_size) - full_q_len = query_states.size(2) # full_q_len = seq_length - else: - full_q_len = q_len - - # Because the input can be padded, the absolute sequence length depends on the max position id. - if position_embeddings is None: - cos, sin = self.rotary_emb(value_states, position_ids) - else: - cos, sin = position_embeddings - - query_states, key_states = apply_multimodal_rotary_pos_emb(query_states, key_states, cos, sin, - self.rope_scaling["mrope_section"]) - dropout_rate = 0.0 if not self.training else self.attention_dropout - - # Reashape to the expected shape for Flash Attention - query_states = query_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - - if (self.config.use_sliding_window and getattr(self.config, "sliding_window", None) is not None and - self.layer_idx >= self.config.max_window_layers): - sliding_window = self.config.sliding_window - else: - sliding_window = None - - attn_output = flash_attention_forward( - query_states, - key_states, - value_states, - attention_mask, - full_q_len, - dropout=dropout_rate, - sliding_window=sliding_window, - is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, - position_ids=position_ids, # important: pass position ids - ) # (batch_size, seq_length, num_head / sp_size, head_size) - if ulysses_sp_size > 1: - attn_output = gather_heads_scatter_seq(attn_output, head_dim=2, seq_dim=1) - - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() - attn_output = self.o_proj(attn_output) - return attn_output, None, None diff --git a/verl/models/weight_loader_registry.py b/verl/models/weight_loader_registry.py deleted file mode 100644 index a700761268..0000000000 --- a/verl/models/weight_loader_registry.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def get_weight_loader(arch: str): - from verl.models.llama.megatron.checkpoint_utils.llama_loader import load_state_dict_to_megatron_llama - from verl.models.qwen2.megatron.checkpoint_utils.qwen2_loader import load_state_dict_to_megatron_qwen2 - from verl.models.mcore.loader import load_state_dict_to_megatron_gptmodel - _MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY = { - 'LlamaForCausalLM': load_state_dict_to_megatron_gptmodel, - 'Qwen2ForCausalLM': load_state_dict_to_megatron_gptmodel, - } - - if arch in _MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY: - return _MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY[arch] - raise ValueError(f"Model architectures {arch} loader are not supported for now. " - f"Supported architectures: {_MODEL_WEIGHT_MEGATRON_LOADER_REGISTRY.keys()}") - - -def get_weight_saver(arch: str): - from verl.models.llama.megatron.checkpoint_utils.llama_saver import merge_megatron_ckpt_llama - from verl.models.qwen2.megatron.checkpoint_utils.qwen2_saver import merge_megatron_ckpt_qwen2 - from verl.models.mcore.saver import merge_megatron_ckpt_gptmodel - _MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY = { - 'LlamaForCausalLM': merge_megatron_ckpt_gptmodel, - 'Qwen2ForCausalLM': merge_megatron_ckpt_gptmodel, - } - if arch in _MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY: - return _MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY[arch] - raise ValueError(f"Model architectures {arch} saver are not supported for now. " - f"Supported architectures: {_MODEL_WEIGHT_MEGATRON_SAVER_REGISTRY.keys()}") diff --git a/verl/protocol.py b/verl/protocol.py deleted file mode 100644 index 27a2d57d15..0000000000 --- a/verl/protocol.py +++ /dev/null @@ -1,784 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Implement base data transfer protocol between any two functions, modules. -We can subclass Protocol to define more detailed batch info with specific keys -""" - -import pickle -import numpy as np -import pandas as pd -import copy -from dataclasses import dataclass, field -from typing import Callable, Dict, List, Union - -import torch -import tensordict -from tensordict import TensorDict -from torch.utils.data import DataLoader, Dataset - -from verl.utils.py_functional import union_two_dict - -__all__ = ['DataProto', 'union_tensor_dict'] - -try: - tensordict.set_lazy_legacy(False).set() -except: - pass - - -def pad_dataproto_to_divisor(data: 'DataProto', size_divisor: int): - """Pad a DataProto to size divisible by size_divisor - - Args: - size_divisor (int): size divisor - - Returns: - data: (DataProto): the padded DataProto - pad_size (int) - """ - assert isinstance(data, DataProto), 'data must be a DataProto' - if len(data) % size_divisor != 0: - pad_size = size_divisor - len(data) % size_divisor - padding_protos = [] - remaining_pad = pad_size - while remaining_pad > 0: - take_size = min(remaining_pad, len(data)) - padding_protos.append(data[:take_size]) - remaining_pad -= take_size - data_padded = DataProto.concat([data] + padding_protos) - else: - pad_size = 0 - data_padded = data - return data_padded, pad_size - - -def unpad_dataproto(data: 'DataProto', pad_size): - if pad_size != 0: - data = data[:-pad_size] - return data - - -def union_tensor_dict(tensor_dict1: TensorDict, tensor_dict2: TensorDict) -> TensorDict: - """Union two tensordicts.""" - assert tensor_dict1.batch_size == tensor_dict2.batch_size, \ - f'Two tensor dict must have identical batch size. Got {tensor_dict1.batch_size} and {tensor_dict2.batch_size}' - for key in tensor_dict2.keys(): - if key not in tensor_dict1.keys(): - tensor_dict1[key] = tensor_dict2[key] - else: - assert tensor_dict1[key].equal(tensor_dict2[key]), \ - f'{key} in tensor_dict1 and tensor_dict2 are not the same object' - - return tensor_dict1 - - -def union_numpy_dict(tensor_dict1: dict[str, np.ndarray], tensor_dict2: dict[str, np.ndarray]) -> dict[str, np.ndarray]: - for key, val in tensor_dict2.items(): - if key in tensor_dict1: - assert isinstance(tensor_dict2[key], np.ndarray) - assert isinstance(tensor_dict1[key], np.ndarray) - # to properly deal with nan and object type - assert pd.DataFrame(tensor_dict2[key]).equals(pd.DataFrame(tensor_dict1[key])), \ - f'{key} in tensor_dict1 and tensor_dict2 are not the same object' - tensor_dict1[key] = val - - return tensor_dict1 - - -def list_of_dict_to_dict_of_list(list_of_dict: list[dict]): - if len(list_of_dict) == 0: - return {} - keys = list_of_dict[0].keys() - output = {key: [] for key in keys} - for data in list_of_dict: - for key, item in data.items(): - assert key in output - output[key].append(item) - return output - - -def fold_batch_dim(data: 'DataProto', new_batch_size): - """ - Fold a batch dim from [bsz, xxx] into [new_bsz, bsz // new_bsz, xxx] - """ - batch_size = data.batch.batch_size[0] - - assert batch_size % new_batch_size == 0 - - tensor: TensorDict = data.batch - non_tensor = data.non_tensor_batch - - tensor = tensor.view(new_batch_size, -1) - tensor.auto_batch_size_(batch_dims=1) - - for key, val in non_tensor.items(): - non_tensor[key] = np.reshape(val, newshape=(new_batch_size, -1, *val.shape[1:])) - - return DataProto(batch=tensor, non_tensor_batch=non_tensor, meta_info=data.meta_info) - - -def unfold_batch_dim(data: 'DataProto', batch_dims=2): - """ - Unfold the first n dims as new batch dim - """ - tensor: TensorDict = data.batch - non_tensor = data.non_tensor_batch - tensor.auto_batch_size_(batch_dims=batch_dims) - tensor = tensor.view(-1) - - batch_size = tensor.batch_size[0] - - non_tensor_new = {} - - for key, val in non_tensor.items(): - non_tensor_new[key] = np.reshape(val, newshape=(batch_size, *val.shape[batch_dims:])) - - return DataProto(batch=tensor, non_tensor_batch=non_tensor_new, meta_info=data.meta_info) - - -def collate_fn(x: list['DataProtoItem']): - batch = [] - non_tensor_batch = [] - for data in x: - batch.append(data.batch) - non_tensor_batch.append(data.non_tensor_batch) - batch = torch.stack(batch).contiguous() - non_tensor_batch = list_of_dict_to_dict_of_list(non_tensor_batch) - for key, val in non_tensor_batch.items(): - non_tensor_batch[key] = np.array(val, dtype=object) - return DataProto(batch=batch, non_tensor_batch=non_tensor_batch) - - -@dataclass -class DataProtoItem: - # TODO(zhangchi.usc1992) add consistency check - batch: TensorDict = None - non_tensor_batch: Dict = field(default_factory=dict) - meta_info: Dict = field(default_factory=dict) - - def to_dict(self, with_meta_info=False): - ret = { - **self.batch.to_dict(), - **self.non_tensor_batch, - } - if with_meta_info: - ret["meta_info"] = self.meta_info - return ret - - -@dataclass -class DataProto: - """ - A DataProto is a data structure that aims to provide a standard protocol for data exchange between functions. - It contains a batch (TensorDict) and a meta_info (Dict). The batch is a TensorDict https://pytorch.org/tensordict/. - TensorDict allows you to manipulate a dictionary of Tensors like a single Tensor. Ideally, the tensors with the - same batch size should be put inside batch. - """ - batch: TensorDict = None - non_tensor_batch: Dict = field(default_factory=dict) - meta_info: Dict = field(default_factory=dict) - - def __post_init__(self): - # perform necessary checking - self.check_consistency() - - def __len__(self): - if self.batch is not None: - return self.batch.batch_size[0] - elif self.non_tensor_batch is not None and len(self.non_tensor_batch) > 0: - random_key = list(self.non_tensor_batch.keys())[0] - return self.non_tensor_batch[random_key].shape[0] - else: - return 0 - - def __getitem__(self, item): - """ - Enhanced indexing for DataProto objects. - - Args: - item: Can be one of: - - int: A single index - - slice: A slice object (start:stop:step) - - list: A list of indices - - numpy.ndarray: An array of indices - - torch.Tensor: A tensor of indices - - Returns: - DataProto: For all indexing types except single integers - DataProtoItem: Only for single integer indices - """ - # Case 1: Slice object - use the slice method - if isinstance(item, slice): - return self.slice(item.start, item.stop, item.step) - - # Case 2: List, numpy array, or torch tensor - use sel_idxs - elif isinstance(item, (list, np.ndarray, torch.Tensor)): - return self.select_idxs(item) - - # Case 3: Single integer - return DataProtoItem for backward compatibility - elif isinstance(item, (int, np.integer)): - tensor_data = self.batch[item] - non_tensor_data = {key: val[item] for key, val in self.non_tensor_batch.items()} - return_type = DataProto if isinstance(item, slice) else DataProtoItem - return return_type(batch=tensor_data, non_tensor_batch=non_tensor_data, meta_info=self.meta_info) - - # Case 4: Unsupported type - else: - raise TypeError(f"Indexing with {type(item)} is not supported") - - def __getstate__(self): - import io - buffer = io.BytesIO() - if tensordict.__version__ >= '0.5.0' and self.batch is not None: - self.batch = self.batch.contiguous() - self.batch = self.batch.consolidate() - torch.save(self.batch, buffer) - buffer_bytes = buffer.getvalue() - return buffer_bytes, self.non_tensor_batch, self.meta_info - - def __setstate__(self, data): - import io - batch_deserialized_bytes, non_tensor_batch, meta_info = data - batch_deserialized = io.BytesIO(initial_bytes=batch_deserialized_bytes) - batch = torch.load(batch_deserialized, - weights_only=False, - map_location='cpu' if not torch.cuda.is_available() else None) - self.batch = batch - self.non_tensor_batch = non_tensor_batch - self.meta_info = meta_info - - def save_to_disk(self, filepath): - with open(filepath, 'wb') as f: - pickle.dump(self, f) - - @staticmethod - def load_from_disk(filepath) -> 'DataProto': - with open(filepath, 'rb') as f: - data = pickle.load(f) - return data - - def print_size(self, prefix=""): - size_of_tensordict = 0 - for key, tensor in self.batch.items(): - size_of_tensordict += tensor.element_size() * tensor.numel() - size_of_numpy_array = 0 - for key, numpy_array in self.non_tensor_batch.items(): - size_of_numpy_array += numpy_array.nbytes - - size_of_numpy_array /= 1024**3 - size_of_tensordict /= 1024**3 - - message = f'Size of tensordict: {size_of_tensordict} GB, size of non_tensor_batch: {size_of_numpy_array} GB' - - if prefix: - message = f'{prefix}, ' + message - print(message) - - def check_consistency(self): - """Check the consistency of the DataProto. Mainly for batch and non_tensor_batch - We expose this function as a public one so that user can call themselves directly - """ - if self.batch is not None: - assert len(self.batch.batch_size) == 1, 'only support num_batch_dims=1' - - if self.non_tensor_batch is not None: - for key, val in self.non_tensor_batch.items(): - assert isinstance(val, np.ndarray) - - if self.batch is not None and len(self.non_tensor_batch) != 0: - # TODO: we can actually lift this restriction if needed - assert len(self.batch.batch_size) == 1, 'only support num_batch_dims=1 when non_tensor_batch is not empty.' - - batch_size = self.batch.batch_size[0] - for key, val in self.non_tensor_batch.items(): - assert isinstance( - val, np.ndarray - ), f'data in the non_tensor_batch must be a numpy.array with dtype=object, but for {key=}, got {type(val)=}' - assert val.shape[ - 0] == batch_size, f'key {key} length {len(val)} is not equal to batch size {batch_size}' - - @classmethod - def from_single_dict(cls, data: Dict[str, Union[torch.Tensor, np.ndarray]], meta_info=None): - tensors = {} - non_tensors = {} - - for key, val in data.items(): - if isinstance(val, torch.Tensor): - tensors[key] = val - elif isinstance(val, np.ndarray): - non_tensors[key] = val - else: - raise ValueError(f'Unsupported type in data {type(val)}') - - return DataProto.from_dict(tensors=tensors, non_tensors=non_tensors, meta_info=meta_info) - - @classmethod - def from_dict(cls, tensors: Dict[str, torch.Tensor], non_tensors=None, meta_info=None, num_batch_dims=1): - """Create a DataProto from a dict of tensors. This assumes that - 1. All the tensor in tensors have the same dim0 - 2. Only dim0 is the batch dim - """ - assert len(tensors) > 0, 'tensors must not be empty' - assert num_batch_dims > 0, 'num_batch_dims must be greater than zero' - if non_tensors is not None: - assert num_batch_dims == 1, 'only support num_batch_dims=1 when non_tensors is not None.' - - if meta_info is None: - meta_info = {} - if non_tensors is None: - non_tensors = {} - - assert isinstance(non_tensors, dict) - - # get and check batch size - batch_size = None - pivot_key = None - for key, tensor in tensors.items(): - if batch_size is None: - batch_size = tensor.shape[:num_batch_dims] - pivot_key = key - else: - current_batch = tensor.shape[:num_batch_dims] - assert batch_size == current_batch, \ - f'Not all the tensor in tensors have the same batch size with batch_dims={num_batch_dims}. Got {pivot_key} has {batch_size}, {key} has {current_batch}' - - for key, val in non_tensors.items(): - non_tensors[key] = np.array(val, dtype=object) - - tensor_dict = TensorDict(source=tensors, batch_size=batch_size) - return cls(batch=tensor_dict, non_tensor_batch=non_tensors, meta_info=meta_info) - - def to(self, device) -> 'DataProto': - """move the batch to device - - Args: - device (torch.device, str): torch device - - Returns: - DataProto: the current DataProto - - """ - if self.batch is not None: - self.batch = self.batch.to(device) - return self - - def select(self, batch_keys=None, non_tensor_batch_keys=None, meta_info_keys=None, deepcopy=False) -> 'DataProto': - """Select a subset of the DataProto via batch_keys and meta_info_keys - - Args: - batch_keys (list, optional): a list of strings indicating the keys in batch to select - meta_info_keys (list, optional): a list of keys indicating the meta info to select - - Returns: - DataProto: the DataProto with the selected batch_keys and meta_info_keys - """ - # TODO (zhangchi.usc1992) whether to copy - if batch_keys is not None: - batch_keys = tuple(batch_keys) - sub_batch = self.batch.select(*batch_keys) - else: - sub_batch = self.batch - - if non_tensor_batch_keys is not None: - non_tensor_batch = {key: val for key, val in self.non_tensor_batch.items() if key in non_tensor_batch_keys} - else: - non_tensor_batch = self.non_tensor_batch - - if deepcopy: - non_tensor_batch = copy.deepcopy(non_tensor_batch) - - if meta_info_keys is not None: - sub_meta_info = {key: val for key, val in self.meta_info.items() if key in meta_info_keys} - else: - sub_meta_info = self.meta_info - - if deepcopy: - sub_meta_info = copy.deepcopy(sub_meta_info) - - return DataProto(batch=sub_batch, non_tensor_batch=non_tensor_batch, meta_info=sub_meta_info) - - def select_idxs(self, idxs): - """ - Select specific indices from the DataProto. - - Args: - idxs (torch.Tensor or numpy.ndarray or list): Indices to select - - Returns: - DataProto: A new DataProto containing only the selected indices - """ - if isinstance(idxs, list): - idxs = torch.tensor(idxs, dtype=torch.int32) - - if isinstance(idxs, np.ndarray): - idxs_np = idxs - idxs_torch = torch.from_numpy(idxs) - else: # torch.Tensor - idxs_torch = idxs - idxs_np = idxs.detach().cpu().numpy() - - if self.batch is not None: - # Use TensorDict's built-in indexing capabilities - selected_batch = TensorDict(source={ - key: tensor[idxs_torch] for key, tensor in self.batch.items() - }, - batch_size=(idxs_torch.shape[0],)) - else: - selected_batch = None - - selected_non_tensor = {} - for key, val in self.non_tensor_batch.items(): - selected_non_tensor[key] = val[idxs_np] - - return DataProto(batch=selected_batch, non_tensor_batch=selected_non_tensor, meta_info=self.meta_info) - - def slice(self, start=None, end=None, step=None): - """ - Slice the DataProto and return a new DataProto object. - This is an improved version of direct slicing which returns a DataProtoItem. - - Args: - start (int, optional): Start index. Defaults to None (start from beginning). - end (int, optional): End index (exclusive). Defaults to None (go to end). - step (int, optional): Step size. Defaults to None (step=1). - - Returns: - DataProto: A new DataProto containing the sliced data - - Examples: - # Using the slice method directly - sliced_data = data_proto.slice(10, 20) - - # Using enhanced indexing (returns DataProto) - sliced_data = data_proto[10:20] - sliced_data = data_proto[::2] # Every other element - - # Using list indexing (returns DataProto) - indices = [1, 5, 10] - selected_data = data_proto[indices] - - # Single index still returns DataProtoItem - single_item = data_proto[5] - """ - # Create a slice object - slice_obj = slice(start, end, step) - - # Handle the batch data - if self.batch is not None: - # Use TensorDict's built-in slicing capabilities - sliced_batch = self.batch[slice_obj] - else: - sliced_batch = None - - # Handle the non-tensor batch data - sliced_non_tensor = {} - for key, val in self.non_tensor_batch.items(): - sliced_non_tensor[key] = val[slice_obj] - - # Return a new DataProto object - return DataProto(batch=sliced_batch, non_tensor_batch=sliced_non_tensor, meta_info=self.meta_info) - - def pop(self, batch_keys=None, non_tensor_batch_keys=None, meta_info_keys=None) -> 'DataProto': - """Pop a subset of the DataProto via `batch_keys` and `meta_info_keys` - - Args: - batch_keys (list, optional): a list of strings indicating the keys in batch to pop - meta_info_keys (list, optional): a list of keys indicating the meta info to pop - - Returns: - DataProto: the DataProto with the poped batch_keys and meta_info_keys - """ - assert batch_keys is not None - if meta_info_keys is None: - meta_info_keys = [] - if non_tensor_batch_keys is None: - non_tensor_batch_keys = [] - - tensors = {} - # tensor batch - for key in batch_keys: - assert key in self.batch.keys() - tensors[key] = self.batch.pop(key) - non_tensors = {} - # non tensor batch - for key in non_tensor_batch_keys: - assert key in self.non_tensor_batch.keys() - non_tensors[key] = self.non_tensor_batch.pop(key) - meta_info = {} - for key in meta_info_keys: - assert key in self.meta_info.keys() - meta_info[key] = self.meta_info.pop(key) - return DataProto.from_dict(tensors=tensors, non_tensors=non_tensors, meta_info=meta_info) - - def rename(self, old_keys=None, new_keys=None) -> 'DataProto': - """ - Note that this function only rename the key in the batch - """ - - def validate_input(keys): - if keys is not None: - if isinstance(keys, str): - keys = [keys] - elif isinstance(keys, list): - pass - else: - raise TypeError(f'keys must be a list or a string, but got {type(keys)}') - return keys - - old_keys = validate_input(old_keys) - new_keys = validate_input(new_keys) - - if len(new_keys) != len(old_keys): - raise ValueError( - f'new_keys and old_keys must have the same length, but got {len(new_keys)} and {len(old_keys)}') - - self.batch.rename_key_(tuple(old_keys), tuple(new_keys)) - - return self - - def union(self, other: 'DataProto') -> 'DataProto': - """Union with another DataProto. Union batch and meta_info separately. - Throw an error if - - - there are conflict keys in batch and they are not equal - - the batch size of two data batch is not the same - - there are conflict keys in meta_info and they are not the same. - - Args: - other (DataProto): another DataProto to union - - Returns: - DataProto: the DataProto after union - """ - self.batch = union_tensor_dict(self.batch, other.batch) - self.non_tensor_batch = union_numpy_dict(self.non_tensor_batch, other.non_tensor_batch) - self.meta_info = union_two_dict(self.meta_info, other.meta_info) - return self - - def make_iterator(self, mini_batch_size, epochs, seed=None, dataloader_kwargs=None): - r"""Make an iterator from the DataProto. This is built upon that TensorDict can be used as a normal Pytorch - dataset. See https://pytorch.org/tensordict/tutorials/data_fashion for more details. - - - Args: - mini_batch_size (int): mini-batch size when iterating the dataset. We require that ``batch.batch_size[0] % mini_batch_size == 0``. - epochs (int): number of epochs when iterating the dataset. - dataloader_kwargs (Any): internally, it returns a DataLoader over the batch. The dataloader_kwargs is the kwargs passed to the DataLoader. - - Returns: - Iterator: an iterator that yields a mini-batch data at a time. The total number of iteration steps is ``self.batch.batch_size * epochs // mini_batch_size`` - """ - assert self.batch.batch_size[0] % mini_batch_size == 0, f"{self.batch.batch_size[0]} % {mini_batch_size} != 0" - # we can directly create a dataloader from TensorDict - if dataloader_kwargs is None: - dataloader_kwargs = {} - - if seed is not None: - generator = torch.Generator() - generator.manual_seed(seed) - else: - generator = None - - assert isinstance(dataloader_kwargs, Dict) - train_dataloader = DataLoader(dataset=self, - batch_size=mini_batch_size, - collate_fn=collate_fn, - generator=generator, - **dataloader_kwargs) - - def get_data(): - for _ in range(epochs): - for d in train_dataloader: - d.meta_info = self.meta_info - yield d - - return iter(get_data()) - - def chunk(self, chunks: int) -> List['DataProto']: - """Split the batch among dim=0 into chunks. The meta_info is passed to each DataProto after split. - - Args: - chunks (int): the number of chunks to split on dim=0 - - Returns: - List[DataProto]: a list of DataProto after splitting - """ - assert len( - self) % chunks == 0, f'only support equal chunk. Got size of DataProto {len(self)} and chunk {chunks}.' - - if self.batch is not None: - batch_lst = self.batch.chunk(chunks=chunks, dim=0) - else: - batch_lst = [None for _ in range(chunks)] - - non_tensor_batch_lst = [{} for _ in range(chunks)] - for key, val in self.non_tensor_batch.items(): - assert isinstance(val, np.ndarray) - non_tensor_lst = np.array_split(val, chunks) - assert len(non_tensor_lst) == chunks - for i in range(chunks): - non_tensor_batch_lst[i][key] = non_tensor_lst[i] - - output = [] - for i in range(chunks): - output.append( - DataProto(batch=batch_lst[i], non_tensor_batch=non_tensor_batch_lst[i], meta_info=self.meta_info)) - - return output - - @staticmethod - def concat(data: List['DataProto']) -> 'DataProto': - """Concat a list of DataProto. The batch is concatenated among dim=0. - The meta_info is assumed to be identical and will use the first one. - - Args: - data (List[DataProto]): list of DataProto - - Returns: - DataProto: concatenated DataProto - """ - batch_lst = [] - for batch in data: - batch_lst.append(batch.batch) - if batch_lst[0] is not None: - new_batch = torch.cat(batch_lst, dim=0) - else: - new_batch = None - - non_tensor_batch = list_of_dict_to_dict_of_list(list_of_dict=[d.non_tensor_batch for d in data]) - for key, val in non_tensor_batch.items(): - non_tensor_batch[key] = np.concatenate(val, axis=0) - - return DataProto(batch=new_batch, non_tensor_batch=non_tensor_batch, meta_info=data[0].meta_info) - - def reorder(self, indices): - """ - Note that this operation is in-place - """ - indices_np = indices.detach().numpy() - self.batch = self.batch[indices] - self.non_tensor_batch = {key: val[indices_np] for key, val in self.non_tensor_batch.items()} - - def repeat(self, repeat_times=2, interleave=True): - """ - Repeat the batch data a specified number of times. - - Args: - repeat_times (int): Number of times to repeat the data. - interleave (bool): Whether to interleave the repeated data. - - Returns: - DataProto: A new DataProto with repeated data. - """ - if self.batch is not None: - if interleave: - # Interleave the data - repeated_tensors = { - key: tensor.repeat_interleave(repeat_times, dim=0) for key, tensor in self.batch.items() - } - else: - # Stack the data - repeated_tensors = { - key: tensor.unsqueeze(0).expand(repeat_times, *tensor.shape).reshape(-1, *tensor.shape[1:]) - for key, tensor in self.batch.items() - } - - repeated_batch = TensorDict( - source=repeated_tensors, - batch_size=(self.batch.batch_size[0] * repeat_times,), - ) - else: - repeated_batch = None - - repeated_non_tensor_batch = {} - for key, val in self.non_tensor_batch.items(): - if interleave: - repeated_non_tensor_batch[key] = np.repeat(val, repeat_times, axis=0) - else: - repeated_non_tensor_batch[key] = np.tile(val, (repeat_times,) + (1,) * (val.ndim - 1)) - - return DataProto( - batch=repeated_batch, - non_tensor_batch=repeated_non_tensor_batch, - meta_info=self.meta_info, - ) - - -import ray - - -@dataclass -class DataProtoFuture: - """ - DataProtoFuture aims to eliminate actual data fetching on driver. By doing so, the driver doesn't have to wait - for data so that asynchronous execution becomes possible. - DataProtoFuture contains a list of futures from another WorkerGroup of size world_size. - - collect_fn is a Callable that reduces the list of futures to a DataProto - - dispatch_fn is a Callable that partitions the DataProto into a list of DataProto of size world_size and then select - - Potential issue: we can optimize dispatch_fn(collect_fn) such that only needed data is fetched on destination - - DataProtoFuture only supports directly passing from the output of a method to another input. You can't perform any - operation on the DataProtoFuture in driver. - """ - collect_fn: Callable - futures: List[ray.ObjectRef] - dispatch_fn: Callable = None - - @staticmethod - def concat(data: List[ray.ObjectRef]) -> 'DataProtoFuture': - output = DataProtoFuture(collect_fn=DataProto.concat, futures=data) - return output - - def chunk(self, chunks: int) -> List['DataProtoFuture']: - from functools import partial - - arg_future_lst = [] - for i in range(chunks): - # note that we can't directly pass i and chunks - def dispatch_fn(x, i, chunks): - return x.chunk(chunks=chunks)[i] - - arg_future = DataProtoFuture(collect_fn=self.collect_fn, - dispatch_fn=partial(dispatch_fn, i=i, chunks=chunks), - futures=self.futures) - arg_future_lst.append(arg_future) - return arg_future_lst - - def get(self): - output = ray.get(self.futures) # dp_size. - for o in output: - assert isinstance(o, DataProto) - output = self.collect_fn(output) # select dp, concat - if self.dispatch_fn is not None: - output = self.dispatch_fn(output) # split in batch dim, select using dp - return output - - -from verl.utils.torch_functional import allgather_dict_tensors -import torch.distributed - - -def all_gather_data_proto(data: DataProto, process_group): - # Note that this is an inplace operator just like torch.distributed.all_gather - group_size = torch.distributed.get_world_size(group=process_group) - assert isinstance(data, DataProto) - prev_device = data.batch.device - data.batch = data.batch.cuda(device=torch.cuda.current_device()) - data.batch = allgather_dict_tensors(data.batch.contiguous(), size=group_size, group=process_group, dim=0) - data.batch = data.batch.to(prev_device) - # all gather non_tensor_batch - all_non_tensor_batch = [None for _ in range(group_size)] - torch.distributed.all_gather_object(all_non_tensor_batch, data.non_tensor_batch, group=process_group) - data.non_tensor_batch = {k: np.concatenate([d[k] for d in all_non_tensor_batch]) for k in data.non_tensor_batch} diff --git a/verl/single_controller/__init__.py b/verl/single_controller/__init__.py deleted file mode 100644 index cb11fa5559..0000000000 --- a/verl/single_controller/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__))) - -# Note(haibin.lin): single_controller.__version__ is deprecated -with open(os.path.join(os.path.join(version_folder, os.pardir), 'version/version')) as f: - __version__ = f.read().strip() - -from . import base -from .base import * - -__all__ = base.__all__ \ No newline at end of file diff --git a/verl/single_controller/base/__init__.py b/verl/single_controller/base/__init__.py deleted file mode 100644 index d91f827dfd..0000000000 --- a/verl/single_controller/base/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .worker import Worker -from .worker_group import WorkerGroup, ClassWithInitArgs, ResourcePool - -__all__ = ['Worker', 'WorkerGroup', 'ClassWithInitArgs', 'ResourcePool'] \ No newline at end of file diff --git a/verl/single_controller/base/decorator.py b/verl/single_controller/base/decorator.py deleted file mode 100644 index 6fdacb6d97..0000000000 --- a/verl/single_controller/base/decorator.py +++ /dev/null @@ -1,410 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from enum import Enum -from functools import wraps -from typing import Dict, List, Tuple -from types import FunctionType -from verl.protocol import DataProtoFuture - -# here we add a magic number of avoid user-defined function already have this attribute -MAGIC_ATTR = 'attrs_3141562937' - - -class Dispatch(Enum): - RANK_ZERO = 0 - ONE_TO_ALL = 1 - ALL_TO_ALL = 2 - MEGATRON_COMPUTE = 3 - MEGATRON_PP_AS_DP = 4 - MEGATRON_PP_ONLY = 5 - MEGATRON_COMPUTE_PROTO = 6 - MEGATRON_PP_AS_DP_PROTO = 7 - DP_COMPUTE = 8 - DP_COMPUTE_PROTO = 9 - DP_COMPUTE_PROTO_WITH_FUNC = 10 - DP_COMPUTE_METRIC = 11 - - -class Execute(Enum): - ALL = 0 - RANK_ZERO = 1 - - -def _split_args_kwargs_data_proto(chunks, *args, **kwargs): - from verl.protocol import DataProto, DataProtoFuture - splitted_args = [] - for arg in args: - assert isinstance(arg, (DataProto, DataProtoFuture)) - splitted_args.append(arg.chunk(chunks=chunks)) - - splitted_kwargs = {} - for key, val in kwargs.items(): - assert isinstance(val, (DataProto, DataProtoFuture)) - splitted_kwargs[key] = val.chunk(chunks=chunks) - - return splitted_args, splitted_kwargs - - -def dispatch_one_to_all(worker_group, *args, **kwargs): - args = tuple([arg] * worker_group.world_size for arg in args) - kwargs = {k: [v] * worker_group.world_size for k, v in kwargs.items()} - return args, kwargs - - -def dispatch_all_to_all(worker_group, *args, **kwargs): - return args, kwargs - - -def collect_all_to_all(worker_group, output): - return output - - -def dispatch_megatron_compute(worker_group, *args, **kwargs): - """ - User passes in dp data. The data is dispatched to all tp/pp ranks with the same dp - """ - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, - MegatronWorkerGroup), f'worker_group must be MegatronWorkerGroup, Got {type(worker_group)}' - - all_args = [] - for arg in args: - assert isinstance(arg, (Tuple, List)) and len(arg) == worker_group.dp_size - transformed_args = [] - for i in range(worker_group.world_size): - local_dp_rank = worker_group.get_megatron_rank_info(rank=i).dp_rank - transformed_args.append(arg[local_dp_rank]) - all_args.append(transformed_args) - all_args = tuple(all_args) - - all_kwargs = {} - for k, v in kwargs.items(): - assert isinstance(v, (Tuple, List)) and len(v) == worker_group.dp_size - transformed_v = [] - for i in range(worker_group.world_size): - local_dp_rank = worker_group.get_megatron_rank_info(rank=i).dp_rank - transformed_v.append(v[local_dp_rank]) - all_kwargs[k] = transformed_v - return all_args, all_kwargs - - -def collect_megatron_compute(worker_group, output): - """ - Only collect the data from the tp=0 and pp=last and every dp ranks - """ - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, MegatronWorkerGroup) - output_in_dp = [] - pp_size = worker_group.get_megatron_global_info().pp_size - for global_rank in range(worker_group.world_size): - local_rank_info = worker_group.get_megatron_rank_info(rank=global_rank) - if local_rank_info.tp_rank == 0 and local_rank_info.pp_rank == pp_size - 1: - output_in_dp.append(output[global_rank]) - return output_in_dp - - -def dispatch_megatron_compute_data_proto(worker_group, *args, **kwargs): - """ - All the args and kwargs must be DataProto. The batch will be chunked by dp_size and passed to each rank - """ - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, MegatronWorkerGroup) - - splitted_args, splitted_kwargs = _split_args_kwargs_data_proto(worker_group.dp_size, *args, **kwargs) - return dispatch_megatron_compute(worker_group, *splitted_args, **splitted_kwargs) - - -def _concat_data_proto_or_future(output: List): - from verl.protocol import DataProto, DataProtoFuture - import ray - - # make sure all the elements in output has the same type - for o in output: - assert type(o) == type(output[0]) - - o = output[0] - - if isinstance(o, DataProto): - return DataProto.concat(output) - elif isinstance(o, ray.ObjectRef): - return DataProtoFuture.concat(output) - else: - raise NotImplementedError - - -def collect_megatron_compute_data_proto(worker_group, output): - """ - Each output must be a DataProto. We concat the dim=0 of output - """ - from verl.protocol import DataProto - import ray - - output = collect_megatron_compute(worker_group, output) - for o in output: - assert isinstance(o, (DataProto, ray.ObjectRef)), f"expecting {o} to be DataProto, but got {type(o)}" - - return _concat_data_proto_or_future(output) - - -def dispatch_megatron_pp_as_dp(worker_group, *args, **kwargs): - """ - treat pp as dp. - """ - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, MegatronWorkerGroup) - - pp_size = worker_group.pp_size - dp_size = worker_group.dp_size - - pp_dp_size = pp_size * dp_size - - all_args = [] - for arg in args: - assert isinstance(arg, (List, Tuple)) and len(arg) == pp_dp_size - transformed_args = [] - for i in range(worker_group.world_size): - local_dp_rank = worker_group.get_megatron_rank_info(rank=i).dp_rank - local_pp_rank = worker_group.get_megatron_rank_info(rank=i).pp_rank - # compute the rank in arg. Note that the order is dp then pp - # Also note that the outputs within a pp group will be firstly allgathered, then only the output of pp0 will be collected. - # For pp=2 dp=4, a batch of data "ABCDEFGH" should be dispatched and collected in below order: - # dispatch: pp_allgther: collect: - # dp 0 1 2 3 dp 0 1 2 3 - # pp +---------+ pp +-------------+ - # 0 | A C E G | 0 | AB CD EF GH | ABCDEFGH - # 1 | B D F H | 1 | AB CD EF GH | - # +---------+ +-------------+ - arg_rank = local_dp_rank * worker_group.pp_size + local_pp_rank - - transformed_args.append(arg[arg_rank]) - all_args.append(transformed_args) - all_args = tuple(all_args) - - all_kwargs = {} - for k, v in kwargs.items(): - assert isinstance(v, (List, Tuple)) and len(v) == pp_dp_size, f'expect len(v)=={pp_dp_size}, got {len(v)}' - transformed_v = [] - for i in range(worker_group.world_size): - local_dp_rank = worker_group.get_megatron_rank_info(rank=i).dp_rank - local_pp_rank = worker_group.get_megatron_rank_info(rank=i).pp_rank - # compute the rank in arg. Note that the order is dp then pp - arg_rank = local_dp_rank * worker_group.pp_size + local_pp_rank - transformed_v.append(v[arg_rank]) - all_kwargs[k] = transformed_v - return all_args, all_kwargs - - -def collect_megatron_pp_as_dp(worker_group, output): - """ - treat pp as dp. Only collect data on tp=0 - """ - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, MegatronWorkerGroup) - output_in_dp = [] - for global_rank in range(worker_group.world_size): - local_rank_info = worker_group.get_megatron_rank_info(rank=global_rank) - if local_rank_info.tp_rank == 0 and local_rank_info.pp_rank == 0: - output_in_dp.append(output[global_rank]) - return output_in_dp - - -def collect_megatron_pp_only(worker_group, output): - """ - Only collect output of megatron pp. This is useful when examine weight names as they are identical in tp/dp - """ - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, MegatronWorkerGroup) - output_in_pp = [] - for global_rank in range(worker_group.world_size): - local_rank_info = worker_group.get_megatron_rank_info(rank=global_rank) - if local_rank_info.tp_rank == 0 and local_rank_info.dp_rank == 0: - output_in_pp.append(output[global_rank]) - return output_in_pp - - -def dispatch_megatron_pp_as_dp_data_proto(worker_group, *args, **kwargs): - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, MegatronWorkerGroup) - - pp_dp_size = worker_group.dp_size * worker_group.pp_size - splitted_args, splitted_kwargs = _split_args_kwargs_data_proto(pp_dp_size, *args, **kwargs) - return dispatch_megatron_pp_as_dp(worker_group, *splitted_args, **splitted_kwargs) - - -def collect_megatron_pp_as_dp_data_proto(worker_group, output): - from verl.protocol import DataProto - from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - assert isinstance(worker_group, MegatronWorkerGroup) - - output = collect_megatron_pp_as_dp(worker_group, output) - return _concat_data_proto_or_future(output) - - -def dispatch_dp_compute(worker_group, *args, **kwargs): - from verl.single_controller.base.worker_group import WorkerGroup - assert isinstance(worker_group, WorkerGroup) - for arg in args: - assert isinstance(arg, (Tuple, List)) and len(arg) == worker_group.world_size - for k, v in kwargs.items(): - assert isinstance(v, (Tuple, List)) and len(v) == worker_group.world_size - return args, kwargs - - -def collect_dp_compute(worker_group, output): - from verl.single_controller.base.worker_group import WorkerGroup - assert isinstance(worker_group, WorkerGroup) - assert len(output) == worker_group.world_size - return output - - -def dispatch_dp_compute_data_proto(worker_group, *args, **kwargs): - from verl.single_controller.base.worker_group import WorkerGroup - assert isinstance(worker_group, WorkerGroup) - splitted_args, splitted_kwargs = _split_args_kwargs_data_proto(worker_group.world_size, *args, **kwargs) - return splitted_args, splitted_kwargs - - -def dispatch_dp_compute_data_proto_with_func(worker_group, *args, **kwargs): - from verl.single_controller.base.worker_group import WorkerGroup - assert isinstance(worker_group, WorkerGroup) - assert type(args[0]) == FunctionType # NOTE: The first one args is a function! - - splitted_args, splitted_kwargs = _split_args_kwargs_data_proto(worker_group.world_size, *args[1:], **kwargs) - splitted_args_with_func = [[args[0]] * worker_group.world_size] + splitted_args - return splitted_args_with_func, splitted_kwargs - - -def collect_dp_compute_data_proto(worker_group, output): - from verl.protocol import DataProto - import ray - - for o in output: - assert isinstance(o, (DataProto, ray.ObjectRef)), f"expecting {o} to be DataProto, but got {type(o)}" - - output = collect_dp_compute(worker_group, output) - return _concat_data_proto_or_future(output) - - -def get_predefined_dispatch_fn(dispatch_mode): - predefined_dispatch_mode_fn = { - Dispatch.ONE_TO_ALL: { - 'dispatch_fn': dispatch_one_to_all, - 'collect_fn': collect_all_to_all, - }, - Dispatch.ALL_TO_ALL: { - 'dispatch_fn': dispatch_all_to_all, - 'collect_fn': collect_all_to_all, - }, - Dispatch.MEGATRON_COMPUTE: { - 'dispatch_fn': dispatch_megatron_compute, - 'collect_fn': collect_megatron_compute, - }, - Dispatch.MEGATRON_PP_AS_DP: { - 'dispatch_fn': dispatch_megatron_pp_as_dp, - 'collect_fn': collect_megatron_pp_as_dp, - }, - Dispatch.MEGATRON_PP_ONLY: { - 'dispatch_fn': dispatch_one_to_all, - 'collect_fn': collect_megatron_pp_only - }, - Dispatch.MEGATRON_COMPUTE_PROTO: { - 'dispatch_fn': dispatch_megatron_compute_data_proto, - 'collect_fn': collect_megatron_compute_data_proto - }, - Dispatch.MEGATRON_PP_AS_DP_PROTO: { - 'dispatch_fn': dispatch_megatron_pp_as_dp_data_proto, - 'collect_fn': collect_megatron_pp_as_dp_data_proto - }, - Dispatch.DP_COMPUTE: { - 'dispatch_fn': dispatch_dp_compute, - 'collect_fn': collect_dp_compute - }, - Dispatch.DP_COMPUTE_PROTO: { - 'dispatch_fn': dispatch_dp_compute_data_proto, - 'collect_fn': collect_dp_compute_data_proto - }, - Dispatch.DP_COMPUTE_PROTO_WITH_FUNC: { - 'dispatch_fn': dispatch_dp_compute_data_proto_with_func, - 'collect_fn': collect_dp_compute_data_proto - }, - Dispatch.DP_COMPUTE_METRIC: { - 'dispatch_fn': dispatch_dp_compute_data_proto, - 'collect_fn': collect_dp_compute - } - } - return predefined_dispatch_mode_fn[dispatch_mode] - - -def get_predefined_execute_fn(execute_mode): - """ - Note that here we only asks execute_all and execute_rank_zero to be implemented - Leave the choice of how these two functions handle argument 'blocking' to users - """ - predefined_execute_mode_fn = { - Execute.ALL: { - 'execute_fn_name': 'execute_all' - }, - Execute.RANK_ZERO: { - 'execute_fn_name': 'execute_rank_zero' - } - } - return predefined_execute_mode_fn[execute_mode] - - -def _check_dispatch_mode(dispatch_mode): - assert isinstance(dispatch_mode, - (Dispatch, Dict)), f'dispatch_mode must be a Dispatch or a Dict. Got {dispatch_mode}' - if isinstance(dispatch_mode, Dict): - necessary_keys = ['dispatch_fn', 'collect_fn'] - for key in necessary_keys: - assert key in dispatch_mode, f'key {key} should be in dispatch_mode if it is a dictionary' - - -def _check_execute_mode(execute_mode): - assert isinstance(execute_mode, Execute), f'execute_mode must be a Execute. Got {execute_mode}' - - -def _materialize_futures(*args, **kwargs): - new_args = [] - for arg in args: - if isinstance(arg, DataProtoFuture): - arg = arg.get() - # add more type to materialize - new_args.append(arg) - for k, v in kwargs.items(): - if isinstance(v, DataProtoFuture): - kwargs[k] = v.get() - - new_args = tuple(new_args) - return new_args, kwargs - - -def register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.ALL, blocking=True, materialize_futures=True): - _check_dispatch_mode(dispatch_mode=dispatch_mode) - _check_execute_mode(execute_mode=execute_mode) - - def decorator(func): - - @wraps(func) - def inner(*args, **kwargs): - if materialize_futures: - args, kwargs = _materialize_futures(*args, **kwargs) - return func(*args, **kwargs) - - attrs = {'dispatch_mode': dispatch_mode, 'execute_mode': execute_mode, 'blocking': blocking} - setattr(inner, MAGIC_ATTR, attrs) - return inner - - return decorator diff --git a/verl/single_controller/base/megatron/__init__.py b/verl/single_controller/base/megatron/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/single_controller/base/megatron/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/single_controller/base/megatron/worker.py b/verl/single_controller/base/megatron/worker.py deleted file mode 100644 index 452d464481..0000000000 --- a/verl/single_controller/base/megatron/worker.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl.single_controller.base.worker import Worker, DistRankInfo, DistGlobalInfo - - -class MegatronWorker(Worker): - - def __init__(self, cuda_visible_devices=None) -> None: - super().__init__(cuda_visible_devices) - - def get_megatron_global_info(self): - from megatron.core import parallel_state as mpu - tp_size = mpu.get_tensor_model_parallel_world_size() - dp_size = mpu.get_data_parallel_world_size() - pp_size = mpu.get_pipeline_model_parallel_world_size() - info = DistGlobalInfo(tp_size=tp_size, dp_size=dp_size, pp_size=pp_size) - return info - - def get_megatron_rank_info(self): - from megatron.core import parallel_state as mpu - tp_rank = mpu.get_tensor_model_parallel_rank() - dp_rank = mpu.get_data_parallel_rank() - pp_rank = mpu.get_pipeline_model_parallel_rank() - info = DistRankInfo(tp_rank=tp_rank, dp_rank=dp_rank, pp_rank=pp_rank) - return info \ No newline at end of file diff --git a/verl/single_controller/base/megatron/worker_group.py b/verl/single_controller/base/megatron/worker_group.py deleted file mode 100644 index 67c21d309b..0000000000 --- a/verl/single_controller/base/megatron/worker_group.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict - -from .worker import DistRankInfo, DistGlobalInfo -from verl.single_controller.base import ResourcePool, WorkerGroup - - -class MegatronWorkerGroup(WorkerGroup): - - def __init__(self, resource_pool: ResourcePool, **kwargs): - super().__init__(resource_pool=resource_pool, **kwargs) - self._megatron_rank_info = None - self._megatron_global_info: DistGlobalInfo = None - - def init_megatron(self, default_megatron_kwargs: Dict = None): - raise NotImplementedError(f"MegatronWorkerGroup.init_megatron should be overwritten") - - def get_megatron_rank_info(self, rank: int) -> DistRankInfo: - assert 0 <= rank < self.world_size, f'rank must be from [0, world_size), Got {rank}' - return self._megatron_rank_info[rank] - - @property - def tp_size(self): - assert self._megatron_global_info is not None, "MegatronWorkerGroup._megatron_global_info must be initialized" - return self._megatron_global_info.tp_size - - @property - def dp_size(self): - assert self._megatron_global_info is not None, "MegatronWorkerGroup._megatron_global_info must be initialized" - return self._megatron_global_info.dp_size - - @property - def pp_size(self): - assert self._megatron_global_info is not None, "MegatronWorkerGroup._megatron_global_info must be initialized" - return self._megatron_global_info.pp_size - - def get_megatron_global_info(self): - return self._megatron_global_info diff --git a/verl/single_controller/base/register_center/__init__.py b/verl/single_controller/base/register_center/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/single_controller/base/register_center/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/single_controller/base/register_center/ray.py b/verl/single_controller/base/register_center/ray.py deleted file mode 100644 index 430290cf26..0000000000 --- a/verl/single_controller/base/register_center/ray.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ray - - -@ray.remote -class WorkerGroupRegisterCenter: - - def __init__(self, rank_zero_info): - self.rank_zero_info = rank_zero_info - - def get_rank_zero_info(self): - return self.rank_zero_info - - -def create_worker_group_register_center(name, info): - return WorkerGroupRegisterCenter.options(name=name).remote(info) diff --git a/verl/single_controller/base/worker.py b/verl/single_controller/base/worker.py deleted file mode 100644 index dbe4cc600f..0000000000 --- a/verl/single_controller/base/worker.py +++ /dev/null @@ -1,217 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -the class for Worker -""" -import os -import socket -from dataclasses import dataclass -from .decorator import register, Dispatch, Execute - - -@dataclass -class DistRankInfo: - tp_rank: int - dp_rank: int - pp_rank: int - - -@dataclass -class DistGlobalInfo: - tp_size: int - dp_size: int - pp_size: int - - -class WorkerHelper: - - def _get_node_ip(self): - - def get_node_ip_by_sdk(): - if os.getenv("WG_BACKEND", None) == "ray": - import ray - return ray._private.services.get_node_ip_address() - else: - raise NotImplementedError("WG_BACKEND now just support ray mode.") - - host_ipv4 = os.getenv("MY_HOST_IP", None) - host_ipv6 = os.getenv("MY_HOST_IPV6", None) - host_ip_by_env = host_ipv4 or host_ipv6 - host_ip_by_sdk = get_node_ip_by_sdk() - - host_ip = host_ip_by_env or host_ip_by_sdk - return host_ip - - def _get_free_port(self): - with socket.socket() as sock: - sock.bind(('', 0)) - return sock.getsockname()[1] - - def get_availale_master_addr_port(self): - return self._get_node_ip(), str(self._get_free_port()) - - def _get_pid(self): - return - - -class WorkerMeta: - keys = [ - "WORLD_SIZE", "RANK", "LOCAL_WORLD_SIZE", "LOCAL_RANK", "MASTER_ADDR", "MASTER_PORT", "CUDA_VISIBLE_DEVICES" - ] - - def __init__(self, store) -> None: - self._store = store - - def to_dict(self): - return {f"_{key.lower()}": self._store.get(f"_{key.lower()}", None) for key in WorkerMeta.keys} - - -# we assume that in each WorkerGroup, there is a Master Worker -class Worker(WorkerHelper): - """A (distributed) worker.""" - - def __new__(cls, *args, **kwargs): - instance = super().__new__(cls) - - # note that here we use int to distinguish - disable_worker_init = int(os.environ.get('DISABLE_WORKER_INIT', 0)) - if disable_worker_init: - return instance - - rank = os.environ.get("RANK", None) - worker_group_prefix = os.environ.get("WG_PREFIX", None) - - # when decorator @ray.remote applies, __new__ will be called while we don't want to apply _configure_before_init - if None not in [rank, worker_group_prefix] and 'ActorClass(' not in cls.__name__: - instance._configure_before_init(f"{worker_group_prefix}_register_center", int(rank)) - - return instance - - def _configure_before_init(self, register_center_name: str, rank: int): - assert isinstance(rank, int), f"rank must be int, instead of {type(rank)}" - - if rank == 0: - master_addr, master_port = self.get_availale_master_addr_port() - rank_zero_info = { - "MASTER_ADDR": master_addr, - "MASTER_PORT": master_port, - } - - if os.getenv("WG_BACKEND", None) == "ray": - from verl.single_controller.base.register_center.ray import create_worker_group_register_center - self.register_center = create_worker_group_register_center(name=register_center_name, - info=rank_zero_info) - - os.environ.update(rank_zero_info) - - def __init__(self, cuda_visible_devices=None) -> None: - # construct a meta from envrionment variable. Note that the import must be inside the class because it is executed remotely - import os - - ### - # [SUPPORT AMD: torch] - import torch - ### - - ### - # [SUPPORT AMD: torch] - if "AMD" in torch.cuda.get_device_name(): - os.environ['CUDA_VISIBLE_DEVICES'] = os.environ.get('ROCR_VISIBLE_DEVICES') - os.environ['LOCAL_RANK'] = os.environ.get('RAY_LOCAL_RANK') - ### - - world_size = int(os.environ['WORLD_SIZE']) - rank = int(os.environ['RANK']) - self._rank = rank - self._world_size = world_size - - master_addr = os.environ["MASTER_ADDR"] - master_port = os.environ["MASTER_PORT"] - - local_world_size = int(os.getenv("LOCAL_WORLD_SIZE", "1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - - ### - # [SUPPORT AMD: torch] - if "AMD" in torch.cuda.get_device_name(): - self.local_rank = int(os.environ['LOCAL_RANK']) - ### - - ### - # [SUPPORT AMD: torch] - if "AMD" in torch.cuda.get_device_name(): - cuda_visible_devices = str(local_rank) - ### - - store = { - '_world_size': world_size, - '_rank': rank, - '_local_world_size': local_world_size, - '_local_rank': local_rank, - '_master_addr': master_addr, - '_master_port': master_port - } - if cuda_visible_devices is not None: - store['_cuda_visible_devices'] = cuda_visible_devices - - meta = WorkerMeta(store=store) - self._configure_with_meta(meta=meta) - - ### - # [SUPPORT AMD: torch] - # torch.cuda.set_device(local_rank) - if "AMD" in torch.cuda.get_device_name(): - torch.cuda.set_device(int(cuda_visible_devices)) - ### - - def _configure_with_meta(self, meta: WorkerMeta): - """ - This function should only be called inside by WorkerGroup - """ - assert isinstance(meta, WorkerMeta) - self.__dict__.update(meta.to_dict()) # this is hacky - # print(f"__dict__: {self.__dict__}") - for key in WorkerMeta.keys: - val = self.__dict__.get(f"_{key.lower()}", None) - if val is not None: - # print(f"set {key} to {val}") - os.environ[key] = str(val) - os.environ["REDIS_STORE_SERVER_HOST"] = str(self._master_addr).replace("[", "").replace( - "]", "") if self._master_addr else "" - - def get_master_addr_port(self): - return self._master_addr, self._master_port - - def get_cuda_visible_devices(self): - import os - cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "not set") - return cuda_visible_devices - - @property - def world_size(self): - return self._world_size - - @property - def rank(self): - return self._rank - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO_WITH_FUNC) - def execute_with_func_generator(self, func, *args, **kwargs): - ret_proto = func(self, *args, **kwargs) - return ret_proto - - @register(dispatch_mode=Dispatch.ALL_TO_ALL, execute_mode=Execute.RANK_ZERO) - def execute_func_rank_zero(self, func, *args, **kwargs): - result = func(*args, **kwargs) - return result diff --git a/verl/single_controller/base/worker_group.py b/verl/single_controller/base/worker_group.py deleted file mode 100644 index a17d5a50fa..0000000000 --- a/verl/single_controller/base/worker_group.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -the class of WorkerGroup -""" -import logging -import threading -import signal -import time -from typing import List, Any, Callable, Dict - -from .decorator import MAGIC_ATTR, Dispatch, get_predefined_dispatch_fn, get_predefined_execute_fn - - -class ResourcePool: - """The resource pool with meta info such as world_size.""" - - def __init__(self, process_on_nodes=None, max_collocate_count: int = 10, n_gpus_per_node=8) -> None: - if process_on_nodes is None: - process_on_nodes = [] - self._store = process_on_nodes - self.max_collocate_count = max_collocate_count - self.n_gpus_per_node = n_gpus_per_node # this is left for future huawei GPU that contains 16 GPUs per node - - def add_node(self, process_count): - self._store.append(process_count) - - @property - def world_size(self): - return sum(self._store) - - def __call__(self) -> Any: - return self._store - - @property - def store(self): - return self._store - - def local_world_size_list(self) -> List[int]: - nested_local_world_size_list = [ - [local_world_size for _ in range(local_world_size)] for local_world_size in self._store - ] - return [item for row in nested_local_world_size_list for item in row] - - def local_rank_list(self) -> List[int]: - nested_local_rank_list = [[i for i in range(local_world_size)] for local_world_size in self._store] - return [item for row in nested_local_rank_list for item in row] - - -class ClassWithInitArgs: - """ - This class stores a class constructor and the args/kwargs to construct the class. - It is used to instantiate the remote class. - """ - - def __init__(self, cls, *args, **kwargs) -> None: - self.cls = cls - self.args = args - self.kwargs = kwargs - - # def add_arg(self, arg): - # self.args += (arg,) - - # def add_kwarg(self, key, value): - # self.kwargs[key] = value - - def __call__(self) -> Any: - return self.cls(*self.args, **self.kwargs) - - -def check_workers_alive(workers: List, is_alive: Callable, gap_time: float = 1) -> None: - import time - while True: - for worker in workers: - if not is_alive(worker): - logging.warning(f"worker {worker} is not alive" + " sending signal to main thread") - signal.raise_signal(signal.SIGABRT) - time.sleep(gap_time) - - -class WorkerGroup: - """A group of workers""" - - def __init__(self, resource_pool: ResourcePool, **kwargs) -> None: - self._is_init_with_detached_workers = True if resource_pool is None else False - - if resource_pool is not None: - # handle the case when WorkGroup is attached to an existing one - self._procecss_dispatch_config = resource_pool() - else: - self._procecss_dispatch_config = None - - self._workers = [] - self._worker_names = [] - - self._master_addr = None - self._master_port = None - - self._checker_thread: threading.Thread = None - - def _is_worker_alive(self, worker): - raise NotImplementedError(f"WorkerGroup._is_worker_alive called, should be implemented in derived class.") - - def _block_until_all_workers_alive(self) -> None: - while True: - all_state = [self._is_worker_alive(worker) for worker in self._workers] - if False in all_state: - time.sleep(1) - else: - break - - def start_worker_aliveness_check(self, every_n_seconds=1) -> None: - # before starting checking worker aliveness, make sure all workers are already alive - self._block_until_all_workers_alive() - - self._checker_thread = threading.Thread(target=check_workers_alive, - args=(self._workers, self._is_worker_alive, every_n_seconds)) - self._checker_thread.start() - - @property - def world_size(self): - return len(self._workers) - - # execute_all_async and execute_rank_zero_async should be implemented by RayWorkerGroup, TorchRPCWorkerGroup, - # MegatronWorkerGroup, XperfWorkerGroup should skip - - def _bind_worker_method(self, user_defined_cls, func_generator): - """ - Bind the worker method to the WorkerGroup - """ - - for method_name in dir(user_defined_cls): - - try: - method = getattr(user_defined_cls, method_name) - assert callable(method), f"{method_name} in {user_defined_cls} is not callable" - except Exception as e: - # if it is a property, it will fail because Class doesn't have instance property - continue - - if hasattr(method, MAGIC_ATTR): - # this method is decorated by register - attribute = getattr(method, MAGIC_ATTR) - assert isinstance(attribute, Dict), f'attribute must be a dictionary. Got {type(attribute)}' - assert 'dispatch_mode' in attribute, f'attribute must contain dispatch_mode in its key' - - dispatch_mode = attribute['dispatch_mode'] - execute_mode = attribute['execute_mode'] - blocking = attribute['blocking'] - - # get dispatch fn - if isinstance(dispatch_mode, Dispatch): - # get default dispatch fn - fn = get_predefined_dispatch_fn(dispatch_mode=dispatch_mode) - dispatch_fn = fn['dispatch_fn'] - collect_fn = fn['collect_fn'] - else: - assert isinstance(dispatch_mode, dict) - assert 'dispatch_fn' in dispatch_mode - assert 'collect_fn' in dispatch_mode - dispatch_fn = dispatch_mode['dispatch_fn'] - collect_fn = dispatch_mode['collect_fn'] - - # get execute_fn_name - execute_mode = get_predefined_execute_fn(execute_mode=execute_mode) - wg_execute_fn_name = execute_mode['execute_fn_name'] - - # get execute_fn from string - try: - execute_fn = getattr(self, wg_execute_fn_name) - assert callable(execute_fn), 'execute_fn must be callable' - except Exception as e: - print(f'execute_fn {wg_execute_fn_name} is invalid') - raise - - # bind a new method to the RayWorkerGroup - func = func_generator(self, - method_name, - dispatch_fn=dispatch_fn, - collect_fn=collect_fn, - execute_fn=execute_fn, - blocking=blocking) - - try: - setattr(self, method_name, func) - except Exception as e: - raise ValueError(f'Fail to set method_name {method_name}') diff --git a/verl/single_controller/ray/__init__.py b/verl/single_controller/ray/__init__.py deleted file mode 100644 index 2b567d0cfa..0000000000 --- a/verl/single_controller/ray/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base import RayResourcePool, RayClassWithInitArgs, RayWorkerGroup, create_colocated_worker_cls \ No newline at end of file diff --git a/verl/single_controller/ray/base.py b/verl/single_controller/ray/base.py deleted file mode 100644 index 4a060a3a8a..0000000000 --- a/verl/single_controller/ray/base.py +++ /dev/null @@ -1,504 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time -from typing import Dict, List, Any, Tuple - -import ray -from ray.util import list_named_actors -from ray.util.placement_group import placement_group, PlacementGroup -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy, NodeAffinitySchedulingStrategy -from ray.experimental.state.api import get_actor - -from verl.single_controller.base import WorkerGroup, ResourcePool, ClassWithInitArgs, Worker - -__all__ = ['Worker'] - - -def get_random_string(length: int) -> str: - import random - import string - letters_digits = string.ascii_letters + string.digits - return ''.join(random.choice(letters_digits) for _ in range(length)) - - -def func_generator(self, method_name, dispatch_fn, collect_fn, execute_fn, blocking): - - def func(*args, **kwargs): - args, kwargs = dispatch_fn(self, *args, **kwargs) - output = execute_fn(method_name, *args, **kwargs) - if blocking: - output = ray.get(output) - output = collect_fn(self, output) - return output - - return func - - -def sort_placement_group_by_node_ip(pgs: List[PlacementGroup]) -> List[PlacementGroup]: - """ - Sort the placement groups by node ip, all bundles in a single placement group should be on the same node. - - FSDPCheckpointManager saves sharded model states and optimizer states in local storage, which requires RANK - to be consistent across nodes when resume from checkpoint. - - With this function, if there's only one resource pool and there's no node change, RANK should be consistent - across nodes in multiple ray jobs, even if the whole ray cluster is restarted. - """ - node_ip = {node["NodeID"]: node["NodeManagerAddress"] for node in ray.nodes()} - pg_ip = {} - for pg in pgs: - specs = ray._private.state.state.placement_group_table(pg.id) - # all bunles should be on the same node - node_id = specs["bundles_to_node_id"][0] - pg_ip[pg.id] = node_ip[node_id] - return sorted(pgs, key=lambda pg: pg_ip[pg.id]) - - -class RayResourcePool(ResourcePool): - - def __init__(self, - process_on_nodes: List[int] = None, - use_gpu: bool = True, - name_prefix: str = "", - max_colocate_count: int = 5, - detached=False) -> None: - super().__init__(process_on_nodes, max_colocate_count) - self.use_gpu = use_gpu - # print(f"in RayProcessDispatchConfiguration: name_prefix = {name_prefix}") - self.name_prefix = name_prefix - self.pgs = None - self.detached = detached - - def get_placement_groups(self, strategy="STRICT_PACK", name=None): - if self.pgs is not None: - return self.pgs - - pg_name_prefix = name if name else \ - f"{self.name_prefix}verl_group_{'_'.join([str(count) for count in self._store])}:" - # print(f"pg_name_prefix = {pg_name_prefix}") - pg_scheme = [[{ - "CPU": self.max_collocate_count, - "GPU": 1 - } if self.use_gpu else { - "CPU": self.max_collocate_count - } for _ in range(process_count)] for process_count in self._store] - - lifetime = 'detached' if self.detached else None - - pgs = [ - placement_group(bundles=bundles, strategy=strategy, name=pg_name_prefix + str(idx), lifetime=lifetime) - for idx, bundles in enumerate(pg_scheme) - ] - - ray.get([pg.ready() for pg in pgs]) - - self.pgs = pgs - return pgs - - -def extract_pg_from_exist(resource_pools: Dict[str, RayResourcePool], src_role_names: List[str], - resource_pool: RayResourcePool) -> List: - - src_pgs = [ - pg for role_name, resource_pool in resource_pools.items() for pg in resource_pool.get_placement_groups() - if role_name in src_role_names - ] - - sorted_src_pgs = sorted(src_pgs, key=lambda pg: pg.bundle_count, reverse=True) - sorted_process_on_nodes = sorted([(val, idx) for idx, val in enumerate(resource_pool.store)], reverse=True) - - unsorted_pgs: List[Tuple[int, PlacementGroup]] = [] - searching_idx = 0 - for request_process, original_idx in sorted_process_on_nodes: - assert searching_idx < len(sorted_src_pgs), f"no enough nodes for request: searching {searching_idx} th node" - assert request_process <= sorted_src_pgs[searching_idx].bundle_count, \ - f"requesting {request_process} processes, bundle count cannot satisfy" - unsorted_pgs.append((original_idx, sorted_src_pgs[searching_idx])) - searching_idx += 1 - - return [pg for _, pg in sorted(unsorted_pgs)] - - -def merge_resource_pool(rp1: RayResourcePool, rp2: RayResourcePool) -> RayResourcePool: - assert rp1.use_gpu == rp2.use_gpu, 'Both RayResourcePool must either use_gpu or not' - assert rp1.max_collocate_count == rp2.max_collocate_count, 'Both RayResourcePool must has the same max_collocate_count' - assert rp1.n_gpus_per_node == rp2.n_gpus_per_node, 'Both RayResourcePool must has the same n_gpus_per_node' - assert rp1.detached == rp2.detached, 'Detached ResourcePool cannot be merged with non-detached ResourcePool' - - new_store = rp1.store + rp2.store - - merged = RayResourcePool(new_store, rp1.use_gpu, f"{rp1.name_prefix}_{rp2.name_prefix}") - merged.pgs = rp1.get_placement_groups() + rp2.get_placement_groups() - - return merged - - -class RayClassWithInitArgs(ClassWithInitArgs): - - def __init__(self, cls, *args, **kwargs) -> None: - # self._options = kwargs.pop('options', dict()) - super().__init__(cls, *args, **kwargs) - self._options = {} - self._additional_resource = {} - - def set_additional_resource(self, additional_resource): - self._additional_resource = additional_resource - - def update_options(self, options: Dict): - self._options.update(options) - - def __call__(self, - placement_group, - placement_group_bundle_idx, - use_gpu: bool = True, - num_gpus=1, - sharing_with=None) -> Any: - if sharing_with is not None: - target_node_id = ray.get(sharing_with.get_node_id.remote()) - cuda_visible_devices = ray.get(sharing_with.get_cuda_visible_devices.remote()) - options = {"scheduling_strategy": NodeAffinitySchedulingStrategy(node_id=target_node_id, soft=False)} - return self.cls.options(**options).remote(*self.args, - cuda_visible_devices=cuda_visible_devices, - **self.kwargs) - - options = { - "scheduling_strategy": - PlacementGroupSchedulingStrategy(placement_group=placement_group, - placement_group_bundle_index=placement_group_bundle_idx) - } - options.update(self._options) - - if use_gpu: - options["num_gpus"] = num_gpus - - if len(self._additional_resource) > 1: - for k, v in self._additional_resource.items(): - options[k] = v - - # print("cls:", self.cls) - # print("args: ", self.args) - # print("kwargs: ", self.kwargs) - return self.cls.options(**options).remote(*self.args, **self.kwargs) - - -class RayWorkerGroup(WorkerGroup): - - def __init__(self, - resource_pool: RayResourcePool = None, - ray_cls_with_init: RayClassWithInitArgs = None, - bin_pack: bool = True, - name_prefix: str = None, - detached=False, - worker_names=None, - **kwargs) -> None: - """ - This class allows placing one RayClassWithInitArgs on a resource pool. - steps: - 1. create workers on resource pool, record their names. - 2. bind the method of the worker to the WorkerGroup, with a dispatch function. - Only methods decorated with @register will be bound. Dispatch function is specified in args of @register - - When initializing without specifying resource_pool arg, will be init with detached workers. - This means it will gather all workers by worker names. - Args: - resource_pool: - ray_cls_with_init: - bin_pack: - name_prefix: - detached: - worker_names: - **kwargs: - """ - super().__init__(resource_pool=resource_pool, **kwargs) - self.ray_cls_with_init = ray_cls_with_init - self.name_prefix = get_random_string(length=6) if name_prefix is None else name_prefix - - if worker_names is not None: - assert self._is_init_with_detached_workers - self._worker_names = worker_names - - if self._is_init_with_detached_workers: - self._init_with_detached_workers(worker_names=worker_names) - else: - self._init_with_resource_pool(resource_pool=resource_pool, - ray_cls_with_init=ray_cls_with_init, - bin_pack=bin_pack, - detached=detached) - - if ray_cls_with_init is not None: - self._bind_worker_method(self.ray_cls_with_init.cls, func_generator) - - def _is_worker_alive(self, worker: ray.actor.ActorHandle): - worker_state_dict = get_actor(worker._actor_id.hex()) - return worker_state_dict.get("state", "undefined") == "ALIVE" if worker_state_dict is not None else False - - def _init_with_detached_workers(self, worker_names): - workers = [ray.get_actor(name=name) for name in worker_names] - self._workers = workers - self._world_size = len(worker_names) - - def _init_with_resource_pool(self, resource_pool, ray_cls_with_init, bin_pack, detached): - use_gpu = resource_pool.use_gpu - - strategy = "PACK" - if bin_pack: - strategy = "STRICT_PACK" - pgs = resource_pool.get_placement_groups(strategy=strategy) - world_size = resource_pool.world_size - self._world_size = world_size - # cia.add_kwarg("_world_size", world_size) - num_gpus = 1 / resource_pool.max_collocate_count - - rank = -1 - local_world_size = resource_pool.store[0] - for pg_idx, pg in enumerate(sort_placement_group_by_node_ip(pgs)): - assert local_world_size <= pg.bundle_count, \ - f"when generating for {self.name_prefix}, for the " - for local_rank in range(local_world_size): - rank += 1 - - # we pass in environment variable at option so that Worker can use environment variable to set - env_vars = { - 'WORLD_SIZE': str(world_size), - 'RANK': str(rank), - 'WG_PREFIX': self.name_prefix, - 'WG_BACKEND': 'ray', - 'RAY_LOCAL_WORLD_SIZE': str(local_world_size), - 'RAY_LOCAL_RANK': str(local_rank), - } - if rank != 0: - env_vars['MASTER_ADDR'] = self._master_addr - env_vars['MASTER_PORT'] = self._master_port - - import re - cia_name = type(ray_cls_with_init.cls).__name__ - match = re.search(r"ActorClass\(([^)]+)\)", cia_name) # ray.remote(Obj) -> "ActorClass(Obj)" - cia_name = match.group(1) if match else cia_name # "ActorClass(Obj)" -> "Obj" - name = f"{self.name_prefix}{cia_name}_{pg_idx}:{local_rank}" # e.g. Worker_2:5 - - env_vars['VLLM_ATTENTION_BACKEND'] = 'XFORMERS' - - ray_cls_with_init.update_options({'runtime_env': {'env_vars': env_vars}, 'name': name}) - - if detached: - ray_cls_with_init.update_options({'lifetime': 'detached'}) - - # create a worker - worker = ray_cls_with_init(placement_group=pg, - placement_group_bundle_idx=local_rank, - use_gpu=use_gpu, - num_gpus=num_gpus) - self._workers.append(worker) - self._worker_names.append(name) - - if rank == 0: - register_center_actor = None - for _ in range(120): - if f"{self.name_prefix}_register_center" not in list_named_actors(): - time.sleep(1) - else: - register_center_actor = ray.get_actor(f"{self.name_prefix}_register_center") - break - assert register_center_actor is not None, f"failed to get register_center_actor: {self.name_prefix}_register_center in {list_named_actors(all_namespaces=True)}" - rank_zero_info = ray.get(register_center_actor.get_rank_zero_info.remote()) - self._master_addr, self._master_port = rank_zero_info['MASTER_ADDR'], rank_zero_info['MASTER_PORT'] - # print(f"rank_zero_info: {rank_zero_info}") - # print(f"master_addr: {self._master_addr}, master_port: {self._master_port}") - - @property - def worker_names(self): - return self._worker_names - - @classmethod - def from_detached(cls, worker_names=None, ray_cls_with_init=None): - worker_group = cls(resource_pool=None, - ray_cls_with_init=ray_cls_with_init, - name_prefix=None, - worker_names=worker_names) - return worker_group - - def spawn(self, prefix_set): - """ - spawn to a dictionary of worker groups, each with a subset of method with prefix. - This method will create some new worker groups according to the prefix set. - Each worker group will re-bind all functions from the old wg with that prefix to this new wg without prefix. - """ - - def _rebind_actor_methods(worker_group, actor_name): - """ - bind the method with actor_prefix to its original name - """ - prefix: str = actor_name + '_' - for method_name in dir(worker_group): - if method_name.startswith(prefix): - # only valid when Python >= 3.9 - original_method_name = method_name.removeprefix(prefix) - method = getattr(worker_group, method_name) - setattr(worker_group, original_method_name, method) - - new_worker_group_dict = {} - for prefix in prefix_set: - new_worker_group = self.from_detached(worker_names=self._worker_names, - ray_cls_with_init=self.ray_cls_with_init) - - _rebind_actor_methods(new_worker_group, prefix) - new_worker_group_dict[prefix] = new_worker_group - return new_worker_group_dict - - def execute_rank_zero_sync(self, method_name: str, *args, **kwargs): - return ray.get(self.execute_rank_zero_async(method_name, *args, **kwargs)) - - def execute_rank_zero_async(self, method_name: str, *args, **kwargs): - remote_call = getattr(self._workers[0], method_name) - return remote_call.remote(*args, **kwargs) - - def execute_rank_zero(self, method_name: str, *args, **kwargs): - return self.execute_rank_zero_async(method_name, *args, **kwargs) - - def execute_all(self, method_name: str, *args, **kwargs): - return self.execute_all_async(method_name, *args, **kwargs) - - def execute_all_sync(self, method_name: str, *args, **kwargs): - return ray.get(self.execute_all_async(method_name, *args, **kwargs)) - - def execute_all_async(self, method_name: str, *args, **kwargs): - # Here, we assume that if all arguments in args and kwargs are lists, and their lengths match len(self._workers), - # we'll distribute each element in these lists to the corresponding worker - # print(f"execute_all_async: method {method_name}({args}, {kwargs})") - length = len(self._workers) - if all(isinstance(arg, list) for arg in args) and all(isinstance(kwarg, list) for kwarg in kwargs.values()): - if all(len(arg) == length for arg in args) and all(len(kwarg) == length for kwarg in kwargs.values()): - # print(f"splitting args and kwargs into {length} shards") - result = [] - for i in range(length): - sliced_args = tuple(arg[i] for arg in args) - sliced_kwargs = {k: v[i] for k, v in kwargs.items()} - remote_call = getattr(self._workers[i], method_name) - result.append(remote_call.remote(*sliced_args, **sliced_kwargs)) - return result - - return [getattr(worker, method_name).remote(*args, **kwargs) for worker in self._workers] - - @property - def master_address(self): - return self._master_addr - - @property - def master_port(self): - return self._master_port - - @property - def workers(self): - return self._workers - - @property - def world_size(self): - return self._world_size - - -""" -Utilities that enables creating workers inside the same ray.Actor, -with code written in separate ray.Actors. -""" - -from unittest.mock import patch -from verl.single_controller.base.decorator import MAGIC_ATTR -import os - - -def _bind_workers_method_to_parent(cls, key, user_defined_cls): - """ - Binds the methods of each worker to the WorkerDict. - Note that we only bind public methods that are decorated by register - """ - for method_name in dir(user_defined_cls): - try: - method = getattr(user_defined_cls, method_name) - assert callable(method), f"{method_name} in {user_defined_cls} is not callable" - except Exception as e: - # if it is a property, it will fail because Class doesn't have instance property - continue - - if hasattr(method, MAGIC_ATTR): - - def generate_function(name): - - def func(self, *args, **kwargs): - # dispatch to the actual worker - return getattr(self.worker_dict[key], name)(*args, **kwargs) - - return func - - func = generate_function(method_name) - # pass MAGIC_ATTR for outer worker group - setattr(func, MAGIC_ATTR, getattr(method, MAGIC_ATTR)) - try: - method_name_with_prefix = key + '_' + method_name - setattr(cls, method_name_with_prefix, func) - # print(f'Binding {method_name_with_prefix}') - except Exception as e: - raise ValueError(f'Fail to set method_name {method_name}') - - -def _unwrap_ray_remote(cls): - if hasattr(cls, '__ray_actor_class__'): - cls = cls.__ray_actor_class__ - return cls - - -def create_colocated_worker_cls(class_dict: dict[str, RayClassWithInitArgs]): - """ - This function should return a class instance that delegates the calls to every - cls in cls_dict - This function aggregates multiple RayClassWithInitArgs into one RayClassWithInitArgs, - all methods are bound to this new cls with keys as prefixes respectively. - Only method decorated with @register is bound. - """ - cls_dict = {} - init_args_dict = {} - worker_cls = None - for key, cls in class_dict.items(): - if worker_cls == None: - worker_cls = cls.cls.__ray_actor_class__.__base__ - else: - assert worker_cls == cls.cls.__ray_actor_class__.__base__, \ - 'the worker class should be the same when share the same process' - cls_dict[key] = cls.cls - init_args_dict[key] = {'args': cls.args, 'kwargs': cls.kwargs} - - assert cls_dict.keys() == init_args_dict.keys() - - # TODO: create a class with customizable name - class WorkerDict(worker_cls): - - def __init__(self): - super().__init__() - self.worker_dict = {} - for key, user_defined_cls in cls_dict.items(): - user_defined_cls = _unwrap_ray_remote(user_defined_cls) - # directly instantiate the class without remote - # in worker class, e.g. when DISABLE_WORKER_INIT == 1 it will return immediately - with patch.dict(os.environ, {'DISABLE_WORKER_INIT': '1'}): - self.worker_dict[key] = user_defined_cls(*init_args_dict[key].get('args', ()), - **init_args_dict[key].get('kwargs', {})) - - # now monkey-patch the methods from inner class to WorkerDict - for key, user_defined_cls in cls_dict.items(): - user_defined_cls = _unwrap_ray_remote(user_defined_cls) - _bind_workers_method_to_parent(WorkerDict, key, user_defined_cls) - - remote_cls = ray.remote(WorkerDict) - remote_cls = RayClassWithInitArgs(cls=remote_cls) - return remote_cls diff --git a/verl/single_controller/ray/megatron.py b/verl/single_controller/ray/megatron.py deleted file mode 100644 index 3ccb23a155..0000000000 --- a/verl/single_controller/ray/megatron.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Optional - -import ray - -from .base import RayWorkerGroup, RayResourcePool, RayClassWithInitArgs -from verl.single_controller.base.megatron.worker import DistRankInfo, DistGlobalInfo -from verl.single_controller.base.megatron.worker_group import MegatronWorkerGroup - - -# NOTE(sgm): for open-source megatron-core -class NVMegatronRayWorkerGroup(RayWorkerGroup, MegatronWorkerGroup): - """ - MegatronWorkerGroup will query each worker of its megatron rank info and store it inside the WorkerGroup - so that the dispatcher can use it to dispatch data. - """ - - def __init__(self, resource_pool: RayResourcePool, ray_cls_with_init: RayClassWithInitArgs, **kwargs): - super().__init__(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init, **kwargs) - self._megatron_rank_info: DistRankInfo = self.execute_all_sync(method_name='get_megatron_rank_info') - self._megatron_global_info: DistGlobalInfo = ray.get( - self.execute_rank_zero_async(method_name='get_megatron_global_info')) - - -class MegatronRayWorkerGroup(RayWorkerGroup, MegatronWorkerGroup): - """ - MegatronWorkerGroup will query each worker of its megatron rank info and store it inside the WorkerGroup - so that the dispatcher can use it to dispatch data. - """ - - def __init__(self, - resource_pool: RayResourcePool, - ray_cls_with_init: RayClassWithInitArgs, - default_megatron_kwargs: Dict = None, - **kwargs): - super().__init__(resource_pool=resource_pool, - ray_cls_with_init=ray_cls_with_init, - default_megatron_kwargs=default_megatron_kwargs, - **kwargs) - self.init_megatron(default_megatron_kwargs=default_megatron_kwargs) - self._megatron_rank_info: DistRankInfo = self.execute_all_sync(method_name='get_megatron_rank_info') - self._megatron_global_info: DistGlobalInfo = ray.get( - self.execute_rank_zero_async(method_name='get_megatron_global_info')) - - def init_megatron(self, default_megatron_kwargs: Optional[Dict] = None): - # after super, we will call init of each worker - if not self._is_init_with_detached_workers: - # only init_megatron if the WorkerGroup is created from scratch - self.execute_all_sync(method_name='init_megatron', default_megatron_kwargs=default_megatron_kwargs) diff --git a/verl/third_party/__init__.py b/verl/third_party/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/third_party/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/third_party/sglang/__init__.py b/verl/third_party/sglang/__init__.py deleted file mode 100644 index 93d16900e3..0000000000 --- a/verl/third_party/sglang/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/verl/third_party/sglang/parallel_state.py b/verl/third_party/sglang/parallel_state.py deleted file mode 100644 index 30bbea3b31..0000000000 --- a/verl/third_party/sglang/parallel_state.py +++ /dev/null @@ -1,328 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The SGlang team. -# Adapted from -# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -"""Model and data parallel groups.""" -import os -from typing import Optional - -import torch -import torch.distributed -import sglang.srt.distributed.parallel_state as ps -from sglang.srt.distributed.parallel_state import ( - get_pp_group, - get_world_group, - init_distributed_environment, - init_model_parallel_group, -) -""" -This version is strongly tied with Megatron to implement HybridEngine and weight sharing between vllm and Megatron. -- We assume the Megatron tp+dp+pp world is already established before calling this function. - -""" - -# Device mesh for using DTensor -_DEVICE_MESH = None - -# Tensor model parallel group that the current rank belongs to. -_TP = None -# Pipeline model parallel group that the current rank belongs to. -_PP = None - - -# This method is for initializing the ParallelGroup when using HybridEngine -# NOTE(linjunrong): this function is for megatron -def initialize_parallel_state( - distributed_init_method: str = "env://", - backend: str = "nccl", - tensor_model_parallel_size: int = 1, - num_tp_per_train_tp: int = 1, - pipeline_model_parallel_size: int = 1, -): - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. - rank = int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - init_distributed_environment(world_size, rank, distributed_init_method, local_rank, backend) - if torch.distributed.get_world_size() > 1: - # NOTE: build a sepearate inference group with infer tp & micro dp - initialize_model_parallel_for_sglang( - tensor_model_parallel_size=tensor_model_parallel_size, - num_tensor_model_parallel_groups_per_train_tp=num_tp_per_train_tp, - ) - else: - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - - -# NOTE(linjunrong): After init SGLang rollout using class EngineFragment, user should always remember to call -# this function to sync the _TP, _PP define at the beginning of this file. Otherwise, only the conterparts -# inside sglang.srt.distributed are init as ProcessGroup, the symbols defined in this file remain as None. -# It could be weird to maintain two _TP and _PP, I follow the same way to maintain an extra ones for -# veRL itself as how it was done in verl.third_party.vllm.parallel_state. Note that the process is a little -# bit different -def ensure_model_parallel_initialized( - tensor_model_parallel_size: int, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """Helper to initialize model parallel groups if they are not initialized, - or ensure tensor-parallel and pipeline-parallel sizes are equal to expected - values if the model parallel groups are initialized. - """ - # get the backend of _DEVICE_WORLD_GROUP - backend = backend or torch.distributed.get_backend(get_world_group().device_group) - if not model_parallel_is_initialized(): - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - return - - assert get_tensor_model_parallel_world_size() == tensor_model_parallel_size, ( - "tensor parallel group already initialized, but of unexpected size: " - f"{get_tensor_model_parallel_world_size()=} vs. " - f"{tensor_model_parallel_size=}") - pp_world_size = get_pp_group().world_size - assert pp_world_size == pipeline_model_parallel_size, ( - "pipeline parallel group already initialized, but of unexpected size: " - f"{pp_world_size=} vs. " - f"{pipeline_model_parallel_size=}") - - -# TODO(sgm): deviate from the v0.5.4, not pp now -# NOTE(linjunrong): the SGLang version using _TP instead of ps._TP -def model_parallel_is_initialized(): - """Check if tensor and pipeline parallel groups are initialized.""" - return _TP is not None - # and _PIPELINE_MODEL_PARALLEL_GROUP is not None) - - -def initialize_model_parallel_for_sglang( - tensor_model_parallel_size: int, - num_tensor_model_parallel_groups_per_train_tp: int = 1, - pipeline_model_parallel_size: int = 1, -) -> None: - pass - - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - - assert isinstance(tensor_model_parallel_size, int) - - # assert num_tensor_model_parallel_groups_per_train_tp == 1 and not different_tp_group - # assert num_tensor_model_parallel_groups_per_train_tp > 1 and different_tp_group - - # Build the tensor model-parallel groups. - assert ps._TP is None, "tensor model parallel group is already initialized" - - global _TP - - world_size: int = torch.distributed.get_world_size() - - rank = torch.distributed.get_rank() - - backend = torch.distributed.get_backend() - - num_tensor_model_parallel_groups = world_size // tensor_model_parallel_size - - if num_tensor_model_parallel_groups_per_train_tp == 1: - # if tensor_model_parallel_size == train_tensor_parallel_size: - # using the same tp group as Megatron/vllm - assert _TP is None, "tensor model parallel group is already initialized" - group_ranks = [] - for i in range(num_tensor_model_parallel_groups): - ranks = range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) - group_ranks.append(ranks) - _TP = init_model_parallel_group( - group_ranks=group_ranks, - local_rank=get_world_group().local_rank, - backend=backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True, - ) - ps._TP = _TP - # _MICRO_DATA_PARALLEL_GROUP is move to hybrid engine - else: - # initialize a micro_dp group and a tp group - # assume training tp=4, infer tp=2, then, weight is partitioned as - # [1], [2], [3], [4] for training and [1,2], [1,2], [3,4], [3,4] for inference - - # Build the inference tp groups - # train_tp = train_tensor_parallel_size - train_tp = num_tensor_model_parallel_groups_per_train_tp * tensor_model_parallel_size - # num_tensor_model_parallel_groups_per_train_tp = train_tp // tensor_model_parallel_size - assert _TP is None, "tensor model parallel group is already initialized" - group_ranks = [] - for i in range(num_tensor_model_parallel_groups // num_tensor_model_parallel_groups_per_train_tp): - start = train_tp * i - end = train_tp * (i + 1) - for j in range(num_tensor_model_parallel_groups_per_train_tp): - ranks = list(range(start, end, num_tensor_model_parallel_groups_per_train_tp)) - for i in range(len(ranks)): - ranks[i] += j - group_ranks.append(ranks) - _TP = init_model_parallel_group( - group_ranks=group_ranks, - local_rank=get_world_group().local_rank, - backend=backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True, - ) - ps._TP = _TP - - # Build the pipeline model-parallel groups. - # global _PIPELINE_MODEL_PARALLEL_GROUP - # global _PIPELINE_GLOBAL_RANKS - # assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized") - - # ps._PIPELINE_MODEL_PARALLEL_GROUP = mpu.get_pipeline_model_parallel_group() - # ps._PIPELINE_GLOBAL_RANKS = mpu.get_pipeline_model_parallel_ranks() - - # TODO: init using device mesh (not support hybrid engine now) - # Build the pipeline model-parallel groups. - num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size - global _PP - assert _PP is None, "pipeline model parallel group is already initialized" - group_ranks = [] - for i in range(num_pipeline_model_parallel_groups): - ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) - group_ranks.append(ranks) - # pipeline parallel does not need custom allreduce - _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) - ps._PP = _PP # for verl - - -def initialize_model_parallel( - tensor_model_parallel_size: int = 1, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """ - NOTE: This method is a hack from the open-sourced version without - asertion of world_size = tp * pp - - Initialize model parallel groups. - - Arguments: - tensor_model_parallel_size: number of GPUs used for tensor model - parallelism. - pipeline_model_parallel_size: number of GPUs used for pipeline model - parallelism. - - Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we - use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize - the model pipeline. The present function will - create 4 tensor model-parallel groups and 2 pipeline model-parallel groups: - 4 tensor model-parallel groups: - [g0, g1], [g2, g3], [g4, g5], [g6, g7] - 2 pipeline model-parallel groups: - [g0, g2, g4, g6], [g1, g3, g5, g7] - Note that for efficiency, the caller should make sure adjacent ranks - are on the same DGX box. For example if we are using 2 DGX-1 boxes - with a total of 16 GPUs, rank 0 to 7 belong to the first box and - ranks 8 to 15 belong to the second box. - """ - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - world_size: int = torch.distributed.get_world_size() - backend = backend or torch.distributed.get_backend(ps.get_world_group().device_group) - - # NOTE(sgm) we don't assert world_size == tp * pp - # DP is not managed by vllm but by the VeRL WorkerGroup - # if (world_size != - # tensor_model_parallel_size * pipeline_model_parallel_size): - # raise RuntimeError( - # f"world_size ({world_size}) is not equal to " - # f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " - # f"pipeline_model_parallel_size ({pipeline_model_parallel_size})") - - num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size - - global _TP - assert _TP is None, "tensor model parallel group is already initialized" - group_ranks = [] - for i in range(num_tensor_model_parallel_groups): - ranks = list(range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size)) - group_ranks.append(ranks) - - # message queue broadcaster is only used in tensor model parallel group - if ps._TP is not None: - _TP = ps._TP - else: - _TP = init_model_parallel_group( - group_ranks, - get_world_group().local_rank, - backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True, - ) - ps._TP = _TP - - # TODO: init using device mesh (not support hybrid engine now) - # Build the pipeline model-parallel groups. - num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size - global _PP - assert _PP is None, "pipeline model parallel group is already initialized" - group_ranks = [] - for i in range(num_pipeline_model_parallel_groups): - ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) - group_ranks.append(ranks) - # pipeline parallel does not need custom allreduce - if ps._TP is not None: - _PP = ps._TP - else: - _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) - ps._PP = _PP - - -""" -Device mesh utilities -""" - - -def get_device_mesh(): - assert _DEVICE_MESH is not None, "device mesh is not initialized" - return _DEVICE_MESH - - -""" -Tensor model parallel utilities -""" - - -# NOTE(linjunrong): In the vllm version parallel_state.py. veRL created its own _TP and _PP as veRL want to use -# the process group for some extra purpose. Under the hood, there is no difference between them and the original -# one in vllm.distributed.parallel_state. However, the implementation need to hack the init process of inference -# engine, as we do not maintain another SGLang here, I just use the original _TP and _PP directly. -def get_tensor_model_parallel_group(): - """Get the tensor model parallel group the caller rank belongs to.""" - - assert _TP is not None, "tensor model parallel group is not initialized" - return _TP.device_group - - -def get_tensor_model_parallel_world_size(): - """Return world size for the tensor model parallel group.""" - return torch.distributed.get_world_size(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_rank(): - """Return my rank for the tensor model parallel group.""" - return torch.distributed.get_rank(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_src_rank(): - """Calculate the global rank corresponding to the first local rank - in the tensor model parallel group.""" - global_rank = torch.distributed.get_rank() - local_world_size = get_tensor_model_parallel_world_size() - return (global_rank // local_world_size) * local_world_size diff --git a/verl/third_party/vllm/__init__.py b/verl/third_party/vllm/__init__.py deleted file mode 100644 index 34d61e7149..0000000000 --- a/verl/third_party/vllm/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from importlib.metadata import version, PackageNotFoundError -from packaging import version as vs -from verl.utils.import_utils import is_sglang_available - - -def get_version(pkg): - try: - return version(pkg) - except PackageNotFoundError: - return None - - -package_name = 'vllm' -package_version = get_version(package_name) -vllm_version = None - -if package_version == '0.3.1': - vllm_version = '0.3.1' - from .vllm_v_0_3_1.llm import LLM - from .vllm_v_0_3_1.llm import LLMEngine - from .vllm_v_0_3_1 import parallel_state -elif package_version == '0.4.2': - vllm_version = '0.4.2' - from .vllm_v_0_4_2.llm import LLM - from .vllm_v_0_4_2.llm import LLMEngine - from .vllm_v_0_4_2 import parallel_state -elif package_version == '0.5.4': - vllm_version = '0.5.4' - from .vllm_v_0_5_4.llm import LLM - from .vllm_v_0_5_4.llm import LLMEngine - from .vllm_v_0_5_4 import parallel_state -elif package_version == '0.6.3': - vllm_version = '0.6.3' - from .vllm_v_0_6_3.llm import LLM - from .vllm_v_0_6_3.llm import LLMEngine - from .vllm_v_0_6_3 import parallel_state -elif package_version == '0.6.3+rocm624': - vllm_version = '0.6.3' - from .vllm_v_0_6_3.llm import LLM - from .vllm_v_0_6_3.llm import LLMEngine - from .vllm_v_0_6_3 import parallel_state -elif vs.parse(package_version) >= vs.parse('0.7.0'): - # From 0.6.6.post2 on, vllm supports SPMD inference - # See https://github.com/vllm-project/vllm/pull/12071 - - from vllm import LLM - from vllm.distributed import parallel_state -else: - if not is_sglang_available(): - raise ValueError( - f'vllm version {package_version} not supported and SGLang also not Found. Currently supported vllm versions are 0.3.1, 0.4.2, 0.5.4, 0.6.3 and 0.7.0+' - ) diff --git a/verl/third_party/vllm/vllm_v_0_3_1/__init__.py b/verl/third_party/vllm/vllm_v_0_3_1/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/third_party/vllm/vllm_v_0_3_1/arg_utils.py b/verl/third_party/vllm/vllm_v_0_3_1/arg_utils.py deleted file mode 100644 index 1ae8f3b8f6..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/arg_utils.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py -import argparse -import dataclasses -from dataclasses import dataclass -from typing import Dict, Optional, Tuple - -import torch.nn as nn -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig, ParallelConfig, SchedulerConfig, LoRAConfig) -from transformers import PretrainedConfig -from .config import ModelConfig - - -@dataclass -class EngineArgs: - """Arguments for vLLM engine.""" - model_hf_config: PretrainedConfig = None - dtype: str = 'auto' - kv_cache_dtype: str = 'auto' - seed: int = 0 - max_model_len: Optional[int] = None - worker_use_ray: bool = False - pipeline_parallel_size: int = 1 - tensor_parallel_size: int = 1 - max_parallel_loading_workers: Optional[int] = None - block_size: int = 16 - swap_space: int = 4 # GiB - gpu_memory_utilization: float = 0.90 - max_num_batched_tokens: Optional[int] = None - max_num_seqs: int = 256 - max_paddings: int = 256 - disable_log_stats: bool = False - revision: Optional[str] = None - tokenizer_revision: Optional[str] = None - quantization: Optional[str] = None - load_format: str = 'model' - enforce_eager: bool = False - max_context_len_to_capture: int = 8192 - disable_custom_all_reduce: bool = False - enable_lora: bool = False - max_loras: int = 1 - max_lora_rank: int = 16 - lora_extra_vocab_size: int = 256 - lora_dtype = 'auto' - max_cpu_loras: Optional[int] = None - device: str = 'cuda' - - @staticmethod - def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - """Shared CLI arguments for vLLM engine.""" - # Model arguments - # TODO(shengguangming): delete the unused args - parser.add_argument('--model', - type=str, - default='facebook/opt-125m', - help='name or path of the huggingface model to use') - parser.add_argument('--tokenizer', - type=str, - default=EngineArgs.tokenizer, - help='name or path of the huggingface tokenizer to use') - parser.add_argument('--revision', - type=str, - default=None, - help='the specific model version to use. It can be a branch ' - 'name, a tag name, or a commit id. If unspecified, will use ' - 'the default version.') - parser.add_argument('--tokenizer-revision', - type=str, - default=None, - help='the specific tokenizer version to use. It can be a branch ' - 'name, a tag name, or a commit id. If unspecified, will use ' - 'the default version.') - parser.add_argument('--tokenizer-mode', - type=str, - default=EngineArgs.tokenizer_mode, - choices=['auto', 'slow'], - help='tokenizer mode. "auto" will use the fast ' - 'tokenizer if available, and "slow" will ' - 'always use the slow tokenizer.') - parser.add_argument('--trust-remote-code', action='store_true', help='trust remote code from huggingface') - parser.add_argument('--download-dir', - type=str, - default=EngineArgs.download_dir, - help='directory to download and load the weights, ' - 'default to the default cache dir of ' - 'huggingface') - parser.add_argument('--load-format', - type=str, - default=EngineArgs.load_format, - choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'], - help='The format of the model weights to load. ' - '"auto" will try to load the weights in the safetensors format ' - 'and fall back to the pytorch bin format if safetensors format ' - 'is not available. ' - '"pt" will load the weights in the pytorch bin format. ' - '"safetensors" will load the weights in the safetensors format. ' - '"npcache" will load the weights in pytorch format and store ' - 'a numpy cache to speed up the loading. ' - '"dummy" will initialize the weights with random values, ' - 'which is mainly for profiling.') - parser.add_argument('--dtype', - type=str, - default=EngineArgs.dtype, - choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'], - help='data type for model weights and activations. ' - 'The "auto" option will use FP16 precision ' - 'for FP32 and FP16 models, and BF16 precision ' - 'for BF16 models.') - parser.add_argument('--max-model-len', - type=int, - default=None, - help='model context length. If unspecified, ' - 'will be automatically derived from the model.') - # Parallel arguments - parser.add_argument('--worker-use-ray', - action='store_true', - help='use Ray for distributed serving, will be ' - 'automatically set when using more than 1 GPU') - parser.add_argument('--pipeline-parallel-size', - '-pp', - type=int, - default=EngineArgs.pipeline_parallel_size, - help='number of pipeline stages') - parser.add_argument('--tensor-parallel-size', - '-tp', - type=int, - default=EngineArgs.tensor_parallel_size, - help='number of tensor parallel replicas') - # KV cache arguments - parser.add_argument('--block-size', - type=int, - default=EngineArgs.block_size, - choices=[8, 16, 32], - help='token block size') - # TODO(woosuk): Support fine-grained seeds (e.g., seed per request). - parser.add_argument('--seed', type=int, default=EngineArgs.seed, help='random seed') - parser.add_argument('--swap-space', - type=int, - default=EngineArgs.swap_space, - help='CPU swap space size (GiB) per GPU') - parser.add_argument('--gpu-memory-utilization', - type=float, - default=EngineArgs.gpu_memory_utilization, - help='the percentage of GPU memory to be used for' - 'the model executor') - parser.add_argument('--max-num-batched-tokens', - type=int, - default=EngineArgs.max_num_batched_tokens, - help='maximum number of batched tokens per ' - 'iteration') - parser.add_argument('--max-num-seqs', - type=int, - default=EngineArgs.max_num_seqs, - help='maximum number of sequences per iteration') - parser.add_argument('--disable-log-stats', action='store_true', help='disable logging statistics') - # Quantization settings. - parser.add_argument('--quantization', - '-q', - type=str, - choices=['awq', None], - default=None, - help='Method used to quantize the weights') - return parser - - @classmethod - def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs': - # Get the list of attributes of this dataclass. - attrs = [attr.name for attr in dataclasses.fields(cls)] - # Set the attributes from the parsed arguments. - engine_args = cls(**{attr: getattr(args, attr) for attr in attrs}) - return engine_args - - def create_engine_configs( - self, - ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]: - device_config = DeviceConfig(self.device) - model_config = ModelConfig(self.model_hf_config, self.dtype, self.seed, self.load_format, self.revision, - self.tokenizer_revision, self.max_model_len, self.quantization, self.enforce_eager, - self.max_context_len_to_capture) - cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization, self.swap_space, self.kv_cache_dtype, - model_config.get_sliding_window()) - parallel_config = ParallelConfig(self.pipeline_parallel_size, self.tensor_parallel_size, self.worker_use_ray, - self.max_parallel_loading_workers, self.disable_custom_all_reduce) - scheduler_config = SchedulerConfig(self.max_num_batched_tokens, self.max_num_seqs, model_config.max_model_len, - self.max_paddings) - lora_config = LoRAConfig(max_lora_rank=self.max_lora_rank, - max_loras=self.max_loras, - lora_extra_vocab_size=self.lora_extra_vocab_size, - lora_dtype=self.lora_dtype, - max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras and self.max_cpu_loras > 0 else - None) if self.enable_lora else None - return (model_config, cache_config, parallel_config, scheduler_config, device_config, lora_config) - - -@dataclass -class AsyncEngineArgs(EngineArgs): - """Arguments for asynchronous vLLM engine.""" - engine_use_ray: bool = False - disable_log_requests: bool = False - max_log_len: Optional[int] = None - - @staticmethod - def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - parser = EngineArgs.add_cli_args(parser) - parser.add_argument('--engine-use-ray', - action='store_true', - help='use Ray to start the LLM engine in a ' - 'separate process as the server process.') - parser.add_argument('--disable-log-requests', action='store_true', help='disable logging requests') - parser.add_argument('--max-log-len', - type=int, - default=None, - help='max number of prompt characters or prompt ' - 'ID numbers being printed in log. ' - 'Default: unlimited.') - return parser diff --git a/verl/third_party/vllm/vllm_v_0_3_1/config.py b/verl/third_party/vllm/vllm_v_0_3_1/config.py deleted file mode 100644 index 1e1fead862..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/config.py +++ /dev/null @@ -1,577 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py - -from typing import Optional, Union, ClassVar -from dataclasses import dataclass -import torch -from transformers import PretrainedConfig -from packaging.version import Version - -from vllm.logger import init_logger -from vllm.transformers_utils.config import get_config -from vllm.utils import get_cpu_memory, is_hip, get_nvcc_cuda_version - -logger = init_logger(__name__) - -_GB = 1 << 30 - - -class ModelConfig: - """Configuration for the model. - - Args: - model: Name or path of the huggingface model to use. - tokenizer: Name or path of the huggingface tokenizer to use. - tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if - available, and "slow" will always use the slow tokenizer. - trust_remote_code: Trust remote code (e.g., from HuggingFace) when - downloading the model and tokenizer. - download_dir: Directory to download and load the weights, default to the - default cache directory of huggingface. - load_format: The format of the model weights to load: - "auto" will try to load the weights in the safetensors format and - fall back to the pytorch bin format if safetensors format is - not available. - "pt" will load the weights in the pytorch bin format. - "safetensors" will load the weights in the safetensors format. - "npcache" will load the weights in pytorch format and store - a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is - mainly for profiling. - dtype: Data type for model weights and activations. The "auto" option - will use FP16 precision for FP32 and FP16 models, and BF16 precision - for BF16 models. - seed: Random seed for reproducibility. - revision: The specific model version to use. It can be a branch name, - a tag name, or a commit id. If unspecified, will use the default - version. - tokenizer_revision: The specific tokenizer version to use. It can be a - branch name, a tag name, or a commit id. If unspecified, will use - the default version. - max_model_len: Maximum length of a sequence (including prompt and - output). If None, will be derived from the model. - quantization: Quantization method that was used to quantize the model - weights. If None, we assume the model weights are not quantized. - enforce_eager: Whether to enforce eager execution. If True, we will - disable CUDA graph and always execute the model in eager mode. - If False, we will use CUDA graph and eager execution in hybrid. - max_context_len_to_capture: Maximum context len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode. - """ - - def __init__( - self, - hf_config: PretrainedConfig, - dtype: str, - seed: int, - load_format: str = 'model', - revision: Optional[str] = None, - tokenizer_revision: Optional[str] = None, - max_model_len: Optional[int] = None, - quantization: Optional[str] = None, - trust_remote_code: Optional[bool] = True, - enforce_eager: bool = False, - max_context_len_to_capture: Optional[int] = None, - ) -> None: - self.model = hf_config._name_or_path - self.tokenizer = hf_config._name_or_path - self.load_format = load_format - self.seed = seed - self.revision = revision - self.tokenizer_revision = tokenizer_revision - self.quantization = quantization - self.trust_remote_code = trust_remote_code - self.enforce_eager = enforce_eager - self.max_context_len_to_capture = max_context_len_to_capture - - # self.hf_config = get_config(model, trust_remote_code, revision) - self.hf_config = hf_config - self.dtype = _get_and_verify_dtype(self.hf_config, dtype) - self.max_model_len = _get_and_verify_max_len(self.hf_config, max_model_len) - # self._verify_load_format() - # self._verify_tokenizer_mode() - self._verify_quantization() - self._verify_cuda_graph() - - def _verify_load_format(self) -> None: - load_format = self.load_format.lower() - if load_format not in ["auto", "pt", "safetensors", "npcache", "dummy", "model"]: - raise ValueError(f"Unknown load format: {self.load_format}. Must be one of " - "'auto', 'pt', 'safetensors', 'npcache', 'dummy' or 'model'.") - self.load_format = load_format - - # def _verify_tokenizer_mode(self) -> None: - # tokenizer_mode = self.tokenizer_mode.lower() - # if tokenizer_mode not in ["auto", "slow"]: - # raise ValueError( - # f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be " - # "either 'auto' or 'slow'.") - # self.tokenizer_mode = tokenizer_mode - - def _verify_quantization(self) -> None: - supported_quantization = ["awq", "gptq", "squeezellm"] - rocm_not_supported_quantization = ["awq", "gptq"] - if self.quantization is not None: - self.quantization = self.quantization.lower() - - # Parse quantization method from the HF model config, if available. - hf_quant_config = getattr(self.hf_config, "quantization_config", None) - if hf_quant_config is not None: - hf_quant_method = str(hf_quant_config["quant_method"]).lower() - if self.quantization is None: - self.quantization = hf_quant_method - elif self.quantization != hf_quant_method: - raise ValueError("Quantization method specified in the model config " - f"({hf_quant_method}) does not match the quantization " - f"method specified in the `quantization` argument " - f"({self.quantization}).") - - if self.quantization is not None: - if self.quantization not in supported_quantization: - raise ValueError(f"Unknown quantization method: {self.quantization}. Must " - f"be one of {supported_quantization}.") - if is_hip() and self.quantization in rocm_not_supported_quantization: - raise ValueError(f"{self.quantization} quantization is currently not supported " - f"in ROCm.") - logger.warning(f"{self.quantization} quantization is not fully " - "optimized yet. The speed can be slower than " - "non-quantized models.") - - def _verify_cuda_graph(self) -> None: - if self.max_context_len_to_capture is None: - self.max_context_len_to_capture = self.max_model_len - self.max_context_len_to_capture = min(self.max_context_len_to_capture, self.max_model_len) - if (self.quantization in ["gptq", "squeezellm"] and not self.enforce_eager): - # Related issue: https://github.com/vllm-project/vllm/issues/2147 - logger.warning(f"{self.quantization} does not support CUDA graph " - "yet. Disabling CUDA graph.") - self.enforce_eager = True - - def verify_with_parallel_config( - self, - parallel_config: "ParallelConfig", - ) -> None: - total_num_attention_heads = self.hf_config.num_attention_heads - tensor_parallel_size = parallel_config.tensor_parallel_size - if total_num_attention_heads % tensor_parallel_size != 0: - raise ValueError(f"Total number of attention heads ({total_num_attention_heads})" - " must be divisible by tensor parallel size " - f"({tensor_parallel_size}).") - - total_num_hidden_layers = self.hf_config.num_hidden_layers - pipeline_parallel_size = parallel_config.pipeline_parallel_size - if total_num_hidden_layers % pipeline_parallel_size != 0: - raise ValueError(f"Total number of hidden layers ({total_num_hidden_layers}) " - "must be divisible by pipeline parallel size " - f"({pipeline_parallel_size}).") - - def get_sliding_window(self) -> Optional[int]: - return getattr(self.hf_config, "sliding_window", None) - - def get_vocab_size(self) -> int: - return self.hf_config.vocab_size - - def get_hidden_size(self) -> int: - return self.hf_config.hidden_size - - def get_head_size(self) -> int: - # FIXME(woosuk): This may not be true for all models. - return self.hf_config.hidden_size // self.hf_config.num_attention_heads - - def get_total_num_kv_heads(self) -> int: - """Returns the total number of KV heads.""" - # For GPTBigCode & Falcon: - # NOTE: for falcon, when new_decoder_architecture is True, the - # multi_query flag is ignored and we use n_head_kv for the number of - # KV heads. - falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"] - new_decoder_arch_falcon = (self.hf_config.model_type in falcon_model_types and - getattr(self.hf_config, "new_decoder_architecture", False)) - if not new_decoder_arch_falcon and getattr(self.hf_config, "multi_query", False): - # Multi-query attention, only one KV head. - # Currently, tensor parallelism is not supported in this case. - return 1 - - attributes = [ - # For Falcon: - "n_head_kv", - "num_kv_heads", - # For LLaMA-2: - "num_key_value_heads", - # For ChatGLM: - "multi_query_group_num", - ] - for attr in attributes: - num_kv_heads = getattr(self.hf_config, attr, None) - if num_kv_heads is not None: - return num_kv_heads - - # For non-grouped-query attention models, the number of KV heads is - # equal to the number of attention heads. - return self.hf_config.num_attention_heads - - def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int: - """Returns the number of KV heads per GPU.""" - total_num_kv_heads = self.get_total_num_kv_heads() - # If tensor parallelism is used, we divide the number of KV heads by - # the tensor parallel size. We will replicate the KV heads in the - # case where the number of KV heads is smaller than the tensor - # parallel size so each GPU has at least one KV head. - return max(1, total_num_kv_heads // parallel_config.tensor_parallel_size) - - def get_num_layers(self, parallel_config: "ParallelConfig") -> int: - total_num_hidden_layers = self.hf_config.num_hidden_layers - return total_num_hidden_layers // parallel_config.pipeline_parallel_size - - -class CacheConfig: - """Configuration for the KV cache. - - Args: - block_size: Size of a cache block in number of tokens. - gpu_memory_utilization: Fraction of GPU memory to use for the - vLLM execution. - swap_space: Size of the CPU swap space per GPU (in GiB). - cache_dtype: Data type for kv cache storage. - """ - - def __init__( - self, - block_size: int, - gpu_memory_utilization: float, - swap_space: int, - cache_dtype: str, - sliding_window: Optional[int] = None, - ) -> None: - self.block_size = block_size - self.gpu_memory_utilization = gpu_memory_utilization - self.swap_space_bytes = swap_space * _GB - self.cache_dtype = cache_dtype - self.sliding_window = sliding_window - self._verify_args() - self._verify_cache_dtype() - - # Will be set after profiling. - self.num_gpu_blocks = None - self.num_cpu_blocks = None - - def _verify_args(self) -> None: - if self.gpu_memory_utilization > 1.0: - raise ValueError("GPU memory utilization must be less than 1.0. Got " - f"{self.gpu_memory_utilization}.") - - def _verify_cache_dtype(self) -> None: - if self.cache_dtype == "auto": - pass - elif self.cache_dtype == "fp8_e5m2": - nvcc_cuda_version = get_nvcc_cuda_version() - if nvcc_cuda_version < Version("11.8"): - raise ValueError("FP8 is not supported when cuda version is lower than 11.8.") - device_name = torch.cuda.get_device_name() - if "AMD" in device_name: - raise NotImplementedError("FP8_E5M2 KV Cache on AMD GPU has not been supported yet.") - logger.info("Using fp8_e5m2 data type to store kv cache. It reduces " - "the GPU memory footprint and boosts the performance. " - "But it may cause slight accuracy drop. " - "Currently we only support fp8 without scaling factors and " - "make e5m2 as a default format.") - else: - raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}") - - def verify_with_parallel_config( - self, - parallel_config: "ParallelConfig", - ) -> None: - total_cpu_memory = get_cpu_memory() - # FIXME(woosuk): Here, it is assumed that the GPUs in a tensor parallel - # group are in the same node. However, the GPUs may span multiple nodes. - num_gpus_per_node = parallel_config.tensor_parallel_size - cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node - - msg = (f"{cpu_memory_usage / _GB:.2f} GiB out of " - f"the {total_cpu_memory / _GB:.2f} GiB total CPU memory is " - "allocated for the swap space.") - if cpu_memory_usage > 0.7 * total_cpu_memory: - raise ValueError("Too large swap space. " + msg) - elif cpu_memory_usage > 0.4 * total_cpu_memory: - logger.warning("Possibly too large swap space. " + msg) - - -class ParallelConfig: - """Configuration for the distributed execution. - - Args: - pipeline_parallel_size: Number of pipeline parallel groups. - tensor_parallel_size: Number of tensor parallel groups. - worker_use_ray: Whether to use Ray for model workers. Will be set to - True if either pipeline_parallel_size or tensor_parallel_size is - greater than 1. - max_parallel_loading_workers: Maximum number of multiple batches - when load model sequentially. To avoid RAM OOM when using tensor - parallel and large models. - disable_custom_all_reduce: Disable the custom all-reduce kernel and - fall back to NCCL. - """ - - def __init__( - self, - pipeline_parallel_size: int, - tensor_parallel_size: int, - worker_use_ray: bool, - max_parallel_loading_workers: Optional[int] = None, - disable_custom_all_reduce: bool = False, - ) -> None: - self.pipeline_parallel_size = pipeline_parallel_size - self.tensor_parallel_size = tensor_parallel_size - self.worker_use_ray = worker_use_ray - self.max_parallel_loading_workers = max_parallel_loading_workers - self.disable_custom_all_reduce = disable_custom_all_reduce - - self.world_size = pipeline_parallel_size * tensor_parallel_size - if self.world_size > 1: - self.worker_use_ray = True - self._verify_args() - - def _verify_args(self) -> None: - if self.pipeline_parallel_size > 1: - raise NotImplementedError("Pipeline parallelism is not supported yet.") - if not self.disable_custom_all_reduce and self.world_size > 1: - if is_hip(): - self.disable_custom_all_reduce = True - logger.info("Disabled the custom all-reduce kernel because it is not " - "supported on AMD GPUs.") - elif self.pipeline_parallel_size > 1: - self.disable_custom_all_reduce = True - logger.info("Disabled the custom all-reduce kernel because it is not " - "supported with pipeline parallelism.") - - # FIXME(woosuk): Fix the stability issues and re-enable the custom - # all-reduce kernel. - if not self.disable_custom_all_reduce and self.world_size > 1: - self.disable_custom_all_reduce = True - logger.info("Custom all-reduce kernels are temporarily disabled due to " - "stability issues. We will re-enable them once the issues are " - "resolved.") - - -class SchedulerConfig: - """Scheduler configuration. - - Args: - max_num_batched_tokens: Maximum number of tokens to be processed in - a single iteration. - max_num_seqs: Maximum number of sequences to be processed in a single - iteration. - max_model_len: Maximum length of a sequence (including prompt - and generated text). - max_paddings: Maximum number of paddings to be added to a batch. - """ - - def __init__( - self, - max_num_batched_tokens: Optional[int], - max_num_seqs: int, - max_model_len: int, - max_paddings: int, - ) -> None: - if max_num_batched_tokens is not None: - self.max_num_batched_tokens = max_num_batched_tokens - else: - # If max_model_len is too short, use 2048 as the default value for - # higher throughput. - self.max_num_batched_tokens = max(max_model_len, 2048) - self.max_num_seqs = max_num_seqs - self.max_model_len = max_model_len - self.max_paddings = max_paddings - self._verify_args() - - def _verify_args(self) -> None: - if self.max_num_batched_tokens < self.max_model_len: - raise ValueError(f"max_num_batched_tokens ({self.max_num_batched_tokens}) is " - f"smaller than max_model_len ({self.max_model_len}). " - "This effectively limits the maximum sequence length to " - "max_num_batched_tokens and makes vLLM reject longer " - "sequences. Please increase max_num_batched_tokens or " - "decrease max_model_len.") - if self.max_num_batched_tokens < self.max_num_seqs: - raise ValueError(f"max_num_batched_tokens ({self.max_num_batched_tokens}) must " - "be greater than or equal to max_num_seqs " - f"({self.max_num_seqs}).") - - -class DeviceConfig: - - def __init__(self, device: str = "cuda") -> None: - self.device = torch.device(device) - - -@dataclass -class LoRAConfig: - max_lora_rank: int - max_loras: int - max_cpu_loras: Optional[int] = None - lora_dtype: Optional[torch.dtype] = None - lora_extra_vocab_size: int = 256 - # This is a constant. - lora_vocab_padding_size: ClassVar[int] = 256 - - def __post_init__(self): - # Keep this in sync with csrc/punica/bgmv/bgmv_config.h - possible_max_ranks = (8, 16, 32, 64) - possible_lora_extra_vocab_size = (0, 256, 512) - if self.max_lora_rank not in possible_max_ranks: - raise ValueError(f"max_lora_rank ({self.max_lora_rank}) must be one of " - f"{possible_max_ranks}.") - if self.lora_extra_vocab_size not in possible_lora_extra_vocab_size: - raise ValueError(f"lora_extra_vocab_size ({self.lora_extra_vocab_size}) " - f"must be one of {possible_lora_extra_vocab_size}.") - if self.max_loras < 1: - raise ValueError(f"max_loras ({self.max_loras}) must be >= 1.") - if self.max_cpu_loras is None: - self.max_cpu_loras = self.max_loras - elif self.max_cpu_loras < self.max_loras: - raise ValueError(f"max_cpu_loras ({self.max_cpu_loras}) must be >= " - f"max_loras ({self.max_loras})") - - def verify_with_model_config(self, model_config: ModelConfig): - if self.lora_dtype in (None, "auto"): - self.lora_dtype = model_config.dtype - elif isinstance(self.lora_dtype, str): - self.lora_dtype = getattr(torch, self.lora_dtype) - if model_config.quantization is not None: - raise ValueError("LoRA is not supported with quantized models yet.") - - def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig): - if scheduler_config.max_num_batched_tokens > 65528: - raise ValueError("Due to limitations of the custom LoRA CUDA kernel, " - "max_num_batched_tokens must be <= 65528 when " - "LoRA is enabled.") - - -_STR_DTYPE_TO_TORCH_DTYPE = { - "half": torch.float16, - "float16": torch.float16, - "float": torch.float32, - "float32": torch.float32, - "bfloat16": torch.bfloat16, -} - -_ROCM_NOT_SUPPORTED_DTYPE = ["float", "float32"] - - -def _get_and_verify_dtype( - config: PretrainedConfig, - dtype: Union[str, torch.dtype], -) -> torch.dtype: - # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct - # because config.torch_dtype can be None. - config_dtype = getattr(config, "torch_dtype", None) - if config_dtype is None: - config_dtype = torch.float32 - - if isinstance(dtype, str): - dtype = dtype.lower() - if dtype == "auto": - if config_dtype == torch.float32: - # Following the common practice, we use float16 for float32 - # models. - torch_dtype = torch.float16 - else: - torch_dtype = config_dtype - else: - if dtype not in _STR_DTYPE_TO_TORCH_DTYPE: - raise ValueError(f"Unknown dtype: {dtype}") - torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype] - elif isinstance(dtype, torch.dtype): - torch_dtype = dtype - else: - raise ValueError(f"Unknown dtype: {dtype}") - - if is_hip() and torch_dtype == torch.float32: - rocm_supported_dtypes = [ - k for k, v in _STR_DTYPE_TO_TORCH_DTYPE.items() if (k not in _ROCM_NOT_SUPPORTED_DTYPE) - ] - raise ValueError(f"dtype \'{dtype}\' is not supported in ROCm. " - f"Supported dtypes are {rocm_supported_dtypes}") - - # Verify the dtype. - if torch_dtype != config_dtype: - if torch_dtype == torch.float32: - # Upcasting to float32 is allowed. - pass - elif config_dtype == torch.float32: - # Downcasting from float32 to float16 or bfloat16 is allowed. - pass - else: - # Casting between float16 and bfloat16 is allowed with a warning. - logger.warning(f"Casting {config_dtype} to {torch_dtype}.") - - return torch_dtype - - -def _get_and_verify_max_len( - hf_config: PretrainedConfig, - max_model_len: Optional[int], -) -> int: - """Get and verify the model's maximum length.""" - derived_max_model_len = float("inf") - possible_keys = [ - # OPT - "max_position_embeddings", - # GPT-2 - "n_positions", - # MPT - "max_seq_len", - # ChatGLM2 - "seq_length", - # Others - "max_sequence_length", - "max_seq_length", - "seq_len", - ] - for key in possible_keys: - max_len_key = getattr(hf_config, key, None) - if max_len_key is not None: - derived_max_model_len = min(derived_max_model_len, max_len_key) - if derived_max_model_len == float("inf"): - if max_model_len is not None: - # If max_model_len is specified, we use it. - return max_model_len - - default_max_len = 2048 - logger.warning("The model's config.json does not contain any of the following " - "keys to determine the original maximum length of the model: " - f"{possible_keys}. Assuming the model's maximum length is " - f"{default_max_len}.") - derived_max_model_len = default_max_len - - rope_scaling = getattr(hf_config, "rope_scaling", None) - if rope_scaling is not None: - assert "factor" in rope_scaling - scaling_factor = rope_scaling["factor"] - if rope_scaling["type"] == "yarn": - derived_max_model_len = rope_scaling["original_max_position_embeddings"] - derived_max_model_len *= scaling_factor - - if max_model_len is None: - max_model_len = derived_max_model_len - elif max_model_len > derived_max_model_len: - raise ValueError(f"User-specified max_model_len ({max_model_len}) is greater than " - f"the derived max_model_len ({max_len_key}={derived_max_model_len}" - " in model's config.json). This may lead to incorrect model " - "outputs or CUDA errors. Make sure the value is correct and " - "within the model context size.") - return int(max_model_len) diff --git a/verl/third_party/vllm/vllm_v_0_3_1/llm.py b/verl/third_party/vllm/vllm_v_0_3_1/llm.py deleted file mode 100644 index 8d2475998c..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/llm.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/llm.py - -from typing import Dict, List, Optional, Tuple, Union - -from tqdm import tqdm -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast -from transformers import PretrainedConfig -import torch.nn as nn -from .arg_utils import EngineArgs -from .llm_engine_sp import LLMEngine -from vllm.lora.request import LoRARequest -from vllm.outputs import RequestOutput -from vllm.sampling_params import SamplingParams -from vllm.utils import Counter -import torch -from torch.nn.utils.rnn import pad_sequence -from verl.workers.rollout.tokenizer import HybridEngineBaseTokenizer - - -class LLM: - """An LLM for generating texts from given prompts and sampling parameters. - - This class includes a tokenizer, a language model (possibly distributed - across multiple GPUs), and GPU memory space allocated for intermediate - states (aka KV cache). Given a batch of prompts and sampling parameters, - this class generates texts from the model, using an intelligent batching - mechanism and efficient memory management. - - NOTE: This class is intended to be used for offline inference. For online - serving, use the `AsyncLLMEngine` class instead. - NOTE: For the comprehensive list of arguments, see `EngineArgs`. - - Args: - model: A HuggingFace Transformers model instance. - tokenizer: A HuggingFace Transformers tokenizer instance. - tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer - if available, and "slow" will always use the slow tokenizer. - trust_remote_code: Trust remote code (e.g., from HuggingFace) when - downloading the model and tokenizer. - tensor_parallel_size: The number of GPUs to use for distributed - execution with tensor parallelism. - dtype: The data type for the model weights and activations. Currently, - we support `float32`, `float16`, and `bfloat16`. If `auto`, we use - the `torch_dtype` attribute specified in the model config file. - However, if the `torch_dtype` in the config is `float32`, we will - use `float16` instead. - quantization: The method used to quantize the model weights. Currently, - we support "awq". If None, we assume the model weights are not - quantized and use `dtype` to determine the data type of the weights. - revision: The specific model version to use. It can be a branch name, - a tag name, or a commit id. - tokenizer_revision: The specific tokenizer version to use. It can be a - branch name, a tag name, or a commit id. - seed: The seed to initialize the random number generator for sampling. - gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to - reserve for the model weights, activations, and KV cache. Higher - values will increase the KV cache size and thus improve the model's - throughput. However, if the value is too high, it may cause out-of- - memory (OOM) errors. - swap_space: The size (GiB) of CPU memory per GPU to use as swap space. - This can be used for temporarily storing the states of the requests - when their `best_of` sampling parameters are larger than 1. If all - requests will have `best_of=1`, you can safely set this to 0. - Otherwise, too small values may cause out-of-memory (OOM) errors. - enforce_eager: Whether to enforce eager execution. If True, we will - disable CUDA graph and always execute the model in eager mode. - If False, we will use CUDA graph and eager execution in hybrid. - max_context_len_to_capture: Maximum context len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode. - disable_custom_all_reduce: See ParallelConfig - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer], - model_hf_config: PretrainedConfig, - tokenizer_mode: str = "auto", - trust_remote_code: bool = False, - tensor_parallel_size: int = 1, - dtype: str = "auto", - quantization: Optional[str] = None, - revision: Optional[str] = None, - tokenizer_revision: Optional[str] = None, - seed: int = 0, - gpu_memory_utilization: float = 0.9, - swap_space: int = 4, - enforce_eager: bool = False, - max_context_len_to_capture: int = 8192, - disable_custom_all_reduce: bool = False, - **kwargs, - ) -> None: - if "disable_log_stats" not in kwargs: - kwargs["disable_log_stats"] = True - engine_args = EngineArgs( - model_hf_config=model_hf_config, - tensor_parallel_size=tensor_parallel_size, - dtype=dtype, - quantization=quantization, - revision=revision, - tokenizer_revision=tokenizer_revision, - seed=seed, - gpu_memory_utilization=gpu_memory_utilization, - swap_space=swap_space, - enforce_eager=enforce_eager, - max_context_len_to_capture=max_context_len_to_capture, - disable_custom_all_reduce=disable_custom_all_reduce, - **kwargs, - ) - tokenizer_cls = (PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer) - if not isinstance(tokenizer, tokenizer_cls): - raise ValueError( - f"Unexpected tokenizer type: {type(tokenizer)}. Must be" - "one of the following: PreTrainedTokenizer, PreTrainedTokenizerFast, verl.workers.rollout.HybridEngineBaseTokenizer" - ) - self.llm_engine = LLMEngine.from_engine_args(model, tokenizer, engine_args) - self.request_counter = Counter() - - def init_cache_engine(self): - self.llm_engine.init_cache_engine() - - def free_cache_engine(self): - self.llm_engine.free_cache_engine() - - def get_tokenizer(self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: - return self.llm_engine.tokenizer - - def set_tokenizer( - self, - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], - ) -> None: - self.llm_engine.tokenizer = tokenizer - - def generate( - self, - prompts: Optional[Union[str, List[str]]] = None, - sampling_params: Optional[SamplingParams] = None, - prompt_token_ids: Optional[List[List[int]]] = None, - prefix_pos: Optional[Union[int, List[int]]] = None, - use_tqdm: bool = True, - lora_request: Optional[LoRARequest] = None, - ) -> List[RequestOutput]: - """Generates the completions for the input prompts. - - NOTE: This class automatically batches the given prompts, considering - the memory constraint. For the best performance, put all of your prompts - into a single list and pass it to this method. - - Args: - prompts: A list of prompts to generate completions for. - sampling_params: The sampling parameters for text generation. If - None, we use the default sampling parameters. - prompt_token_ids: A list of token IDs for the prompts. If None, we - use the tokenizer to convert the prompts to token IDs. - use_tqdm: Whether to use tqdm to display the progress bar. - - Returns: - A list of `RequestOutput` objects containing the generated - completions in the same order as the input prompts. - """ - if prompts is None and prompt_token_ids is None: - raise ValueError("Either prompts or prompt_token_ids must be " - "provided.") - if isinstance(prompts, str): - # Convert a single prompt to a list. - prompts = [prompts] - if prompts is not None and prompt_token_ids is not None: - if len(prompts) != len(prompt_token_ids): - raise ValueError("The lengths of prompts and prompt_token_ids " - "must be the same.") - if sampling_params is None: - # Use default sampling params. - sampling_params = SamplingParams() - - # Add requests to the engine. - num_requests = len(prompts) if prompts is not None else len(prompt_token_ids) - for i in range(num_requests): - prompt = prompts[i] if prompts is not None else None - prefix_pos_i = prefix_pos[i] if prefix_pos is not None else None - token_ids = None if prompt_token_ids is None else prompt_token_ids[i] - if not isinstance(token_ids, list): - # NOTE(shengguangming): convert the rollout input into List[str] - token_ids = self._pre_process_inputs(token_ids) - self._add_request(prompt, sampling_params, token_ids, lora_request=lora_request, prefix_pos=prefix_pos_i) - return self._run_engine(use_tqdm) - - def _add_request( - self, - prompt: Optional[str], - sampling_params: SamplingParams, - prompt_token_ids: Optional[List[int]], - lora_request: Optional[LoRARequest] = None, - prefix_pos: Optional[int] = None, - ) -> None: - request_id = str(next(self.request_counter)) - self.llm_engine.add_request(request_id, - prompt, - sampling_params, - prompt_token_ids, - lora_request=lora_request, - prefix_pos=prefix_pos) - - def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]: - # Initialize tqdm. - if use_tqdm: - num_requests = self.llm_engine.get_num_unfinished_requests() - pbar = tqdm(total=num_requests, desc="Processed prompts") - # Run the engine. - outputs: List[RequestOutput] = [] - while self.llm_engine.has_unfinished_requests(): - step_outputs = self.llm_engine.step() - for output in step_outputs: - if output.finished: - outputs.append(output) - if use_tqdm: - pbar.update(1) - if use_tqdm: - pbar.close() - # Sort the outputs by request ID. - # This is necessary because some requests may be finished earlier than - # its previous requests. - outputs = sorted(outputs, key=lambda x: int(x.request_id)) - # TODO(shengguangming): maybe we can hack the autoregressive logics without only apply post process for better performance - return self._post_process_outputs(outputs) - - # NOTE(shengguangming): add for verl - # TODO(sgm): we can optimize it by making the dataloader yield List[int] without padding. - def _pre_process_inputs(self, prompt_token_ids: torch.Tensor) -> List[int]: - # remove the left padding in the prompt token_id - pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = prompt_token_ids[non_pad_index:].tolist() - return token_ids - - # NOTE(shengguangming): add for verl - def _post_process_outputs(self, outputs: List[RequestOutput]) -> Tuple[torch.Tensor, torch.Tensor]: - output_token_ids = [] - logprobs = [] - for output in outputs: # List[RequestOutput] - output = output.outputs - for output in output: # List[CompletionOutput], usually len == 1 - output_token_ids.append(torch.tensor(output.token_ids)) - # TODO(shengguangming): can be optimzied by rewrite the Sampler._get_logprobs() logits - logprobs_dicts = output.logprobs - if logprobs_dicts is not None: - logprob = [] - for logprobs_dict, id in zip(logprobs_dicts, output.token_ids): - logprob.append(logprobs_dict[id]) - logprobs.append(torch.tensor(logprob)) - - pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - output_token_ids = pad_sequence(output_token_ids, batch_first=True, padding_value=pad_token_id) - if len(logprobs) > 0: - logprobs = pad_sequence(logprobs, batch_first=True, padding_value=pad_token_id) - return output_token_ids, logprobs - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor]) -> None: - self.llm_engine.sync_model_weights(actor_weights=actor_weights) - - def offload_model_weights(self) -> None: - self.llm_engine.offload_model_weights() diff --git a/verl/third_party/vllm/vllm_v_0_3_1/llm_engine_sp.py b/verl/third_party/vllm/vllm_v_0_3_1/llm_engine_sp.py deleted file mode 100644 index e264a8585b..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/llm_engine_sp.py +++ /dev/null @@ -1,765 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/llm_engine.py - -import os -import socket -import time -import torch -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union - -from vllm.lora.request import LoRARequest -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig, ParallelConfig, SchedulerConfig, LoRAConfig) -from vllm.core.scheduler import Scheduler, SchedulerOutputs -from vllm.logger import init_logger -from vllm.outputs import RequestOutput -from vllm.sampling_params import SamplingParams -from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup, SequenceGroupMetadata, SequenceGroupOutput, - SequenceOutput, SequenceStatus) -from vllm.transformers_utils.tokenizer import detokenize_incrementally -from vllm.engine.metrics import StatLogger, Stats -from vllm.utils import Counter -import torch.nn as nn -from .arg_utils import EngineArgs -from .tokenizer import TokenizerGroup - -logger = init_logger(__name__) -_LOCAL_LOGGING_INTERVAL_SEC = 5 - - -class LLMEngine: - """An LLM engine that receives requests and generates texts. - - This is the main class for the vLLM engine. It receives requests - from clients and generates texts from the LLM. It includes a tokenizer, a - language model (possibly distributed across multiple GPUs), and GPU memory - space allocated for intermediate states (aka KV cache). This class utilizes - iteration-level scheduling and efficient memory management to maximize the - serving throughput. - - The `LLM` class wraps this class for offline batched inference and the - `AsyncLLMEngine` class wraps this class for online serving. - - NOTE: The config arguments are derived from the `EngineArgs` class. For the - comprehensive list of arguments, see `EngineArgs`. - - Args: - model_config: The configuration related to the LLM model. - cache_config: The configuration related to the KV cache memory - management. - parallel_config: The configuration related to distributed execution. - scheduler_config: The configuration related to the request scheduler. - distributed_init_method: The initialization method for distributed - execution. See `torch.distributed.init_process_group` for details. - placement_group: Ray placement group for distributed execution. - Required for distributed execution. - log_stats: Whether to log statistics. - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: nn.Module, - model_config: ModelConfig, - cache_config: CacheConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], - distributed_init_method: str, - placement_group: Optional[None], - log_stats: bool, - ) -> None: - logger.info("Initializing an LLM engine with config: " - f"model={model_config.model!r}, " - f"tokenizer={model_config.tokenizer!r}, " - # f"tokenizer_mode={model_config.tokenizer_mode}, " - f"revision={model_config.revision}, " - f"tokenizer_revision={model_config.tokenizer_revision}, " - # f"trust_remote_code={model_config.trust_remote_code}, " - f"dtype={model_config.dtype}, " - f"max_seq_len={model_config.max_model_len}, " - # f"download_dir={model_config.download_dir!r}, " - # f"load_format={model_config.load_format}, " - f"disable_custom_all_reduce={parallel_config.disable_custom_all_reduce}, " - f"tensor_parallel_size={parallel_config.tensor_parallel_size}, " - f"quantization={model_config.quantization}, " - f"seed={model_config.seed})") - # TODO(woosuk): Print more configs in debug mode. - - self.model_config = model_config # TODO: currently is hfconfig - self.cache_config = cache_config - self.lora_config = lora_config - assert self.cache_config.sliding_window == getattr(self.model_config.hf_config, "sliding_window", None) - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.log_stats = log_stats - self._verify_args() - - # self.model = model # should not store the model, it should be deleted - # TODO(shengguangming): maybe we can choose init here or from arguments - self._init_tokenizer(tokenizer) - - self.seq_counter = Counter() - - # Create the parallel GPU workers. - self._init_workers_sp(model, distributed_init_method) - - # Profile the memory usage and initialize the cache. - self._init_cache_sp() - - # Create the scheduler. - # NOTE(shengguangming): each process will have independent scheduler - self.scheduler = Scheduler(scheduler_config, cache_config, lora_config) - - # Metric Logging. - if self.log_stats: - self.stat_logger = StatLogger(local_interval=_LOCAL_LOGGING_INTERVAL_SEC) - - # Logging. - self.last_logging_time = 0.0 - # List of (timestamp, num_tokens) - self.num_prompt_tokens: List[Tuple[float, int]] = [] - # List of (timestamp, num_tokens) - self.num_generation_tokens: List[Tuple[float, int]] = [] - - def _init_tokenizer(self, tokenizer, **tokenizer_init_kwargs): - init_kwargs = dict(enable_lora=bool(self.lora_config), - max_num_seqs=self.scheduler_config.max_num_seqs, - max_input_length=None) - init_kwargs.update(tokenizer_init_kwargs) - self.tokenizer: TokenizerGroup = TokenizerGroup(tokenizer, **init_kwargs) - - # TODO: check get_lora_tokenizer func - def get_tokenizer_for_seq(self, sequence: Sequence): - return self.tokenizer.get_lora_tokenizer(sequence.lora_request) - - def _init_workers_sp(self, model, distributed_init_method: str): - # Lazy import the Worker to avoid importing torch.cuda/xformers - # before CUDA_VISIBLE_DEVICES is set in the Worker - from .worker import Worker # pylint: disable=import-outside-toplevel - - rank = int(os.getenv("RANK")) - - self.worker = Worker( - model, - self.model_config, - self.parallel_config, - self.scheduler_config, - self.device_config, - rank, - distributed_init_method, - lora_config=self.lora_config, - kv_cache_dtype=self.cache_config.cache_dtype, - ) - - # NOTE(shengguangming): torch.distributed.init_process_group will be called inside the init_model() - self.worker.init_model() - self.worker.load_model() - - def _verify_args(self) -> None: - self.model_config.verify_with_parallel_config(self.parallel_config) - self.cache_config.verify_with_parallel_config(self.parallel_config) - - def _init_cache_sp(self) -> None: - """Profiles the memory usage and initializes the KV cache.""" - # Get the maximum number of blocks that can be allocated on GPU and CPU. - num_blocks = self.worker.profile_num_available_blocks( - block_size=self.cache_config.block_size, - gpu_memory_utilization=self.cache_config.gpu_memory_utilization, - cpu_swap_space=self.cache_config.swap_space_bytes, - cache_dtype=self.cache_config.cache_dtype, - ) - - # NOTE(shengguangming): Now we don't use a shared centralized controler but each process will - # have its own scheduler - num_gpu_blocks = num_blocks[0] - num_cpu_blocks = num_blocks[1] - - # FIXME(woosuk): Change to debug log. - logger.info(f"# GPU blocks: {num_gpu_blocks}, " - f"# CPU blocks: {num_cpu_blocks}") - - if num_gpu_blocks <= 0: - raise ValueError("No available memory for the cache blocks. " - "Try increasing `gpu_memory_utilization` when " - "initializing the engine.") - - max_seq_len = self.cache_config.block_size * num_gpu_blocks - if self.model_config.max_model_len > max_seq_len: - raise ValueError(f"The model's max seq len ({self.model_config.max_model_len}) " - "is larger than the maximum number of tokens that can be " - f"stored in KV cache ({max_seq_len}). Try increasing " - "`gpu_memory_utilization` or decreasing `max_model_len` when " - "initializing the engine.") - - self.cache_config.num_gpu_blocks = num_gpu_blocks - self.cache_config.num_cpu_blocks = num_cpu_blocks - - # Initialize the cache. - self.worker.init_cache_engine(cache_config=self.cache_config) - self.worker.warm_up_model() - - def init_cache_engine(self): - self.worker.init_cache_engine(cache_config=self.cache_config) - - def free_cache_engine(self): - self.worker.free_cache_engine() - - @classmethod - def from_engine_args(cls, model, tokenizer, engine_args: EngineArgs) -> "LLMEngine": - """Creates an LLM engine from the engine arguments.""" - # Create the engine configs. - engine_configs = engine_args.create_engine_configs() - parallel_config = engine_configs[2] - # Initialize the cluster. - distributed_init_method, placement_group = initialize_cluster(parallel_config) - # Create the LLM engine. - engine = cls(model, - tokenizer, - *engine_configs, - distributed_init_method, - placement_group, - log_stats=not engine_args.disable_log_stats) - return engine - - def add_request( - self, - request_id: str, - prompt: Optional[str], - sampling_params: SamplingParams, - prompt_token_ids: Optional[List[int]] = None, - arrival_time: Optional[float] = None, - lora_request: Optional[LoRARequest] = None, - prefix_pos: Optional[int] = None, - ) -> None: - """Add a request to the engine's request pool. - - The request is added to the request pool and will be processed by the - scheduler as `engine.step()` is called. The exact scheduling policy is - determined by the scheduler. - - Args: - request_id: The unique ID of the request. - prompt: The prompt string. Can be None if prompt_token_ids is - provided. - sampling_params: The sampling parameters for text generation. - prompt_token_ids: The token IDs of the prompt. If None, we - use the tokenizer to convert the prompts to token IDs. - arrival_time: The arrival time of the request. If None, we use - the current monotonic time. - prefix_pos: If not None, we use the given position as the prefix - position for each prompt. We will cache the prefix's KV - cache and reuse it for the next request with the same prefix. - This is an experimental feature, and may be replaced with - automatic prefix caching in the future. - - Details: - - Set arrival_time to the current time if it is None. - - Set prompt_token_ids to the encoded prompt if it is None. - - Create `best_of` number of :class:`~vllm.Sequence` objects. - - Create a :class:`~vllm.SequenceGroup` object - from the list of :class:`~vllm.Sequence`. - - Add the :class:`~vllm.SequenceGroup` object to the scheduler. - - Example: - >>> # initialize engine - >>> engine = LLMEngine.from_engine_args(engine_args) - >>> # set request arguments - >>> example_prompt = "Who is the president of the United States?" - >>> sampling_params = SamplingParams(temperature=0.0) - >>> request_id = 0 - >>> - >>> # add the request to the engine - >>> engine.add_request( - >>> str(request_id), - >>> example_prompt, - >>> SamplingParams(temperature=0.0)) - >>> # continue the request processing - >>> ... - """ - if lora_request is not None and not self.lora_config: - raise ValueError(f"Got lora_request {lora_request} but LoRA is " - "not enabled!") - if arrival_time is None: - arrival_time = time.monotonic() - if prompt_token_ids is None: - assert prompt is not None - prompt_token_ids = self.tokenizer.encode(prompt) - - # Create the sequences. - block_size = self.cache_config.block_size - seq_id = next(self.seq_counter) - seq = Sequence(seq_id, prompt, prompt_token_ids, block_size, lora_request) - - # Check whether the input specifies prefix - prefix = self.scheduler.prefix_pool.add_or_get_prefix(prompt_token_ids[:prefix_pos], lora_request.lora_int_id if - lora_request else 0) if prefix_pos is not None else None - - # Create the sequence group. - seq_group = SequenceGroup(request_id, [seq], sampling_params, arrival_time, lora_request, prefix) - - # Add the sequence group to the scheduler. - self.scheduler.add_seq_group(seq_group) - - def abort_request(self, request_id: Union[str, Iterable[str]]) -> None: - """Aborts a request(s) with the given ID. - - Args: - request_id: The ID(s) of the request to abort. - - Details: - - Refer to the - :meth:`~vllm.core.scheduler.Scheduler.abort_seq_group` - from class :class:`~vllm.core.scheduler.Scheduler`. - - Example: - >>> # initialize engine and add a request with request_id - >>> request_id = str(0) - >>> # abort the request - >>> engine.abort_request(request_id) - """ - self.scheduler.abort_seq_group(request_id) - - def get_model_config(self) -> ModelConfig: - """Gets the model configuration.""" - return self.model_config - - def get_num_unfinished_requests(self) -> int: - """Gets the number of unfinished requests.""" - return self.scheduler.get_num_unfinished_seq_groups() - - def has_unfinished_requests(self) -> bool: - """Returns True if there are unfinished requests.""" - return self.scheduler.has_unfinished_seqs() - - def _check_beam_search_early_stopping( - self, - early_stopping: Union[bool, str], - sampling_params: SamplingParams, - best_running_seq: Sequence, - current_worst_seq: Sequence, - ) -> bool: - assert sampling_params.use_beam_search - length_penalty = sampling_params.length_penalty - if early_stopping is True: - return True - - current_worst_score = (current_worst_seq.get_beam_search_score( - length_penalty=length_penalty, eos_token_id=self.get_tokenizer_for_seq(current_worst_seq).eos_token_id)) - if early_stopping is False: - highest_attainable_score = (best_running_seq.get_beam_search_score( - length_penalty=length_penalty, eos_token_id=self.get_tokenizer_for_seq(best_running_seq).eos_token_id)) - else: - assert early_stopping == "never" - if length_penalty > 0.0: - # If length_penalty > 0.0, beam search will prefer longer - # sequences. The highest attainable score calculation is - # based on the longest possible sequence length in this case. - max_possible_length = max(best_running_seq.get_prompt_len() + sampling_params.max_tokens, - self.scheduler_config.max_model_len) - highest_attainable_score = (best_running_seq.get_beam_search_score( - length_penalty=length_penalty, - eos_token_id=self.get_tokenizer_for_seq(best_running_seq).eos_token_id, - seq_len=max_possible_length)) - else: - # Otherwise, beam search will prefer shorter sequences. The - # highest attainable score calculation is based on the current - # sequence length. - highest_attainable_score = (best_running_seq.get_beam_search_score( - length_penalty=length_penalty, - eos_token_id=self.get_tokenizer_for_seq(best_running_seq).eos_token_id)) - - def _process_sequence_group_outputs(self, seq_group: SequenceGroup, outputs: SequenceGroupOutput) -> None: - - # Process prompt logprobs - prompt_logprobs = outputs.prompt_logprobs - if prompt_logprobs is not None: - seq_group.prompt_logprobs = prompt_logprobs - - # Process samples - samples = outputs.samples - parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING) - existing_finished_seqs = seq_group.get_finished_seqs() - parent_child_dict = {parent_seq.seq_id: [] for parent_seq in parent_seqs} - for sample in samples: - parent_child_dict[sample.parent_seq_id].append(sample) - # List of (child, parent) - child_seqs: List[Tuple[Sequence, Sequence]] = [] - - # Process the child samples for each parent sequence - for parent in parent_seqs: - child_samples: List[SequenceOutput] = parent_child_dict[parent.seq_id] - if len(child_samples) == 0: - # This parent sequence has no children samples. Remove - # the parent sequence from the sequence group since it will - # not be used in the future iterations. - parent.status = SequenceStatus.FINISHED_ABORTED - seq_group.remove(parent.seq_id) - self.scheduler.free_seq(parent) - continue - # Fork the parent sequence if there are multiple child samples. - for child_sample in child_samples[:-1]: - new_child_seq_id = next(self.seq_counter) - child = parent.fork(new_child_seq_id) - child.append_token_id(child_sample.output_token, child_sample.logprobs) - child_seqs.append((child, parent)) - # Continue the parent sequence for the last child sample. - # We reuse the parent sequence here to reduce redundant memory - # copies, especially when using non-beam search sampling methods. - last_child_sample = child_samples[-1] - parent.append_token_id(last_child_sample.output_token, last_child_sample.logprobs) - child_seqs.append((parent, parent)) - - for seq, _ in child_seqs: - # self._decode_sequence(seq, seq_group.sampling_params) - self._check_stop(seq, seq_group.sampling_params) - - # Non-beam search case - if not seq_group.sampling_params.use_beam_search: - # For newly created child sequences, add them to the sequence group - # and fork them in block manager if they are not finished. - for seq, parent in child_seqs: - if seq is not parent: - seq_group.add(seq) - if not seq.is_finished(): - self.scheduler.fork_seq(parent, seq) - - # Free the finished and selected parent sequences' memory in block - # manager. Keep them in the sequence group as candidate output. - # NOTE: we need to fork the new sequences before freeing the - # old sequences. - for seq, parent in child_seqs: - if seq is parent and seq.is_finished(): - self.scheduler.free_seq(seq) - return - - # Beam search case - # Select the child sequences to keep in the sequence group. - selected_child_seqs = [] - unselected_child_seqs = [] - beam_width = seq_group.sampling_params.best_of - length_penalty = seq_group.sampling_params.length_penalty - - # Select the newly finished sequences with the highest scores - # to replace existing finished sequences. - # Tuple of (seq, parent, is_new) - existing_finished_seqs = [(seq, None, False) for seq in existing_finished_seqs] - new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs if seq.is_finished()] - all_finished_seqs = existing_finished_seqs + new_finished_seqs - # Sort the finished sequences by their scores. - all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score( - length_penalty=length_penalty, eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id), - reverse=True) - for seq, parent, is_new in all_finished_seqs[:beam_width]: - if is_new: - # A newly generated child sequence finishes and has a high - # score, so we will add it into the sequence group. - selected_child_seqs.append((seq, parent)) - for seq, parent, is_new in all_finished_seqs[beam_width:]: - if is_new: - # A newly generated child sequence finishes but has a low - # score, so we will not add it into the sequence group. - # Additionally, if this sequence is a continuation of a - # parent sequence, we will need remove the parent sequence - # from the sequence group. - unselected_child_seqs.append((seq, parent)) - else: - # An existing finished sequence has a low score, so we will - # remove it from the sequence group. - seq_group.remove(seq.seq_id) - - # select the top beam_width sequences from the running - # sequences for the next iteration to continue the beam - # search. - running_child_seqs = [(seq, parent) for seq, parent in child_seqs if not seq.is_finished()] - # Sort the running sequences by their scores. - running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score( - length_penalty=length_penalty, eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id), - reverse=True) - - # Check if we can stop the beam search. - if len(running_child_seqs) == 0: - # No running sequences, stop the beam search. - stop_beam_search = True - elif len(all_finished_seqs) < beam_width: - # Not enough finished sequences, continue the beam search. - stop_beam_search = False - else: - # Check the early stopping criteria - best_running_seq = running_child_seqs[0][0] - current_worst_seq = all_finished_seqs[beam_width - 1][0] - stop_beam_search = self._check_beam_search_early_stopping(seq_group.sampling_params.early_stopping, - seq_group.sampling_params, best_running_seq, - current_worst_seq) - - if stop_beam_search: - # Stop the beam search and remove all the running sequences from - # the sequence group. - unselected_child_seqs.extend(running_child_seqs) - else: - # Continue the beam search and select the top beam_width sequences - # to continue the beam search. - selected_child_seqs.extend(running_child_seqs[:beam_width]) - # The remaining running sequences will not be used in the next - # iteration. Again, if these sequences are continuations of - # parent sequences, we will need to remove the parent sequences - # from the sequence group. - unselected_child_seqs.extend(running_child_seqs[beam_width:]) - - # For newly created child sequences, add them to the sequence group - # and fork them in block manager if they are not finished. - for seq, parent in selected_child_seqs: - if seq is not parent: - seq_group.add(seq) - if not seq.is_finished(): - self.scheduler.fork_seq(parent, seq) - - # Free the finished and selected parent sequences' memory in block - # manager. Keep them in the sequence group as candidate output. - for seq, parent in selected_child_seqs: - if seq is parent and seq.is_finished(): - self.scheduler.free_seq(seq) - - # Remove the unselected parent sequences from the sequence group and - # free their memory in block manager. - for seq, parent in unselected_child_seqs: - if seq is parent: - # Remove the parent sequence if it is not selected for next - # iteration - seq_group.remove(seq.seq_id) - self.scheduler.free_seq(seq) - - def _process_model_outputs(self, output: SamplerOutput, scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]: - # Update the scheduled sequence groups with the model outputs. - scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups - for seq_group, outputs in zip(scheduled_seq_groups, output): - self._process_sequence_group_outputs(seq_group, outputs) - - # Free the finished sequence groups. - self.scheduler.free_finished_seq_groups() - - # Create the outputs. - request_outputs: List[RequestOutput] = [] - for seq_group in scheduled_seq_groups: - request_output = RequestOutput.from_seq_group(seq_group) - request_outputs.append(request_output) - for seq_group in scheduler_outputs.ignored_seq_groups: - request_output = RequestOutput.from_seq_group(seq_group) - request_outputs.append(request_output) - - # Update prefix state, now all the uncomputed prefixes are computed. - for seq_group in scheduled_seq_groups: - if (seq_group.prefix is not None and seq_group.prefix.allocated and not seq_group.prefix.computed): - seq_group.prefix.computed = True - - # Log stats. - if self.log_stats: - self.stat_logger.log(self._get_stats(scheduler_outputs)) - - return request_outputs - - def step(self) -> List[RequestOutput]: - """Performs one decoding iteration and returns newly generated results. - - This function performs one decoding iteration of the engine. It first - schedules the sequences to be executed in the next iteration and the - token blocks to be swapped in/out/copy. Then, it executes the model - and updates the scheduler with the model outputs. Finally, it decodes - the sequences and returns the newly generated results. - """ - seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule() - if not scheduler_outputs.is_empty(): - output = self.worker.execute_model( - seq_group_metadata_list=seq_group_metadata_list, # TODO: check this input - blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in, - blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out, - blocks_to_copy=scheduler_outputs.blocks_to_copy,) - else: - return [RequestOutput.from_seq_group(seq_group) for seq_group in scheduler_outputs.ignored_seq_groups] - - return self._process_model_outputs(output, scheduler_outputs) - - def do_log_stats(self) -> None: - """Forced log when no requests active.""" - if self.log_stats: - self.stat_logger.log(self._get_stats(scheduler_outputs=None)) - - def _get_stats(self, scheduler_outputs: Optional[SchedulerOutputs]) -> Stats: - """Get Stats to be Logged to Prometheus.""" - now = time.monotonic() - - # KV Cache Usage in %. - num_total_gpu = self.cache_config.num_gpu_blocks - num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks() - gpu_cache_usage = 1.0 - (num_free_gpu / num_total_gpu) - - num_total_cpu = self.cache_config.num_cpu_blocks - cpu_cache_usage = 0. - if num_total_cpu > 0: - num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks() - cpu_cache_usage = 1.0 - (num_free_cpu / num_total_cpu) - - # Scheduler State - num_running = len(self.scheduler.running) - num_swapped = len(self.scheduler.swapped) - num_waiting = len(self.scheduler.waiting) - - # Iteration stats if we have scheduler output. - num_prompt_tokens = 0 - num_generation_tokens = 0 - time_to_first_tokens = [] - time_per_output_tokens = [] - time_e2e_requests = [] - if scheduler_outputs is not None: - prompt_run = scheduler_outputs.prompt_run - - # Number of Tokens. - if prompt_run: - num_prompt_tokens = scheduler_outputs.num_batched_tokens - else: - num_generation_tokens = scheduler_outputs.num_batched_tokens - - # Latency Timings. - time_last_iters = [] - for seq_group in scheduler_outputs.scheduled_seq_groups: - # Time since last token. (n.b. updates seq_group.last_token_time) - time_last_iters.append(seq_group.get_last_latency(now)) - # Time since arrival for all finished requests. - if seq_group.is_finished(): - time_e2e_requests.append(now - seq_group.arrival_time) - - time_to_first_tokens = time_last_iters if prompt_run else [] - time_per_output_tokens = [] if prompt_run else time_last_iters - - return Stats( - now=now, - num_running=num_running, - num_swapped=num_swapped, - num_waiting=num_waiting, - gpu_cache_usage=gpu_cache_usage, - cpu_cache_usage=cpu_cache_usage, - num_prompt_tokens=num_prompt_tokens, - num_generation_tokens=num_generation_tokens, - time_to_first_tokens=time_to_first_tokens, - time_per_output_tokens=time_per_output_tokens, - time_e2e_requests=time_e2e_requests, - ) - - # TODO: we may not need to decode - def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None: - """Decodes the new token for a sequence.""" - (new_tokens, new_output_text, prefix_offset, read_offset) = detokenize_incrementally( - self.get_tokenizer_for_seq(seq), - all_input_ids=seq.get_token_ids(), - prev_tokens=seq.tokens, - prefix_offset=seq.prefix_offset, - read_offset=seq.read_offset, - skip_special_tokens=prms.skip_special_tokens, - spaces_between_special_tokens=prms.spaces_between_special_tokens, - ) - if seq.tokens is None: - seq.tokens = new_tokens - else: - seq.tokens.extend(new_tokens) - seq.prefix_offset = prefix_offset - seq.read_offset = read_offset - seq.output_text += new_output_text - - def _check_stop(self, seq: Sequence, sampling_params: SamplingParams) -> None: - """Stop the finished sequences.""" - # for stop_str in sampling_params.stop: - # if seq.output_text.endswith(stop_str): - # self._finalize_sequence(seq, sampling_params, stop_str) - # seq.status = SequenceStatus.FINISHED_STOPPED - # return - # if seq.get_last_token_id() in sampling_params.stop_token_ids: - # stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(seq.get_last_token_id()) - # self._finalize_sequence(seq, sampling_params, stop_str) - # seq.status = SequenceStatus.FINISHED_STOPPED - # return - - # Check if the sequence has reached max_model_len. - if seq.get_len() > self.scheduler_config.max_model_len: - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED - return - - # Check if the sequence has reached max_tokens. - if seq.get_output_len() == sampling_params.max_tokens: - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED - return - - # Check if the sequence has generated the EOS token. - if ((not sampling_params.ignore_eos) and - seq.get_last_token_id() == self.get_tokenizer_for_seq(seq).eos_token_id): - seq.status = SequenceStatus.FINISHED_STOPPED - return - - def _finalize_sequence(self, seq: Sequence, sampling_params: SamplingParams, stop_string: str) -> None: - if not sampling_params.include_stop_str_in_output and stop_string: - # Truncate the output text so that the stop string is - # not included in the output. - seq.output_text = seq.output_text[:-len(stop_string)] - - def add_lora(self, lora_request: LoRARequest) -> bool: - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0." - return self.worker.add_lora(lora_request) - - def remove_lora(self, lora_id: int) -> bool: - assert lora_id > 0, "lora_id must be greater than 0." - return self.worker.remove_lora(lora_id) - - def list_loras(self) -> List[int]: - return self.worker.list_loras() - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor]) -> None: - self.worker.sync_model_weights(actor_weights=actor_weights) - - def offload_model_weights(self) -> None: - self.worker.offload_model_weights() - - -def initialize_cluster( - parallel_config: ParallelConfig, - engine_use_ray: bool = False, - ray_address: Optional[str] = None, -) -> Tuple[str, Optional[None]]: - """Initialize the distributed cluster probably with Ray. - - Args: - parallel_config: The configurations for parallel execution. - engine_use_ray: Whether to use Ray for async engine. - ray_address: The address of the Ray cluster. If None, uses - the default Ray cluster address. - - Returns: - A tuple of (`distributed_init_method`, `placement_group`). The - `distributed_init_method` is the address for initializing the - distributed backend. `placement_group` includes the specification - of the resources for each distributed worker. - """ - - # Initialize cluster locally. - port = get_open_port() - # We need to setup the distributed init method to make sure - # the distributed megatron code (e.g., get world size) works correctly. - distributed_init_method = f"tcp://localhost:{port}" - return distributed_init_method, None - - -def get_open_port(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - return s.getsockname()[1] diff --git a/verl/third_party/vllm/vllm_v_0_3_1/model_loader.py b/verl/third_party/vllm/vllm_v_0_3_1/model_loader.py deleted file mode 100644 index 450e2f4b49..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/model_loader.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/model_loader -"""Utilities for selecting and loading models.""" -import contextlib -from typing import Dict, Type, Union - -import torch -import torch.nn as nn -from transformers import PretrainedConfig, PreTrainedModel -from megatron.core.tensor_parallel.utils import VocabUtility - -from vllm.model_executor.models import ModelRegistry -from vllm.model_executor.weight_utils import (get_quant_config, initialize_dummy_weights) - -from .config import ModelConfig -from vllm.config import DeviceConfig, LoRAConfig -from .weight_loaders import * -from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors -from vllm.sequence import SamplerOutput -from typing import Optional -from vllm.model_executor.layers.sampler import Sampler -from vllm.model_executor.layers.sampler import _prune_hidden_states, _apply_logits_processors, _apply_penalties, _apply_top_k_top_p, _apply_min_p, _apply_penalties, _sample, _get_logprobs, _build_sampler_output - - -@contextlib.contextmanager -def _set_default_torch_dtype(dtype: torch.dtype): - """Sets the default torch dtype to the given dtype.""" - old_dtype = torch.get_default_dtype() - torch.set_default_dtype(dtype) - yield - torch.set_default_dtype(old_dtype) - - -def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]: - architectures = getattr(config, "architectures", []) - for arch in architectures: - model_cls = ModelRegistry.load_model_cls(arch) - if model_cls is not None: - return model_cls - raise ValueError(f"Model architectures {architectures} are not supported for now. " - f"Supported architectures: {ModelRegistry.get_supported_archs()}") - - -from vllm.model_executor.layers.linear import * -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead -from vllm.model_executor.layers.activation import ScaledActivation - -__LAYER_WEIGHT_LOADER_REGISTRY__ = { - ColumnParallelLinear: parallel_weight_loader, - MergedColumnParallelLinear: parallel_weight_loader, - QKVParallelLinear: parallel_weight_loader, - RowParallelLinear: parallel_weight_loader, - VocabParallelEmbedding: parallel_weight_loader, - ParallelLMHead: parallel_weight_loader - # "ScaledActivation.weight_loader": ScaledActivation, # TODO(shengguangming): latest commit in vllm fix awq for this function and add load_weights - # "default_weight_loader": default_weight_loader -} - -# NOTE(gmsheng): change the weight_loader function in runtime -for layer_class, weight_loader in __LAYER_WEIGHT_LOADER_REGISTRY__.items(): - layer_class.weight_loader = weight_loader - -__MODEL_WEIGHT_LOADER_REGISTRY__ = { - 'GPT2LMHeadModel': gpt2_weight_loader, - 'LlamaForCausalLM': llama_weight_loader, - 'LLaMAForCausalLM': llama_weight_loader, - 'MistralForCausalLM': mistral_weight_loader, -} - -# FIXME(shengguangming): the vLLM vocab will pad to 64, which may incur out of bounds -# so we need to rewrite the init function of vocab -DEFAULT_VOCAB_PADDING_SIZE = 64 - - -def vocab_init(self, - num_embeddings: int, - embedding_dim: int, - params_dtype: Optional[torch.dtype] = None, - org_num_embeddings: Optional[int] = None, - padding_size: int = DEFAULT_VOCAB_PADDING_SIZE): - super(VocabParallelEmbedding, self).__init__() - - # Keep the input dimensions. - # TODO (pad to be divided by 4) - self.num_embeddings = num_embeddings - self.org_vocab_size = org_num_embeddings or num_embeddings - - # self.num_embeddings_padded = pad_vocab_size(num_embeddings, - # padding_size) - self.embedding_dim = embedding_dim - if params_dtype is None: - params_dtype = torch.get_default_dtype() - self.tp_size = get_tensor_model_parallel_world_size() - # Divide the weight matrix along the vocaburaly dimension. - - self.vocab_start_index, self.vocab_end_index = (VocabUtility.vocab_range_from_global_vocab_size( - self.num_embeddings, get_tensor_model_parallel_rank(), self.tp_size)) - self.num_embeddings_per_partition = (self.vocab_end_index - self.vocab_start_index) - self.weight = Parameter( - torch.empty( - self.num_embeddings_per_partition, - self.embedding_dim, - # device=torch.cuda.current_device(), - dtype=params_dtype)) - set_weight_attrs(self.weight, {"parallel_dim": 0, "weight_loader": self.weight_loader}) - - -VocabParallelEmbedding.__init__ = vocab_init - - -def _get_model_weight_loader(arch: str): - if arch in __MODEL_WEIGHT_LOADER_REGISTRY__: - return __MODEL_WEIGHT_LOADER_REGISTRY__[arch] - raise ValueError(f"Model architectures {arch} are not supported for now. " - f"Supported architectures: {ModelRegistry.get_supported_archs()}") - - -def get_model(actor_model: Union[PreTrainedModel, Dict], - model_config: ModelConfig, - device_config: DeviceConfig, - lora_config: Optional[LoRAConfig] = None) -> nn.Module: - model_class = _get_model_architecture(model_config.hf_config) - - # Get the quantization config. - linear_method = None - quant_config = None - if model_config.quantization is not None: - quant_config = get_quant_config(model_config.quantization, model_config.model, model_config.hf_config, - model_config.download_dir) - capability = torch.cuda.get_device_capability() - capability = capability[0] * 10 + capability[1] - if capability < quant_config.get_min_capability(): - raise ValueError(f"The quantization method {model_config.quantization} is not " - "supported for the current GPU. " - f"Minimum capability: {quant_config.get_min_capability()}. " - f"Current capability: {capability}.") - supported_dtypes = quant_config.get_supported_act_dtypes() - if model_config.dtype not in supported_dtypes: - raise ValueError(f"{model_config.dtype} is not supported for quantization " - f"method {model_config.quantization}. Supported dtypes: " - f"{supported_dtypes}") - linear_method = quant_config.get_linear_method() - - with _set_default_torch_dtype(model_config.dtype): - # Create a model instance. - # The weights will be initialized as empty tensors. - # with torch.device(device_config.device): - # NOTE(sgm): init the model in cpu - model = model_class(model_config.hf_config, linear_method) - - if model_config.load_format == "dummy": - model = model.cuda() - # NOTE(woosuk): For accurate performance evaluation, we assign - # random values to the weights. - initialize_dummy_weights(model) - elif model_config.load_format == 'model' or model_config.load_format == 'auto': - # NOTE(shengguangming) Load the weights from the actor model - if isinstance(actor_model, nn.Module): - load_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), vllm_model=model) - else: - load_weights(actor_weights=actor_model, vllm_model=model) - - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -# the actor model is .state_dict() -def load_weights(actor_weights: Dict, vllm_model: nn.Module): - weight_loader = _get_model_weight_loader(vllm_model.__class__.__name__) - weight_loader(actor_weights, vllm_model) - # NOTE(sgm) to reduce peak memory usage, we offload vllm model to cpu - # after init, and we need this after sync model weights for in first iter. - vllm_model = vllm_model.cuda() - - -# FIXME(sgm): hack the Sampler function in vllm v0.3.1 -# as they use ray, the sampler result will only need to return to the driver node, -# therefore gather is enough. However, we use SPMD instead of a central scheduler, -# all_gather is required (aligned with v0.2.6) -def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor, - embedding_bias: Optional[torch.Tensor]) -> torch.Tensor: - # Get the logits for the next tokens. - logits = torch.matmul(hidden_states, embedding.t()) - if embedding_bias is not None: - logits += embedding_bias - logits = tensor_model_parallel_all_gather(logits) - # Remove paddings in vocab (if any). - if logits is not None: - logits = logits[:, :self.org_vocab_size] - return logits - - -def forward( - self, - embedding: torch.Tensor, - hidden_states: torch.Tensor, - sampling_metadata: SamplingMetadata, - embedding_bias: Optional[torch.Tensor] = None, -) -> Optional[SamplerOutput]: - # Get the hidden states that we use for sampling. - hidden_states = _prune_hidden_states(hidden_states, sampling_metadata) - - # Get the logits for the next tokens. - logits = self._get_logits(hidden_states, embedding, embedding_bias) - # save origin logprobs for sampler_output - origin_logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float) - - # Only perform sampling in the driver worker. - # Note: `_get_logits` is still distributed across TP workers because - # the `embedding` weight is distributed across TP workers. - # TODO(zhuohan): Change the get_logits part to a separate stage. - if not sampling_metadata.perform_sampling: - return None - - assert logits is not None - _, vocab_size = logits.shape - - # Apply logits processors (if any). - logits = _apply_logits_processors(logits, sampling_metadata) - - # Prepare sampling tensors with pinned memory to avoid blocking. - (sampling_tensors, do_penalties, do_top_p_top_k, - do_min_p) = SamplingTensors.from_sampling_metadata(sampling_metadata, vocab_size, logits.device, logits.dtype) - - # Apply presence and frequency penalties. - if do_penalties: - logits = _apply_penalties(logits, sampling_tensors.prompt_tokens, sampling_tensors.output_tokens, - sampling_tensors.presence_penalties, sampling_tensors.frequency_penalties, - sampling_tensors.repetition_penalties) - - # Apply temperature scaling. - # Use in-place division to avoid creating a new tensor. - logits.div_(sampling_tensors.temperatures.unsqueeze_(dim=1)) - - if do_top_p_top_k: - logits = _apply_top_k_top_p(logits, sampling_tensors.top_ps, sampling_tensors.top_ks) - - if do_min_p: - logits = _apply_min_p(logits, sampling_tensors.min_ps) - - # We use float32 for probabilities and log probabilities. - # Compute the probabilities. - probs = torch.softmax(logits, dim=-1, dtype=torch.float) - # Compute the log probabilities. - # Use log_softmax to ensure numerical stability. - logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float) - - # Sample the next tokens. - sample_results = _sample(probs, logprobs, sampling_metadata) - - # Get the logprobs query results. - # prompt_logprobs, sample_logprobs = _get_logprobs( - # logprobs, sampling_metadata, sample_results) - prompt_logprobs, sample_logprobs = _get_logprobs(origin_logprobs, sampling_metadata, sample_results) - - return _build_sampler_output(sample_results, sampling_metadata, prompt_logprobs, sample_logprobs) - - -from vllm.model_executor.layers.sampler import Sampler - -Sampler._get_logits = _get_logits -Sampler.forward = forward diff --git a/verl/third_party/vllm/vllm_v_0_3_1/model_runner.py b/verl/third_party/vllm/vllm_v_0_3_1/model_runner.py deleted file mode 100644 index 4acf3422d4..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/model_runner.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/model_runner.py - -from typing import Dict, List, Optional, Tuple, Set, Union -import contextlib -import time -import numpy as np -import torch -import torch.nn as nn - -from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig) -from vllm.logger import init_logger -from vllm.model_executor import InputMetadata, SamplingMetadata -from vllm.sampling_params import SamplingParams, SamplingType -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata -from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager -from vllm.lora.layers import LoRAMapping -from vllm.lora.request import LoRARequest -from vllm.utils import in_wsl -from vllm.worker.model_runner import ModelRunner, CUDAGraphRunner, _async_h2d - -from .model_loader import get_model - -logger = init_logger(__name__) - -KVCache = Tuple[torch.Tensor, torch.Tensor] -_PAD_SLOT_ID = -1 -LORA_WARMUP_RANK = 8 -# Capture graphs for batch size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256. -# NOTE: _get_graph_batch_size needs to be updated if this list is changed. -_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [8 * i for i in range(1, 33)] - - -class ModelRunner(ModelRunner): - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], - kv_cache_dtype: Optional[str] = "auto", - ): - self.model_config = model_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.lora_config = lora_config - - # model_config can be None in tests/samplers/test_sampler.py. - # FIXME(woosuk): This is a hack to make the tests work. Refactor this. - self.sliding_window = (model_config.get_sliding_window() if model_config is not None else None) - - self.device_config = (device_config if device_config is not None else DeviceConfig()) - self.device = self.device_config.device - - self.model = model # this will be replaced by get_model() - self.block_size = None # Set after initial profiling. - self.lora_manager = None - - self.graph_runners: Dict[int, CUDAGraphRunner] = {} - self.graph_memory_pool = None # Set during graph capture. - - self.max_context_len_to_capture = (self.model_config.max_context_len_to_capture - if self.model_config is not None else 0) - # When using CUDA graph, the input block tables must be padded to - # max_context_len_to_capture. However, creating the block table in - # Python can be expensive. To optimize this, we cache the block table - # in numpy and only copy the actual input content at every iteration. - # The shape of the cached block table will be - # (max batch size to capture, max context len to capture / block size). - self.graph_block_tables = None # Set after initial profiling. - # cache in_wsl result - self.in_wsl = in_wsl() - self.kv_cache_dtype = kv_cache_dtype - - def load_model(self) -> None: - self.model = get_model(actor_model=self.model, - model_config=self.model_config, - device_config=self.device_config, - lora_config=self.lora_config) - vocab_size = self.model.config.vocab_size - - if self.lora_config: - assert hasattr( - self.model, - "supported_lora_modules") and self.model.supported_lora_modules, "Model does not support LoRA" - assert hasattr(self.model, "embedding_modules"), "Model does not have embedding_modules" - assert hasattr(self.model, "embedding_padding_modules"), "Model does not have embedding_padding_modules" - self.lora_manager = LRUCacheWorkerLoRAManager( - self.scheduler_config.max_num_seqs, - self.scheduler_config.max_num_batched_tokens + self.scheduler_config.max_paddings, vocab_size, - self.lora_config, self.device, self.model.embedding_modules, self.model.embedding_padding_modules) - self.model = self.lora_manager.create_lora_manager(self.model) - - def _prepare_sample( - self, - seq_group_metadata_list: List[SequenceGroupMetadata], - prompt_lens: List[int], - subquery_lens: Optional[List[int]], - ) -> SamplingMetadata: - seq_groups: List[Tuple[List[int], SamplingParams]] = [] - selected_token_indices: List[int] = [] - selected_token_start_idx = 0 - categorized_sample_indices = {t: [] for t in SamplingType} - categorized_sample_indices_start_idx = 0 - - max_subquery_len = max(subquery_lens) if subquery_lens else 1 - for i, seq_group_metadata in enumerate(seq_group_metadata_list): - seq_ids = list(seq_group_metadata.seq_data.keys()) - sampling_params = seq_group_metadata.sampling_params - seq_groups.append((seq_ids, sampling_params)) - - if seq_group_metadata.is_prompt: - assert len(seq_ids) == 1 - assert subquery_lens is not None - subquery_len = subquery_lens[i] - if sampling_params.prompt_logprobs is not None: - # NOTE: prompt token positions do not need sample, skip - categorized_sample_indices_start_idx += subquery_len - 1 - - categorized_sample_indices[sampling_params.sampling_type].append(categorized_sample_indices_start_idx) - categorized_sample_indices_start_idx += 1 - - if sampling_params.prompt_logprobs is not None: - selected_token_indices.extend( - range(selected_token_start_idx, selected_token_start_idx + subquery_len - 1)) - selected_token_indices.append(selected_token_start_idx + subquery_len - 1) - selected_token_start_idx += max_subquery_len - else: - num_seqs = len(seq_ids) - selected_token_indices.extend(range(selected_token_start_idx, selected_token_start_idx + num_seqs)) - selected_token_start_idx += num_seqs - - categorized_sample_indices[sampling_params.sampling_type].extend( - range(categorized_sample_indices_start_idx, categorized_sample_indices_start_idx + num_seqs)) - categorized_sample_indices_start_idx += num_seqs - - selected_token_indices = _async_h2d(selected_token_indices, - dtype=torch.long, - target_device=self.device, - pin_memory=not self.in_wsl) - categorized_sample_indices = { - t: _async_h2d(seq_ids, dtype=torch.int, target_device=self.device, pin_memory=not self.in_wsl) - for t, seq_ids in categorized_sample_indices.items() - } - - seq_data: Dict[int, SequenceData] = {} - for seq_group_metadata in seq_group_metadata_list: - seq_data.update(seq_group_metadata.seq_data) - - sampling_metadata = SamplingMetadata( - seq_groups=seq_groups, - seq_data=seq_data, - prompt_lens=prompt_lens, - selected_token_indices=selected_token_indices, - categorized_sample_indices=categorized_sample_indices, - ) - return sampling_metadata - - def prepare_input_tensors( - self, - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]], - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata, Set[int], LoRAMapping]: - # NOTE: We assume that all sequences in the group are all prompts or - # all decodes. - is_prompt = seq_group_metadata_list[0].is_prompt - # Prepare input tensors. - if is_prompt: - (input_tokens, input_positions, input_metadata, prompt_lens, subquery_lens, lora_index_mapping, - lora_prompt_mapping, lora_requests) = self._prepare_prompt(seq_group_metadata_list) - else: - (input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, - lora_requests) = self._prepare_decode(seq_group_metadata_list) - prompt_lens = [] - subquery_lens = None - sampling_metadata = self._prepare_sample(seq_group_metadata_list, prompt_lens, subquery_lens) - if self.lora_config: - flat_lora_index_mapping = [item for sublist in lora_index_mapping for item in sublist] - lora_mapping = LoRAMapping( - flat_lora_index_mapping, - lora_prompt_mapping, - ) - else: - lora_mapping = None - - return (input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping) - - @torch.inference_mode() - def execute_model( - self, - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]], - kv_caches: List[Tuple[torch.Tensor, torch.Tensor]], - ) -> Optional[SamplerOutput]: - (input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, - lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list) - - if self.lora_config: - self.set_active_loras(lora_requests, lora_mapping) - - # Execute the model. - if input_metadata.use_cuda_graph: - graph_batch_size = input_tokens.shape[0] - model_executable = self.graph_runners[graph_batch_size] - else: - model_executable = self.model - hidden_states = model_executable( - input_ids=input_tokens, - positions=input_positions, - kv_caches=kv_caches, - input_metadata=input_metadata, - ) - - # Sample the next token. - output = self.model.sample( - hidden_states=hidden_states, - sampling_metadata=sampling_metadata, - ) - return output - - @torch.inference_mode() - def profile_run(self) -> None: - # Enable top-k sampling to reflect the accurate memory usage. - vocab_size = self.model_config.get_vocab_size() - # FIXME(sgm): this sampling params will call cumsum(), causing the - # deterministic cumsum throw error - sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1) - max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens - max_num_seqs = self.scheduler_config.max_num_seqs - - # This represents the maximum number of different requests - # that will have unique loras, an therefore the max amount of memory - # consumption create dummy lora request copies from the lora request - # passed in, which contains a lora from the lora warmup path. - dummy_lora_requests = [] - dummy_lora_requests_per_seq = [] - if self.lora_config: - for idx in range(self.lora_config.max_loras): - lora_id = idx + 1 - dummy_lora_request = LoRARequest( - lora_name=f"warmup_{lora_id}", - lora_int_id=lora_id, - lora_local_path="/not/a/real/path", - ) - self.lora_manager.add_dummy_lora(dummy_lora_request, rank=LORA_WARMUP_RANK) - dummy_lora_requests.append(dummy_lora_request) - dummy_lora_requests_per_seq = [ - dummy_lora_requests[idx % len(dummy_lora_requests)] for idx in range(max_num_seqs) - ] - - # Profile memory usage with max_num_sequences sequences and the total - # number of tokens equal to max_num_batched_tokens. - seqs: List[SequenceGroupMetadata] = [] - for group_id in range(max_num_seqs): - seq_len = (max_num_batched_tokens // max_num_seqs + (group_id < max_num_batched_tokens % max_num_seqs)) - seq_data = SequenceData([0] * seq_len) - seq = SequenceGroupMetadata( - request_id=str(group_id), - is_prompt=True, - seq_data={group_id: seq_data}, - sampling_params=sampling_params, - block_tables=None, - lora_request=dummy_lora_requests_per_seq[group_id] if dummy_lora_requests_per_seq else None, - ) - seqs.append(seq) - - # Run the model with the dummy inputs. - num_layers = self.model_config.get_num_layers(self.parallel_config) - kv_caches = [(None, None)] * num_layers - self.execute_model(seqs, kv_caches) - torch.cuda.synchronize() - return diff --git a/verl/third_party/vllm/vllm_v_0_3_1/parallel_state.py b/verl/third_party/vllm/vllm_v_0_3_1/parallel_state.py deleted file mode 100644 index c3b7a45c8d..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/parallel_state.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Adapted from -# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -"""Model and data parallel groups.""" - -import torch -import torch.distributed - -import vllm.model_executor.parallel_utils.parallel_state as ps -""" -This version is strongly tied with Megatron to implement HybridEngine and weight sharing between vllm and Megatron. -- We assume the Megatron tp+dp+pp world is already established before calling this function. - -""" - -# Tensor model parallel group that the current rank belongs to. -_TENSOR_MODEL_PARALLEL_GROUP = None - -# Micro Data parallel group. Micro data parallel group is additional dp group that origins from splitting training tp -# into infer_tp and micro_tp. By default, we use order micro_dp - tp -_MICRO_DATA_PARALLEL_GROUP = None - - -def initialize_model_parallel_from_megatron( - tensor_model_parallel_size=None # we set None for backward compatibility to set infer_tp = train_tp -) -> None: - from megatron.core import parallel_state as mpu - from megatron.distributed import new_group - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - - if tensor_model_parallel_size is None: - tensor_model_parallel_size = mpu.get_tensor_model_parallel_world_size() - else: - assert isinstance(tensor_model_parallel_size, int) - - # Build the tensor model-parallel groups. - assert ps._TENSOR_MODEL_PARALLEL_GROUP is None, ("tensor model parallel group is already initialized") - - assert tensor_model_parallel_size <= mpu.get_tensor_model_parallel_world_size( - ), 'Not implemented for infer_tp > train_tp' - - global _TENSOR_MODEL_PARALLEL_GROUP - global _MICRO_DATA_PARALLEL_GROUP - - assert mpu.get_tensor_model_parallel_world_size() % tensor_model_parallel_size == 0 - - micro_dp_size = mpu.get_tensor_model_parallel_world_size() // tensor_model_parallel_size - - world_size: int = torch.distributed.get_world_size() - - num_micro_dp_groups = world_size // micro_dp_size - - rank = torch.distributed.get_rank() - - # Build the micro dp groups. - assert _MICRO_DATA_PARALLEL_GROUP is None, ("micro data parallel group is already initialized") - for i in range(num_micro_dp_groups): - ranks = range(i * micro_dp_size, (i + 1) * micro_dp_size) - group = new_group(rank=rank, ranks=ranks, group_type='micro_dp') - if rank in ranks: - _MICRO_DATA_PARALLEL_GROUP = group - - if tensor_model_parallel_size == mpu.get_tensor_model_parallel_world_size(): - # using the same tp group as Megatron - ps._TENSOR_MODEL_PARALLEL_GROUP = mpu.get_tensor_model_parallel_group() - - _TENSOR_MODEL_PARALLEL_GROUP = mpu.get_tensor_model_parallel_group() - # no _MICRO_DATA_PARALLEL_GROUP - else: - # initialize a micro_dp group and a tp group - # assume training tp=4, infer tp=2, then, weight is partitioned as - # [1], [2], [3], [4] for training and [1,2], [1,2], [3,4], [3,4] for inference - - # Build the inference tp groups - train_tp = mpu.get_tensor_model_parallel_world_size() - num_tensor_model_parallel_groups_per_train_tp = train_tp // tensor_model_parallel_size - num_tensor_model_parallel_groups = world_size // tensor_model_parallel_size - assert _TENSOR_MODEL_PARALLEL_GROUP is None, ("tensor model parallel group is already initialized") - for i in range(num_tensor_model_parallel_groups // num_tensor_model_parallel_groups_per_train_tp): - start = train_tp * i - end = train_tp * (i + 1) - for j in range(num_tensor_model_parallel_groups_per_train_tp): - ranks = list(range(start, end, num_tensor_model_parallel_groups_per_train_tp)) - for i in range(len(ranks)): - ranks[i] += j - # group = torch.distributed.new_group(ranks) - group = new_group(rank=rank, ranks=ranks, group_type='infer_tp') - if rank in ranks: - _TENSOR_MODEL_PARALLEL_GROUP = group - ps._TENSOR_MODEL_PARALLEL_GROUP = _TENSOR_MODEL_PARALLEL_GROUP - # Build the pipeline model-parallel groups. - # global _PIPELINE_MODEL_PARALLEL_GROUP - # global _PIPELINE_GLOBAL_RANKS - # assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized") - - # ps._PIPELINE_MODEL_PARALLEL_GROUP = mpu.get_pipeline_model_parallel_group() - # ps._PIPELINE_GLOBAL_RANKS = mpu.get_pipeline_model_parallel_ranks() - - -""" -Tensor model parallel utilities -""" - - -def get_tensor_model_parallel_group(): - """Get the tensor model parallel group the caller rank belongs to.""" - assert _TENSOR_MODEL_PARALLEL_GROUP is not None, ("tensor model parallel group is not initialized") - return _TENSOR_MODEL_PARALLEL_GROUP - - -def get_tensor_model_parallel_world_size(): - """Return world size for the tensor model parallel group.""" - return torch.distributed.get_world_size(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_rank(): - """Return my rank for the tensor model parallel group.""" - return torch.distributed.get_rank(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_src_rank(): - """Calculate the global rank corresponding to the first local rank - in the tensor model parallel group.""" - global_rank = torch.distributed.get_rank() - local_world_size = get_tensor_model_parallel_world_size() - return (global_rank // local_world_size) * local_world_size - - -""" -Micro Data parallel group -""" - - -def get_micro_data_parallel_group(): - assert _MICRO_DATA_PARALLEL_GROUP is not None - return _MICRO_DATA_PARALLEL_GROUP - - -def get_micro_data_parallel_world_size(): - return torch.distributed.get_world_size(group=get_micro_data_parallel_group()) - - -def get_micro_data_parallel_rank(): - return torch.distributed.get_rank(group=get_micro_data_parallel_group()) diff --git a/verl/third_party/vllm/vllm_v_0_3_1/tokenizer.py b/verl/third_party/vllm/vllm_v_0_3_1/tokenizer.py deleted file mode 100644 index b8de24afb8..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/tokenizer.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/tokenizer_group/tokenizer_group.py - -from typing import List, Optional, Tuple, Union - -from transformers import (AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast) - -from vllm.lora.request import LoRARequest -from vllm.utils import make_async, LRUCache -from vllm.transformers_utils.tokenizers import * - - -class TokenizerGroup: - """A group of tokenizers that can be used for LoRA adapters.""" - - def __init__(self, tokenizer: PreTrainedTokenizer, enable_lora: bool, max_num_seqs: int, - max_input_length: Optional[int]): - self.enable_lora = enable_lora - self.max_input_length = max_input_length - self.tokenizer = tokenizer - if enable_lora: - self.lora_tokenizers = LRUCache(capacity=max_num_seqs) - else: - self.lora_tokenizers = None - - def encode(self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None) -> List[int]: - tokenizer = self.get_lora_tokenizer(lora_request) - return tokenizer.encode(prompt) - - async def encode_async(self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None) -> List[int]: - tokenizer = await self.get_lora_tokenizer_async(lora_request) - return tokenizer.encode(prompt) - - def get_lora_tokenizer(self, lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer": - if not lora_request or not self.enable_lora: - return self.tokenizer - if lora_request.lora_int_id not in self.lora_tokenizers: - # TODO(sgm): the lora tokenizer is also passed, but may be different - tokenizer = self.tokenizer - # tokenizer = (get_lora_tokenizer( - # lora_request, **self.tokenizer_config) or self.tokenizer) - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) - return tokenizer - else: - return self.lora_tokenizers.get(lora_request.lora_int_id) - - # FIXME(sgm): for simplicity, we assign the special token here - @property - def pad_token_id(self): - return self.tokenizer.pad_token_id - - @property - def eos_token_id(self): - return self.tokenizer.eos_token_id diff --git a/verl/third_party/vllm/vllm_v_0_3_1/weight_loaders.py b/verl/third_party/vllm/vllm_v_0_3_1/weight_loaders.py deleted file mode 100644 index 72aa26d060..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/weight_loaders.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models - -from typing import Dict -import torch -import torch.nn as nn - - -# NOTE(shengguangming): replace the origin weight loader function in the class -def parallel_weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Parallel Linear weight loader.""" - assert param.size() == loaded_weight.size( - ), 'the parameter size is not align with the loaded weight size, param size: {}, loaded_weight size: {}'.format( - param.size(), loaded_weight.size()) - assert param.data.dtype == loaded_weight.data.dtype, "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Default weight loader.""" - assert param.size() == loaded_weight.size() - assert param.data.dtype == loaded_weight.data.dtype, "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def gpt2_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "lm_head.weight" in name: - # GPT-2 ties the weights of the embedding layer and the final - # linear layer. - continue - if ".attn.bias" in name or ".attn.masked_bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - if not name.startswith("transformer."): - name = "transformer." + name - param = params_dict[name] - # The HF's GPT-2 implementation uses Conv1D instead of Linear. - # Because of this, we need to transpose the weights. - # Note(zhuohan): the logic below might break quantized models. - for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]: - if conv1d_weight_name not in name: - continue - if not name.endswith(".weight"): - continue - # TODO: check megatron - loaded_weight = loaded_weight.t() - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # NOTE(shengguangming): the megatron llama may have this prefix - prefix = '0.module.module.' - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if name[:len(prefix)] == prefix: - name = name[len(prefix):] - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def mistral_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # TODO: need to implement a general way to deal with prefix - prefix = '0.module.module.' - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if name[:len(prefix)] == prefix: - name = name[len(prefix):] - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) diff --git a/verl/third_party/vllm/vllm_v_0_3_1/worker.py b/verl/third_party/vllm/vllm_v_0_3_1/worker.py deleted file mode 100644 index 50eebd70b8..0000000000 --- a/verl/third_party/vllm/vllm_v_0_3_1/worker.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/worker.py -"""A GPU worker class.""" -import os -import gc -from typing import Dict, List, Tuple, Optional, Union, Set - -import torch -import torch.distributed -import torch.nn as nn - -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig, ParallelConfig, SchedulerConfig, LoRAConfig) -from vllm.model_executor import InputMetadata, set_random_seed -from vllm.model_executor.parallel_utils.parallel_state import (initialize_model_parallel) -from vllm.sampling_params import SamplingParams, SamplingType -from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata -from vllm.worker.cache_engine import CacheEngine -from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar -from vllm.model_executor.parallel_utils.parallel_state import get_tensor_model_parallel_group - -from .model_runner import ModelRunner -from .model_loader import load_weights -from .parallel_state import initialize_model_parallel_from_megatron -from vllm.lora.request import LoRARequest - - -class Worker: - """A worker class that executes (a partition of) the model on a GPU. - - Each worker is associated with a single GPU. The worker is responsible for - maintaining the KV cache and executing the model on the GPU. In case of - distributed inference, each worker is assigned a partition of the model. - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - rank: Optional[int] = None, - distributed_init_method: Optional[str] = None, - lora_config: Optional[LoRAConfig] = None, - kv_cache_dtype: Optional[str] = "auto", - ) -> None: - # self.model = model # will be replaced in the init_model - self.model_config = model_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.rank = rank - self.distributed_init_method = distributed_init_method - self.lora_config = lora_config - - self.model_runner = ModelRunner( - model, - model_config, - parallel_config, - scheduler_config, - device_config, - lora_config=self.lora_config, - kv_cache_dtype=kv_cache_dtype, - ) - - # Uninitialized cache engine. Will be initialized by - # self.init_cache_engine(). - self.cache_config = None - self.block_size = None - self.sliding_window = None - self.cache_engine = None - self.cache_events = None - self.gpu_cache = None - - # For offloading inference engine params - self.cpu_model = None - - def init_model(self, cupy_port: Optional[int] = None): - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # Env vars will be set by TORCHRUN. - self.rank = self.rank if self.rank is not None else int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.device = torch.device(f"cuda:{local_rank}") - if self.rank < 0: - raise ValueError("Invalid or unspecified rank.") - torch.cuda.set_device(self.device) - - _check_if_gpu_supports_dtype(self.model_config.dtype) - - # Initialize the distributed environment. - # TODO: do not use cupy - _init_distributed_environment(self.parallel_config, self.rank, self.distributed_init_method) - if not self.parallel_config.disable_custom_all_reduce: - init_custom_ar() - # Initialize the model. - set_random_seed(self.model_config.seed) - # self.model = get_model(actor_model=self.model, model_config=self.model_config) - - def load_model(self): - self.model_runner.load_model() - - @torch.inference_mode() - def profile_num_available_blocks( - self, - block_size: int, - gpu_memory_utilization: float, - cpu_swap_space: int, - cache_dtype: str, - ) -> Tuple[int, int]: - # Profile the memory usage of the model and get the maximum number of - # cache blocks that can be allocated with the remaining free memory. - torch.cuda.empty_cache() - # torch.cuda.reset_peak_memory_stats() - - # Execute a forward pass with dummy inputs to profile the memory usage - # of the model. - self.model_runner.profile_run() - - # Calculate the number of blocks that can be allocated with the - # profiled peak memory. - torch.cuda.synchronize() - free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info() - peak_memory = total_gpu_memory - free_gpu_memory - - cache_block_size = CacheEngine.get_cache_block_size(block_size, cache_dtype, self.model_config, - self.parallel_config) - # NOTE(sgm) use the remaining memory - num_gpu_blocks = int((free_gpu_memory * gpu_memory_utilization) // cache_block_size) - # num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization - peak_memory) // cache_block_size) - num_cpu_blocks = int(cpu_swap_space // cache_block_size) - num_gpu_blocks = max(num_gpu_blocks, 0) - num_cpu_blocks = max(num_cpu_blocks, 0) - if self.model_runner.lora_manager: - self.model_runner.remove_all_loras() - gc.collect() - torch.cuda.empty_cache() - # Synchronize number of blocks with all the rank - num_gpu_blocks = torch.tensor([num_gpu_blocks], device='cuda') - num_cpu_blocks = torch.tensor([num_cpu_blocks], device='cuda') - torch.distributed.all_reduce(num_gpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group()) - torch.distributed.all_reduce(num_cpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group()) - num_gpu_blocks = num_gpu_blocks.item() - num_cpu_blocks = num_cpu_blocks.item() - return num_gpu_blocks, num_cpu_blocks - - def init_cache_engine(self, cache_config: CacheConfig) -> None: - if self.cache_engine is None and self.gpu_cache is None: - self.cache_config = cache_config - self.cache_engine = CacheEngine(self.cache_config, self.model_config, self.parallel_config) - self.cache_events = self.cache_engine.events - self.gpu_cache = self.cache_engine.gpu_cache - self.model_runner.set_block_size(self.cache_engine.block_size) - - def free_cache_engine(self): - # ensure `enforce_eager=True` - self.cache_engine = None - self.gpu_cache = None - - def warm_up_model(self) -> None: - if not self.model_config.enforce_eager: - self.model_runner.capture_model(self.gpu_cache) - # Reset the seed to ensure that the random state is not affected by - # the model initialization and profiling. - set_random_seed(self.model_config.seed) - - def cache_swap( - self, - blocks_to_swap_in: Dict[int, int], - blocks_to_swap_out: Dict[int, int], - blocks_to_copy: Dict[int, List[int]], - ) -> None: - # Issue cache operations. - issued_cache_op = False - if blocks_to_swap_in: - self.cache_engine.swap_in(blocks_to_swap_in) - issued_cache_op = True - if blocks_to_swap_out: - self.cache_engine.swap_out(blocks_to_swap_out) - issued_cache_op = True - if blocks_to_copy: - self.cache_engine.copy(blocks_to_copy) - issued_cache_op = True - - cache_events = self.cache_events if issued_cache_op else None - - # Wait for cache operations to finish. - # TODO(woosuk): Profile swapping overhead and optimize if needed. - if cache_events is not None: - for event in cache_events: - event.wait() - - @torch.inference_mode() - def execute_model( - self, - seq_group_metadata_list: List[SequenceGroupMetadata], - blocks_to_swap_in: Dict[int, int], - blocks_to_swap_out: Dict[int, int], - blocks_to_copy: Dict[int, List[int]], - ) -> SamplerOutput: - num_seq_groups = len(seq_group_metadata_list) - self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy) - - # If there is no input, we don't need to execute the model. - if num_seq_groups == 0: - return {} - output = self.model_runner.execute_model(seq_group_metadata_list, self.gpu_cache) - return output - - # # Prepare input tensors. - # # NOTE(shengguangming): currently we pad in our dataloader and unpad it in pre_process_input, j - # # we can just input un-padded sequence for better performance - # input_tokens, input_positions, input_metadata = self._prepare_inputs(seq_group_metadata_list) - - # # Execute the model. - # output = self.model( - # input_ids=input_tokens, - # positions=input_positions, - # kv_caches=self.gpu_cache, - # input_metadata=input_metadata, - # cache_events=cache_events, - # ) - # return output - - # assume the input is .state_dict() - def sync_model_weights(self, actor_weights: Dict): - load_weights(actor_weights, self.model_runner.model) - - def offload_model_weights(self) -> None: - if self.cpu_model == None: - self.cpu_model = {} - for name, params in self.model_runner.model.named_parameters(): - self.cpu_model[name] = torch.empty_like(params, device='cpu') - params.data = self.cpu_model[name] - else: - for name, params in self.model_runner.model.named_parameters(): - params.data = self.cpu_model[name] - - def add_lora(self, lora_request: LoRARequest) -> bool: - return self.model_runner.add_lora(lora_request) - - def remove_lora(self, lora_id: int) -> bool: - return self.model_runner.remove_lora(lora_id) - - def list_loras(self) -> Set[int]: - return self.model_runner.list_loras() - - -def _init_distributed_environment( - parallel_config: ParallelConfig, - rank: int, - distributed_init_method: Optional[str] = None, -) -> None: - """Initialize the distributed environment.""" - if torch.distributed.is_initialized(): - print('The distributed environment has been initialized before vLLM') - elif not distributed_init_method: - raise ValueError("distributed_init_method must be set if torch.distributed " - "is not already initialized") - else: - torch.distributed.init_process_group( - backend="nccl", - world_size=parallel_config.world_size, - rank=rank, - # init_method=distributed_init_method, - ) - - # A small all_reduce for warmup. - torch.distributed.all_reduce(torch.zeros(1).cuda()) - # TODO (shengguangming): maybe we should also flag the megatron is initialized - if torch.distributed.get_world_size() > 1: - initialize_model_parallel_from_megatron(tensor_model_parallel_size=parallel_config.tensor_parallel_size) - else: - initialize_model_parallel() - - -def _pad_to_alignment(x: List[int], multiple_of: int, pad: int) -> List[int]: - return x + [pad] * ((-len(x)) % multiple_of) - - -def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]: - return x + [pad] * (max_len - len(x)) - - -def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype): - # Check if the GPU supports the dtype. - if torch_dtype == torch.bfloat16: - compute_capability = torch.cuda.get_device_capability() - if compute_capability[0] < 8: - gpu_name = torch.cuda.get_device_name() - raise ValueError("Bfloat16 is only supported on GPUs with compute capability " - f"of at least 8.0. Your {gpu_name} GPU has compute capability " - f"{compute_capability[0]}.{compute_capability[1]}.") diff --git a/verl/third_party/vllm/vllm_v_0_4_2/__init__.py b/verl/third_party/vllm/vllm_v_0_4_2/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/third_party/vllm/vllm_v_0_4_2/arg_utils.py b/verl/third_party/vllm/vllm_v_0_4_2/arg_utils.py deleted file mode 100644 index 089bbd748b..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/arg_utils.py +++ /dev/null @@ -1,320 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py - -import os -import argparse -import dataclasses -from dataclasses import dataclass -from typing import List, Optional, Union - -import torch.nn as nn - -from transformers import PretrainedConfig -from .config import ModelConfig, LoadConfig - -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, EngineConfig, LoRAConfig, ParallelConfig, - SchedulerConfig, SpeculativeConfig, TokenizerPoolConfig, VisionLanguageConfig) -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS -from vllm.utils import str_to_int_tuple - - -def nullable_str(val: str): - if not val or val == "None": - return None - return val - - -@dataclass -class EngineArgs: - """Arguments for vLLM engine.""" - model_hf_config: PretrainedConfig = None - skip_tokenizer_init: bool = False - served_model_name: Optional[Union[str, List[str]]] = None # TODO - download_dir: Optional[str] = None - load_format: str = 'auto' - dtype: str = 'auto' - kv_cache_dtype: str = 'auto' - quantization_param_path: Optional[str] = None - seed: int = 0 - max_model_len: Optional[int] = None - worker_use_ray: bool = False - pipeline_parallel_size: int = 1 - tensor_parallel_size: int = 1 - max_parallel_loading_workers: Optional[int] = None - block_size: int = 16 - enable_prefix_caching: bool = False - use_v2_block_manager: bool = False - swap_space: int = 4 # GiB - gpu_memory_utilization: float = 0.90 - max_num_batched_tokens: Optional[int] = None - max_num_seqs: int = 256 - max_logprobs: int = 5 # OpenAI default value - disable_log_stats: bool = False - revision: Optional[str] = None - code_revision: Optional[str] = None - tokenizer_revision: Optional[str] = None - quantization: Optional[str] = None - enforce_eager: bool = False - max_context_len_to_capture: Optional[int] = None - max_seq_len_to_capture: int = 8192 - disable_custom_all_reduce: bool = False - tokenizer_pool_size: int = 0 - tokenizer_pool_type: str = "ray" - tokenizer_pool_extra_config: Optional[dict] = None - enable_lora: bool = False - max_loras: int = 1 - max_lora_rank: int = 16 - fully_sharded_loras: bool = False - lora_extra_vocab_size: int = 256 - lora_dtype = 'auto' - max_cpu_loras: Optional[int] = None - device: str = 'auto' - ray_workers_use_nsight: bool = False - num_gpu_blocks_override: Optional[int] = None - num_lookahead_slots: int = 0 - model_loader_extra_config: Optional[dict] = None - - # Related to Vision-language models such as llava - image_input_type: Optional[str] = None - image_token_id: Optional[int] = None - image_input_shape: Optional[str] = None - image_feature_size: Optional[int] = None - scheduler_delay_factor: float = 0.0 - enable_chunked_prefill: bool = False - - guided_decoding_backend: str = 'outlines' - # Speculative decoding configuration. - speculative_model: Optional[str] = None - num_speculative_tokens: Optional[int] = None - speculative_max_model_len: Optional[int] = None - ngram_prompt_lookup_max: Optional[int] = None - ngram_prompt_lookup_min: Optional[int] = None - - @staticmethod - def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - """Shared CLI arguments for vLLM engine.""" - # Model arguments - # TODO(shengguangming): delete the unused args - parser.add_argument('--model', - type=str, - default='facebook/opt-125m', - help='name or path of the huggingface model to use') - parser.add_argument('--tokenizer', - type=str, - default=EngineArgs.tokenizer, - help='name or path of the huggingface tokenizer to use') - parser.add_argument('--revision', - type=str, - default=None, - help='the specific model version to use. It can be a branch ' - 'name, a tag name, or a commit id. If unspecified, will use ' - 'the default version.') - parser.add_argument('--tokenizer-revision', - type=str, - default=None, - help='the specific tokenizer version to use. It can be a branch ' - 'name, a tag name, or a commit id. If unspecified, will use ' - 'the default version.') - parser.add_argument('--tokenizer-mode', - type=str, - default=EngineArgs.tokenizer_mode, - choices=['auto', 'slow'], - help='tokenizer mode. "auto" will use the fast ' - 'tokenizer if available, and "slow" will ' - 'always use the slow tokenizer.') - parser.add_argument('--trust-remote-code', action='store_true', help='trust remote code from huggingface') - parser.add_argument('--download-dir', - type=str, - default=EngineArgs.download_dir, - help='directory to download and load the weights, ' - 'default to the default cache dir of ' - 'huggingface') - parser.add_argument('--load-format', - type=str, - default=EngineArgs.load_format, - choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'], - help='The format of the model weights to load. ' - '"auto" will try to load the weights in the safetensors format ' - 'and fall back to the pytorch bin format if safetensors format ' - 'is not available. ' - '"pt" will load the weights in the pytorch bin format. ' - '"safetensors" will load the weights in the safetensors format. ' - '"npcache" will load the weights in pytorch format and store ' - 'a numpy cache to speed up the loading. ' - '"dummy" will initialize the weights with random values, ' - 'which is mainly for profiling.') - parser.add_argument('--dtype', - type=str, - default=EngineArgs.dtype, - choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'], - help='data type for model weights and activations. ' - 'The "auto" option will use FP16 precision ' - 'for FP32 and FP16 models, and BF16 precision ' - 'for BF16 models.') - parser.add_argument('--max-model-len', - type=int, - default=None, - help='model context length. If unspecified, ' - 'will be automatically derived from the model.') - # Parallel arguments - parser.add_argument('--worker-use-ray', - action='store_true', - help='use Ray for distributed serving, will be ' - 'automatically set when using more than 1 GPU') - parser.add_argument('--pipeline-parallel-size', - '-pp', - type=int, - default=EngineArgs.pipeline_parallel_size, - help='number of pipeline stages') - parser.add_argument('--tensor-parallel-size', - '-tp', - type=int, - default=EngineArgs.tensor_parallel_size, - help='number of tensor parallel replicas') - # KV cache arguments - parser.add_argument('--block-size', - type=int, - default=EngineArgs.block_size, - choices=[8, 16, 32], - help='token block size') - # TODO(woosuk): Support fine-grained seeds (e.g., seed per request). - parser.add_argument('--seed', type=int, default=EngineArgs.seed, help='random seed') - parser.add_argument('--swap-space', - type=int, - default=EngineArgs.swap_space, - help='CPU swap space size (GiB) per GPU') - parser.add_argument('--gpu-memory-utilization', - type=float, - default=EngineArgs.gpu_memory_utilization, - help='the percentage of GPU memory to be used for' - 'the model executor') - parser.add_argument('--max-num-batched-tokens', - type=int, - default=EngineArgs.max_num_batched_tokens, - help='maximum number of batched tokens per ' - 'iteration') - parser.add_argument('--max-num-seqs', - type=int, - default=EngineArgs.max_num_seqs, - help='maximum number of sequences per iteration') - parser.add_argument('--disable-log-stats', action='store_true', help='disable logging statistics') - # Quantization settings. - parser.add_argument('--quantization', - '-q', - type=str, - choices=['awq', None], - default=None, - help='Method used to quantize the weights') - return parser - - @classmethod - def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs': - # Get the list of attributes of this dataclass. - attrs = [attr.name for attr in dataclasses.fields(cls)] - # Set the attributes from the parsed arguments. - engine_args = cls(**{attr: getattr(args, attr) for attr in attrs}) - return engine_args - - def create_engine_config( - self, - ) -> EngineConfig: - device_config = DeviceConfig(self.device) - # NOTE(sgm): we only modify ModelConfig, other configs are import from vllm - model_config = ModelConfig(self.model_hf_config, self.dtype, self.seed, self.revision, self.code_revision, - self.tokenizer_revision, self.max_model_len, self.quantization, - self.quantization_param_path, self.enforce_eager, self.max_context_len_to_capture, - self.max_seq_len_to_capture, self.max_logprobs, self.skip_tokenizer_init, - self.served_model_name) - cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization, - self.swap_space, self.kv_cache_dtype, self.num_gpu_blocks_override, - model_config.get_sliding_window(), self.enable_prefix_caching) - parallel_config = ParallelConfig( - self.pipeline_parallel_size, self.tensor_parallel_size, self.worker_use_ray, - self.max_parallel_loading_workers, self.disable_custom_all_reduce, - TokenizerPoolConfig.create_config( - self.tokenizer_pool_size, - self.tokenizer_pool_type, - self.tokenizer_pool_extra_config, - ), self.ray_workers_use_nsight) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - parallel_config.world_size = world_size - - # TODO: spec config - speculative_config = SpeculativeConfig.maybe_create_spec_config( - target_model_config=model_config, - target_parallel_config=parallel_config, - target_dtype=self.dtype, - speculative_model=self.speculative_model, - num_speculative_tokens=self.num_speculative_tokens, - speculative_max_model_len=self.speculative_max_model_len, - enable_chunked_prefill=self.enable_chunked_prefill, - use_v2_block_manager=self.use_v2_block_manager, - ngram_prompt_lookup_max=self.ngram_prompt_lookup_max, - ngram_prompt_lookup_min=self.ngram_prompt_lookup_min, - ) - - scheduler_config = SchedulerConfig( - self.max_num_batched_tokens, - self.max_num_seqs, - model_config.max_model_len, - self.use_v2_block_manager, - num_lookahead_slots=(self.num_lookahead_slots - if speculative_config is None else speculative_config.num_lookahead_slots), - delay_factor=self.scheduler_delay_factor, - enable_chunked_prefill=self.enable_chunked_prefill, - ) - - lora_config = LoRAConfig(max_lora_rank=self.max_lora_rank, - max_loras=self.max_loras, - fully_sharded_loras=self.fully_sharded_loras, - lora_extra_vocab_size=self.lora_extra_vocab_size, - lora_dtype=self.lora_dtype, - max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras and self.max_cpu_loras > 0 else - None) if self.enable_lora else None - - load_config = LoadConfig( - load_format=self.load_format, - download_dir=self.download_dir, - model_loader_extra_config=self.model_loader_extra_config, - ) - - if self.image_input_type: - if (not self.image_token_id or not self.image_input_shape or not self.image_feature_size): - raise ValueError('Specify `image_token_id`, `image_input_shape` and ' - '`image_feature_size` together with `image_input_type`.') - vision_language_config = VisionLanguageConfig( - image_input_type=VisionLanguageConfig.get_image_input_enum_type(self.image_input_type), - image_token_id=self.image_token_id, - image_input_shape=str_to_int_tuple(self.image_input_shape), - image_feature_size=self.image_feature_size, - ) - else: - vision_language_config = None - - decoding_config = DecodingConfig(guided_decoding_backend=self.guided_decoding_backend) - - return EngineConfig(model_config=model_config, - cache_config=cache_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - device_config=device_config, - lora_config=lora_config, - vision_language_config=vision_language_config, - speculative_config=speculative_config, - load_config=load_config, - decoding_config=decoding_config) diff --git a/verl/third_party/vllm/vllm_v_0_4_2/config.py b/verl/third_party/vllm/vllm_v_0_4_2/config.py deleted file mode 100644 index 6af04417b4..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/config.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py - -import enum -import json -from typing import List, Optional, Union -from dataclasses import dataclass, field, fields - -from transformers import PretrainedConfig - -from vllm.logger import init_logger -from vllm.model_executor.layers.quantization import get_quantization_config -from vllm.transformers_utils.config import get_hf_text_config -from vllm.utils import is_hip -# Add for verl -from vllm.config import ModelConfig, _get_and_verify_dtype, _get_and_verify_max_len - -GPTQMarlinConfig = get_quantization_config("gptq_marlin") - -logger = init_logger(__name__) - -_GB = 1 << 30 - - -class ModelConfig(ModelConfig): - """Configuration for the model. - - Args: - model: Name or path of the huggingface model to use. - tokenizer: Name or path of the huggingface tokenizer to use. - tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if - available, and "slow" will always use the slow tokenizer. - trust_remote_code: Trust remote code (e.g., from HuggingFace) when - downloading the model and tokenizer. - download_dir: Directory to download and load the weights, default to the - default cache directory of huggingface. - load_format: The format of the model weights to load: - "auto" will try to load the weights in the safetensors format and - fall back to the pytorch bin format if safetensors format is - not available. - "pt" will load the weights in the pytorch bin format. - "safetensors" will load the weights in the safetensors format. - "npcache" will load the weights in pytorch format and store - a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is - mainly for profiling. - dtype: Data type for model weights and activations. The "auto" option - will use FP16 precision for FP32 and FP16 models, and BF16 precision - for BF16 models. - seed: Random seed for reproducibility. - revision: The specific model version to use. It can be a branch name, - a tag name, or a commit id. If unspecified, will use the default - version. - code_revision: The specific revision to use for the model code on - Hugging Face Hub. It can be a branch name, a tag name, or a - commit id. If unspecified, will use the default version. - tokenizer_revision: The specific tokenizer version to use. It can be a - branch name, a tag name, or a commit id. If unspecified, will use - the default version. - max_model_len: Maximum length of a sequence (including prompt and - output). If None, will be derived from the model. - quantization: Quantization method that was used to quantize the model - weights. If None, we assume the model weights are not quantized. - quantization_param_path: Path to JSON file containing scaling factors. - Used to load KV cache scaling factors into the model when KV cache - type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also - be used to load activation and weight scaling factors when the - model dtype is FP8_E4M3 on ROCm. - enforce_eager: Whether to enforce eager execution. If True, we will - disable CUDA graph and always execute the model in eager mode. - If False, we will use CUDA graph and eager execution in hybrid. - max_context_len_to_capture: Maximum context len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode (DEPRECATED. Use max_seq_len_to_capture instead). - max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode - skip_tokenizer_init: If true, skip initialization of tokenizer and - detokenizer. - served_model_name: The model name used in metrics tag `model_name`, - matches the model name exposed via the APIs. If multiple model - names provided, the first name will be used. If not specified, - the model name will be the same as `model`. - """ - - def __init__( - self, - hf_config: PretrainedConfig, - dtype: str, - seed: int, - revision: Optional[str] = None, - code_revision: Optional[str] = None, - tokenizer_revision: Optional[str] = None, - max_model_len: Optional[int] = None, - quantization: Optional[str] = None, - quantization_param_path: Optional[str] = None, - enforce_eager: bool = False, - max_context_len_to_capture: Optional[int] = None, - max_seq_len_to_capture: Optional[int] = None, - max_logprobs: int = 5, - skip_tokenizer_init: bool = False, - served_model_name: Optional[Union[str, List[str]]] = None, - ) -> None: - self.model = hf_config._name_or_path - self.tokenizer = hf_config._name_or_path - self.seed = seed - self.revision = revision - self.code_revision = code_revision - self.tokenizer_revision = tokenizer_revision - self.quantization = quantization - self.quantization_param_path = quantization_param_path - self.enforce_eager = enforce_eager - self.max_context_len_to_capture = max_context_len_to_capture - if self.max_context_len_to_capture is not None: - raise ValueError("`max_context_len_to_capture` is deprecated. " - "Use `max_seq_len_to_capture` instead.") - self.max_seq_len_to_capture = (max_seq_len_to_capture or max_context_len_to_capture) - self.max_logprobs = max_logprobs - self.skip_tokenizer_init = skip_tokenizer_init - - # self.hf_config = get_config(model, trust_remote_code, revision) - self.hf_config = hf_config - self.hf_text_config = get_hf_text_config(hf_config) - # TODO: for multimodal model - self.dtype = _get_and_verify_dtype(self.hf_config, dtype) - self.max_model_len = _get_and_verify_max_len(self.hf_config, max_model_len) - # self.served_model_name = get_served_model_name(model, - # served_model_name) - # self._verify_load_format() - # self._verify_tokenizer_mode() - self._verify_quantization() - self._verify_cuda_graph() - - -class LoadFormat(str, enum.Enum): - AUTO = 'auto' - MEGATRON = "megatron" - HF = "hf" - DTENSOR = 'dtensor' - DUMMY_HF = 'dummy_hf' - DUMMY_MEGATRON = 'dummy_megatron' - DUMMY_DTENSOR = 'dummy_dtensor' - - -@dataclass -class LoadConfig: - """ - download_dir: Directory to download and load the weights, default to the - default cache directory of huggingface. - load_format: The format of the model weights to load: - "auto" will try to load the weights in the safetensors format and - fall back to the pytorch bin format if safetensors format is - not available. - "pt" will load the weights in the pytorch bin format. - "safetensors" will load the weights in the safetensors format. - "npcache" will load the weights in pytorch format and store - a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is - mainly for profiling. - "tensorizer" will use CoreWeave's tensorizer library for - fast weight loading. - """ - - load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO - download_dir: Optional[str] = None - model_loader_extra_config: Optional[Union[str, dict]] = field(default_factory=dict) - - def __post_init__(self): - model_loader_extra_config = self.model_loader_extra_config or {} - if isinstance(model_loader_extra_config, str): - self.model_loader_extra_config = json.loads(model_loader_extra_config) - self._verify_load_format() - - def _verify_load_format(self) -> None: - if not isinstance(self.load_format, str): - return - - load_format = self.load_format.lower() - self.load_format = LoadFormat(load_format) - - rocm_not_supported_load_format: List[str] = [] - if is_hip() and load_format in rocm_not_supported_load_format: - rocm_supported_load_format = [ - f for f in LoadFormat.__members__ if (f not in rocm_not_supported_load_format) - ] - raise ValueError(f"load format '{load_format}' is not supported in ROCm. " - f"Supported load formats are " - f"{rocm_supported_load_format}") diff --git a/verl/third_party/vllm/vllm_v_0_4_2/dtensor_weight_loaders.py b/verl/third_party/vllm/vllm_v_0_4_2/dtensor_weight_loaders.py deleted file mode 100644 index 6668b75091..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/dtensor_weight_loaders.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models - -from typing import Dict, Iterable, Tuple -import torch -import torch.nn as nn -from torch.distributed._tensor import DTensor, Shard, Replicate - -from vllm.model_executor.layers.linear import * -from vllm.model_executor.models import ModelRegistry -from vllm.model_executor.model_loader.weight_utils import default_weight_loader - - -def gemma_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - for (param_name, shard_name, shard_id) in stacked_params_mapping: - if shard_name not in name: - continue - stacked_name = name.replace(shard_name, param_name) - # Skip loading extra bias for GPTQ models. - if stacked_name.endswith(".bias") and stacked_name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[stacked_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # lm_head is not used in vllm as it is tied with embed_token. - # To prevent errors, skip loading lm_head.weight. - if "lm_head.weight" in name: - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # GemmaRMSNorm is different from Llama's in that it multiplies - # (1 + weight) to the output, instead of just weight. - if "norm.weight" in name: - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - - norm_weight = local_loaded_weight + 1.0 - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, norm_weight.to(dtype=param.dtype)) - else: - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def gptbigcode_dtensor_load_weights(actor_weights: Dict, vllm_model: nn.Module): - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "lm_head.weight" in name: - continue - if ".attn.bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def starcoder2_dtensor_load_weights(actor_weights: Dict, vllm_model: nn.Module): - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - - for (param_name, weight_name, shard_id) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def llama_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if ("rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name): - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - # With tie_word_embeddings, we can skip lm_head.weight - # The weight might appear unnecessarily in the files if the model is - # processed with quantization, LoRA, fine-tuning, etc. - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - for (param_name, weight_name, shard_id) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight) - - -def qwen2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - for (param_name, weight_name, shard_id) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def gpt2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - pass - - -def redistribute_dtensor(param_name: str, loaded_weights: DTensor, parallelize_plan: Dict = None): - param_name = _process_parameter_names(name=param_name) - if parallelize_plan is not None: - assert param_name in parallelize_plan.keys(), \ - f"param name: {param_name} not in parallelize_plan :{parallelize_plan.keys()}" - placement = parallelize_plan[param_name] - local_loaded_weights = loaded_weights.redistribute(device_mesh=loaded_weights.device_mesh, - placements=placement).to_local() - else: - local_loaded_weights = loaded_weights.full_tensor() - return local_loaded_weights - - -def _process_parameter_names(name): - # Remove '.weight' if it exists at the end of the string - if name.endswith(".weight"): - name = name[:-7] - - # Remove 'model.layers.x.' or 'model.' prefix - if "model.layers" in name: - parts = name.split('.') - # Reconstruct the string without 'model.layers.x.' - name = '.'.join(parts[3:]) # parts[0] is 'model', parts[1] is 'layers', parts[2] is 'x' - elif name.startswith("model."): - name = name[6:] # Remove 'model.' - - return name - - -__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__ = { - 'GPT2LMHeadModel': gpt2_dtensor_weight_loader, - 'LlamaForCausalLM': llama_dtensor_weight_loader, - 'LLaMAForCausalLM': llama_dtensor_weight_loader, - 'MistralForCausalLM': llama_dtensor_weight_loader, # mistral is the same as llama in vLLM - 'InternLMForCausalLM': llama_dtensor_weight_loader, - 'AquilaModel': llama_dtensor_weight_loader, - 'AquilaForCausalLM': llama_dtensor_weight_loader, - 'Phi3ForCausalLM': llama_dtensor_weight_loader, - 'GemmaForCausalLM': gemma_dtensor_weight_loader, - 'GPTBigCodeForCausalLM': gptbigcode_dtensor_load_weights, - 'Starcoder2ForCausalLM': starcoder2_dtensor_load_weights, - 'Qwen2ForCausalLM': qwen2_dtensor_weight_loader -} - - -# the actor model is .state_dict() -# Load dtensor weights -def load_dtensor_weights(actor_weights: Dict, vllm_model: nn.Module): - weight_loader = _get_model_weight_loader(vllm_model.__class__.__name__) - weight_loader(actor_weights, vllm_model) - # NOTE(sgm) to reduce peak memory usage, we offload vllm model to cpu - # after init, and we need this after sync model weights for in first iter. - vllm_model = vllm_model.cuda() - - -def _get_model_weight_loader(arch: str): - if arch in __MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__: - return __MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__[arch] - raise ValueError(f"Model architectures {arch} are not supported for now. " - f"Supported architectures: {__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__.keys()}") - - -# NOTE(sgm): we use per-parameter weight loader in each vllm sub -def update_dtensor_weight_loader(): - pass diff --git a/verl/third_party/vllm/vllm_v_0_4_2/hf_weight_loader.py b/verl/third_party/vllm/vllm_v_0_4_2/hf_weight_loader.py deleted file mode 100644 index 0d562e596b..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/hf_weight_loader.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models - -from typing import Dict, Union, Optional, Iterable, Tuple - -import torch -import torch.nn as nn - -from vllm.model_executor.model_loader.utils import set_default_torch_dtype -from vllm.model_executor.model_loader.weight_utils import default_weight_loader - - -def update_hf_weight_loader(): - from vllm.model_executor.models.gemma import GemmaForCausalLM - GemmaForCausalLM.load_weights = gemma_load_weights - - -def gemma_load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params = set() - for name, loaded_weight in weights: - for (param_name, shard_name, shard_id) in stacked_params_mapping: - if shard_name not in name: - continue - name = name.replace(shard_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # lm_head is not used in vllm as it is tied with embed_token. - # To prevent errors, skip loading lm_head.weight. - if "lm_head.weight" in name: - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # GemmaRMSNorm is different from Llama's in that it multiplies - # (1 + weight) to the output, instead of just weight. - if "norm.weight" in name: - norm_weight = loaded_weight + 1.0 # prevent inplace modify actor weights - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, norm_weight) - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - unloaded_params = params_dict.keys() - loaded_params - if unloaded_params: - raise RuntimeError("Some weights are not initialized from checkpoints: " - f"{unloaded_params}") - - -def load_hf_weights(actor_weights: Dict, vllm_model: nn.Module): - assert isinstance(actor_weights, Dict) - with set_default_torch_dtype(next(vllm_model.parameters()).dtype): # TODO - vllm_model.load_weights(actor_weights.items()) - for _, module in vllm_model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - vllm_model = vllm_model.cuda() diff --git a/verl/third_party/vllm/vllm_v_0_4_2/llm.py b/verl/third_party/vllm/vllm_v_0_4_2/llm.py deleted file mode 100644 index 94623a4142..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/llm.py +++ /dev/null @@ -1,306 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/llm.py - -from typing import Dict, List, Optional, Tuple, Union - -from tqdm import tqdm -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast -from transformers import PretrainedConfig -import torch.nn as nn -from .arg_utils import EngineArgs -from .llm_engine_sp import LLMEngine -from vllm.lora.request import LoRARequest -from vllm.outputs import RequestOutput -from vllm.sampling_params import SamplingParams -from vllm.sequence import MultiModalData -from vllm.usage.usage_lib import UsageContext -from vllm.utils import Counter -import torch -from torch.nn.utils.rnn import pad_sequence -from verl.workers.rollout.tokenizer import HybridEngineBaseTokenizer - - -class LLM: - """An LLM for generating texts from given prompts and sampling parameters. - - This class includes a tokenizer, a language model (possibly distributed - across multiple GPUs), and GPU memory space allocated for intermediate - states (aka KV cache). Given a batch of prompts and sampling parameters, - this class generates texts from the model, using an intelligent batching - mechanism and efficient memory management. - - NOTE: This class is intended to be used for offline inference. For online - serving, use the `AsyncLLMEngine` class instead. - NOTE: For the comprehensive list of arguments, see `EngineArgs`. - - Args: - model: A HuggingFace Transformers model instance. - tokenizer: A HuggingFace Transformers tokenizer instance. - tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer - if available, and "slow" will always use the slow tokenizer. - trust_remote_code: Trust remote code (e.g., from HuggingFace) when - downloading the model and tokenizer. - tensor_parallel_size: The number of GPUs to use for distributed - execution with tensor parallelism. - dtype: The data type for the model weights and activations. Currently, - we support `float32`, `float16`, and `bfloat16`. If `auto`, we use - the `torch_dtype` attribute specified in the model config file. - However, if the `torch_dtype` in the config is `float32`, we will - use `float16` instead. - quantization: The method used to quantize the model weights. Currently, - we support "awq". If None, we assume the model weights are not - quantized and use `dtype` to determine the data type of the weights. - revision: The specific model version to use. It can be a branch name, - a tag name, or a commit id. - tokenizer_revision: The specific tokenizer version to use. It can be a - branch name, a tag name, or a commit id. - seed: The seed to initialize the random number generator for sampling. - gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to - reserve for the model weights, activations, and KV cache. Higher - values will increase the KV cache size and thus improve the model's - throughput. However, if the value is too high, it may cause out-of- - memory (OOM) errors. - swap_space: The size (GiB) of CPU memory per GPU to use as swap space. - This can be used for temporarily storing the states of the requests - when their `best_of` sampling parameters are larger than 1. If all - requests will have `best_of=1`, you can safely set this to 0. - Otherwise, too small values may cause out-of-memory (OOM) errors. - enforce_eager: Whether to enforce eager execution. If True, we will - disable CUDA graph and always execute the model in eager mode. - If False, we will use CUDA graph and eager execution in hybrid. - max_context_len_to_capture: Maximum context len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode. - disable_custom_all_reduce: See ParallelConfig - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer], - model_hf_config: PretrainedConfig, - tokenizer_mode: str = "auto", - trust_remote_code: bool = False, - tensor_parallel_size: int = 1, - dtype: str = "auto", - quantization: Optional[str] = None, - revision: Optional[str] = None, - tokenizer_revision: Optional[str] = None, - seed: int = 0, - gpu_memory_utilization: float = 0.9, - swap_space: int = 4, - enforce_eager: bool = False, - max_context_len_to_capture: int = None, - disable_custom_all_reduce: bool = False, - load_format = 'auto', - **kwargs, - ) -> None: - if "disable_log_stats" not in kwargs: - kwargs["disable_log_stats"] = True - engine_args = EngineArgs( - model_hf_config=model_hf_config, - tensor_parallel_size=tensor_parallel_size, - dtype=dtype, - quantization=quantization, - revision=revision, - tokenizer_revision=tokenizer_revision, - seed=seed, - gpu_memory_utilization=gpu_memory_utilization, - swap_space=swap_space, - enforce_eager=enforce_eager, - max_context_len_to_capture=max_context_len_to_capture, - disable_custom_all_reduce=disable_custom_all_reduce, - load_format=load_format, - **kwargs, - ) - tokenizer_cls = (PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer) - if not isinstance(tokenizer, tokenizer_cls): - raise ValueError( - f"Unexpected tokenizer type: {type(tokenizer)}. Must be" - "one of the following: PreTrainedTokenizer, PreTrainedTokenizerFast, verl.workers.rollout.HybridEngineBaseTokenizer" - ) - self.llm_engine = LLMEngine.from_engine_args(model, tokenizer, engine_args) - self.request_counter = Counter() - - def init_cache_engine(self): - self.llm_engine.init_cache_engine() - - def free_cache_engine(self): - self.llm_engine.free_cache_engine() - - def get_tokenizer(self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: - return self.llm_engine.tokenizer - - def set_tokenizer( - self, - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], - ) -> None: - self.llm_engine.tokenizer = tokenizer - - def generate( - self, - prompts: Optional[Union[str, List[str]]] = None, - sampling_params: Optional[Union[SamplingParams, List[SamplingParams]]] = None, - prompt_token_ids: Optional[List[List[int]]] = None, - use_tqdm: bool = True, - lora_request: Optional[LoRARequest] = None, - multi_modal_data: Optional[MultiModalData] = None, - ) -> List[RequestOutput]: - """Generates the completions for the input prompts. - - NOTE: This class automatically batches the given prompts, considering - the memory constraint. For the best performance, put all of your prompts - into a single list and pass it to this method. - - Args: - prompts: A list of prompts to generate completions for. - sampling_params: The sampling parameters for text generation. If - None, we use the default sampling parameters. - When it is a single value, it is applied to every prompt. - When it is a list, the list must have the same length as the - prompts and it is paired one by one with the prompt. - prompt_token_ids: A list of token IDs for the prompts. If None, we - use the tokenizer to convert the prompts to token IDs. - use_tqdm: Whether to use tqdm to display the progress bar. - lora_request: LoRA request to use for generation, if any. - multi_modal_data: Multi modal data. - - Returns: - A list of `RequestOutput` objects containing the generated - completions in the same order as the input prompts. - """ - if prompts is None and prompt_token_ids is None: - raise ValueError("Either prompts or prompt_token_ids must be " - "provided.") - if self.llm_engine.model_config.skip_tokenizer_init \ - and prompts is not None: - raise ValueError("prompts must be None if skip_tokenizer_init " - "is True") - if isinstance(prompts, str): - # Convert a single prompt to a list. - prompts = [prompts] - if (prompts is not None and prompt_token_ids is not None and len(prompts) != len(prompt_token_ids)): - raise ValueError("The lengths of prompts and prompt_token_ids " - "must be the same.") - - if prompts is not None: - num_requests = len(prompts) - else: - assert prompt_token_ids is not None - num_requests = len(prompt_token_ids) - - if sampling_params is None: - # Use default sampling params. - sampling_params = SamplingParams() - - elif isinstance(sampling_params, list) and len(sampling_params) != num_requests: - raise ValueError("The lengths of prompts and sampling_params " - "must be the same.") - if multi_modal_data: - multi_modal_data.data = multi_modal_data.data.to(torch.float16) - - # Add requests to the engine. - for i in range(num_requests): - prompt = prompts[i] if prompts is not None else None - token_ids = None if prompt_token_ids is None else prompt_token_ids[i] - if not isinstance(token_ids, list): - # NOTE(shengguangming): convert the rollout input into List[str] - token_ids = self._pre_process_inputs(token_ids) - self._add_request( - prompt, - sampling_params[i] if isinstance(sampling_params, list) else sampling_params, - token_ids, - lora_request=lora_request, - # Get ith image while maintaining the batch dim. - multi_modal_data=MultiModalData(type=multi_modal_data.type, data=multi_modal_data.data[i].unsqueeze(0)) - if multi_modal_data else None, - ) - return self._run_engine(use_tqdm) - - def _add_request( - self, - prompt: Optional[str], - sampling_params: SamplingParams, - prompt_token_ids: Optional[List[int]], - lora_request: Optional[LoRARequest] = None, - multi_modal_data: Optional[MultiModalData] = None, - ) -> None: - request_id = str(next(self.request_counter)) - self.llm_engine.add_request(request_id, - prompt, - sampling_params, - prompt_token_ids, - lora_request=lora_request, - multi_modal_data=multi_modal_data) - - def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]: - # Initialize tqdm. - if use_tqdm: - num_requests = self.llm_engine.get_num_unfinished_requests() - pbar = tqdm(total=num_requests, desc="Processed prompts", dynamic_ncols=True) - # Run the engine. - outputs: List[RequestOutput] = [] - while self.llm_engine.has_unfinished_requests(): - step_outputs = self.llm_engine.step() - for output in step_outputs: - if output.finished: - outputs.append(output) - if use_tqdm: - pbar.update(1) - if use_tqdm: - pbar.close() - # Sort the outputs by request ID. - # This is necessary because some requests may be finished earlier than - # its previous requests. - outputs = sorted(outputs, key=lambda x: int(x.request_id)) - # TODO(shengguangming): maybe we can hack the autoregressive logics without only apply post process for better performance - return self._post_process_outputs(outputs) - - # NOTE(shengguangming): add for verl - # TODO(sgm): we can optimize it by making the dataloader yield List[int] without padding. - def _pre_process_inputs(self, prompt_token_ids: torch.Tensor) -> List[int]: - # remove the left padding in the prompt token_id - pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = prompt_token_ids[non_pad_index:].tolist() - return token_ids - - # NOTE(shengguangming): add for verl - def _post_process_outputs(self, request_outputs: List[RequestOutput]) -> Tuple[torch.Tensor, torch.Tensor]: - output_token_ids = [] - logprobs = [] - for request_output in request_outputs: # List[RequestOutput] - outputs = request_output.outputs - for output in outputs: # List[CompletionOutput], usually len == 1 - output_token_ids.append(torch.tensor(output.token_ids)) - # TODO(shengguangming): can be optimzied by rewrite the Sampler._get_logprobs() logits - logprobs_dicts = output.logprobs - if logprobs_dicts is not None: - logprob = [] - for logprobs_dict, id in zip(logprobs_dicts, output.token_ids): - logprob.append(logprobs_dict[id].logprob) - logprobs.append(torch.tensor(logprob)) - - pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - output_token_ids = pad_sequence(output_token_ids, batch_first=True, padding_value=pad_token_id) - if len(logprobs) > 0: - logprobs = pad_sequence(logprobs, batch_first=True, padding_value=pad_token_id) - return output_token_ids, logprobs - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.llm_engine.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - def offload_model_weights(self) -> None: - self.llm_engine.offload_model_weights() diff --git a/verl/third_party/vllm/vllm_v_0_4_2/llm_engine_sp.py b/verl/third_party/vllm/vllm_v_0_4_2/llm_engine_sp.py deleted file mode 100644 index 75bf11ab31..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/llm_engine_sp.py +++ /dev/null @@ -1,283 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/llm_engine.py - -import torch -from typing import Dict, Optional, Union, Type - -import vllm -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoRAConfig, ParallelConfig, SchedulerConfig, - SpeculativeConfig, VisionLanguageConfig) -from vllm.core.scheduler import Scheduler -from vllm.engine.output_processor.interfaces import (SequenceGroupOutputProcessor) -from vllm.engine.output_processor.stop_checker import StopChecker -from vllm.executor.executor_base import ExecutorBase -from vllm.logger import init_logger -from vllm.transformers_utils.detokenizer import Detokenizer -from vllm.engine.metrics import StatLogger -from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled, usage_message) -from vllm.utils import Counter -from vllm.engine.llm_engine import _load_generation_config_dict -from vllm.engine.llm_engine import LLMEngine - -import torch.nn as nn -from .arg_utils import EngineArgs -from .tokenizer import TokenizerGroup -from .config import ModelConfig, LoadConfig - -logger = init_logger(__name__) -_LOCAL_LOGGING_INTERVAL_SEC = 5 - - -class LLMEngine(LLMEngine): - """An LLM engine that receives requests and generates texts. - - This is the main class for the vLLM engine. It receives requests - from clients and generates texts from the LLM. It includes a tokenizer, a - language model (possibly distributed across multiple GPUs), and GPU memory - space allocated for intermediate states (aka KV cache). This class utilizes - iteration-level scheduling and efficient memory management to maximize the - serving throughput. - - The `LLM` class wraps this class for offline batched inference and the - `AsyncLLMEngine` class wraps this class for online serving. - - NOTE: The config arguments are derived from the `EngineArgs` class. For the - comprehensive list of arguments, see `EngineArgs`. - - Args: - model: the actor model initialize outside vllm (add for verl) - tokenizer: the initialized tokenizer (add for verl) - model_config: The configuration related to the LLM model. - cache_config: The configuration related to the KV cache memory - management. - parallel_config: The configuration related to distributed execution. - scheduler_config: The configuration related to the request scheduler. - distributed_init_method: The initialization method for distributed - execution. See `torch.distributed.init_process_group` for details. - placement_group: Ray placement group for distributed execution. - Required for distributed execution. - log_stats: Whether to log statistics. - """ - - def __init__( - self, - # NOTE(sgm): first two arguments are added for verl - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: nn.Module, - # NOTE(sgm): vllm original arguments - model_config: ModelConfig, - cache_config: CacheConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - vision_language_config: Optional[VisionLanguageConfig], - speculative_config: Optional[SpeculativeConfig], - decoding_config: Optional[DecodingConfig], - executor_class: Type[ExecutorBase], - log_stats: bool, - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, - ) -> None: - logger.info( - "Initializing an LLM engine (v%s) with config: " - "model=%r, speculative_config=%r, tokenizer=%r, " - "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, " - "tokenizer_revision=%s, trust_remote_code=%s, dtype=%s, " - "max_seq_len=%d, download_dir=%r, load_format=%s, " - "tensor_parallel_size=%d, disable_custom_all_reduce=%s, " - "quantization=%s, enforce_eager=%s, kv_cache_dtype=%s, " - "quantization_param_path=%s, device_config=%s, " - "decoding_config=%r, seed=%d, served_model_name=%s)", - vllm.__version__, - model_config.model, - speculative_config, - model_config.tokenizer, - model_config.skip_tokenizer_init, - # model_config.tokenizer_mode, - model_config.revision, - model_config.tokenizer_revision, - # model_config.trust_remote_code, - model_config.dtype, - model_config.max_model_len, - load_config.download_dir, - load_config.load_format, - parallel_config.tensor_parallel_size, - parallel_config.disable_custom_all_reduce, - model_config.quantization, - model_config.enforce_eager, - cache_config.cache_dtype, - model_config.quantization_param_path, - device_config.device, - decoding_config, - model_config.seed, - # model_config.served_model_name, - ) - # TODO(woosuk): Print more configs in debug mode. - - self.model_config = model_config # TODO: currently is hfconfig - self.cache_config = cache_config - self.lora_config = lora_config - self.vision_language_config = vision_language_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.speculative_config = speculative_config - self.load_config = load_config - self.decoding_config = decoding_config or DecodingConfig() - self.log_stats = log_stats - - # self.model = model # should not store the model, it should be deleted - # TODO(shengguangming): maybe we can choose init here or from arguments - if not self.model_config.skip_tokenizer_init: - # TODO: check tokenizer class - self._init_tokenizer(tokenizer) - self.detokenizer = Detokenizer(self.tokenizer) - else: - self.detokenizer = None - self.tokenizer = None - - self.seq_counter = Counter() - # TODO: don't know what's the usage - self.generation_config_fields = _load_generation_config_dict(model_config) - - self.model_executor = executor_class( - model=model, # add for spmd_gpu_executor - model_config=model_config, - cache_config=cache_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - device_config=device_config, - lora_config=lora_config, - vision_language_config=vision_language_config, - speculative_config=speculative_config, - load_config=load_config, - ) - - # Profile the memory usage and initialize the cache. - self._initialize_kv_caches() - - # If usage stat is enabled, collect relevant info. - if is_usage_stats_enabled(): - from vllm.model_executor.model_loader import (get_architecture_class_name) - usage_message.report_usage( - get_architecture_class_name(model_config), - usage_context, - extra_kvs={ - # Common configuration - "dtype": str(model_config.dtype), - "tensor_parallel_size": parallel_config.tensor_parallel_size, - "block_size": cache_config.block_size, - "gpu_memory_utilization": cache_config.gpu_memory_utilization, - - # Quantization - "quantization": model_config.quantization, - "kv_cache_dtype": cache_config.cache_dtype, - - # Feature flags - "enable_lora": bool(lora_config), - "enable_prefix_caching": cache_config.enable_prefix_caching, - "enforce_eager": model_config.enforce_eager, - "disable_custom_all_reduce": parallel_config.disable_custom_all_reduce, - }) - - if self.tokenizer: - # Ping the tokenizer to ensure liveness if it runs in a - # different process. - self.tokenizer.ping() - - # Create the scheduler. - # NOTE: the cache_config here have been updated with the numbers of - # GPU and CPU blocks, which are profiled in the distributed executor. - # NOTE(shengguangming): each process will have independent scheduler - self.scheduler = Scheduler(scheduler_config, cache_config, lora_config) - - # Metric Logging. - if self.log_stats: - self.stat_logger = StatLogger(local_interval=_LOCAL_LOGGING_INTERVAL_SEC, - labels=dict(model_name=model_config.served_model_name), - max_model_len=self.model_config.max_model_len) - self.stat_logger.info("cache_config", self.cache_config) - - # Create sequence output processor, e.g. for beam search or - # speculative decoding. - self.output_processor = (SequenceGroupOutputProcessor.create_output_processor( - self.scheduler_config, - self.detokenizer, - self.scheduler, - self.seq_counter, - self.get_tokenizer_for_seq, - stop_checker=StopChecker( - self.scheduler_config.max_model_len, - self.get_tokenizer_for_seq, - ), - )) - - # TODO(sgm): add for verl but we may not tokenizer in Rollout - def _init_tokenizer(self, tokenizer, **tokenizer_init_kwargs): - init_kwargs = dict(enable_lora=bool(self.lora_config), - max_num_seqs=self.scheduler_config.max_num_seqs, - max_input_length=None) - init_kwargs.update(tokenizer_init_kwargs) - self.tokenizer: TokenizerGroup = TokenizerGroup(tokenizer, **init_kwargs) - - def init_cache_engine(self): - # TODO: check whether we should rebuild the CUDAGraph every iter when offload/load KVCache - # Re-capture CUDAGraph would be time-consuming - self.model_executor.init_cache_engine() - - def free_cache_engine(self): - self.model_executor.free_cache_engine() - - # NOTE(sgm): currently, we only support GPU executor - # The GPUExecutor remove the Ray dependency - @classmethod - def from_engine_args( - cls, - model, - tokenizer, - engine_args: EngineArgs, - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, - ) -> "LLMEngine": - """Creates an LLM engine from the engine arguments.""" - # Create the engine configs. - engine_config = engine_args.create_engine_config() - - # Initialize the cluster and specify the executor class. - assert engine_config.device_config.device_type == "cuda", \ - "Currently, the vllm in verl only support running on GPU" - - if engine_config.parallel_config.world_size == 1: - engine_config.load_config.load_format = "dummy_hf" - - from .spmd_gpu_executor import SPMDGPUExecutor - executor_class = SPMDGPUExecutor - - # Create the LLM engine. - engine = cls( - model, - tokenizer, - **engine_config.to_dict(), - executor_class=executor_class, - log_stats=not engine_args.disable_log_stats, - usage_context=usage_context, - ) - return engine - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.model_executor.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - def offload_model_weights(self) -> None: - self.model_executor.offload_model_weights() diff --git a/verl/third_party/vllm/vllm_v_0_4_2/megatron_weight_loaders.py b/verl/third_party/vllm/vllm_v_0_4_2/megatron_weight_loaders.py deleted file mode 100644 index 1a7c2e2cfe..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/megatron_weight_loaders.py +++ /dev/null @@ -1,348 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models - -from typing import Dict -import torch -import torch.nn as nn - -from vllm.model_executor.layers.linear import * -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead -from vllm.model_executor.layers.activation import ScaledActivation -from vllm.model_executor.models import ModelRegistry - - -# NOTE(shengguangming): replace the origin weight loader function in the class -def parallel_weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Parallel Linear weight loader.""" - assert param.size() == loaded_weight.size( - ), 'the parameter size is not align with the loaded weight size, param size: {}, loaded_weight size: {}'.format( - param.size(), loaded_weight.size()) - assert param.data.dtype == loaded_weight.data.dtype, "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Default weight loader.""" - assert param.size() == loaded_weight.size() - assert param.data.dtype == loaded_weight.data.dtype, "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def gpt2_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "lm_head.weight" in name: - # GPT-2 ties the weights of the embedding layer and the final - # linear layer. - continue - if ".attn.bias" in name or ".attn.masked_bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - if not name.startswith("transformer."): - name = "transformer." + name - param = params_dict[name] - # The HF's GPT-2 implementation uses Conv1D instead of Linear. - # Because of this, we need to transpose the weights. - # Note(zhuohan): the logic below might break quantized models. - for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]: - if conv1d_weight_name not in name: - continue - if not name.endswith(".weight"): - continue - # TODO: check megatron - loaded_weight = loaded_weight.t() - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_te_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv.layer_norm_weight", "input_layernorm.weight"), - ("self_attention.linear_qkv.layer_norm_bias", "input_layernorm.bias"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1.layer_norm_weight', 'post_attention_layernorm.weight'), - ('mlp.linear_fc1.layer_norm_bias', 'post_attention_layernorm.bias'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ( - 'input_layernorm', - 'input_layernorm', - ), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def _replace_name(megatron_name, name_mapping): - for m_name, v_name in name_mapping: - if m_name not in megatron_name: - continue - if 'layers' in megatron_name: # deal with decoder layers - megatron_name = megatron_name.replace('decoder', 'model') - megatron_name_list = megatron_name.split('.') - if 'layer_norm_weight' in megatron_name_list or 'layer_norm_bias' in megatron_name_list: - param_name_list = megatron_name_list[:3] - param_name_list.append(v_name) - param_name = '.'.join(param_name_list) - else: - param_name_list = megatron_name_list[:3] - weight_or_bias = megatron_name_list[-1] - param_name_list.append(v_name) - param_name_list.append(weight_or_bias) - param_name = '.'.join(param_name_list) - return param_name - else: - param_name = megatron_name.replace(m_name, v_name) - return param_name - - -def llama_megatron_core_te_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv.layer_norm_weight", "input_layernorm.weight"), - ("self_attention.linear_qkv.layer_norm_bias", "input_layernorm.bias"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1.layer_norm_weight', 'post_attention_layernorm.weight'), - ('mlp.linear_fc1.layer_norm_bias', 'post_attention_layernorm.bias'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ( - 'input_layernorm', - 'input_layernorm', - ), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def _replace_name(megatron_name, name_mapping): - for m_name, v_name in name_mapping: - if m_name not in megatron_name: - continue - if 'layers' in megatron_name: # deal with decoder layers - megatron_name = megatron_name.replace('decoder', 'model') - megatron_name_list = megatron_name.split('.') - if 'layer_norm_weight' in megatron_name_list or 'layer_norm_bias' in megatron_name_list: - param_name_list = megatron_name_list[:3] - param_name_list.append(v_name) - param_name = '.'.join(param_name_list) - else: - param_name_list = megatron_name_list[:3] - weight_or_bias = megatron_name_list[-1] - param_name_list.append(v_name) - param_name_list.append(weight_or_bias) - param_name = '.'.join(param_name_list) - return param_name - else: - param_name = megatron_name.replace(m_name, v_name) - return param_name - - -def mistral_megatron_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # TODO: need to implement a general way to deal with prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -__LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__ = { - ColumnParallelLinear: parallel_weight_loader, - MergedColumnParallelLinear: parallel_weight_loader, - QKVParallelLinear: parallel_weight_loader, - RowParallelLinear: parallel_weight_loader, - VocabParallelEmbedding: parallel_weight_loader, - ParallelLMHead: parallel_weight_loader - # "ScaledActivation.weight_loader": ScaledActivation, # TODO(shengguangming): latest commit in vllm fix awq for this function and add load_weights - # "default_weight_loader": default_weight_loader -} - -# for layer_class, weight_loader in __LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__.items(): -# # setattr(layer_class, 'megatron_weight_loader', weight_loader) -# layer_class.weight_loader = weight_loader - -__MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__ = { - 'GPT2LMHeadModel': gpt2_weight_loader, - 'LlamaForCausalLM': llama_megatron_core_te_weight_loader, # use te backend for open-source megatron - 'LLaMAForCausalLM': llama_megatron_core_te_weight_loader, - 'MistralForCausalLM': mistral_megatron_weight_loader, -} - - -# the actor model is .state_dict() -# Load megatron weights -def load_megatron_weights(actor_weights: Dict, vllm_model: nn.Module): - weight_loader = _get_model_weight_loader(vllm_model.__class__.__name__) - weight_loader(actor_weights, vllm_model) - # NOTE(sgm) to reduce peak memory usage, we offload vllm model to cpu - # after init, and we need this after sync model weights for in first iter. - vllm_model = vllm_model.cuda() - - -def _get_model_weight_loader(arch: str): - if arch in __MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__: - return __MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__[arch] - raise ValueError(f"Model architectures {arch} are not supported for now. " - f"Supported architectures: {ModelRegistry.get_supported_archs()}") - - -def update_megatron_weight_loader(): - for layer_class, weight_loader in __LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__.items(): - layer_class.weight_loader = weight_loader - VocabParallelEmbedding.__init__ = vocab_init - - -# FIXME(shengguangming): the vLLM vocab will pad to 64, which may incur out of bounds -# so we need to rewrite the init function of vocab -DEFAULT_VOCAB_PADDING_SIZE = 64 - - -def vocab_init(self, - num_embeddings: int, - embedding_dim: int, - params_dtype: Optional[torch.dtype] = None, - org_num_embeddings: Optional[int] = None, - padding_size: int = DEFAULT_VOCAB_PADDING_SIZE): - super(VocabParallelEmbedding, self).__init__() - - # Keep the input dimensions. - # TODO (pad to be divided by 4) - self.num_embeddings = num_embeddings - self.org_vocab_size = org_num_embeddings or num_embeddings - - # self.num_embeddings_padded = pad_vocab_size(num_embeddings, - # padding_size) - self.embedding_dim = embedding_dim - if params_dtype is None: - params_dtype = torch.get_default_dtype() - self.tp_size = get_tensor_model_parallel_world_size() - # Divide the weight matrix along the vocaburaly dimension. - - # TODO: remove dependencies from megatron - from megatron.core.tensor_parallel.utils import VocabUtility - self.vocab_start_index, self.vocab_end_index = (VocabUtility.vocab_range_from_global_vocab_size( - self.num_embeddings, get_tensor_model_parallel_rank(), self.tp_size)) - self.num_embeddings_per_partition = (self.vocab_end_index - self.vocab_start_index) - self.weight = Parameter( - torch.empty( - self.num_embeddings_per_partition, - self.embedding_dim, - # device=torch.cuda.current_device(), - dtype=params_dtype)) - set_weight_attrs(self.weight, {"parallel_dim": 0, "weight_loader": self.weight_loader}) diff --git a/verl/third_party/vllm/vllm_v_0_4_2/model_loader.py b/verl/third_party/vllm/vllm_v_0_4_2/model_loader.py deleted file mode 100644 index 5f4013451f..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/model_loader.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/model_loader -"""Utilities for selecting and loading models.""" -from typing import Dict, Union, Optional, Iterable, Tuple - -import torch -import torch.nn as nn -from transformers import PreTrainedModel - -from vllm.config import (DeviceConfig, LoRAConfig, ParallelConfig, SchedulerConfig, VisionLanguageConfig) -from vllm.model_executor.model_loader import BaseModelLoader -from vllm.model_executor.model_loader.loader import _initialize_model -from vllm.model_executor.model_loader.utils import set_default_torch_dtype -from vllm.distributed.communication_op import tensor_model_parallel_all_gather - -from .config import ModelConfig, LoadFormat, LoadConfig -from .megatron_weight_loaders import load_megatron_weights, update_megatron_weight_loader -from .dtensor_weight_loaders import load_dtensor_weights, update_dtensor_weight_loader -from .hf_weight_loader import update_hf_weight_loader - - -def get_model(actor_model: Union[PreTrainedModel, Dict], model_config: ModelConfig, load_config: LoadConfig, - device_config: DeviceConfig, parallel_config: ParallelConfig, scheduler_config: SchedulerConfig, - lora_config: Optional[LoRAConfig], vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module: - loader = get_model_loader(load_config) - if load_config.load_format.startswith('dummy'): - return loader.load_model(model_config=model_config, - device_config=device_config, - lora_config=lora_config, - vision_language_config=vision_language_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config) - else: - return loader.load_model(actor_model=actor_model, - model_config=model_config, - device_config=device_config, - lora_config=lora_config, - vision_language_config=vision_language_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config) - - -def get_model_loader(load_config: LoadConfig) -> BaseModelLoader: - """Get a model loader based on the load format.""" - - if isinstance(load_config.load_format, type): - return load_config.load_format(load_config) - - if load_config.load_format == LoadFormat.AUTO: - update_megatron_weight_loader() - return MegatronLoader(load_config) - - # NOTE(sgm): change the weight_loader function in runtime - if load_config.load_format == LoadFormat.MEGATRON: - update_megatron_weight_loader() - return MegatronLoader(load_config) - - if load_config.load_format == LoadFormat.HF: - update_hf_weight_loader() - return HFLoader(load_config) - - if load_config.load_format == LoadFormat.DTENSOR: - update_dtensor_weight_loader() - return DTensorLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_HF: - update_hf_weight_loader() - return DummyModelLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_MEGATRON: - update_megatron_weight_loader() - return DummyModelLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_DTENSOR: - update_dtensor_weight_loader() - return DummyModelLoader(load_config) - - raise ValueError('load format not supported in verl: {}, only support {} and {}'.format( - load_config.load_format, LoadFormat.MEGATRON, LoadFormat.HF)) - - -class DummyModelLoader(BaseModelLoader): - """Model loader that will set model weights to random values.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def load_model(self, *, model_config: ModelConfig, device_config: DeviceConfig, lora_config: Optional[LoRAConfig], - vision_language_config: Optional[VisionLanguageConfig], parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, vision_language_config) - # NOTE(woosuk): For accurate performance evaluation, we assign - # random values to the weights. - # initialize_dummy_weights(model) - return model.eval() - - -class MegatronLoader(BaseModelLoader): - """Model loader that can load the model weights from partitioned megatron model.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def _get_weights_iterator(actor_model: Union[PreTrainedModel, Dict]): - # NOTE(shengguangming) Load the weights from the actor model - pass - # if isinstance(actor_model, nn.Module): - # load_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), vllm_model=model) - # else: - # load_weights(actor_weights=actor_model, vllm_model=model) - # return actor_model - - def load_model(self, actor_model: Union[PreTrainedModel, - Dict], model_config: ModelConfig, device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], vision_language_config: Optional[VisionLanguageConfig], - parallel_config: ParallelConfig, scheduler_config: SchedulerConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, vision_language_config) - - # TODO(sgm): This is a hack, we need to register the load_weight() func for each model in vllm - if isinstance(actor_model, nn.Module): - load_megatron_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), - vllm_model=model) - else: - load_megatron_weights(actor_weights=actor_model, vllm_model=model) - - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -class HFLoader(BaseModelLoader): - """Model loader that can load the model weights from model's full params.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def _get_weights_iterator(self, actor_model: Union[PreTrainedModel, Dict]): - if isinstance(actor_model, Dict): - return actor_model.items() - elif isinstance(actor_model, nn.Module): - return dict(actor_model.named_parameters()).items() - else: - raise ValueError(f'actor model should be Dict or nn.Module, but get {type(actor_model)}') - - def load_model(self, actor_model: Union[PreTrainedModel, - Dict], model_config: ModelConfig, device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], vision_language_config: Optional[VisionLanguageConfig], - parallel_config: ParallelConfig, scheduler_config: SchedulerConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - # with torch.device(device_config.device): - # NOTE(sgm): init the model in cpu - model = _initialize_model(model_config, self.load_config, lora_config, vision_language_config) - model.load_weights(self._get_weights_iterator(actor_model)) - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -class DTensorLoader(BaseModelLoader): - """Model loader that can load the model weights from partitioned megatron model.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def _get_weights_iterator(actor_model: Union[PreTrainedModel, Dict]): - # NOTE(shengguangming) Load the weights from the actor model - pass - # if isinstance(actor_model, nn.Module): - # load_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), vllm_model=model) - # else: - # load_weights(actor_weights=actor_model, vllm_model=model) - # return actor_model - - def load_model(self, actor_model: Union[PreTrainedModel, - Dict], model_config: ModelConfig, device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], vision_language_config: Optional[VisionLanguageConfig], - parallel_config: ParallelConfig, scheduler_config: SchedulerConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, vision_language_config) - - # TODO(sgm): This is a hack, we need to register the load_weight() func for each model in vllm - if isinstance(actor_model, nn.Module): - load_dtensor_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), - vllm_model=model) - else: - load_dtensor_weights(actor_weights=actor_model, vllm_model=model) - - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -# FIXME(sgm): hack the _get_logits function in vllm v0.4.2 -# as they use ray, the _get_logits result will only need to return to the driver node, -# therefore gather is enough. However, we use SPMD instead of a central scheduler, -# all_gather is required (aligned with v0.2.6) -def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor, - embedding_bias: Optional[torch.Tensor]) -> torch.Tensor: - # Get the logits for the next tokens. - logits = torch.matmul(hidden_states, embedding.t()) - if embedding_bias is not None: - logits += embedding_bias - logits = tensor_model_parallel_all_gather(logits) - # Remove paddings in vocab (if any). - if logits is not None: - logits = logits[:, :self.org_vocab_size] - return logits - - -from vllm.model_executor.layers.logits_processor import LogitsProcessor - -LogitsProcessor._get_logits = _get_logits diff --git a/verl/third_party/vllm/vllm_v_0_4_2/model_runner.py b/verl/third_party/vllm/vllm_v_0_4_2/model_runner.py deleted file mode 100644 index 1604b03630..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/model_runner.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/model_runner.py - -import torch -import torch.nn as nn -from enum import IntEnum -from typing import Dict, List, Optional, Set, Tuple, Union - -from vllm.attention import (AttentionMetadata, get_attn_backend) -from vllm.config import (DeviceConfig, LoRAConfig, ParallelConfig, SchedulerConfig, VisionLanguageConfig) -from vllm.logger import init_logger -from vllm.lora.layers import LoRAMapping -from vllm.lora.request import LoRARequest -from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager -from vllm.model_executor import SamplingMetadata -from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData, SequenceGroupMetadata) -from vllm.utils import (CudaMemoryProfiler, is_hip, is_pin_memory_available) -from vllm.worker.model_runner import ModelRunner, CUDAGraphRunner - -from .model_loader import get_model -from .config import ModelConfig, LoadConfig - -logger = init_logger(__name__) - - -# How batches are constructed. -class BatchType(IntEnum): - # Every batch is prefill. - PREFILL = 0 - # Every batch is decode. - DECODE = 1 - # Batch is a mixture of prefill and decode. - MIXED = 2 - - -class ModelRunner(ModelRunner): - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - kv_cache_dtype: Optional[str] = "auto", - vision_language_config: Optional[VisionLanguageConfig] = None, - ): - self.model_config = model_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.lora_config = lora_config - self.load_config = load_config - - # model_config can be None in tests/samplers/test_sampler.py. - # FIXME(woosuk): This is a hack to make the tests work. Refactor this. - self.sliding_window = (model_config.get_sliding_window() if model_config is not None else None) - self.device_config = (device_config if device_config is not None else DeviceConfig()) - self.device = self.device_config.device - - # NOTE(sgm): add for verl - self.model = model # this will be replaced by get_model() - - # Set after load_model. - self.lora_manager: LRUCacheWorkerLoRAManager = None - - self.graph_runners: Dict[int, CUDAGraphRunner] = {} - self.graph_memory_pool: Optional[Tuple[int, int]] = None # Set during graph capture. - - self.max_seq_len_to_capture = (self.model_config.max_seq_len_to_capture if self.model_config is not None else 0) - - self.pin_memory = is_pin_memory_available() - self.kv_cache_dtype = kv_cache_dtype - self.vision_language_config = vision_language_config - - self.attn_backend = get_attn_backend(self.model_config.dtype if model_config is not None else None) - - # Lazy initialization - self.block_size: int # Set after initial profiling. - # When using CUDA graph, the input block tables must be padded to - # max_seq_len_to_capture. However, creating the block table in - # Python can be expensive. To optimize this, we cache the block table - # in numpy and only copy the actual input content at every iteration. - # The shape of the cached block table will be - # (max batch size to capture, max context len to capture / block size). - self.graph_block_tables: torch.Tensor # Set after initial profiling. - - # Set if the backend is flashinfer. - self.flashinfer_workspace_buffer: torch.Tensor - - # NOTE(sgm): initialize model using the actor model - def load_model(self) -> None: - with CudaMemoryProfiler() as m: - self.model = get_model(actor_model=self.model, - model_config=self.model_config, - device_config=self.device_config, - lora_config=self.lora_config, - load_config=self.load_config, - parallel_config=self.parallel_config, - scheduler_config=self.scheduler_config, - vision_language_config=self.vision_language_config) - self.model_memory_usage = m.consumed_memory - logger.info("Loading model weights took %.4f GB", self.model_memory_usage / float(2**30)) - - if self.lora_config: - assert hasattr(self.model, "supported_lora_modules") and self.model.supported_lora_modules, ( - "Model does not support LoRA") - assert hasattr(self.model, "embedding_modules"), "Model does not have embedding_modules" - assert hasattr(self.model, "embedding_padding_modules"), "Model does not have embedding_padding_modules" - self.lora_manager = LRUCacheWorkerLoRAManager(self.scheduler_config.max_num_seqs, - self.scheduler_config.max_num_batched_tokens, self.vocab_size, - self.lora_config, self.device, self.model.embedding_modules, - self.model.embedding_padding_modules) - self.model = self.lora_manager.create_lora_manager(self.model) - - if self.kv_cache_dtype == "fp8" and is_hip(): - # Currently scaled KV cache is only enabled on ROCm - if self.model_config.quantization_param_path is not None: - if callable(getattr(self.model, "load_kv_cache_scales", None)): - self.model.load_kv_cache_scales(self.model_config.quantization_param_path) - else: - raise RuntimeError( - "Using FP8 KV cache and scaling factors provided but " - "model %s does not support loading scaling factors.", self.model.__class__) - else: - logger.warning("Using FP8 KV cache but no scaling factors " - "provided. Defaulting to scaling factors of 1.0. " - "This may lead to less accurate results!") - elif self.model_config.quantization_param_path is not None: - logger.warning("KV cache scaling factors provided, " - "but the KV cache data type is not FP8. " - "KV cache scaling factors will not be used.") - - def prepare_input_tensors( - self, - seq_group_metadata_list: List[SequenceGroupMetadata], - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata, Set[LoRARequest], LoRAMapping, - torch.Tensor]: - # NOTE(sgm): all workers prepare the input in the same way - prefill_reqs = [] - decode_reqs = [] - for seq_group_meta in seq_group_metadata_list: - if seq_group_meta.is_prompt: - prefill_reqs.append(seq_group_meta) - else: - decode_reqs.append(seq_group_meta) - - # Prepare input tensors. - ( - input_tokens, - input_positions, - prefill_attn_metadata, - seq_lens, - query_lens, - lora_index_mapping, - lora_prompt_mapping, - lora_requests, - multi_modal_input, - slot_mapping, - ) = self._prepare_prompt(prefill_reqs) - ( - decode_input_tokens, - decode_input_positions, - decode_attn_metadata, - decode_lora_index_mapping, - decode_lora_prompt_mapping, - decode_lora_requests, - decode_slot_mapping, - ) = self._prepare_decode(decode_reqs) - sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list, seq_lens, query_lens, self.device, - self.pin_memory) - - if not self.scheduler_config.chunked_prefill_enabled: - assert (len(prefill_reqs) and len(decode_reqs)) == 0 - - num_prefills = len(seq_lens) - num_prefill_tokens = len(input_tokens) - num_decode_tokens = len(decode_input_tokens) - - # Coalesce tensors. Note that attn_metadata is currently not - # coalesced for simplicity. - input_tokens.extend(decode_input_tokens) - input_positions.extend(decode_input_positions) - slot_mapping.extend(decode_slot_mapping) - lora_index_mapping.extend(decode_lora_index_mapping) - lora_prompt_mapping.extend(decode_lora_prompt_mapping) - lora_requests.update(decode_lora_requests) - - input_tokens = torch.tensor(input_tokens, dtype=torch.long, device=self.device) - input_positions = torch.tensor(input_positions, dtype=torch.long, device=self.device) - slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=self.device) - - if self.lora_config: - lora_mapping = LoRAMapping( - lora_index_mapping, - lora_prompt_mapping, - ) - else: - lora_mapping = None - - # Broadcast the metadata. - # If batch contains both prefill and decode, it sends 2 broadcasts. - # If it only contains 1 type, it triggers a single broadcast. - if (prefill_attn_metadata is not None and decode_attn_metadata is not None): - batch_type = BatchType.MIXED - elif prefill_attn_metadata is not None: - batch_type = BatchType.PREFILL - else: - batch_type = BatchType.DECODE - - attn_metadata = AttentionMetadata( - num_prefills=num_prefills, - slot_mapping=slot_mapping, - num_prefill_tokens=num_prefill_tokens, - num_decode_tokens=num_decode_tokens, - prefill_metadata=prefill_attn_metadata, - decode_metadata=decode_attn_metadata, - kv_cache_dtype=self.kv_cache_dtype, - ) - - return (input_tokens, input_positions, attn_metadata, sampling_metadata, lora_requests, lora_mapping, - multi_modal_input) - - @torch.inference_mode() - def execute_model( - self, - seq_group_metadata_list: List[SequenceGroupMetadata], - kv_caches: List[torch.Tensor], - ) -> Optional[SamplerOutput]: - (input_tokens, input_positions, attn_metadata, sampling_metadata, lora_requests, lora_mapping, - multi_modal_input) = self.prepare_input_tensors(seq_group_metadata_list) - - if self.lora_config: - self.set_active_loras(lora_requests, lora_mapping) - - # Currently cuda graph is only supported by the decode phase. - prefill_meta = attn_metadata.prefill_metadata - decode_meta = attn_metadata.decode_metadata - if prefill_meta is None and decode_meta.use_cuda_graph: - graph_batch_size = input_tokens.shape[0] - model_executable = self.graph_runners[graph_batch_size] - else: - model_executable = self.model - execute_model_kwargs = { - "input_ids": input_tokens, - "positions": input_positions, - "kv_caches": kv_caches, - "attn_metadata": attn_metadata, - } - if self.vision_language_config: - execute_model_kwargs.update({"image_input": multi_modal_input}) - hidden_states = model_executable(**execute_model_kwargs) - - # Compute the logits. - logits = self.model.compute_logits(hidden_states, sampling_metadata) - - # Only perform sampling in the driver worker. - # if not self.is_driver_worker: - # return None - - # TODO(sgm): perform sampling on rank 0 - # Sample the next token. - output = self.model.sample( - logits=logits, - sampling_metadata=sampling_metadata, - ) - - return output diff --git a/verl/third_party/vllm/vllm_v_0_4_2/parallel_state.py b/verl/third_party/vllm/vllm_v_0_4_2/parallel_state.py deleted file mode 100644 index 9336089ed3..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/parallel_state.py +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Adapted from -# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -"""Model and data parallel groups.""" -import os -import torch -import torch.distributed -from typing import Optional - -import vllm.distributed.parallel_state as ps - -import vllm.envs as envs -from vllm.logger import init_logger - -from torch.distributed.device_mesh import init_device_mesh - -logger = init_logger(__name__) -""" -This version is strongly tied with Megatron to implement HybridEngine and weight sharing between vllm and Megatron. -- We assume the Megatron tp+dp+pp world is already established before calling this function. - -""" - -# Device mesh for using DTensor -_DEVICE_MESH = None - -# Tensor model parallel group that the current rank belongs to. -_TP_DEVICE_GROUP = None -_TP_CPU_GROUP = None - - -# This method is for initializing the ParallelGroup when using HybridEngine -def initialize_parallel_state( - distributed_init_method: str = "env://", - backend: str = "nccl", - tensor_model_parallel_size: int = 1, - num_tp_per_train_tp: int = 1, - pipeline_model_parallel_size: int = 1, -): - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. - rank = int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - ps.init_distributed_environment(world_size, rank, distributed_init_method, local_rank, backend) - if torch.distributed.get_world_size() > 1: - # NOTE: build a sepearate inference group with infer tp & micro dp - initialize_model_parallel_for_vllm(tensor_model_parallel_size=tensor_model_parallel_size, - num_tensor_model_parallel_groups_per_train_tp=num_tp_per_train_tp) - else: - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - - -def ensure_model_parallel_initialized( - tensor_model_parallel_size: int, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """Helper to initialize model parallel groups if they are not initialized, - or ensure tensor-parallel and pipeline-parallel sizes are equal to expected - values if the model parallel groups are initialized. - """ - # get the backend of _DEVICE_WORLD_GROUP - backend = backend or torch.distributed.get_backend() - if not model_parallel_is_initialized(): - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - return - - assert (get_tensor_model_parallel_world_size() == tensor_model_parallel_size), ( - "tensor parallel group already initialized, but of unexpected size: " - f"{get_tensor_model_parallel_world_size()=} vs. " - f"{tensor_model_parallel_size=}") - # assert (get_pipeline_model_parallel_world_size( - # ) == pipeline_model_parallel_size), ( - # "pipeline parallel group already initialized, but of unexpected size: " - # f"{get_pipeline_model_parallel_world_size()=} vs. " - # f"{pipeline_model_parallel_size=}") - - -def model_parallel_is_initialized(): - """Check if tensor and pipeline parallel groups are initialized.""" - return (ps._TP_DEVICE_GROUP is not None) - # and _PIPELINE_MODEL_PARALLEL_GROUP is not None) - - -def initialize_model_parallel_for_vllm(tensor_model_parallel_size: int, - num_tensor_model_parallel_groups_per_train_tp: int = 1) -> None: - from torch.distributed import new_group - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - - assert isinstance(tensor_model_parallel_size, int) - - # assert num_tensor_model_parallel_groups_per_train_tp == 1 and not different_tp_group - # assert num_tensor_model_parallel_groups_per_train_tp > 1 and different_tp_group - - # Build the tensor model-parallel groups. - assert ps._TP_DEVICE_GROUP is None, ("tensor model parallel group is already initialized") - - global _TP_DEVICE_GROUP - global _TP_CPU_GROUP - global _DEVICE_MESH - - world_size: int = torch.distributed.get_world_size() - - rank = torch.distributed.get_rank() - - backend = torch.distributed.get_backend() - - num_tensor_model_parallel_groups = world_size // tensor_model_parallel_size - - if num_tensor_model_parallel_groups_per_train_tp == 1: - # if tensor_model_parallel_size == train_tensor_parallel_size: - # using the same tp group as Megatron/vllm - for i in range(num_tensor_model_parallel_groups): - ranks = range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) - group = torch.distributed.new_group(ranks, backend=backend) - cpu_group = torch.distributed.new_group(ranks, backend="gloo") - if rank in ranks: - _TP_DEVICE_GROUP = group - _TP_CPU_GROUP = cpu_group - ps._TP_DEVICE_GROUP = group - ps._TP_CPU_GROUP = cpu_group - - # no _MICRO_DATA_PARALLEL_GROUP - else: - # initialize a micro_dp group and a tp group - # assume training tp=4, infer tp=2, then, weight is partitioned as - # [1], [2], [3], [4] for training and [1,2], [1,2], [3,4], [3,4] for inference - - # Build the inference tp groups - # train_tp = train_tensor_parallel_size - train_tp = num_tensor_model_parallel_groups_per_train_tp * tensor_model_parallel_size - # num_tensor_model_parallel_groups_per_train_tp = train_tp // tensor_model_parallel_size - assert _TP_DEVICE_GROUP is None, ("tensor model parallel group is already initialized") - for i in range(num_tensor_model_parallel_groups // num_tensor_model_parallel_groups_per_train_tp): - start = train_tp * i - end = train_tp * (i + 1) - for j in range(num_tensor_model_parallel_groups_per_train_tp): - ranks = list(range(start, end, num_tensor_model_parallel_groups_per_train_tp)) - for i in range(len(ranks)): - ranks[i] += j - group = torch.distributed.new_group(ranks) - cpu_group = torch.distributed.new_group(ranks, backend='gloo') - if rank in ranks: - _TP_DEVICE_GROUP = group - _TP_CPU_GROUP = cpu_group - ps._TP_DEVICE_GROUP = _TP_DEVICE_GROUP - ps._TP_CPU_GROUP = cpu_group - - # Build the pipeline model-parallel groups. - # global _PIPELINE_MODEL_PARALLEL_GROUP - # global _PIPELINE_GLOBAL_RANKS - # assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized") - - # ps._PIPELINE_MODEL_PARALLEL_GROUP = mpu.get_pipeline_model_parallel_group() - # ps._PIPELINE_GLOBAL_RANKS = mpu.get_pipeline_model_parallel_ranks() - - -def initialize_model_parallel( - tensor_model_parallel_size: int = 1, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """ - NOTE: This method is a hack from the open-sourced version without - asertion of world_size = tp * pp - - Initialize model parallel groups. - - Arguments: - tensor_model_parallel_size: number of GPUs used for tensor model - parallelism. - pipeline_model_parallel_size: number of GPUs used for pipeline model - parallelism. - - Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we - use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize - the model pipeline. The present function will - create 4 tensor model-parallel groups and 2 pipeline model-parallel groups: - 4 tensor model-parallel groups: - [g0, g1], [g2, g3], [g4, g5], [g6, g7] - 2 pipeline model-parallel groups: - [g0, g2, g4, g6], [g1, g3, g5, g7] - Note that for efficiency, the caller should make sure adjacent ranks - are on the same DGX box. For example if we are using 2 DGX-1 boxes - with a total of 16 GPUs, rank 0 to 7 belong to the first box and - ranks 8 to 15 belong to the second box. - """ - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - world_size: int = torch.distributed.get_world_size() - # get the backend of _DEVICE_WORLD_GROUP - backend = backend or torch.distributed.get_backend() - - # NOTE(sgm) we don't assert world_size == tp * pp - # DP is not managed by vllm but by the verl WorkerGroup - - num_tensor_model_parallel_groups: int = (world_size // tensor_model_parallel_size) - num_pipeline_model_parallel_groups: int = (world_size // pipeline_model_parallel_size) - rank = torch.distributed.get_rank() - - # Build device mesh for TP - if num_tensor_model_parallel_groups > 1: - device_mesh = init_device_mesh("cuda", (num_tensor_model_parallel_groups, tensor_model_parallel_size), - mesh_dim_names=("replicate", "tp_shard")) - else: - device_mesh = init_device_mesh("cuda", (tensor_model_parallel_size,), mesh_dim_names=["tp_shard"]) - shard_group = device_mesh.get_group(mesh_dim="tp_shard") - - # Build the tensor model-parallel groups. - global _TP_DEVICE_GROUP, _TP_CPU_GROUP - global _DEVICE_MESH - assert _TP_DEVICE_GROUP is None, ("tensor model parallel group is already initialized") - assert _DEVICE_MESH is None, ("device mesh in vllm is already initialized") - - _DEVICE_MESH = device_mesh - # for i in range(num_tensor_model_parallel_groups): - # ranks = range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) - # group = torch.distributed.new_group(ranks, backend=backend) - # cpu_group = torch.distributed.new_group(ranks, backend="gloo") - # assert torch.distributed.get_process_group_ranks(shard_group) == torch.distributed.get_process_group_ranks(cpu_group) - # ranks = torch.distributed.get_process_group_ranks(shard_group) - # cpu_group = torch.distributed.new_group(ranks, backend="gloo") # TODO: this will hang - # cpu_group = torch.distributed.new_group(, backend="gloo") - # if rank == 0: - # print(f'rank: {rank}') - # print(f'ranks: {ranks}') - # print(f'torch.distributed.get_process_group_ranks(shard_group): {torch.distributed.get_process_group_ranks(shard_group)}') - # if rank in ranks: - _TP_DEVICE_GROUP = shard_group - ps._TP_DEVICE_GROUP = _TP_DEVICE_GROUP - # ps._TP_CPU_GROUP = cpu_group # TODO: will hang when used with device mesh - - # TODO: init using device mesh - # Build the pipeline model-parallel groups. - assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized") - for i in range(num_pipeline_model_parallel_groups): - ranks = range(i, world_size, num_pipeline_model_parallel_groups) - group = torch.distributed.new_group(ranks, backend=backend) - if rank in ranks: - ps._PIPELINE_MODEL_PARALLEL_GROUP = group - ps._PIPELINE_GLOBAL_RANKS = ranks - - -""" -Device mesh utilities -""" - - -def get_device_mesh(): - assert _DEVICE_MESH is not None, ("device mesh is not initialized") - return _DEVICE_MESH - - -""" -Tensor model parallel utilities -""" - - -def get_tensor_model_parallel_group(): - """Get the tensor model parallel group the caller rank belongs to.""" - assert _TP_DEVICE_GROUP is not None, ("tensor model parallel group is not initialized") - return _TP_DEVICE_GROUP - - -def get_tensor_model_parallel_world_size(): - """Return world size for the tensor model parallel group.""" - return torch.distributed.get_world_size(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_rank(): - """Return my rank for the tensor model parallel group.""" - return torch.distributed.get_rank(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_src_rank(): - """Calculate the global rank corresponding to the first local rank - in the tensor model parallel group.""" - global_rank = torch.distributed.get_rank() - local_world_size = get_tensor_model_parallel_world_size() - return (global_rank // local_world_size) * local_world_size diff --git a/verl/third_party/vllm/vllm_v_0_4_2/spmd_gpu_executor.py b/verl/third_party/vllm/vllm_v_0_4_2/spmd_gpu_executor.py deleted file mode 100644 index b97bb600ac..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/spmd_gpu_executor.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/executor/gpu_executor.py -import os -import socket -from typing import Any, Dict, List, Optional, Set, Tuple - -import torch -import vllm.envs as envs -from vllm.executor.executor_base import ExecutorBase, ExecutorAsyncBase -from vllm.logger import init_logger -from vllm.lora.request import LoRARequest -from vllm.sequence import SamplerOutput, ExecuteModelRequest - -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ParallelConfig, SchedulerConfig, SpeculativeConfig, - VisionLanguageConfig) -from .config import ModelConfig, LoadConfig - -logger = init_logger(__name__) - - -class SPMDGPUExecutor(ExecutorBase): - """SPMD-based multi-GPU executor implementations.""" - - def __init__( - self, - model, # pytorch model itself or its parameter dict - model_config: ModelConfig, - cache_config: CacheConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - vision_language_config: Optional[VisionLanguageConfig], - speculative_config: Optional[SpeculativeConfig], - ) -> None: - self.model_config = model_config - self.cache_config = cache_config - self.lora_config = lora_config - self.load_config = load_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.vision_language_config = vision_language_config - self.speculative_config = speculative_config - - distributed_init_method = initialize_cluster(parallel_config) - self._init_executor(model, distributed_init_method) - - # TODO(sgm): verl not support speculative decode now - def _init_executor(self, model, distributed_init_method) -> None: - assert (not self.speculative_config), "Speculative decoding not yet supported for multi-GPU backend." - - # Create the parallel worker for each GPU. - self._init_workers_sp(model, distributed_init_method) - - def _init_workers_sp(self, model, distributed_init_method: str): - # Lazy import the Worker to avoid importing torch.cuda/xformers - # before CUDA_VISIBLE_DEVICES is set in the Worker - from .worker import Worker # pylint: disable=import-outside-toplevel - - rank = int(os.getenv("RANK")) - local_rank = int(os.getenv("LOCAL_RANK")) - print(f'local rank {local_rank}') - - self.worker = Worker( - model, - self.model_config, - self.parallel_config, - self.scheduler_config, - self.device_config, - self.cache_config, - self.load_config, - local_rank, - rank, - distributed_init_method, - lora_config=self.lora_config, - vision_language_config=self.vision_language_config, - ) - - # NOTE(shengguangming): torch.distributed.init_process_group will be called inside the init_model() - self.worker.init_device() - self.worker.load_model() - - def determine_num_available_blocks(self) -> Tuple[int, int]: - """Determine the number of available KV blocks. - - This invokes `determine_num_available_blocks` on each worker and takes - the min of the results, guaranteeing that the selected cache sizes are - compatible with all workers. - - Returns: - - tuple[num_gpu_blocks, num_cpu_blocks] - """ - # Get the maximum number of blocks that can be allocated on GPU and CPU. - num_blocks = self.worker.determine_num_available_blocks() - - # NOTE(shengguangming): Now we don't use a shared centralized controler but each process will - # have its own scheduler - num_gpu_blocks = num_blocks[0] - num_cpu_blocks = num_blocks[1] - - return num_gpu_blocks, num_cpu_blocks - - def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks: int) -> None: - """Initialize the KV cache in all workers. - """ - - # NOTE: We log here to avoid multiple logs when number of workers is - # greater than one. We could log in the engine, but not all executors - # have GPUs. - logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks, num_cpu_blocks) - - self.cache_config.num_gpu_blocks = num_gpu_blocks - self.cache_config.num_cpu_blocks = num_cpu_blocks - - if torch.distributed.get_rank() == 0: - print( - f'before init cache memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB' - ) - self.worker.initialize_cache(num_gpu_blocks=num_gpu_blocks, num_cpu_blocks=num_cpu_blocks) - if torch.distributed.get_rank() == 0: - print( - f'after init cache memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB' - ) - - # NOTE(sgm): This will not profile & capture the model(CUDAGraph) when rebuilding KVCache - def init_cache_engine(self) -> None: - self.worker._init_cache_engine() - - def free_cache_engine(self) -> None: - self.worker.free_cache_engine() - - def execute_model(self, execute_model_req) -> List[SamplerOutput]: - all_outputs = self.worker.execute_model(execute_model_req=execute_model_req) - - # NOTE(sgm): - # Each GPU in vllm under verl has its own spmd_gpu_executor, therefore all GPUs should return the outputs - # In vllm with ray, only the driver worker returns the sampling results. - return all_outputs - - def add_lora(self, lora_request: LoRARequest) -> bool: - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0." - return self.worker.add_lora(lora_request=lora_request) - - def remove_lora(self, lora_id: int) -> bool: - assert lora_id > 0, "lora_id must be greater than 0." - return self.worker.remove_lora(lora_id=lora_id) - - def list_loras(self) -> Set[int]: - return self.worker.list_loras() - - def check_health(self) -> None: - # SPMDExecutor will always be healthy as long as - # it's running. - return - - # NOTE(sgm): add for verl - def offload_model_weights(self) -> None: - self.worker.offload_model_weights() - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.worker.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - -def initialize_cluster( - parallel_config: ParallelConfig, - engine_use_ray: bool = False, - ray_address: Optional[str] = None, -) -> Tuple[str, Optional[None]]: - """Initialize the distributed cluster probably with Ray. - - Args: - parallel_config: The configurations for parallel execution. - - Returns: - The `distributed_init_method` is the address for initializing the - distributed backend. - """ - - # Initialize cluster locally. - port = get_open_port() - # We need to setup the distributed init method to make sure - # the distributed megatron code (e.g., get world size) works correctly. - # distributed_init_method = f"tcp://localhost:{port}" - distributed_init_method = 'env://' - return distributed_init_method - - -def get_open_port(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - return s.getsockname()[1] - - -# TODO(sgm): not implemented async executor yet -class SPMDGPUExecutorAsync(SPMDGPUExecutor, ExecutorAsyncBase): - - async def execute_model_async(self, execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: - """Executes one model step on the given sequences.""" - raise NotImplementedError - - async def check_health_async(self) -> None: - """Checks if the executor is healthy. If not, it should raise an - exception.""" - self.check_health() diff --git a/verl/third_party/vllm/vllm_v_0_4_2/tokenizer.py b/verl/third_party/vllm/vllm_v_0_4_2/tokenizer.py deleted file mode 100644 index aa625a0338..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/tokenizer.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/tokenizer_group/tokenizer_group.py - -from typing import List, Optional, Tuple, Union - -from transformers import (AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast) - -from vllm.lora.request import LoRARequest -from vllm.utils import make_async, LRUCache -from vllm.transformers_utils.tokenizers import * - - -class TokenizerGroup: - """A group of tokenizers that can be used for LoRA adapters.""" - - def __init__(self, tokenizer: PreTrainedTokenizer, enable_lora: bool, max_num_seqs: int, - max_input_length: Optional[int]): - self.enable_lora = enable_lora - self.max_input_length = max_input_length - self.tokenizer = tokenizer - self.lora_tokenizers = LRUCache[PreTrainedTokenizer](capacity=max_num_seqs) if enable_lora else None - - def ping(self) -> bool: - """Check if the tokenizer group is alive.""" - return True - - def get_max_input_len(self, lora_request: Optional[LoRARequest] = None) -> Optional[int]: - """Get the maximum input length for the LoRA request.""" - return self.max_input_length - - def encode(self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None) -> List[int]: - tokenizer = self.get_lora_tokenizer(lora_request) - return tokenizer.encode(prompt) - - async def encode_async(self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None) -> List[int]: - tokenizer = await self.get_lora_tokenizer_async(lora_request) - return tokenizer.encode(prompt) - - def get_lora_tokenizer(self, lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer": - if not lora_request or not self.enable_lora: - return self.tokenizer - if lora_request.lora_int_id not in self.lora_tokenizers: - # TODO(sgm): the lora tokenizer is also passed, but may be different - tokenizer = self.tokenizer - # tokenizer = (get_lora_tokenizer( - # lora_request, **self.tokenizer_config) or self.tokenizer) - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) - return tokenizer - else: - return self.lora_tokenizers.get(lora_request.lora_int_id) - - # FIXME(sgm): for simplicity, we assign the special token here - @property - def pad_token_id(self): - return self.tokenizer.pad_token_id - - @property - def eos_token_id(self): - return self.tokenizer.eos_token_id diff --git a/verl/third_party/vllm/vllm_v_0_4_2/worker.py b/verl/third_party/vllm/vllm_v_0_4_2/worker.py deleted file mode 100644 index 1fab3e41fe..0000000000 --- a/verl/third_party/vllm/vllm_v_0_4_2/worker.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/worker.py -"""A GPU worker class.""" -import os -import gc -from typing import Dict, List, Tuple, Optional, Union - -import torch -import torch.distributed -import torch.nn as nn - -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ParallelConfig, SchedulerConfig, VisionLanguageConfig) -from vllm.model_executor import set_random_seed -from vllm.sequence import SamplerOutput, ExecuteModelRequest -from vllm.worker.cache_engine import CacheEngine -from vllm.distributed.device_communicators import pynccl_utils -from vllm.distributed.device_communicators.custom_all_reduce import (init_custom_ar) -# TODO(sgm): check why vllm has similar file in vllm.model_executor.parallel_utils.parallel_state -from vllm.distributed import get_tensor_model_parallel_cpu_group, init_distributed_environment, get_tensor_model_parallel_group -from vllm.worker.worker import Worker, _check_if_gpu_supports_dtype - -from .model_runner import ModelRunner -from .megatron_weight_loaders import load_megatron_weights -from .hf_weight_loader import load_hf_weights -from .dtensor_weight_loaders import load_dtensor_weights -from .parallel_state import (ensure_model_parallel_initialized) -from .config import ModelConfig, LoadConfig, LoadFormat - - -class Worker(Worker): - """A worker class that executes (a partition of) the model on a GPU. - - Each worker is associated with a single GPU. The worker is responsible for - maintaining the KV cache and executing the model on the GPU. In case of - distributed inference, each worker is assigned a partition of the model. - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - cache_config: CacheConfig, - load_config: LoadConfig, - local_rank: int, - rank: int, - distributed_init_method: str, - lora_config: Optional[LoRAConfig] = None, - vision_language_config: Optional[VisionLanguageConfig] = None, - is_driver_worker: bool = False, - ) -> None: - # self.model = model # will be replaced in the init_model - self.model_config = model_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.cache_config = cache_config - self.local_rank = local_rank - self.rank = rank - self.distributed_init_method = distributed_init_method - self.lora_config = lora_config - self.load_config = load_config - self.is_driver_worker = is_driver_worker - if self.is_driver_worker: - assert self.rank == 0, "The driver worker must have rank 0." - - self.vision_language_config = vision_language_config - if self.vision_language_config: - assert not self.lora_config, ("To be tested: vision language model with LoRA settings.") - - self.model_runner = ModelRunner( - model, - model_config, - parallel_config, - scheduler_config, - device_config, - load_config=load_config, - lora_config=self.lora_config, - kv_cache_dtype=self.cache_config.cache_dtype, - vision_language_config=vision_language_config, - ) - - # Uninitialized cache engine. Will be initialized by - # init_cache_engine. - self.cache_engine: CacheEngine = None - self.gpu_cache: List[torch.Tensor] = None - - # NOTE(sgm): For offloading inference engine params - self.cpu_model = None - - def init_device(self) -> None: - if self.device_config.device.type == "cuda": - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. - self.rank = self.rank if self.rank is not None else int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.device = torch.device(f"cuda:{local_rank}") - if self.rank < 0: - raise ValueError("Invalid or unspecified rank.") - torch.cuda.set_device(self.device) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - self.parallel_config.world_size = world_size - - _check_if_gpu_supports_dtype(self.model_config.dtype) - torch.cuda.empty_cache() - self.init_gpu_memory = torch.cuda.mem_get_info()[0] - else: - raise RuntimeError(f"Not support device type: {self.device_config.device}") - - # Initialize the distributed environment. - init_worker_distributed_environment(self.parallel_config, self.rank, self.distributed_init_method, - self.local_rank) - # Set random seed. - set_random_seed(self.model_config.seed) - # self.model = get_model(actor_model=self.model, model_config=self.model_config) - - @torch.inference_mode() - def determine_num_available_blocks(self) -> Tuple[int, int]: - """Profiles the peak memory usage of the model to determine how many - KV blocks may be allocated without OOMs. - - The engine will first conduct a profiling of the existing memory usage. - Then, it calculate the maximum possible number of GPU and CPU blocks - that can be allocated with the remaining free memory. - - .. tip:: - You may limit the usage of GPU memory - by adjusting the `gpu_memory_utilization` parameter. - """ - # Profile the memory usage of the model and get the maximum number of - # cache blocks that can be allocated with the remaining free memory. - torch.cuda.empty_cache() - # torch.cuda.reset_peak_memory_stats() - - # Execute a forward pass with dummy inputs to profile the memory usage - # of the model. - self.model_runner.profile_run() - - # Calculate the number of blocks that can be allocated with the - # profiled peak memory. - torch.cuda.synchronize() - free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info() - peak_memory = total_gpu_memory - free_gpu_memory - - assert peak_memory > 0, ("Error in memory profiling. This happens when the GPU memory was " - "not properly cleaned up before initializing the vLLM instance.") - - cache_block_size = self.get_cache_block_size_bytes() - - # NOTE(sgm) use the remaining memory - num_gpu_blocks = int((free_gpu_memory * self.cache_config.gpu_memory_utilization) // cache_block_size) - # num_gpu_blocks = int((total_gpu_memory * self.cache_config.gpu_memory_utilization - peak_memory) // cache_block_size) - - num_cpu_blocks = int(self.cache_config.swap_space_bytes // cache_block_size) - num_gpu_blocks = max(num_gpu_blocks, 0) - num_cpu_blocks = max(num_cpu_blocks, 0) - if self.model_runner.lora_manager: - self.model_runner.remove_all_loras() - - # NOTE(sgm): Add for verl, synchronize number of blocks with all the rank - num_gpu_blocks = torch.tensor([num_gpu_blocks], device='cuda') - num_cpu_blocks = torch.tensor([num_cpu_blocks], device='cuda') - torch.distributed.all_reduce(num_gpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group()) - torch.distributed.all_reduce(num_cpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group()) - num_gpu_blocks = num_gpu_blocks.item() - num_cpu_blocks = num_cpu_blocks.item() - gc.collect() - torch.cuda.empty_cache() - return num_gpu_blocks, num_cpu_blocks - - def _init_cache_engine(self): - if self.cache_engine is None and self.gpu_cache is None: - super()._init_cache_engine() - - def free_cache_engine(self): - # ensure `enforce_eager=True` - self.cache_engine = None - self.gpu_cache = None - - @torch.inference_mode() - def execute_model(self, execute_model_req: Optional[ExecuteModelRequest] = None) -> List[SamplerOutput]: - - if execute_model_req is None: - seq_group_metadata_list = None - else: - seq_group_metadata_list = execute_model_req.seq_group_metadata_list - - # NOTE(sgm): each SPMD rank will have identical input - assert seq_group_metadata_list is not None - assert execute_model_req is not None - num_seq_groups = len(seq_group_metadata_list) - blocks_to_swap_in = execute_model_req.blocks_to_swap_in - blocks_to_swap_out = execute_model_req.blocks_to_swap_out - blocks_to_copy = execute_model_req.blocks_to_copy - - self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy) - - # If there is no input, we don't need to execute the model. - if num_seq_groups == 0: - return [] - - output = self.model_runner.execute_model(seq_group_metadata_list, self.gpu_cache) - - # Worker only supports single-step execution. Wrap the output in a list - # to conform to interface. - return [output] - - # assume the input is .state_dict() - def sync_model_weights(self, actor_weights: Dict, load_format: str): - if load_format in [LoadFormat.MEGATRON, LoadFormat.AUTO]: - load_megatron_weights(actor_weights, self.model_runner.model) - elif load_format == LoadFormat.HF: - # full model state dict without no sharding - load_hf_weights(actor_weights, self.model_runner.model) - elif load_format == LoadFormat.DTENSOR: - load_dtensor_weights(actor_weights, self.model_runner.model) - - def offload_model_weights(self) -> None: - if self.cpu_model == None: - self.cpu_model = {} - for name, params in self.model_runner.model.named_parameters(): - self.cpu_model[name] = torch.empty_like(params, device='cpu') - params.data = self.cpu_model[name] - else: - for name, params in self.model_runner.model.named_parameters(): - params.data = self.cpu_model[name] - - -def init_worker_distributed_environment( - parallel_config: ParallelConfig, - rank: int, - distributed_init_method: Optional[str] = "env://", - local_rank: int = -1, -) -> None: - """Initialize the distributed environment.""" - # NOTE(sgm) use tcp://localhost:xxxx will hang in HF setting without megatron - init_distributed_environment(parallel_config.world_size, rank, distributed_init_method, local_rank) - - ensure_model_parallel_initialized(tensor_model_parallel_size=parallel_config.tensor_parallel_size, - pipeline_model_parallel_size=parallel_config.pipeline_parallel_size) - - # TODO(sgm): check whether need this - # if pynccl_utils.is_initialized(): - # pynccl_world_size = pynccl_utils.get_world_size() - # if pynccl_world_size != parallel_config.world_size: - # raise RuntimeError( - # "pynccl is already initialized but the pynccl world " - # "size does not match parallel_config.world_size " - # f"({pynccl_world_size} vs. {parallel_config.world_size}).") - # elif parallel_config.world_size > 1: - # # NOTE(woosuk): We don't initialize pynccl process group when world size - # # is 1. - # # NOTE(kaichao): By default, pynccl is initialized for tp group. - # pynccl_utils.init_process_group( - # group=get_tensor_model_parallel_cpu_group()) - - # # Initialize a custom fast all-reduce implementation. - # if not parallel_config.disable_custom_all_reduce: - # init_custom_ar() - - # A small all_reduce for warmup. - torch.distributed.all_reduce(torch.zeros(1).cuda()) - # if pynccl_utils.is_initialized(): - # pynccl_utils.all_reduce(torch.zeros(1).cuda()) diff --git a/verl/third_party/vllm/vllm_v_0_5_4/__init__.py b/verl/third_party/vllm/vllm_v_0_5_4/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/third_party/vllm/vllm_v_0_5_4/arg_utils.py b/verl/third_party/vllm/vllm_v_0_5_4/arg_utils.py deleted file mode 100644 index 6c577277b8..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/arg_utils.py +++ /dev/null @@ -1,453 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py - -import os -import argparse -import dataclasses -import json -from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union - -import torch.nn as nn - -from transformers import PretrainedConfig -from .config import ModelConfig, LoadConfig - -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, EngineConfig, LoRAConfig, MultiModalConfig, - ObservabilityConfig, ParallelConfig, PromptAdapterConfig, SchedulerConfig, SpeculativeConfig, - TokenizerPoolConfig) -from vllm.executor.executor_base import ExecutorBase -from vllm.logger import init_logger -from vllm.utils import FlexibleArgumentParser -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS -from vllm.utils import str_to_int_tuple - -if TYPE_CHECKING: - from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (BaseTokenizerGroup) - -logger = init_logger(__name__) - - -def nullable_str(val: str): - if not val or val == "None": - return None - return val - - -@dataclass -class EngineArgs: - """Arguments for vLLM engine.""" - model_hf_config: PretrainedConfig = None # for verl - served_model_name = None # TODO(sgm): check this - # tokenizer: Optional[str] = None # TODO(sgm): check this - skip_tokenizer_init: bool = False - tokenizer_mode: str = 'auto' - trust_remote_code: bool = False - download_dir: Optional[str] = None - load_format: str = 'auto' - dtype: str = 'auto' - kv_cache_dtype: str = 'auto' - quantization_param_path: Optional[str] = None - seed: int = 0 - max_model_len: Optional[int] = None - worker_use_ray: bool = False - # Note: Specifying a custom executor backend by passing a class - # is intended for expert use only. The API may change without - # notice. - distributed_executor_backend: Optional[Union[str, Type[ExecutorBase]]] = None - pipeline_parallel_size: int = 1 - tensor_parallel_size: int = 1 - max_parallel_loading_workers: Optional[int] = None - block_size: int = 16 - enable_prefix_caching: bool = False - disable_sliding_window: bool = False - use_v2_block_manager: bool = False - swap_space: int = 4 # GiB - cpu_offload_gb: int = 0 # GiB - gpu_memory_utilization: float = 0.90 - max_num_batched_tokens: Optional[int] = None - max_num_seqs: int = 256 - max_logprobs: int = 20 # Default value for OpenAI Chat Completions API - disable_log_stats: bool = False - revision: Optional[str] = None - code_revision: Optional[str] = None - rope_scaling: Optional[dict] = None - rope_theta: Optional[float] = None - tokenizer_revision: Optional[str] = None - quantization: Optional[str] = None - enforce_eager: bool = False - max_context_len_to_capture: Optional[int] = None - max_seq_len_to_capture: int = 8192 - disable_custom_all_reduce: bool = False - tokenizer_pool_size: int = 0 - # Note: Specifying a tokenizer pool by passing a class - # is intended for expert use only. The API may change without - # notice. - tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray" - tokenizer_pool_extra_config: Optional[dict] = None - enable_lora: bool = False - max_loras: int = 1 - max_lora_rank: int = 16 - enable_prompt_adapter: bool = False - max_prompt_adapters: int = 1 - max_prompt_adapter_token: int = 0 - fully_sharded_loras: bool = False - lora_extra_vocab_size: int = 256 - long_lora_scaling_factors: Optional[Tuple[float]] = None - lora_dtype: str = 'auto' - max_cpu_loras: Optional[int] = None - device: str = 'auto' - ray_workers_use_nsight: bool = False - num_gpu_blocks_override: Optional[int] = None - num_lookahead_slots: int = 0 - model_loader_extra_config: Optional[dict] = None - ignore_patterns: Optional[Union[str, List[str]]] = None - preemption_mode: Optional[str] = None - - scheduler_delay_factor: float = 0.0 - enable_chunked_prefill: Optional[bool] = None - - guided_decoding_backend: str = 'outlines' - # Speculative decoding configuration. - speculative_model: Optional[str] = None - speculative_draft_tensor_parallel_size: Optional[int] = None - num_speculative_tokens: Optional[int] = None - speculative_max_model_len: Optional[int] = None - speculative_disable_by_batch_size: Optional[int] = None - ngram_prompt_lookup_max: Optional[int] = None - ngram_prompt_lookup_min: Optional[int] = None - spec_decoding_acceptance_method: str = 'rejection_sampler' - typical_acceptance_sampler_posterior_threshold: Optional[float] = None - typical_acceptance_sampler_posterior_alpha: Optional[float] = None - qlora_adapter_name_or_path: Optional[str] = None - disable_logprobs_during_spec_decoding: Optional[bool] = None - - otlp_traces_endpoint: Optional[str] = None - - @staticmethod - def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - """Shared CLI arguments for vLLM engine.""" - # Model arguments - # TODO(shengguangming): delete the unused args - parser.add_argument('--model', - type=str, - default='facebook/opt-125m', - help='name or path of the huggingface model to use') - parser.add_argument('--tokenizer', - type=str, - default=EngineArgs.tokenizer, - help='name or path of the huggingface tokenizer to use') - parser.add_argument('--revision', - type=str, - default=None, - help='the specific model version to use. It can be a branch ' - 'name, a tag name, or a commit id. If unspecified, will use ' - 'the default version.') - parser.add_argument('--tokenizer-revision', - type=str, - default=None, - help='the specific tokenizer version to use. It can be a branch ' - 'name, a tag name, or a commit id. If unspecified, will use ' - 'the default version.') - parser.add_argument('--tokenizer-mode', - type=str, - default=EngineArgs.tokenizer_mode, - choices=['auto', 'slow'], - help='tokenizer mode. "auto" will use the fast ' - 'tokenizer if available, and "slow" will ' - 'always use the slow tokenizer.') - parser.add_argument('--trust-remote-code', action='store_true', help='trust remote code from huggingface') - parser.add_argument('--download-dir', - type=str, - default=EngineArgs.download_dir, - help='directory to download and load the weights, ' - 'default to the default cache dir of ' - 'huggingface') - parser.add_argument('--load-format', - type=str, - default=EngineArgs.load_format, - choices=['auto', 'pt', 'safetensors', 'npcache', 'dummy'], - help='The format of the model weights to load. ' - '"auto" will try to load the weights in the safetensors format ' - 'and fall back to the pytorch bin format if safetensors format ' - 'is not available. ' - '"pt" will load the weights in the pytorch bin format. ' - '"safetensors" will load the weights in the safetensors format. ' - '"npcache" will load the weights in pytorch format and store ' - 'a numpy cache to speed up the loading. ' - '"dummy" will initialize the weights with random values, ' - 'which is mainly for profiling.') - parser.add_argument('--dtype', - type=str, - default=EngineArgs.dtype, - choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'], - help='data type for model weights and activations. ' - 'The "auto" option will use FP16 precision ' - 'for FP32 and FP16 models, and BF16 precision ' - 'for BF16 models.') - parser.add_argument('--max-model-len', - type=int, - default=None, - help='model context length. If unspecified, ' - 'will be automatically derived from the model.') - # Parallel arguments - parser.add_argument('--worker-use-ray', - action='store_true', - help='use Ray for distributed serving, will be ' - 'automatically set when using more than 1 GPU') - parser.add_argument('--pipeline-parallel-size', - '-pp', - type=int, - default=EngineArgs.pipeline_parallel_size, - help='number of pipeline stages') - parser.add_argument('--tensor-parallel-size', - '-tp', - type=int, - default=EngineArgs.tensor_parallel_size, - help='number of tensor parallel replicas') - # KV cache arguments - parser.add_argument('--block-size', - type=int, - default=EngineArgs.block_size, - choices=[8, 16, 32], - help='token block size') - # TODO(woosuk): Support fine-grained seeds (e.g., seed per request). - parser.add_argument('--seed', type=int, default=EngineArgs.seed, help='random seed') - parser.add_argument('--swap-space', - type=int, - default=EngineArgs.swap_space, - help='CPU swap space size (GiB) per GPU') - parser.add_argument('--gpu-memory-utilization', - type=float, - default=EngineArgs.gpu_memory_utilization, - help='the percentage of GPU memory to be used for' - 'the model executor') - parser.add_argument('--max-num-batched-tokens', - type=int, - default=EngineArgs.max_num_batched_tokens, - help='maximum number of batched tokens per ' - 'iteration') - parser.add_argument('--max-num-seqs', - type=int, - default=EngineArgs.max_num_seqs, - help='maximum number of sequences per iteration') - parser.add_argument('--disable-log-stats', action='store_true', help='disable logging statistics') - # Quantization settings. - parser.add_argument('--quantization', - '-q', - type=str, - choices=['awq', None], - default=None, - help='Method used to quantize the weights') - return parser - - @classmethod - def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs': - # Get the list of attributes of this dataclass. - attrs = [attr.name for attr in dataclasses.fields(cls)] - # Set the attributes from the parsed arguments. - engine_args = cls(**{attr: getattr(args, attr) for attr in attrs}) - return engine_args - - def create_engine_config( - self, - ) -> EngineConfig: - # bitsandbytes quantization needs a specific model loader - # so we make sure the quant method and the load format are consistent - if (self.quantization == "bitsandbytes" or - self.qlora_adapter_name_or_path is not None) and \ - self.load_format != "bitsandbytes": - raise ValueError("BitsAndBytes quantization and QLoRA adapter only support " - f"'bitsandbytes' load format, but got {self.load_format}") - - if (self.load_format == "bitsandbytes" or - self.qlora_adapter_name_or_path is not None) and \ - self.quantization != "bitsandbytes": - raise ValueError("BitsAndBytes load format and QLoRA adapter only support " - f"'bitsandbytes' quantization, but got {self.quantization}") - - assert self.cpu_offload_gb >= 0, ("CPU offload space must be non-negative" - f", but got {self.cpu_offload_gb}") - - multimodal_config = MultiModalConfig() - device_config = DeviceConfig(self.device) - # NOTE(sgm): we only modify ModelConfig, other configs are import from vllm - model_config = ModelConfig(hf_config=self.model_hf_config, - tokenizer_mode=self.tokenizer_mode, - trust_remote_code=self.trust_remote_code, - dtype=self.dtype, - seed=self.seed, - revision=self.revision, - code_revision=self.code_revision, - rope_scaling=self.rope_scaling, - rope_theta=self.rope_theta, - tokenizer_revision=self.tokenizer_revision, - max_model_len=self.max_model_len, - quantization=self.quantization, - quantization_param_path=self.quantization_param_path, - enforce_eager=self.enforce_eager, - max_context_len_to_capture=self.max_context_len_to_capture, - max_seq_len_to_capture=self.max_seq_len_to_capture, - max_logprobs=self.max_logprobs, - disable_sliding_window=self.disable_sliding_window, - skip_tokenizer_init=self.skip_tokenizer_init, - served_model_name=self.served_model_name, - multimodal_config=multimodal_config) - cache_config = CacheConfig( - block_size=self.block_size, - gpu_memory_utilization=self.gpu_memory_utilization, - swap_space=self.swap_space, - cache_dtype=self.kv_cache_dtype, - num_gpu_blocks_override=self.num_gpu_blocks_override, - sliding_window=model_config.get_sliding_window(), - enable_prefix_caching=self.enable_prefix_caching, - cpu_offload_gb=self.cpu_offload_gb, - ) - parallel_config = ParallelConfig(pipeline_parallel_size=self.pipeline_parallel_size, - tensor_parallel_size=self.tensor_parallel_size, - worker_use_ray=self.worker_use_ray, - max_parallel_loading_workers=self.max_parallel_loading_workers, - disable_custom_all_reduce=self.disable_custom_all_reduce, - tokenizer_pool_config=TokenizerPoolConfig.create_config( - self.tokenizer_pool_size, - self.tokenizer_pool_type, - self.tokenizer_pool_extra_config, - ), - ray_workers_use_nsight=self.ray_workers_use_nsight, - distributed_executor_backend=self.distributed_executor_backend) - - # NOTE[VERL]: Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - parallel_config.world_size = world_size - - max_model_len = model_config.max_model_len - use_long_context = max_model_len > 32768 - if self.enable_chunked_prefill is None: - # If not explicitly set, enable chunked prefill by default for - # long context (> 32K) models. This is to avoid OOM errors in the - # initial memory profiling phase. - if use_long_context: - is_gpu = device_config.device_type == "cuda" - use_sliding_window = (model_config.get_sliding_window() is not None) - use_spec_decode = self.speculative_model is not None - has_seqlen_agnostic_layers = (model_config.contains_seqlen_agnostic_layers(parallel_config)) - if (is_gpu and not use_sliding_window and not use_spec_decode and not self.enable_lora and - not self.enable_prompt_adapter and not self.enable_prefix_caching and - not has_seqlen_agnostic_layers): - self.enable_chunked_prefill = True - logger.warning("Chunked prefill is enabled by default for models with " - "max_model_len > 32K. Currently, chunked prefill might " - "not work with some features or models. If you " - "encounter any issues, please disable chunked prefill " - "by setting --enable-chunked-prefill=False.") - if self.enable_chunked_prefill is None: - self.enable_chunked_prefill = False - - if not self.enable_chunked_prefill and use_long_context: - logger.warning( - "The model has a long context length (%s). This may cause OOM " - "errors during the initial memory profiling phase, or result " - "in low performance due to small KV cache space. Consider " - "setting --max-model-len to a smaller value.", max_model_len) - - # TODO: spec config - speculative_config = SpeculativeConfig.maybe_create_spec_config( - target_model_config=model_config, - target_parallel_config=parallel_config, - target_dtype=self.dtype, - speculative_model=self.speculative_model, - speculative_draft_tensor_parallel_size = \ - self.speculative_draft_tensor_parallel_size, - num_speculative_tokens=self.num_speculative_tokens, - speculative_disable_by_batch_size=self. - speculative_disable_by_batch_size, - speculative_max_model_len=self.speculative_max_model_len, - enable_chunked_prefill=self.enable_chunked_prefill, - use_v2_block_manager=self.use_v2_block_manager, - disable_log_stats=self.disable_log_stats, - ngram_prompt_lookup_max=self.ngram_prompt_lookup_max, - ngram_prompt_lookup_min=self.ngram_prompt_lookup_min, - draft_token_acceptance_method=\ - self.spec_decoding_acceptance_method, - typical_acceptance_sampler_posterior_threshold=self. - typical_acceptance_sampler_posterior_threshold, - typical_acceptance_sampler_posterior_alpha=self. - typical_acceptance_sampler_posterior_alpha, - disable_logprobs=self.disable_logprobs_during_spec_decoding, - ) - - scheduler_config = SchedulerConfig( - max_num_batched_tokens=self.max_num_batched_tokens, - max_num_seqs=self.max_num_seqs, - max_model_len=model_config.max_model_len, - use_v2_block_manager=self.use_v2_block_manager, - num_lookahead_slots=(self.num_lookahead_slots - if speculative_config is None else speculative_config.num_lookahead_slots), - delay_factor=self.scheduler_delay_factor, - enable_chunked_prefill=self.enable_chunked_prefill, - embedding_mode=model_config.embedding_mode, - preemption_mode=self.preemption_mode, - ) - lora_config = LoRAConfig(max_lora_rank=self.max_lora_rank, - max_loras=self.max_loras, - fully_sharded_loras=self.fully_sharded_loras, - lora_extra_vocab_size=self.lora_extra_vocab_size, - long_lora_scaling_factors=self.long_lora_scaling_factors, - lora_dtype=self.lora_dtype, - max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras and self.max_cpu_loras > 0 else - None) if self.enable_lora else None - - if self.qlora_adapter_name_or_path is not None and \ - self.qlora_adapter_name_or_path != "": - if self.model_loader_extra_config is None: - self.model_loader_extra_config = {} - self.model_loader_extra_config["qlora_adapter_name_or_path"] = self.qlora_adapter_name_or_path - - load_config = LoadConfig( - load_format=self.load_format, - download_dir=self.download_dir, - model_loader_extra_config=self.model_loader_extra_config, - ignore_patterns=self.ignore_patterns, - ) - - prompt_adapter_config = PromptAdapterConfig( - max_prompt_adapters=self.max_prompt_adapters, - max_prompt_adapter_token=self.max_prompt_adapter_token) \ - if self.enable_prompt_adapter else None - - decoding_config = DecodingConfig(guided_decoding_backend=self.guided_decoding_backend) - - observability_config = ObservabilityConfig(otlp_traces_endpoint=self.otlp_traces_endpoint) - - if (model_config.get_sliding_window() is not None and scheduler_config.chunked_prefill_enabled and - not scheduler_config.use_v2_block_manager): - raise ValueError("Chunked prefill is not supported with sliding window. " - "Set --disable-sliding-window to disable sliding window.") - - return EngineConfig( - model_config=model_config, - cache_config=cache_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - device_config=device_config, - lora_config=lora_config, - multimodal_config=multimodal_config, - speculative_config=speculative_config, - load_config=load_config, - decoding_config=decoding_config, - observability_config=observability_config, - prompt_adapter_config=prompt_adapter_config, - ) diff --git a/verl/third_party/vllm/vllm_v_0_5_4/config.py b/verl/third_party/vllm/vllm_v_0_5_4/config.py deleted file mode 100644 index 5fc61e6fe6..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/config.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py - -import enum -import json -from typing import List, Optional, Union -from dataclasses import dataclass, field, fields - -import torch -from transformers import PretrainedConfig - -from vllm.logger import init_logger -from vllm.model_executor.layers.quantization import get_quantization_config -from vllm.transformers_utils.config import get_hf_text_config -from vllm.utils import is_hip, print_warning_once -# Add for verl -from vllm.config import ModelConfig, _get_and_verify_dtype, _get_and_verify_max_len, get_served_model_name - -GPTQMarlinConfig = get_quantization_config("gptq_marlin") - -logger = init_logger(__name__) - -_GB = 1 << 30 - - -class ModelConfig(ModelConfig): - """Configuration for the model. - - Args: - model: Name or path of the huggingface model to use. - tokenizer: Name or path of the huggingface tokenizer to use. - tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if - available, and "slow" will always use the slow tokenizer. - trust_remote_code: Trust remote code (e.g., from HuggingFace) when - downloading the model and tokenizer. - download_dir: Directory to download and load the weights, default to the - default cache directory of huggingface. - load_format: The format of the model weights to load: - "auto" will try to load the weights in the safetensors format and - fall back to the pytorch bin format if safetensors format is - not available. - "pt" will load the weights in the pytorch bin format. - "safetensors" will load the weights in the safetensors format. - "npcache" will load the weights in pytorch format and store - a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is - mainly for profiling. - dtype: Data type for model weights and activations. The "auto" option - will use FP16 precision for FP32 and FP16 models, and BF16 precision - for BF16 models. - seed: Random seed for reproducibility. - revision: The specific model version to use. It can be a branch name, - a tag name, or a commit id. If unspecified, will use the default - version. - code_revision: The specific revision to use for the model code on - Hugging Face Hub. It can be a branch name, a tag name, or a - commit id. If unspecified, will use the default version. - tokenizer_revision: The specific tokenizer version to use. It can be a - branch name, a tag name, or a commit id. If unspecified, will use - the default version. - max_model_len: Maximum length of a sequence (including prompt and - output). If None, will be derived from the model. - quantization: Quantization method that was used to quantize the model - weights. If None, we assume the model weights are not quantized. - quantization_param_path: Path to JSON file containing scaling factors. - Used to load KV cache scaling factors into the model when KV cache - type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also - be used to load activation and weight scaling factors when the - model dtype is FP8_E4M3 on ROCm. - enforce_eager: Whether to enforce eager execution. If True, we will - disable CUDA graph and always execute the model in eager mode. - If False, we will use CUDA graph and eager execution in hybrid. - max_context_len_to_capture: Maximum context len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode (DEPRECATED. Use max_seq_len_to_capture instead). - max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode - skip_tokenizer_init: If true, skip initialization of tokenizer and - detokenizer. - served_model_name: The model name used in metrics tag `model_name`, - matches the model name exposed via the APIs. If multiple model - names provided, the first name will be used. If not specified, - the model name will be the same as `model`. - """ - - def __init__( - self, - hf_config: PretrainedConfig, - tokenizer_mode: str, - trust_remote_code: bool, - dtype: Union[str, torch.dtype], - seed: int, - revision: Optional[str] = None, - code_revision: Optional[str] = None, - rope_scaling: Optional[dict] = None, - rope_theta: Optional[float] = None, - tokenizer_revision: Optional[str] = None, - max_model_len: Optional[int] = None, - quantization: Optional[str] = None, - quantization_param_path: Optional[str] = None, - enforce_eager: bool = False, - max_context_len_to_capture: Optional[int] = None, - max_seq_len_to_capture: Optional[int] = None, - max_logprobs: int = 20, - disable_sliding_window: bool = False, - skip_tokenizer_init: bool = False, - served_model_name: Optional[Union[str, List[str]]] = None, - multimodal_config: Optional["MultiModalConfig"] = None, - ) -> None: - self.model = hf_config._name_or_path - self.tokenizer = hf_config._name_or_path - # NOTE(sgm): same as open-sourced - self.tokenizer_mode = tokenizer_mode - self.trust_remote_code = trust_remote_code - self.seed = seed - self.revision = revision - self.code_revision = code_revision - self.rope_scaling = rope_scaling - self.rope_theta = rope_theta - # The tokenizer version is consistent with the model version by default. - if tokenizer_revision is None: - self.tokenizer_revision = revision - else: - self.tokenizer_revision = tokenizer_revision - self.quantization = quantization - self.quantization_param_path = quantization_param_path - self.enforce_eager = enforce_eager - if max_context_len_to_capture is not None: - raise ValueError("`max_context_len_to_capture` is deprecated. " - "Use `max_seq_len_to_capture` instead.") - self.max_seq_len_to_capture = max_seq_len_to_capture - self.max_logprobs = max_logprobs - self.disable_sliding_window = disable_sliding_window - self.skip_tokenizer_init = skip_tokenizer_init - - # self.hf_config = get_config(model, trust_remote_code, revision) - self.hf_config = hf_config - self.hf_text_config = get_hf_text_config(hf_config) - self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype) - # self.served_model_name = get_served_model_name(model, - # served_model_name) - # self._verify_load_format() - # self._verify_tokenizer_mode() - if (not self.disable_sliding_window and self.hf_text_config.model_type == "gemma2" and - self.hf_text_config.sliding_window is not None): - print_warning_once("Gemma 2 uses sliding window attention for every odd layer, " - "which is currently not supported by vLLM. Disabling sliding " - "window and capping the max length to the sliding window size " - f"({self.hf_text_config.sliding_window}).") - self.disable_sliding_window = True - - self.max_model_len = _get_and_verify_max_len(hf_config=self.hf_text_config, - max_model_len=max_model_len, - disable_sliding_window=self.disable_sliding_window, - sliding_window_len=self.get_hf_config_sliding_window()) - self.served_model_name = get_served_model_name( - self.model, # str - served_model_name) - self.multimodal_config = multimodal_config - - if not self.skip_tokenizer_init: - self._verify_tokenizer_mode() - self._verify_embedding_mode() - self._verify_quantization() - self._verify_cuda_graph() - - -class LoadFormat(str, enum.Enum): - AUTO = 'auto' - MEGATRON = "megatron" - HF = "hf" - DTENSOR = 'dtensor' - DUMMY_HF = 'dummy_hf' - DUMMY_MEGATRON = 'dummy_megatron' - DUMMY_DTENSOR = 'dummy_dtensor' - - -# TODO: check whether this is necessary -@dataclass -class LoadConfig: - """ - download_dir: Directory to download and load the weights, default to the - default cache directory of huggingface. - load_format: The format of the model weights to load: - "auto" will try to load the weights in the safetensors format and - fall back to the pytorch bin format if safetensors format is - not available. - "pt" will load the weights in the pytorch bin format. - "safetensors" will load the weights in the safetensors format. - "npcache" will load the weights in pytorch format and store - a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is - mainly for profiling. - "tensorizer" will use CoreWeave's tensorizer library for - fast weight loading. - "bitsandbytes" will load nf4 type weights. - ignore_patterns: The list of patterns to ignore when loading the model. - Default to "original/**/*" to avoid repeated loading of llama's - checkpoints. - - """ - - load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO - download_dir: Optional[str] = None - model_loader_extra_config: Optional[Union[str, dict]] = field(default_factory=dict) - ignore_patterns: Optional[Union[List[str], str]] = None - - def __post_init__(self): - model_loader_extra_config = self.model_loader_extra_config or {} - if isinstance(model_loader_extra_config, str): - self.model_loader_extra_config = json.loads(model_loader_extra_config) - self._verify_load_format() - - if self.ignore_patterns is not None and len(self.ignore_patterns) > 0: - logger.info("Ignoring the following patterns when downloading weights: %s", self.ignore_patterns) - else: - self.ignore_patterns = ["original/**/*"] - - def _verify_load_format(self) -> None: - if not isinstance(self.load_format, str): - return - - load_format = self.load_format.lower() - self.load_format = LoadFormat(load_format) - - rocm_not_supported_load_format: List[str] = [] - if is_hip() and load_format in rocm_not_supported_load_format: - rocm_supported_load_format = [ - f for f in LoadFormat.__members__ if (f not in rocm_not_supported_load_format) - ] - raise ValueError(f"load format '{load_format}' is not supported in ROCm. " - f"Supported load formats are " - f"{rocm_supported_load_format}") diff --git a/verl/third_party/vllm/vllm_v_0_5_4/dtensor_weight_loaders.py b/verl/third_party/vllm/vllm_v_0_5_4/dtensor_weight_loaders.py deleted file mode 100644 index 732b543db6..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/dtensor_weight_loaders.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models - -from typing import Dict, Iterable, Tuple -import torch -import torch.nn as nn -from torch.distributed._tensor import DTensor, Shard, Replicate - -from vllm.model_executor.layers.linear import * -from vllm.model_executor.models import ModelRegistry -from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm.model_executor.models.utils import is_pp_missing_parameter - - -def gemma_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - for (param_name, shard_name, shard_id) in stacked_params_mapping: - if shard_name not in name: - continue - stacked_name = name.replace(shard_name, param_name) - # Skip loading extra bias for GPTQ models. - if stacked_name.endswith(".bias") and stacked_name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[stacked_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # lm_head is not used in vllm as it is tied with embed_token. - # To prevent errors, skip loading lm_head.weight. - if "lm_head.weight" in name: - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def gptbigcode_dtensor_load_weights(actor_weights: Dict, vllm_model: nn.Module): - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "lm_head.weight" in name: - continue - if ".attn.bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def starcoder2_dtensor_load_weights(actor_weights: Dict, vllm_model: nn.Module): - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - - for (param_name, weight_name, shard_id) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def llama_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if ("rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name): - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - # With tie_word_embeddings, we can skip lm_head.weight - # The weight might appear unnecessarily in the files if the model is - # processed with quantization, LoRA, fine-tuning, etc. - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - for (param_name, weight_name, shard_id) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight) - - -def qwen2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - for (param_name, weight_name, shard_id) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -from vllm.model_executor.layers.fused_moe import FusedMoE - - -def deepseekv2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping(ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=vllm_model.config.n_routed_experts) - - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - for (param_name, weight_name, shard_id) in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if (("mlp.experts." in name) and name not in params_dict): - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, vllm_model): - continue - - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name, vllm_model): - continue - - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, - local_loaded_weight.to(dtype=param.dtype), - weight_name, - shard_id=shard_id, - expert_id=expert_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, vllm_model): - continue - - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def gpt2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - pass - - -def redistribute_dtensor(param_name: str, loaded_weights: DTensor, parallelize_plan: Dict = None): - param_name = _process_parameter_names(name=param_name) - if parallelize_plan is not None: - assert param_name in parallelize_plan.keys(), \ - f"param name: {param_name} not in parallelize_plan :{parallelize_plan.keys()}" - placement = parallelize_plan[param_name] - local_loaded_weights = loaded_weights.redistribute(device_mesh=loaded_weights.device_mesh, - placements=placement).to_local() - else: - local_loaded_weights = loaded_weights.full_tensor() - return local_loaded_weights - - -def _process_parameter_names(name): - # Remove '.weight' if it exists at the end of the string - if name.endswith(".weight"): - name = name[:-7] - - # Remove 'model.layers.x.' or 'model.' prefix - if "model.layers" in name: - parts = name.split('.') - # Reconstruct the string without 'model.layers.x.' - name = '.'.join(parts[3:]) # parts[0] is 'model', parts[1] is 'layers', parts[2] is 'x' - elif name.startswith("model."): - name = name[6:] # Remove 'model.' - - return name - - -__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__ = { - 'GPT2LMHeadModel': gpt2_dtensor_weight_loader, - 'LlamaForCausalLM': llama_dtensor_weight_loader, - 'LLaMAForCausalLM': llama_dtensor_weight_loader, - 'MistralForCausalLM': llama_dtensor_weight_loader, # mistral is the same as llama in vLLM - 'InternLMForCausalLM': llama_dtensor_weight_loader, - 'AquilaModel': llama_dtensor_weight_loader, - 'AquilaForCausalLM': llama_dtensor_weight_loader, - 'Phi3ForCausalLM': llama_dtensor_weight_loader, - 'GemmaForCausalLM': gemma_dtensor_weight_loader, - 'Gemma2ForCausalLM': gemma_dtensor_weight_loader, - 'GPTBigCodeForCausalLM': gptbigcode_dtensor_load_weights, - 'Starcoder2ForCausalLM': starcoder2_dtensor_load_weights, - 'Qwen2ForCausalLM': qwen2_dtensor_weight_loader, - 'DeepseekV2ForCausalLM': deepseekv2_dtensor_weight_loader -} - - -# the actor model is .state_dict() -# Load dtensor weights -def load_dtensor_weights(actor_weights: Dict, vllm_model: nn.Module): - weight_loader = _get_model_weight_loader(vllm_model.__class__.__name__) - weight_loader(actor_weights, vllm_model) - # NOTE(sgm) to reduce peak memory usage, we offload vllm model to cpu - # after init, and we need this after sync model weights for in first iter. - vllm_model = vllm_model.cuda() - - -def _get_model_weight_loader(arch: str): - if arch in __MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__: - return __MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__[arch] - raise ValueError(f"Model architectures {arch} are not supported for now. " - f"Supported architectures: {__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__.keys()}") - - -# NOTE(sgm): we use per-parameter weight loader in each vllm sub -def update_dtensor_weight_loader(): - pass diff --git a/verl/third_party/vllm/vllm_v_0_5_4/hf_weight_loader.py b/verl/third_party/vllm/vllm_v_0_5_4/hf_weight_loader.py deleted file mode 100644 index 7af4953f35..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/hf_weight_loader.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models - -from typing import Dict, Union, Optional, Iterable, Tuple - -import torch -import torch.nn as nn - -from vllm.model_executor.model_loader.utils import set_default_torch_dtype -from vllm.model_executor.model_loader.weight_utils import default_weight_loader - - -def update_hf_weight_loader(): - print('no hf weight loader need to be updated') - return - - -def load_hf_weights(actor_weights: Dict, vllm_model: nn.Module): - assert isinstance(actor_weights, Dict) - with set_default_torch_dtype(next(vllm_model.parameters()).dtype): # TODO - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in actor_weights.keys(): - del actor_weights["lm_head.weight"] - vllm_model.load_weights(actor_weights.items()) - for _, module in vllm_model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - vllm_model = vllm_model.cuda() diff --git a/verl/third_party/vllm/vllm_v_0_5_4/llm.py b/verl/third_party/vllm/vllm_v_0_5_4/llm.py deleted file mode 100644 index 5f56f1e07a..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/llm.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/llm.py - -from contextlib import contextmanager -from typing import ClassVar, List, Optional, Sequence, Union, cast, overload, Dict, Tuple - -from tqdm import tqdm -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast -from transformers import PretrainedConfig -import torch.nn as nn -from .arg_utils import EngineArgs -from .llm_engine_sp import LLMEngine -from vllm import LLM -from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt, parse_and_batch_prompt) -from vllm.logger import init_logger -from vllm.lora.request import LoRARequest -from vllm.model_executor.guided_decoding import (GuidedDecodingRequest, get_local_guided_decoding_logits_processor) -from vllm.model_executor.guided_decoding.guided_fields import LLMGuidedOptions -from vllm.outputs import EmbeddingRequestOutput, RequestOutput -from vllm.pooling_params import PoolingParams -from vllm.prompt_adapter.request import PromptAdapterRequest -from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizer import get_cached_tokenizer -from vllm.usage.usage_lib import UsageContext -from vllm.utils import Counter, deprecate_kwargs -import torch -from torch.nn.utils.rnn import pad_sequence -from verl.workers.rollout.tokenizer import HybridEngineBaseTokenizer - - -class LLM(LLM): - """An LLM for generating texts from given prompts and sampling parameters. - - This class includes a tokenizer, a language model (possibly distributed - across multiple GPUs), and GPU memory space allocated for intermediate - states (aka KV cache). Given a batch of prompts and sampling parameters, - this class generates texts from the model, using an intelligent batching - mechanism and efficient memory management. - - NOTE: This class is intended to be used for offline inference. For online - serving, use the `AsyncLLMEngine` class instead. - NOTE: For the comprehensive list of arguments, see `EngineArgs`. - - Args: - model: A HuggingFace Transformers model instance. - tokenizer: A HuggingFace Transformers tokenizer instance. - tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer - if available, and "slow" will always use the slow tokenizer. - trust_remote_code: Trust remote code (e.g., from HuggingFace) when - downloading the model and tokenizer. - tensor_parallel_size: The number of GPUs to use for distributed - execution with tensor parallelism. - dtype: The data type for the model weights and activations. Currently, - we support `float32`, `float16`, and `bfloat16`. If `auto`, we use - the `torch_dtype` attribute specified in the model config file. - However, if the `torch_dtype` in the config is `float32`, we will - use `float16` instead. - quantization: The method used to quantize the model weights. Currently, - we support "awq". If None, we assume the model weights are not - quantized and use `dtype` to determine the data type of the weights. - revision: The specific model version to use. It can be a branch name, - a tag name, or a commit id. - tokenizer_revision: The specific tokenizer version to use. It can be a - branch name, a tag name, or a commit id. - seed: The seed to initialize the random number generator for sampling. - gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to - reserve for the model weights, activations, and KV cache. Higher - values will increase the KV cache size and thus improve the model's - throughput. However, if the value is too high, it may cause out-of- - memory (OOM) errors. - swap_space: The size (GiB) of CPU memory per GPU to use as swap space. - This can be used for temporarily storing the states of the requests - when their `best_of` sampling parameters are larger than 1. If all - requests will have `best_of=1`, you can safely set this to 0. - Otherwise, too small values may cause out-of-memory (OOM) errors. - enforce_eager: Whether to enforce eager execution. If True, we will - disable CUDA graph and always execute the model in eager mode. - If False, we will use CUDA graph and eager execution in hybrid. - max_context_len_to_capture: Maximum context len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode. - disable_custom_all_reduce: See ParallelConfig - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer], - model_hf_config: PretrainedConfig, - tokenizer_mode: str = "auto", - trust_remote_code: bool = False, - skip_tokenizer_init: bool = False, - tensor_parallel_size: int = 1, - dtype: str = "auto", - quantization: Optional[str] = None, - revision: Optional[str] = None, - tokenizer_revision: Optional[str] = None, - seed: int = 0, - gpu_memory_utilization: float = 0.9, - swap_space: int = 4, - cpu_offload_gb: float = 0, - enforce_eager: bool = False, - max_context_len_to_capture: Optional[int] = None, - max_seq_len_to_capture: int = 8192, - disable_custom_all_reduce: bool = False, - load_format = 'auto', - **kwargs, - ) -> None: - if "disable_log_stats" not in kwargs: - kwargs["disable_log_stats"] = True - engine_args = EngineArgs( - model_hf_config=model_hf_config, - tensor_parallel_size=tensor_parallel_size, - dtype=dtype, - quantization=quantization, - revision=revision, - tokenizer_revision=tokenizer_revision, - seed=seed, - gpu_memory_utilization=gpu_memory_utilization, - swap_space=swap_space, - cpu_offload_gb=cpu_offload_gb, - enforce_eager=enforce_eager, - max_context_len_to_capture=max_context_len_to_capture, - max_seq_len_to_capture=max_seq_len_to_capture, - disable_custom_all_reduce=disable_custom_all_reduce, - load_format=load_format, - skip_tokenizer_init=skip_tokenizer_init, - **kwargs, - ) - tokenizer_cls = (PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer) - if not isinstance(tokenizer, tokenizer_cls): - raise ValueError( - f"Unexpected tokenizer type: {type(tokenizer)}. Must be" - "one of the following: PreTrainedTokenizer, PreTrainedTokenizerFast, verl.workers.rollout.HybridEngineBaseTokenizer" - ) - self.llm_engine = LLMEngine.from_engine_args(model, tokenizer, engine_args) # TODO: check usagecontext - self.request_counter = Counter() - - def init_cache_engine(self): - self.llm_engine.init_cache_engine() - - def free_cache_engine(self): - self.llm_engine.free_cache_engine() - - def get_tokenizer(self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: - return self.llm_engine.tokenizer - - def set_tokenizer( - self, - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], - ) -> None: - self.llm_engine.tokenizer = tokenizer - - def _run_engine(self, *, use_tqdm: bool) -> List[Union[RequestOutput, EmbeddingRequestOutput]]: - # Initialize tqdm. - if use_tqdm: - num_requests = self.llm_engine.get_num_unfinished_requests() - pbar = tqdm( - total=num_requests, - desc="Processed prompts", - dynamic_ncols=True, - postfix=(f"est. speed input: {0:.2f} toks/s, " - f"output: {0:.2f} toks/s"), - ) - # Run the engine. - outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = [] - total_in_toks = 0 - total_out_toks = 0 - while self.llm_engine.has_unfinished_requests(): - step_outputs = self.llm_engine.step() - for output in step_outputs: - if output.finished: - outputs.append(output) - if use_tqdm: - if isinstance(output, RequestOutput): - # Calculate tokens only for RequestOutput - total_in_toks += len(output.prompt_token_ids) - in_spd = total_in_toks / pbar.format_dict["elapsed"] - total_out_toks += sum(len(stp.token_ids) for stp in output.outputs) - out_spd = total_out_toks / pbar.format_dict["elapsed"] - pbar.postfix = (f"est. speed input: {in_spd:.2f} toks/s, " - f"output: {out_spd:.2f} toks/s") - pbar.update(1) - if use_tqdm: - pbar.close() - # Sort the outputs by request ID. - # This is necessary because some requests may be finished earlier than - # its previous requests. - outputs = sorted(outputs, key=lambda x: int(x.request_id)) - return self._post_process_outputs(outputs) - - # # NOTE(shengguangming): add for verl - # # TODO(sgm): we can optimize it by making the dataloader yield List[int] without padding. - # def _pre_process_inputs(self, prompt_token_ids: torch.Tensor) -> List[int]: - # # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - # non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - # token_ids = prompt_token_ids[non_pad_index:].tolist() - # return token_ids - - # NOTE(shengguangming): add for verl - def _post_process_outputs(self, request_outputs: List[RequestOutput]) -> Tuple[torch.Tensor, torch.Tensor]: - output_token_ids = [] - logprobs = [] - for request_output in request_outputs: # List[RequestOutput] - outputs = request_output.outputs - for output in outputs: # List[CompletionOutput], usually len == 1 - output_token_ids.append(torch.tensor(output.token_ids)) - # TODO(shengguangming): can be optimzied by rewrite the Sampler._get_logprobs() logits - logprobs_dicts = output.logprobs - if logprobs_dicts is not None: - logprob = [] - for logprobs_dict, id in zip(logprobs_dicts, output.token_ids): - logprob.append(logprobs_dict[id].logprob) - logprobs.append(torch.tensor(logprob)) - - pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - output_token_ids = pad_sequence(output_token_ids, batch_first=True, padding_value=pad_token_id) - if len(logprobs) > 0: - logprobs = pad_sequence(logprobs, batch_first=True, padding_value=pad_token_id) - return output_token_ids, logprobs - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.llm_engine.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - def offload_model_weights(self) -> None: - self.llm_engine.offload_model_weights() diff --git a/verl/third_party/vllm/vllm_v_0_5_4/llm_engine_sp.py b/verl/third_party/vllm/vllm_v_0_5_4/llm_engine_sp.py deleted file mode 100644 index 8d161e7470..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/llm_engine_sp.py +++ /dev/null @@ -1,328 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/llm_engine.py - -import torch -from typing import Dict, Optional, Union, Type - -import vllm.envs as envs -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, EngineConfig, LoRAConfig, MultiModalConfig, - ObservabilityConfig, ParallelConfig, PromptAdapterConfig, SchedulerConfig, SpeculativeConfig) -from vllm.core.scheduler import Scheduler -from vllm.engine.output_processor.interfaces import (SequenceGroupOutputProcessor) -from vllm.engine.output_processor.stop_checker import StopChecker -from vllm.executor.executor_base import ExecutorBase -from vllm.inputs import INPUT_REGISTRY, LLMInputs, PromptInputs -from vllm.logger import init_logger -from vllm.transformers_utils.detokenizer import Detokenizer -from vllm.engine.metrics import (LoggingStatLogger, PrometheusStatLogger, StatLoggerBase, Stats) -from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context, init_tracer) -from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled, usage_message) -from vllm.utils import Counter -from vllm.engine.llm_engine import _load_generation_config_dict -from vllm.engine.llm_engine import LLMEngine -from vllm.version import __version__ as VLLM_VERSION - -import torch.nn as nn -from .arg_utils import EngineArgs -from .tokenizer import TokenizerGroup -from .config import ModelConfig, LoadConfig - -logger = init_logger(__name__) -_LOCAL_LOGGING_INTERVAL_SEC = 5 - - -class LLMEngine(LLMEngine): - """An LLM engine that receives requests and generates texts. - - This is the main class for the vLLM engine. It receives requests - from clients and generates texts from the LLM. It includes a tokenizer, a - language model (possibly distributed across multiple GPUs), and GPU memory - space allocated for intermediate states (aka KV cache). This class utilizes - iteration-level scheduling and efficient memory management to maximize the - serving throughput. - - The `LLM` class wraps this class for offline batched inference and the - `AsyncLLMEngine` class wraps this class for online serving. - - NOTE: The config arguments are derived from the `EngineArgs` class. For the - comprehensive list of arguments, see `EngineArgs`. - - Args: - model: the actor model initialize outside vllm (add for verl) - tokenizer: the initialized tokenizer (add for verl) - model_config: The configuration related to the LLM model. - cache_config: The configuration related to the KV cache memory - management. - parallel_config: The configuration related to distributed execution. - scheduler_config: The configuration related to the request scheduler. - distributed_init_method: The initialization method for distributed - execution. See `torch.distributed.init_process_group` for details. - placement_group: Ray placement group for distributed execution. - Required for distributed execution. - log_stats: Whether to log statistics. - """ - - def __init__( - self, - # NOTE(sgm): first two arguments are added for verl - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: nn.Module, - # NOTE(sgm): vllm original arguments - model_config: ModelConfig, - cache_config: CacheConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - multimodal_config: Optional[MultiModalConfig], - speculative_config: Optional[SpeculativeConfig], - decoding_config: Optional[DecodingConfig], - observability_config: Optional[ObservabilityConfig], - prompt_adapter_config: Optional[PromptAdapterConfig], - executor_class: Type[ExecutorBase], - log_stats: bool, - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None, - ) -> None: - logger.info( - "Initializing an LLM engine (v%s) with config: " - "model=%r, speculative_config=%r, tokenizer=%r, " - "skip_tokenizer_init=%s, revision=%s, " - "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, " - "trust_remote_code=%s, dtype=%s, max_seq_len=%d, " - "download_dir=%r, load_format=%s, tensor_parallel_size=%d, " - "pipeline_parallel_size=%d, " - "disable_custom_all_reduce=%s, quantization=%s, " - "enforce_eager=%s, kv_cache_dtype=%s, " - "quantization_param_path=%s, device_config=%s, " - "decoding_config=%r, observability_config=%r, " - "seed=%d, served_model_name=%s, use_v2_block_manager=%s, " - "enable_prefix_caching=%s)", - VLLM_VERSION, - model_config.model, - speculative_config, - model_config.tokenizer, - model_config.skip_tokenizer_init, - model_config.revision, - model_config.rope_scaling, - model_config.rope_theta, - model_config.tokenizer_revision, - model_config.trust_remote_code, - model_config.dtype, - model_config.max_model_len, - load_config.download_dir, - load_config.load_format, - parallel_config.tensor_parallel_size, - parallel_config.pipeline_parallel_size, - parallel_config.disable_custom_all_reduce, - model_config.quantization, - model_config.enforce_eager, - cache_config.cache_dtype, - model_config.quantization_param_path, - device_config.device, - decoding_config, - observability_config, - model_config.seed, - model_config.served_model_name, - scheduler_config.use_v2_block_manager, - cache_config.enable_prefix_caching, - ) - # TODO(woosuk): Print more configs in debug mode. - - self.model_config = model_config - self.cache_config = cache_config - self.lora_config = lora_config - self.multimodal_config = multimodal_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.speculative_config = speculative_config - self.load_config = load_config - self.decoding_config = decoding_config or DecodingConfig() - self.prompt_adapter_config = prompt_adapter_config - self.observability_config = observability_config or ObservabilityConfig() - self.log_stats = log_stats - - # self.model = model # should not store the model, it should be deleted - # TODO(shengguangming): maybe we can choose init here or from arguments - if not self.model_config.skip_tokenizer_init: - self.tokenizer = self._init_tokenizer(tokenizer) - self.detokenizer = Detokenizer(self.tokenizer) - else: - self.tokenizer = None - self.detokenizer = None - - self.seq_counter = Counter() - self.generation_config_fields = _load_generation_config_dict(model_config) - - self.input_processor = INPUT_REGISTRY.create_input_processor(self.model_config) - - self.model_executor = executor_class( - model=model, # add for spmd_gpu_executor - model_config=model_config, - cache_config=cache_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - device_config=device_config, - lora_config=lora_config, - multimodal_config=multimodal_config, - speculative_config=speculative_config, - load_config=load_config, - prompt_adapter_config=prompt_adapter_config, - ) - - # Profile the memory usage and initialize the cache. - if not self.model_config.embedding_mode: - self._initialize_kv_caches() - - # If usage stat is enabled, collect relevant info. - if is_usage_stats_enabled(): - from vllm.model_executor.model_loader import (get_architecture_class_name) - usage_message.report_usage( - get_architecture_class_name(model_config), - usage_context, - extra_kvs={ - # Common configuration - "dtype": str(model_config.dtype), - "tensor_parallel_size": parallel_config.tensor_parallel_size, - "block_size": cache_config.block_size, - "gpu_memory_utilization": cache_config.gpu_memory_utilization, - - # Quantization - "quantization": model_config.quantization, - "kv_cache_dtype": str(cache_config.cache_dtype), - - # Feature flags - "enable_lora": bool(lora_config), - "enable_prompt_adapter": bool(prompt_adapter_config), - "enable_prefix_caching": cache_config.enable_prefix_caching, - "enforce_eager": model_config.enforce_eager, - "disable_custom_all_reduce": parallel_config.disable_custom_all_reduce, - }) - - if self.tokenizer: - # Ping the tokenizer to ensure liveness if it runs in a - # different process. - self.tokenizer.ping() - - # Create the scheduler. - # NOTE: the cache_config here have been updated with the numbers of - # GPU and CPU blocks, which are profiled in the distributed executor. - self.scheduler = [ - Scheduler(scheduler_config, cache_config, lora_config, parallel_config.pipeline_parallel_size) - for _ in range(parallel_config.pipeline_parallel_size) - ] - - # Metric Logging. - if self.log_stats: - if stat_loggers is not None: - self.stat_loggers = stat_loggers - else: - self.stat_loggers = { - "logging": - LoggingStatLogger(local_interval=_LOCAL_LOGGING_INTERVAL_SEC), - "prometheus": - PrometheusStatLogger(local_interval=_LOCAL_LOGGING_INTERVAL_SEC, - labels=dict(model_name=model_config.served_model_name), - max_model_len=self.model_config.max_model_len), - } - self.stat_loggers["prometheus"].info("cache_config", self.cache_config) - - self.tracer = None - if self.observability_config.otlp_traces_endpoint: - self.tracer = init_tracer("vllm.llm_engine", self.observability_config.otlp_traces_endpoint) - - # Create sequence output processor, e.g. for beam search or - # speculative decoding. - self.output_processor = (SequenceGroupOutputProcessor.create_output_processor( - self.scheduler_config, - self.detokenizer, - self.scheduler, - self.seq_counter, - self.get_tokenizer_for_seq, - stop_checker=StopChecker( - self.scheduler_config.max_model_len, - self.get_tokenizer_for_seq, - ), - )) - - # TODO(sgm): add for verl but we may not tokenizer in Rollout - def _init_tokenizer(self, tokenizer, **tokenizer_init_kwargs): - init_kwargs = dict(enable_lora=bool(self.lora_config), - max_num_seqs=self.scheduler_config.max_num_seqs, - max_input_length=None) - init_kwargs.update(tokenizer_init_kwargs) - return TokenizerGroup(tokenizer, **init_kwargs) - - def init_cache_engine(self): - # TODO: check whether we should rebuild the CUDAGraph every iter when offload/load KVCache - # Re-capture CUDAGraph would be time-consuming - self.model_executor.init_cache_engine() - - def free_cache_engine(self): - self.model_executor.free_cache_engine() - - # NOTE(sgm): currently, we only support GPU executor - # The GPUExecutor remove the Ray dependency - @classmethod - def _get_executor_cls(cls, engine_config: EngineConfig) -> Type[ExecutorBase]: - assert engine_config.device_config.device_type == "cuda", \ - "Currently, the vllm in verl only support running on GPU" - - if engine_config.parallel_config.world_size == 1: - engine_config.load_config.load_format = "dummy_hf" - - from .spmd_gpu_executor import SPMDGPUExecutor - executor_class = SPMDGPUExecutor - return executor_class - - @classmethod - def from_engine_args( - cls, - model, - tokenizer, - engine_args: EngineArgs, - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None, - ) -> "LLMEngine": - """Creates an LLM engine from the engine arguments.""" - # Create the engine configs. - engine_config = engine_args.create_engine_config() - executor_class = cls._get_executor_cls(engine_config) - # Initialize the cluster and specify the executor class. - assert engine_config.device_config.device_type == "cuda", \ - "Currently, the vllm in verl only support running on GPU" - - from .spmd_gpu_executor import SPMDGPUExecutor - executor_class = SPMDGPUExecutor - - # Create the LLM engine. - engine = cls( - model, - tokenizer, - **engine_config.to_dict(), - executor_class=executor_class, - log_stats=not engine_args.disable_log_stats, - usage_context=usage_context, - stat_loggers=stat_loggers, - ) - return engine - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.model_executor.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - def offload_model_weights(self) -> None: - self.model_executor.offload_model_weights() diff --git a/verl/third_party/vllm/vllm_v_0_5_4/megatron_weight_loaders.py b/verl/third_party/vllm/vllm_v_0_5_4/megatron_weight_loaders.py deleted file mode 100644 index 4f2b19a904..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/megatron_weight_loaders.py +++ /dev/null @@ -1,307 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models - -from typing import Dict -import torch -import torch.nn as nn - -from vllm.model_executor.layers.linear import * -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead -from vllm.model_executor.layers.activation import ScaledActivation -from vllm.model_executor.models import ModelRegistry - - -# NOTE(shengguangming): replace the origin weight loader function in the class -def parallel_weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Parallel Linear weight loader.""" - assert param.size() == loaded_weight.size( - ), 'the parameter size is not align with the loaded weight size, param size: {}, loaded_weight size: {}'.format( - param.size(), loaded_weight.size()) - assert param.data.dtype == loaded_weight.data.dtype, "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Default weight loader.""" - assert param.size() == loaded_weight.size() - assert param.data.dtype == loaded_weight.data.dtype, "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def gpt2_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "lm_head.weight" in name: - # GPT-2 ties the weights of the embedding layer and the final - # linear layer. - continue - if ".attn.bias" in name or ".attn.masked_bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - if not name.startswith("transformer."): - name = "transformer." + name - param = params_dict[name] - # The HF's GPT-2 implementation uses Conv1D instead of Linear. - # Because of this, we need to transpose the weights. - # Note(zhuohan): the logic below might break quantized models. - for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]: - if conv1d_weight_name not in name: - continue - if not name.endswith(".weight"): - continue - # TODO: check megatron - loaded_weight = loaded_weight.t() - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_te_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv.layer_norm_weight", "input_layernorm.weight"), - ("self_attention.linear_qkv.layer_norm_bias", "input_layernorm.bias"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1.layer_norm_weight', 'post_attention_layernorm.weight'), - ('mlp.linear_fc1.layer_norm_bias', 'post_attention_layernorm.bias'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ( - 'input_layernorm', - 'input_layernorm', - ), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def _replace_name(megatron_name, name_mapping): - for m_name, v_name in name_mapping: - if m_name not in megatron_name: - continue - if 'layers' in megatron_name: # deal with decoder layers - megatron_name = megatron_name.replace('decoder', 'model') - megatron_name_list = megatron_name.split('.') - if 'layer_norm_weight' in megatron_name_list or 'layer_norm_bias' in megatron_name_list: - param_name_list = megatron_name_list[:3] - param_name_list.append(v_name) - param_name = '.'.join(param_name_list) - else: - param_name_list = megatron_name_list[:3] - weight_or_bias = megatron_name_list[-1] - param_name_list.append(v_name) - param_name_list.append(weight_or_bias) - param_name = '.'.join(param_name_list) - return param_name - else: - param_name = megatron_name.replace(m_name, v_name) - return param_name - - -def llama_megatron_core_te_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv.layer_norm_weight", "input_layernorm.weight"), - ("self_attention.linear_qkv.layer_norm_bias", "input_layernorm.bias"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1.layer_norm_weight', 'post_attention_layernorm.weight'), - ('mlp.linear_fc1.layer_norm_bias', 'post_attention_layernorm.bias'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", 'self_attn.o_proj'), - ( - 'input_layernorm', - 'input_layernorm', - ), - ('pre_mlp_layernorm', 'post_attention_layernorm'), - ('mlp.linear_fc1', 'mlp.gate_up_proj'), - ('mlp.linear_fc2', 'mlp.down_proj'), - ('decoder.final_layernorm', 'model.norm'), - ('output_layer', 'lm_head'), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith('.bias') and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def _replace_name(megatron_name, name_mapping): - for m_name, v_name in name_mapping: - if m_name not in megatron_name: - continue - if 'layers' in megatron_name: # deal with decoder layers - megatron_name = megatron_name.replace('decoder', 'model') - megatron_name_list = megatron_name.split('.') - if 'layer_norm_weight' in megatron_name_list or 'layer_norm_bias' in megatron_name_list: - param_name_list = megatron_name_list[:3] - param_name_list.append(v_name) - param_name = '.'.join(param_name_list) - else: - param_name_list = megatron_name_list[:3] - weight_or_bias = megatron_name_list[-1] - param_name_list.append(v_name) - param_name_list.append(weight_or_bias) - param_name = '.'.join(param_name_list) - return param_name - else: - param_name = megatron_name.replace(m_name, v_name) - return param_name - - -def mistral_megatron_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # TODO: need to implement a general way to deal with prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -__LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__ = { - ColumnParallelLinear: parallel_weight_loader, - MergedColumnParallelLinear: parallel_weight_loader, - QKVParallelLinear: parallel_weight_loader, - RowParallelLinear: parallel_weight_loader, - VocabParallelEmbedding: parallel_weight_loader, - ParallelLMHead: parallel_weight_loader - # "ScaledActivation.weight_loader": ScaledActivation, # TODO(shengguangming): latest commit in vllm fix awq for this function and add load_weights - # "default_weight_loader": default_weight_loader -} - -# for layer_class, weight_loader in __LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__.items(): -# # setattr(layer_class, 'megatron_weight_loader', weight_loader) -# layer_class.weight_loader = weight_loader - -__MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__ = { - 'GPT2LMHeadModel': gpt2_weight_loader, - 'LlamaForCausalLM': llama_megatron_weight_loader, # use te backend for open-source megatron - 'LLaMAForCausalLM': llama_megatron_weight_loader, - 'MistralForCausalLM': mistral_megatron_weight_loader, -} - - -# the actor model is .state_dict() -# Load megatron weights -def load_megatron_weights(actor_weights: Dict, vllm_model: nn.Module): - weight_loader = _get_model_weight_loader(vllm_model.__class__.__name__) - weight_loader(actor_weights, vllm_model) - # NOTE(sgm) to reduce peak memory usage, we offload vllm model to cpu - # after init, and we need this after sync model weights for in first iter. - vllm_model = vllm_model.cuda() - - -def _get_model_weight_loader(arch: str): - if arch in __MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__: - return __MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__[arch] - raise ValueError(f"Model architectures {arch} are not supported for now. " - f"Supported architectures: {ModelRegistry.get_supported_archs()}") - - -def update_megatron_weight_loader(): - for layer_class, weight_loader in __LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__.items(): - layer_class.weight_loader = weight_loader diff --git a/verl/third_party/vllm/vllm_v_0_5_4/model_loader.py b/verl/third_party/vllm/vllm_v_0_5_4/model_loader.py deleted file mode 100644 index 1b675bb79d..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/model_loader.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/model_loader - -from typing import Dict, Union, Optional, Iterable, Tuple - -import torch -import torch.nn as nn -from transformers import PreTrainedModel - -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, ModelConfig, MultiModalConfig, - ParallelConfig, SchedulerConfig) -from vllm.model_executor.model_loader import BaseModelLoader -from vllm.model_executor.model_loader.loader import _initialize_model -from vllm.model_executor.model_loader.utils import set_default_torch_dtype -from vllm.distributed.communication_op import tensor_model_parallel_all_gather - -from .config import ModelConfig, LoadFormat, LoadConfig -from .megatron_weight_loaders import load_megatron_weights, update_megatron_weight_loader -from .dtensor_weight_loaders import load_dtensor_weights, update_dtensor_weight_loader -from .hf_weight_loader import update_hf_weight_loader - - -def get_model(actor_model: Union[PreTrainedModel, Dict], - model_config: ModelConfig, - load_config: LoadConfig, - device_config: DeviceConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - lora_config: Optional[LoRAConfig], - multimodal_config: Optional[MultiModalConfig], - cache_config: CacheConfig = None) -> nn.Module: - loader = get_model_loader(load_config) - if load_config.load_format.startswith('dummy'): - return loader.load_model(model_config=model_config, - device_config=device_config, - lora_config=lora_config, - multimodal_config=multimodal_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - cache_config=cache_config) - else: - return loader.load_model(actor_model=actor_model, - model_config=model_config, - device_config=device_config, - lora_config=lora_config, - multimodal_config=multimodal_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - cache_config=cache_config) - - -def get_model_loader(load_config: LoadConfig) -> BaseModelLoader: - """Get a model loader based on the load format.""" - - if isinstance(load_config.load_format, type): - return load_config.load_format(load_config) - - if load_config.load_format == LoadFormat.AUTO: - update_megatron_weight_loader() - return MegatronLoader(load_config) - - # NOTE(sgm): change the weight_loader function in runtime - if load_config.load_format == LoadFormat.MEGATRON: - update_megatron_weight_loader() - return MegatronLoader(load_config) - - if load_config.load_format == LoadFormat.HF: - update_hf_weight_loader() - return HFLoader(load_config) - - if load_config.load_format == LoadFormat.DTENSOR: - update_dtensor_weight_loader() - return DTensorLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_HF: - update_hf_weight_loader() - return DummyModelLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_MEGATRON: - update_megatron_weight_loader() - return DummyModelLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_DTENSOR: - update_dtensor_weight_loader() - return DummyModelLoader(load_config) - - raise ValueError('load format not supported in verl: {}, only support {} and {}'.format( - load_config.load_format, LoadFormat.MEGATRON, LoadFormat.HF)) - - -class DummyModelLoader(BaseModelLoader): - """Model loader that will set model weights to random values.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def load_model(self, *, model_config: ModelConfig, device_config: DeviceConfig, lora_config: Optional[LoRAConfig], - multimodal_config: Optional[MultiModalConfig], parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, cache_config: CacheConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, multimodal_config, cache_config, - scheduler_config) - # NOTE(woosuk): For accurate performance evaluation, we assign - # random values to the weights. - # initialize_dummy_weights(model) - return model.eval() - - -class MegatronLoader(BaseModelLoader): - """Model loader that can load the model weights from partitioned megatron model.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def _get_weights_iterator(actor_model: Union[PreTrainedModel, Dict]): - # NOTE(shengguangming) Load the weights from the actor model - pass - # if isinstance(actor_model, nn.Module): - # load_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), vllm_model=model) - # else: - # load_weights(actor_weights=actor_model, vllm_model=model) - # return actor_model - - def load_model(self, actor_model: Union[PreTrainedModel, Dict], model_config: ModelConfig, - device_config: DeviceConfig, lora_config: Optional[LoRAConfig], - multimodal_config: Optional[MultiModalConfig], parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, cache_config: CacheConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, multimodal_config, cache_config, - scheduler_config) - - # TODO(sgm): This is a hack, we need to register the load_weight() func for each model in vllm - if isinstance(actor_model, nn.Module): - load_megatron_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), - vllm_model=model) - else: - load_megatron_weights(actor_weights=actor_model, vllm_model=model) - - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -class HFLoader(BaseModelLoader): - """Model loader that can load the model weights from model's full params.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def _get_weights_iterator(self, actor_model: Union[PreTrainedModel, Dict]): - if isinstance(actor_model, Dict): - return actor_model.items() - elif isinstance(actor_model, nn.Module): - return dict(actor_model.named_parameters()).items() - else: - raise ValueError(f'actor model should be Dict or nn.Module, but get {type(actor_model)}') - - def load_model(self, actor_model: Union[PreTrainedModel, Dict], model_config: ModelConfig, - device_config: DeviceConfig, lora_config: Optional[LoRAConfig], - multimodal_config: Optional[MultiModalConfig], parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, cache_config: CacheConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - # with torch.device(device_config.device): - # NOTE(sgm): init the model in cpu - model = _initialize_model(model_config, self.load_config, lora_config, multimodal_config, cache_config, - scheduler_config) - model.load_weights(self._get_weights_iterator(actor_model)) - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -class DTensorLoader(BaseModelLoader): - """Model loader that can load the model weights from partitioned megatron model.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def _get_weights_iterator(actor_model: Union[PreTrainedModel, Dict]): - # NOTE(shengguangming) Load the weights from the actor model - pass - # if isinstance(actor_model, nn.Module): - # load_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), vllm_model=model) - # else: - # load_weights(actor_weights=actor_model, vllm_model=model) - # return actor_model - - def load_model(self, actor_model: Union[PreTrainedModel, Dict], model_config: ModelConfig, - device_config: DeviceConfig, lora_config: Optional[LoRAConfig], - multimodal_config: Optional[MultiModalConfig], parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, cache_config: CacheConfig) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, multimodal_config, cache_config, - scheduler_config) - - # TODO(sgm): This is a hack, we need to register the load_weight() func for each model in vllm - if isinstance(actor_model, nn.Module): - load_dtensor_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), - vllm_model=model) - else: - load_dtensor_weights(actor_weights=actor_model, vllm_model=model) - - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -# FIXME(sgm): hack the _get_logits function in vllm v0.4.2 -# as they use ray, the _get_logits result will only need to return to the driver node, -# therefore gather is enough. However, we use SPMD instead of a central scheduler, -# all_gather is required (aligned with v0.2.6) -def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor, - embedding_bias: Optional[torch.Tensor]) -> torch.Tensor: - # Get the logits for the next tokens. - logits = torch.matmul(hidden_states, embedding.t()) - if embedding_bias is not None: - logits += embedding_bias - logits = tensor_model_parallel_all_gather(logits) - # Remove paddings in vocab (if any). - if logits is not None: - logits = logits[:, :self.org_vocab_size] - return logits - - -from vllm.model_executor.layers.logits_processor import LogitsProcessor - - -def logitsprocessor_init(self, - vocab_size: int, - org_vocab_size: Optional[int] = None, - scale: float = 1.0, - logits_as_input: bool = False, - soft_cap: Optional[float] = None) -> None: - """ - Args: - scale: A scaling factor to apply to the logits. - """ - super(LogitsProcessor, self).__init__() - self.scale = scale - self.vocab_size = vocab_size - # Whether the input is logits (default is hidden states). - self.logits_as_input = logits_as_input - # original vocabulary size (without LoRA). - self.org_vocab_size = org_vocab_size or vocab_size - # Soft cap the logits. Used in Gemma 2. - self.soft_cap = soft_cap - # Whether to use gather or all-gather to gather the logits. - self.use_gather = False - - -LogitsProcessor.__init__ = logitsprocessor_init # use all_gather diff --git a/verl/third_party/vllm/vllm_v_0_5_4/model_runner.py b/verl/third_party/vllm/vllm_v_0_5_4/model_runner.py deleted file mode 100644 index d6ab232558..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/model_runner.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/model_runner.py - -import torch -import torch.nn as nn -from enum import IntEnum -from typing import Dict, List, Optional, Set, Tuple, Union -import warnings - -import vllm.envs as envs -from vllm.attention import (AttentionMetadata, get_attn_backend) -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, MultiModalConfig, ParallelConfig, PromptAdapterConfig, - SchedulerConfig) -from vllm.logger import init_logger -from vllm.lora.layers import LoRAMapping -from vllm.lora.request import LoRARequest -from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager -from vllm.model_executor import SamplingMetadata -from vllm.model_executor.models.interfaces import (supports_lora, supports_vision) -from vllm.utils import (CudaMemoryProfiler, is_hip, is_pin_memory_available) -from vllm.worker.model_runner import ModelRunner, CUDAGraphRunner -from vllm.prompt_adapter.worker_manager import (LRUCacheWorkerPromptAdapterManager) - -from .model_loader import get_model -from .config import ModelConfig, LoadConfig - -logger = init_logger(__name__) - - -# How batches are constructed. -class BatchType(IntEnum): - # Every batch is prefill. - PREFILL = 0 - # Every batch is decode. - DECODE = 1 - # Batch is a mixture of prefill and decode. - MIXED = 2 - - -class ModelRunner(ModelRunner): - - def __init__( - self, - model: Union[nn.Module, Dict], # [verl] model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - cache_config: CacheConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - kv_cache_dtype: Optional[str] = "auto", - is_driver_worker: bool = False, - prompt_adapter_config: Optional[PromptAdapterConfig] = None, - multimodal_config: Optional[MultiModalConfig] = None, - return_hidden_states: bool = False, - ): - - super().__init__( - model_config, - parallel_config, - scheduler_config, - device_config, - cache_config, - load_config, - lora_config, - kv_cache_dtype, - is_driver_worker=True, # a hack - prompt_adapter_config=prompt_adapter_config, - multimodal_config=multimodal_config, - return_hidden_states=return_hidden_states) - - # NOTE(sgm): add for verl - self.model = model # this will be replaced by get_model() - - # NOTE(sgm): initialize model using the actor model - def load_model(self) -> None: - logger.info("Starting to load model %s...", self.model_config.model) - with CudaMemoryProfiler() as m: - self.model = get_model(actor_model=self.model, - model_config=self.model_config, - device_config=self.device_config, - lora_config=self.lora_config, - load_config=self.load_config, - parallel_config=self.parallel_config, - scheduler_config=self.scheduler_config, - multimodal_config=self.multimodal_config, - cache_config=self.cache_config) - self.model_memory_usage = m.consumed_memory - logger.info("Loading model weights took %.4f GB", self.model_memory_usage / float(2**30)) - - if self.lora_config: - assert supports_lora(self.model), "Model does not support LoRA" - assert not supports_vision(self.model), "To be tested: vision language model with LoRA settings." - - self.lora_manager = LRUCacheWorkerLoRAManager( - self.scheduler_config.max_num_seqs, - self.scheduler_config.max_num_batched_tokens, - self.vocab_size, - self.lora_config, - self.device, - self.model.embedding_modules, - self.model.embedding_padding_modules, - max_position_embeddings=self.model.config.max_position_embeddings, - ) - self.model = self.lora_manager.create_lora_manager(self.model) - - if self.prompt_adapter_config: - self.prompt_adapter_manager = LRUCacheWorkerPromptAdapterManager( - self.scheduler_config.max_num_seqs, self.scheduler_config.max_num_batched_tokens, self.device, - self.prompt_adapter_config) - self.model = (self.prompt_adapter_manager.create_prompt_adapter_manager(self.model)) - - if self.kv_cache_dtype == "fp8" and is_hip(): - # Currently only ROCm accepts kv-cache scaling factors - # via quantization_param_path and this will be deprecated - # in the future. - if self.model_config.quantization_param_path is not None: - if callable(getattr(self.model, "load_kv_cache_scales", None)): - warnings.warn( - "Loading kv cache scaling factor from JSON is " - "deprecated and will be removed. Please include " - "kv cache scaling factors in the model checkpoint.", - FutureWarning, - stacklevel=2) - self.model.load_kv_cache_scales(self.model_config.quantization_param_path) - logger.info("Loaded KV cache scaling factors from %s", self.model_config.quantization_param_path) - else: - raise RuntimeError( - "Using FP8 KV cache and scaling factors provided but " - "model %s does not support loading scaling factors.", self.model.__class__) - else: - logger.warning("Using FP8 KV cache but no scaling factors " - "provided. Defaulting to scaling factors of 1.0. " - "This may lead to less accurate results!") - - if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE: - self.model = torch.compile(self.model, fullgraph=True, backend="eager") diff --git a/verl/third_party/vllm/vllm_v_0_5_4/parallel_state.py b/verl/third_party/vllm/vllm_v_0_5_4/parallel_state.py deleted file mode 100644 index 4d853ba75a..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/parallel_state.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Adapted from -# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -"""Model and data parallel groups.""" -import os -import torch -import torch.distributed -from typing import Optional - -import vllm.distributed.parallel_state as ps -from vllm.distributed.parallel_state import get_pp_group, get_world_group, init_distributed_environment, init_model_parallel_group - -import vllm.envs as envs -from vllm.logger import init_logger - -from torch.distributed.device_mesh import init_device_mesh - -logger = init_logger(__name__) -""" -This version is strongly tied with Megatron to implement HybridEngine and weight sharing between vllm and Megatron. -- We assume the Megatron tp+dp+pp world is already established before calling this function. - -""" - -# Device mesh for using DTensor -_DEVICE_MESH = None - -# Tensor model parallel group that the current rank belongs to. -_TP = None -# Pipeline model parallel group that the current rank belongs to. -_PP = None - - -# This method is for initializing the ParallelGroup when using HybridEngine -def initialize_parallel_state( - distributed_init_method: str = "env://", - backend: str = "nccl", - tensor_model_parallel_size: int = 1, - num_tp_per_train_tp: int = 1, - pipeline_model_parallel_size: int = 1, -): - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. - rank = int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - init_distributed_environment(world_size, rank, distributed_init_method, local_rank, backend) - if torch.distributed.get_world_size() > 1: - # NOTE: build a sepearate inference group with infer tp & micro dp - initialize_model_parallel_for_vllm(tensor_model_parallel_size=tensor_model_parallel_size, - num_tensor_model_parallel_groups_per_train_tp=num_tp_per_train_tp) - else: - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - - -def ensure_model_parallel_initialized( - tensor_model_parallel_size: int, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """Helper to initialize model parallel groups if they are not initialized, - or ensure tensor-parallel and pipeline-parallel sizes are equal to expected - values if the model parallel groups are initialized. - """ - # get the backend of _DEVICE_WORLD_GROUP - backend = backend or torch.distributed.get_backend(get_world_group().device_group) - if not model_parallel_is_initialized(): - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - return - - assert (get_tensor_model_parallel_world_size() == tensor_model_parallel_size), ( - "tensor parallel group already initialized, but of unexpected size: " - f"{get_tensor_model_parallel_world_size()=} vs. " - f"{tensor_model_parallel_size=}") - pp_world_size = get_pp_group().world_size - assert (pp_world_size == pipeline_model_parallel_size), ( - "pipeline parallel group already initialized, but of unexpected size: " - f"{pp_world_size=} vs. " - f"{pipeline_model_parallel_size=}") - - -# TODO(sgm): deviate from the v0.5.4, not pp now -def model_parallel_is_initialized(): - """Check if tensor and pipeline parallel groups are initialized.""" - return (ps._TP is not None) - # and _PIPELINE_MODEL_PARALLEL_GROUP is not None) - - -def initialize_model_parallel_for_vllm(tensor_model_parallel_size: int, - num_tensor_model_parallel_groups_per_train_tp: int = 1, - pipeline_model_parallel_size: int = 1) -> None: - from torch.distributed import new_group - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - - assert isinstance(tensor_model_parallel_size, int) - - # assert num_tensor_model_parallel_groups_per_train_tp == 1 and not different_tp_group - # assert num_tensor_model_parallel_groups_per_train_tp > 1 and different_tp_group - - # Build the tensor model-parallel groups. - assert ps._TP is None, ("tensor model parallel group is already initialized") - - global _TP - - world_size: int = torch.distributed.get_world_size() - - rank = torch.distributed.get_rank() - - backend = torch.distributed.get_backend() - - num_tensor_model_parallel_groups = world_size // tensor_model_parallel_size - - if num_tensor_model_parallel_groups_per_train_tp == 1: - # if tensor_model_parallel_size == train_tensor_parallel_size: - # using the same tp group as Megatron/vllm - assert _TP is None, ("tensor model parallel group is already initialized") - group_ranks = [] - for i in range(num_tensor_model_parallel_groups): - ranks = range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) - group_ranks.append(ranks) - _TP = init_model_parallel_group( - group_ranks=group_ranks, - local_rank=get_world_group().local_rank, - backend=backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True) - ps._TP = _TP - # _MICRO_DATA_PARALLEL_GROUP is move to hybrid engine - else: - # initialize a micro_dp group and a tp group - # assume training tp=4, infer tp=2, then, weight is partitioned as - # [1], [2], [3], [4] for training and [1,2], [1,2], [3,4], [3,4] for inference - - # Build the inference tp groups - # train_tp = train_tensor_parallel_size - train_tp = num_tensor_model_parallel_groups_per_train_tp * tensor_model_parallel_size - # num_tensor_model_parallel_groups_per_train_tp = train_tp // tensor_model_parallel_size - assert _TP is None, ("tensor model parallel group is already initialized") - group_ranks = [] - for i in range(num_tensor_model_parallel_groups // num_tensor_model_parallel_groups_per_train_tp): - start = train_tp * i - end = train_tp * (i + 1) - for j in range(num_tensor_model_parallel_groups_per_train_tp): - ranks = list(range(start, end, num_tensor_model_parallel_groups_per_train_tp)) - for i in range(len(ranks)): - ranks[i] += j - group_ranks.append(ranks) - _TP = init_model_parallel_group( - group_ranks=group_ranks, - local_rank=get_world_group().local_rank, - backend=backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True) - ps._TP = _TP - - # Build the pipeline model-parallel groups. - # global _PIPELINE_MODEL_PARALLEL_GROUP - # global _PIPELINE_GLOBAL_RANKS - # assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized") - - # ps._PIPELINE_MODEL_PARALLEL_GROUP = mpu.get_pipeline_model_parallel_group() - # ps._PIPELINE_GLOBAL_RANKS = mpu.get_pipeline_model_parallel_ranks() - - # TODO: init using device mesh (not support hybrid engine now) - # Build the pipeline model-parallel groups. - num_pipeline_model_parallel_groups: int = (world_size // pipeline_model_parallel_size) - global _PP - assert _PP is None, ("pipeline model parallel group is already initialized") - group_ranks = [] - for i in range(num_pipeline_model_parallel_groups): - ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) - group_ranks.append(ranks) - # pipeline parallel does not need custom allreduce - _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) - ps._PP = _PP # for verl - - -def initialize_model_parallel( - tensor_model_parallel_size: int = 1, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """ - NOTE: This method is a hack from the open-sourced version without - asertion of world_size = tp * pp - - Initialize model parallel groups. - - Arguments: - tensor_model_parallel_size: number of GPUs used for tensor model - parallelism. - pipeline_model_parallel_size: number of GPUs used for pipeline model - parallelism. - - Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we - use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize - the model pipeline. The present function will - create 4 tensor model-parallel groups and 2 pipeline model-parallel groups: - 4 tensor model-parallel groups: - [g0, g1], [g2, g3], [g4, g5], [g6, g7] - 2 pipeline model-parallel groups: - [g0, g2, g4, g6], [g1, g3, g5, g7] - Note that for efficiency, the caller should make sure adjacent ranks - are on the same DGX box. For example if we are using 2 DGX-1 boxes - with a total of 16 GPUs, rank 0 to 7 belong to the first box and - ranks 8 to 15 belong to the second box. - """ - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - world_size: int = torch.distributed.get_world_size() - backend = backend or torch.distributed.get_backend(ps.get_world_group().device_group) - - # NOTE(sgm) we don't assert world_size == tp * pp - # DP is not managed by vllm but by the verl WorkerGroup - # if (world_size != - # tensor_model_parallel_size * pipeline_model_parallel_size): - # raise RuntimeError( - # f"world_size ({world_size}) is not equal to " - # f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " - # f"pipeline_model_parallel_size ({pipeline_model_parallel_size})") - - num_tensor_model_parallel_groups: int = (world_size // tensor_model_parallel_size) - rank = torch.distributed.get_rank() - global _TP - assert _TP is None, ("tensor model parallel group is already initialized") - group_ranks = [] - for i in range(num_tensor_model_parallel_groups): - ranks = list(range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size)) - group_ranks.append(ranks) - - # message queue broadcaster is only used in tensor model parallel group - _TP = init_model_parallel_group( - group_ranks, - get_world_group().local_rank, - backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True) - ps._TP = _TP - - # TODO: init using device mesh (not support hybrid engine now) - # Build the pipeline model-parallel groups. - num_pipeline_model_parallel_groups: int = (world_size // pipeline_model_parallel_size) - global _PP - assert _PP is None, ("pipeline model parallel group is already initialized") - group_ranks = [] - for i in range(num_pipeline_model_parallel_groups): - ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) - group_ranks.append(ranks) - # pipeline parallel does not need custom allreduce - _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) - ps._PP = _PP # for verl - - -""" -Device mesh utilities -""" - - -def get_device_mesh(): - assert _DEVICE_MESH is not None, ("device mesh is not initialized") - return _DEVICE_MESH - - -""" -Tensor model parallel utilities -""" - - -def get_tensor_model_parallel_group(): - """Get the tensor model parallel group the caller rank belongs to.""" - assert _TP is not None, ("tensor model parallel group is not initialized") - return _TP.device_group - - -def get_tensor_model_parallel_world_size(): - """Return world size for the tensor model parallel group.""" - return torch.distributed.get_world_size(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_rank(): - """Return my rank for the tensor model parallel group.""" - return torch.distributed.get_rank(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_src_rank(): - """Calculate the global rank corresponding to the first local rank - in the tensor model parallel group.""" - global_rank = torch.distributed.get_rank() - local_world_size = get_tensor_model_parallel_world_size() - return (global_rank // local_world_size) * local_world_size diff --git a/verl/third_party/vllm/vllm_v_0_5_4/spmd_gpu_executor.py b/verl/third_party/vllm/vllm_v_0_5_4/spmd_gpu_executor.py deleted file mode 100644 index e9040d52b5..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/spmd_gpu_executor.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/executor/gpu_executor.py - -import os -import socket -from typing import Any, Dict, List, Optional, Set, Tuple - -import torch -import vllm.envs as envs -from vllm.executor.executor_base import ExecutorBase, ExecutorAsyncBase -from vllm.logger import init_logger -from vllm.lora.request import LoRARequest -from vllm.sequence import SamplerOutput, ExecuteModelRequest - -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, MultiModalConfig, ParallelConfig, PromptAdapterConfig, - SchedulerConfig, SpeculativeConfig) -from .config import ModelConfig, LoadConfig - -logger = init_logger(__name__) - - -class SPMDGPUExecutor(ExecutorBase): - """SPMD-based multi-GPU executor implementations.""" - - def __init__( - self, - model, # pytorch model itself or its parameter dict - model_config: ModelConfig, - cache_config: CacheConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - multimodal_config: Optional[MultiModalConfig], - speculative_config: Optional[SpeculativeConfig], - prompt_adapter_config: Optional[PromptAdapterConfig], - ) -> None: - self.model_config = model_config - self.cache_config = cache_config - self.lora_config = lora_config - self.load_config = load_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.multimodal_config = multimodal_config - self.speculative_config = speculative_config - self.prompt_adapter_config = prompt_adapter_config - - distributed_init_method = initialize_cluster(parallel_config) - self._init_executor(model, distributed_init_method) - - # TODO(sgm): verl not support speculative decode now - def _init_executor(self, model, distributed_init_method) -> None: - assert (not self.speculative_config), "Speculative decoding not yet supported for multi-GPU backend." - - # Create the parallel worker for each GPU. - self._init_workers_sp(model, distributed_init_method) - - def _init_workers_sp(self, model, distributed_init_method: str): - # Lazy import the Worker to avoid importing torch.cuda/xformers - # before CUDA_VISIBLE_DEVICES is set in the Worker - from .worker import Worker # pylint: disable=import-outside-toplevel - - rank = int(os.getenv("RANK")) - local_rank = int(os.getenv("LOCAL_RANK")) - print(f'local rank {local_rank}') - - # see https://github.com/NVIDIA/nccl/issues/1234 - os.environ['NCCL_CUMEM_ENABLE'] = '0' - - self.worker = Worker( - model, - self.model_config, - self.parallel_config, - self.scheduler_config, - self.device_config, - self.cache_config, - self.load_config, - local_rank, - rank, - distributed_init_method, - lora_config=self.lora_config, - multimodal_config=self.multimodal_config, - speculative_config=None, - prompt_adapter_config=self.speculative_config, - is_driver_worker=True, - model_runner_cls=None, # use the default one - ) - - # NOTE(shengguangming): torch.distributed.init_process_group will be called inside the init_model() - self.worker.init_device() - self.worker.load_model() - - def determine_num_available_blocks(self) -> Tuple[int, int]: - """Determine the number of available KV blocks. - - This invokes `determine_num_available_blocks` on each worker and takes - the min of the results, guaranteeing that the selected cache sizes are - compatible with all workers. - - Returns: - - tuple[num_gpu_blocks, num_cpu_blocks] - """ - # Get the maximum number of blocks that can be allocated on GPU and CPU. - num_blocks = self.worker.determine_num_available_blocks() - - # NOTE(shengguangming): Now we don't use a shared centralized controler but each process will - # have its own scheduler - num_gpu_blocks = num_blocks[0] - num_cpu_blocks = num_blocks[1] - - return num_gpu_blocks, num_cpu_blocks - - def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks: int) -> None: - """Initialize the KV cache in all workers. - """ - - # NOTE: We log here to avoid multiple logs when number of workers is - # greater than one. We could log in the engine, but not all executors - # have GPUs. - logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks, num_cpu_blocks) - - self.cache_config.num_gpu_blocks = num_gpu_blocks - self.cache_config.num_cpu_blocks = num_cpu_blocks - - if torch.distributed.get_rank() == 0: - print( - f'before init cache memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB' - ) - self.worker.initialize_cache(num_gpu_blocks=num_gpu_blocks, num_cpu_blocks=num_cpu_blocks) - if torch.distributed.get_rank() == 0: - print( - f'after init cache memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB' - ) - - # NOTE(sgm): This will not profile & capture the model(CUDAGraph) when rebuilding KVCache - def init_cache_engine(self) -> None: - self.worker._init_cache_engine() - - def free_cache_engine(self) -> None: - self.worker.free_cache_engine() - - def execute_model(self, execute_model_req) -> List[SamplerOutput]: - all_outputs = self.worker.execute_model(execute_model_req=execute_model_req) - - # NOTE(sgm): - # Each GPU in vllm under verl has its own spmd_gpu_executor, therefore all GPUs should return the outputs - # In vllm with ray, only the driver worker returns the sampling results. - return all_outputs - - def add_lora(self, lora_request: LoRARequest) -> bool: - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0." - return self.worker.add_lora(lora_request=lora_request) - - def remove_lora(self, lora_id: int) -> bool: - assert lora_id > 0, "lora_id must be greater than 0." - return self.worker.remove_lora(lora_id=lora_id) - - def list_loras(self) -> Set[int]: - return self.worker.list_loras() - - def check_health(self) -> None: - # SPMDExecutor will always be healthy as long as - # it's running. - return - - # NOTE(sgm) add for verl to pass the abstract class test, not used - from vllm.prompt_adapter.request import PromptAdapterRequest - - def add_prompt_adapter(self, prompt_adapter_request: PromptAdapterRequest) -> bool: - assert prompt_adapter_request.prompt_adapter_id > 0, \ - "prompt_adapter_id must be greater than 0." - return self.worker.add_prompt_adapter(prompt_adapter_request) - - def list_prompt_adapters(self) -> Set[int]: - return self.worker.list_prompt_adapters() - - def pin_lora(self, lora_id: int) -> bool: - assert lora_id > 0, "lora_id must be greater than 0." - return self.worker.pin_lora(lora_id) - - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: - assert prompt_adapter_id > 0, \ - "prompt_adapter_id must be greater than 0." - return self.worker.pin_prompt_adapter(prompt_adapter_id) - - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: - assert prompt_adapter_id > 0, \ - "prompt_adapter_id must be greater than 0." - return self.worker.remove_prompt_adapter(prompt_adapter_id) - - # NOTE(sgm): add for verl - def offload_model_weights(self) -> None: - self.worker.offload_model_weights() - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.worker.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - -def initialize_cluster( - parallel_config: ParallelConfig, - engine_use_ray: bool = False, - ray_address: Optional[str] = None, -) -> Tuple[str, Optional[None]]: - """Initialize the distributed cluster probably with Ray. - - Args: - parallel_config: The configurations for parallel execution. - - Returns: - The `distributed_init_method` is the address for initializing the - distributed backend. - """ - - # Initialize cluster locally. - port = get_open_port() - # We need to setup the distributed init method to make sure - # the distributed megatron code (e.g., get world size) works correctly. - # distributed_init_method = f"tcp://localhost:{port}" - distributed_init_method = 'env://' - return distributed_init_method - - -def get_open_port(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - return s.getsockname()[1] - - -# TODO(sgm): not implemented async executor yet -class SPMDGPUExecutorAsync(SPMDGPUExecutor, ExecutorAsyncBase): - - async def execute_model_async(self, execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: - """Executes one model step on the given sequences.""" - raise NotImplementedError - - async def check_health_async(self) -> None: - """Checks if the executor is healthy. If not, it should raise an - exception.""" - self.check_health() diff --git a/verl/third_party/vllm/vllm_v_0_5_4/tokenizer.py b/verl/third_party/vllm/vllm_v_0_5_4/tokenizer.py deleted file mode 100644 index aa625a0338..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/tokenizer.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/tokenizer_group/tokenizer_group.py - -from typing import List, Optional, Tuple, Union - -from transformers import (AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast) - -from vllm.lora.request import LoRARequest -from vllm.utils import make_async, LRUCache -from vllm.transformers_utils.tokenizers import * - - -class TokenizerGroup: - """A group of tokenizers that can be used for LoRA adapters.""" - - def __init__(self, tokenizer: PreTrainedTokenizer, enable_lora: bool, max_num_seqs: int, - max_input_length: Optional[int]): - self.enable_lora = enable_lora - self.max_input_length = max_input_length - self.tokenizer = tokenizer - self.lora_tokenizers = LRUCache[PreTrainedTokenizer](capacity=max_num_seqs) if enable_lora else None - - def ping(self) -> bool: - """Check if the tokenizer group is alive.""" - return True - - def get_max_input_len(self, lora_request: Optional[LoRARequest] = None) -> Optional[int]: - """Get the maximum input length for the LoRA request.""" - return self.max_input_length - - def encode(self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None) -> List[int]: - tokenizer = self.get_lora_tokenizer(lora_request) - return tokenizer.encode(prompt) - - async def encode_async(self, - prompt: str, - request_id: Optional[str] = None, - lora_request: Optional[LoRARequest] = None) -> List[int]: - tokenizer = await self.get_lora_tokenizer_async(lora_request) - return tokenizer.encode(prompt) - - def get_lora_tokenizer(self, lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer": - if not lora_request or not self.enable_lora: - return self.tokenizer - if lora_request.lora_int_id not in self.lora_tokenizers: - # TODO(sgm): the lora tokenizer is also passed, but may be different - tokenizer = self.tokenizer - # tokenizer = (get_lora_tokenizer( - # lora_request, **self.tokenizer_config) or self.tokenizer) - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer) - return tokenizer - else: - return self.lora_tokenizers.get(lora_request.lora_int_id) - - # FIXME(sgm): for simplicity, we assign the special token here - @property - def pad_token_id(self): - return self.tokenizer.pad_token_id - - @property - def eos_token_id(self): - return self.tokenizer.eos_token_id diff --git a/verl/third_party/vllm/vllm_v_0_5_4/worker.py b/verl/third_party/vllm/vllm_v_0_5_4/worker.py deleted file mode 100644 index a5deb675a1..0000000000 --- a/verl/third_party/vllm/vllm_v_0_5_4/worker.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/worker.py -"""A GPU worker class.""" -import os -import gc -from typing import Dict, List, Tuple, Optional, Union, Type - -import torch -import torch.distributed -import torch.nn as nn - -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, MultiModalConfig, ParallelConfig, PromptAdapterConfig, - SchedulerConfig, SpeculativeConfig) -from vllm.model_executor import set_random_seed -from vllm.sequence import (ExecuteModelRequest, IntermediateTensors, SamplerOutput) -from vllm.worker.cache_engine import CacheEngine -# TODO(sgm): check why vllm has similar file in vllm.model_executor.parallel_utils.parallel_state -from vllm.distributed import (init_distributed_environment, set_custom_all_reduce, get_tensor_model_parallel_group) -from vllm.worker.worker_base import WorkerInput -from vllm.worker.worker import Worker, _check_if_gpu_supports_dtype -from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase -from vllm.worker.embedding_model_runner import EmbeddingModelRunner -from vllm.worker.model_runner import GPUModelRunnerBase -from .model_runner import ModelRunner -from .megatron_weight_loaders import load_megatron_weights -from .hf_weight_loader import load_hf_weights -from .dtensor_weight_loaders import load_dtensor_weights -from .parallel_state import (ensure_model_parallel_initialized) -from .config import ModelConfig, LoadConfig, LoadFormat - - -class Worker(Worker): - """A worker class that executes (a partition of) the model on a GPU. - - Each worker is associated with a single GPU. The worker is responsible for - maintaining the KV cache and executing the model on the GPU. In case of - distributed inference, each worker is assigned a partition of the model. - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - cache_config: CacheConfig, - load_config: LoadConfig, - local_rank: int, - rank: int, - distributed_init_method: str, - lora_config: Optional[LoRAConfig] = None, - multimodal_config: Optional[MultiModalConfig] = None, - speculative_config: Optional[SpeculativeConfig] = None, - prompt_adapter_config: Optional[PromptAdapterConfig] = None, - is_driver_worker: bool = False, - model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None, - ) -> None: - # self.model = model # will be replaced in the init_model - self.model_config = model_config - self.parallel_config = parallel_config - self.parallel_config.rank = rank - self.scheduler_config = scheduler_config - self.device_config = device_config - self.cache_config = cache_config - self.local_rank = local_rank - self.rank = rank - self.distributed_init_method = distributed_init_method - self.lora_config = lora_config - self.load_config = load_config - self.prompt_adapter_config = prompt_adapter_config - self.is_driver_worker = is_driver_worker # TODO: we don't need driver - # if parallel_config and is_driver_worker: - # assert rank % parallel_config.tensor_parallel_size == 0, \ - # "Driver worker should be rank 0 of tensor parallel group." - if self.model_config.trust_remote_code: - # note: lazy import to avoid importing torch before initializing - from vllm.utils import init_cached_hf_modules - init_cached_hf_modules() - self.multimodal_config = multimodal_config - - # Return hidden states from target model if the draft model is an - # mlp_speculator - speculative_args = {} if speculative_config is None \ - or (speculative_config.draft_model_config.model == - model_config.model) \ - or (speculative_config.draft_model_config.hf_config.model_type - not in ["medusa", "mlp_speculator"]) \ - else {"return_hidden_states": True} - - # TODO(sgm): set correct model runner class - ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner - if model_runner_cls is not None: - ModelRunnerClass = model_runner_cls - elif self.model_config.embedding_mode: - ModelRunnerClass = EmbeddingModelRunner - self.model_runner: GPUModelRunnerBase = ModelRunnerClass( - model, # [VERL]: add for verl - model_config, - parallel_config, - scheduler_config, - device_config, - cache_config, - load_config=load_config, - lora_config=self.lora_config, - kv_cache_dtype=self.cache_config.cache_dtype, - is_driver_worker=is_driver_worker, - prompt_adapter_config=prompt_adapter_config, - multimodal_config=multimodal_config, - **speculative_args, - ) - - # Uninitialized cache engine. Will be initialized by - # initialize_cache. - self.cache_engine: List[CacheEngine] = None - # Initialize gpu_cache as embedding models don't initialize kv_caches - self.gpu_cache: Optional[List[List[torch.Tensor]]] = None - - # NOTE(sgm): [VERL] For offloading inference engine params - self.cpu_model = None - - def init_device(self) -> None: - if self.device_config.device.type == "cuda": - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. - self.rank = self.rank if self.rank is not None else int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.device = torch.device(f"cuda:{local_rank}") - if self.rank < 0: - raise ValueError("Invalid or unspecified rank.") - torch.cuda.set_device(self.device) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - self.parallel_config.world_size = world_size - - _check_if_gpu_supports_dtype(self.model_config.dtype) - torch.cuda.empty_cache() - self.init_gpu_memory = torch.cuda.mem_get_info()[0] - else: - raise RuntimeError(f"Not support device type: {self.device_config.device}") - - # Initialize the distributed environment. - init_worker_distributed_environment(self.parallel_config, self.rank, self.distributed_init_method, - self.local_rank) - # Set random seed. - set_random_seed(self.model_config.seed) - # self.model = get_model(actor_model=self.model, model_config=self.model_config) - - @torch.inference_mode() - def determine_num_available_blocks(self) -> Tuple[int, int]: - """Profiles the peak memory usage of the model to determine how many - KV blocks may be allocated without OOMs. - - The engine will first conduct a profiling of the existing memory usage. - Then, it calculate the maximum possible number of GPU and CPU blocks - that can be allocated with the remaining free memory. - - .. tip:: - You may limit the usage of GPU memory - by adjusting the `gpu_memory_utilization` parameter. - """ - # Profile the memory usage of the model and get the maximum number of - # cache blocks that can be allocated with the remaining free memory. - torch.cuda.empty_cache() - # torch.cuda.reset_peak_memory_stats() - - # Execute a forward pass with dummy inputs to profile the memory usage - # of the model. - self.model_runner.profile_run() - - # Calculate the number of blocks that can be allocated with the - # profiled peak memory. - torch.cuda.synchronize() - free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info() - peak_memory = total_gpu_memory - free_gpu_memory - - assert peak_memory > 0, ("Error in memory profiling. This happens when the GPU memory was " - "not properly cleaned up before initializing the vLLM instance.") - - cache_block_size = self.get_cache_block_size_bytes() - - # NOTE(sgm) [VERL] use the remaining memory - num_gpu_blocks = int((free_gpu_memory * self.cache_config.gpu_memory_utilization) // cache_block_size) - # num_gpu_blocks = int((total_gpu_memory * self.cache_config.gpu_memory_utilization - peak_memory) // cache_block_size) - - num_cpu_blocks = int(self.cache_config.swap_space_bytes // cache_block_size) - num_gpu_blocks = max(num_gpu_blocks, 0) - num_cpu_blocks = max(num_cpu_blocks, 0) - if self.model_runner.lora_manager: - self.model_runner.remove_all_loras() - - # NOTE(sgm): Add for [VERL], synchronize number of blocks with all the rank - num_gpu_blocks = torch.tensor([num_gpu_blocks], device='cuda') - num_cpu_blocks = torch.tensor([num_cpu_blocks], device='cuda') - - torch.distributed.all_reduce(num_gpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group().device_group) - torch.distributed.all_reduce(num_cpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group().device_group) - num_gpu_blocks = num_gpu_blocks.item() - num_cpu_blocks = num_cpu_blocks.item() - gc.collect() - torch.cuda.empty_cache() - return num_gpu_blocks, num_cpu_blocks - - def _init_cache_engine(self): - if self.cache_engine is None and self.gpu_cache is None: - super()._init_cache_engine() - - def free_cache_engine(self): - # ensure `enforce_eager=True` - self.cache_engine = None - self.gpu_cache = None - - # NOTE(sgm): [VERL]: adapt from _execute_model_spmd() - def execute_model(self, - execute_model_req: ExecuteModelRequest, - intermediate_tensors: Optional[IntermediateTensors] = None) -> Optional[List[SamplerOutput]]: - """ - Execute model in Single Program Multiple Data (SPMD) fashion. - All workers take the same request, prepare the input and - execute the model. - """ - assert execute_model_req is not None, ("_execute_model_spmd() requires each worker to take in an " - "ExecuteModelRequest") - worker_input: WorkerInput = self.prepare_worker_input(execute_model_req=execute_model_req) - model_input: ModelRunnerInputBase = (self.model_runner.prepare_model_input( - execute_model_req.seq_group_metadata_list)) - - # verl.worker.workerbase.WorkerBase - # swap cache - super().execute_worker(worker_input) - - # If there is no input, we don't need to execute the model. - if worker_input.num_seq_groups == 0: - return [] - - return self.model_runner.execute_model( - model_input, self.kv_cache[worker_input.virtual_engine] if self.kv_cache is not None else None, - intermediate_tensors) - - # assume the input is .state_dict() - def sync_model_weights(self, actor_weights: Dict, load_format: str): - if load_format in [LoadFormat.MEGATRON, LoadFormat.AUTO]: - load_megatron_weights(actor_weights, self.model_runner.model) - elif load_format == LoadFormat.HF: - # full model state dict without no sharding - load_hf_weights(actor_weights, self.model_runner.model) - elif load_format == LoadFormat.DTENSOR: - load_dtensor_weights(actor_weights, self.model_runner.model) - - def offload_model_weights(self) -> None: - if self.cpu_model == None: - self.cpu_model = {} - for name, params in self.model_runner.model.named_parameters(): - self.cpu_model[name] = torch.empty_like(params, device='cpu') - params.data = self.cpu_model[name] - else: - for name, params in self.model_runner.model.named_parameters(): - params.data = self.cpu_model[name] - - -def init_worker_distributed_environment( - parallel_config: ParallelConfig, - rank: int, - distributed_init_method: Optional[str] = "env://", - local_rank: int = -1, -) -> None: - """Initialize the distributed environment.""" - set_custom_all_reduce(not parallel_config.disable_custom_all_reduce) - - # NOTE(sgm) use tcp://localhost:xxxx will hang in HF setting without megatron - init_distributed_environment(parallel_config.world_size, rank, distributed_init_method, local_rank) - - ensure_model_parallel_initialized(tensor_model_parallel_size=parallel_config.tensor_parallel_size, - pipeline_model_parallel_size=parallel_config.pipeline_parallel_size) - - # TODO(sgm): check whether need this - # if pynccl_utils.is_initialized(): - # pynccl_world_size = pynccl_utils.get_world_size() - # if pynccl_world_size != parallel_config.world_size: - # raise RuntimeError( - # "pynccl is already initialized but the pynccl world " - # "size does not match parallel_config.world_size " - # f"({pynccl_world_size} vs. {parallel_config.world_size}).") - # elif parallel_config.world_size > 1: - # # NOTE(woosuk): We don't initialize pynccl process group when world size - # # is 1. - # # NOTE(kaichao): By default, pynccl is initialized for tp group. - # pynccl_utils.init_process_group( - # group=get_tensor_model_parallel_cpu_group()) - - # # Initialize a custom fast all-reduce implementation. - # if not parallel_config.disable_custom_all_reduce: - # init_custom_ar() - - # A small all_reduce for warmup. - torch.distributed.all_reduce(torch.zeros(1).cuda()) - # if pynccl_utils.is_initialized(): - # pynccl_utils.all_reduce(torch.zeros(1).cuda()) diff --git a/verl/third_party/vllm/vllm_v_0_6_3/__init__.py b/verl/third_party/vllm/vllm_v_0_6_3/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/third_party/vllm/vllm_v_0_6_3/arg_utils.py b/verl/third_party/vllm/vllm_v_0_6_3/arg_utils.py deleted file mode 100644 index bc4685c5f7..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/arg_utils.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py - -import os -from dataclasses import dataclass - -from transformers import PretrainedConfig -from vllm.config import EngineConfig -from vllm.engine.arg_utils import EngineArgs - -from .config import LoadConfig, ModelConfig - - -@dataclass -class EngineArgs(EngineArgs): - model_hf_config: PretrainedConfig = None # for verl - - def __post_init__(self): - pass - - def create_model_config(self) -> ModelConfig: - return ModelConfig( - hf_config=self.model_hf_config, - tokenizer_mode=self.tokenizer_mode, - trust_remote_code=self.trust_remote_code, - dtype=self.dtype, - seed=self.seed, - revision=self.revision, - code_revision=self.code_revision, - rope_scaling=self.rope_scaling, - rope_theta=self.rope_theta, - tokenizer_revision=self.tokenizer_revision, - max_model_len=self.max_model_len, - quantization=self.quantization, - quantization_param_path=self.quantization_param_path, - enforce_eager=self.enforce_eager, - max_context_len_to_capture=self.max_context_len_to_capture, - max_seq_len_to_capture=self.max_seq_len_to_capture, - max_logprobs=self.max_logprobs, - disable_sliding_window=self.disable_sliding_window, - skip_tokenizer_init=self.skip_tokenizer_init, - served_model_name=self.served_model_name, - limit_mm_per_prompt=self.limit_mm_per_prompt, - use_async_output_proc=not self.disable_async_output_proc, - override_neuron_config=self.override_neuron_config, - config_format=self.config_format, - mm_processor_kwargs=self.mm_processor_kwargs, - ) - - def create_load_config(self) -> LoadConfig: - return LoadConfig( - load_format=self.load_format, - download_dir=self.download_dir, - model_loader_extra_config=self.model_loader_extra_config, - ignore_patterns=self.ignore_patterns, - ) - - def create_engine_config(self) -> EngineConfig: - engine_config = super().create_engine_config() - - # NOTE[VERL]: Use the world_size set by torchrun - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - engine_config.parallel_config.world_size = world_size - - return engine_config diff --git a/verl/third_party/vllm/vllm_v_0_6_3/config.py b/verl/third_party/vllm/vllm_v_0_6_3/config.py deleted file mode 100644 index d7cee45141..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/config.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py - -import enum -import json -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, List, Optional, Union - -from transformers import PretrainedConfig - -# Add for verl -from vllm.config import ModelConfig -from vllm.logger import init_logger -from vllm.utils import is_hip - -if TYPE_CHECKING: - from vllm.model_executor.model_loader.loader import BaseModelLoader - -logger = init_logger(__name__) - - -class LoadFormat(str, enum.Enum): - AUTO = "auto" - MEGATRON = "megatron" - HF = "hf" - DTENSOR = "dtensor" - DUMMY_HF = "dummy_hf" - DUMMY_MEGATRON = "dummy_megatron" - DUMMY_DTENSOR = "dummy_dtensor" - - -class ModelConfig(ModelConfig): - - def __init__(self, hf_config: PretrainedConfig, *args, **kwargs) -> None: - super().__init__(model=hf_config._name_or_path, tokenizer=hf_config._name_or_path, *args, **kwargs) - self.hf_config = hf_config - - -@dataclass -class LoadConfig: - """ - download_dir: Directory to download and load the weights, default to the - default cache directory of huggingface. - load_format: The format of the model weights to load: - "auto" will try to load the weights in the safetensors format and - fall back to the pytorch bin format if safetensors format is - not available. - "pt" will load the weights in the pytorch bin format. - "safetensors" will load the weights in the safetensors format. - "npcache" will load the weights in pytorch format and store - a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is - mainly for profiling. - "tensorizer" will use CoreWeave's tensorizer library for - fast weight loading. - "bitsandbytes" will load nf4 type weights. - ignore_patterns: The list of patterns to ignore when loading the model. - Default to "original/**/*" to avoid repeated loading of llama's - checkpoints. - - """ - - load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO - download_dir: Optional[str] = None - model_loader_extra_config: Optional[Union[str, dict]] = field(default_factory=dict) - ignore_patterns: Optional[Union[List[str], str]] = None - - def __post_init__(self): - model_loader_extra_config = self.model_loader_extra_config or {} - if isinstance(model_loader_extra_config, str): - self.model_loader_extra_config = json.loads(model_loader_extra_config) - self._verify_load_format() - - if self.ignore_patterns is not None and len(self.ignore_patterns) > 0: - logger.info("Ignoring the following patterns when downloading weights: %s", self.ignore_patterns) - else: - self.ignore_patterns = ["original/**/*"] - - def _verify_load_format(self) -> None: - if not isinstance(self.load_format, str): - return - - load_format = self.load_format.lower() - self.load_format = LoadFormat(load_format) - - rocm_not_supported_load_format: List[str] = [] - if is_hip() and load_format in rocm_not_supported_load_format: - rocm_supported_load_format = [ - f for f in LoadFormat.__members__ if (f not in rocm_not_supported_load_format) - ] - raise ValueError(f"load format '{load_format}' is not supported in ROCm. " - f"Supported load formats are " - f"{rocm_supported_load_format}") diff --git a/verl/third_party/vllm/vllm_v_0_6_3/dtensor_weight_loaders.py b/verl/third_party/vllm/vllm_v_0_6_3/dtensor_weight_loaders.py deleted file mode 100644 index 29859fc095..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/dtensor_weight_loaders.py +++ /dev/null @@ -1,380 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/model_loader - -from typing import Dict - -import torch.nn as nn -from torch.distributed._tensor import DTensor -from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm.model_executor.models.utils import is_pp_missing_parameter - - -def gemma_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - for param_name, shard_name, shard_id in stacked_params_mapping: - if shard_name not in name: - continue - stacked_name = name.replace(shard_name, param_name) - # Skip loading extra bias for GPTQ models. - if stacked_name.endswith(".bias") and stacked_name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[stacked_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # lm_head is not used in vllm as it is tied with embed_token. - # To prevent errors, skip loading lm_head.weight. - if "lm_head.weight" in name: - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def gptbigcode_dtensor_load_weights(actor_weights: Dict, vllm_model: nn.Module): - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "lm_head.weight" in name: - continue - if ".attn.bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def starcoder2_dtensor_load_weights(actor_weights: Dict, vllm_model: nn.Module): - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def llama_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - # With tie_word_embeddings, we can skip lm_head.weight - # The weight might appear unnecessarily in the files if the model is - # processed with quantization, LoRA, fine-tuning, etc. - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight) - - -def qwen2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def qwen2vl_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -from vllm.model_executor.layers.fused_moe import FusedMoE - - -def deepseekv2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=vllm_model.config.n_routed_experts, - ) - - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if ("mlp.experts." in name) and name not in params_dict: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, vllm_model): - continue - - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype), shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name, vllm_model): - continue - - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader( - param, - local_loaded_weight.to(dtype=param.dtype), - weight_name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, vllm_model): - continue - - param = params_dict[name] - local_loaded_weight = redistribute_dtensor(param_name=name, loaded_weights=loaded_weight) - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, local_loaded_weight.to(dtype=param.dtype)) - - -def gpt2_dtensor_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - pass - - -def redistribute_dtensor(param_name: str, loaded_weights: DTensor, parallelize_plan: Dict = None): - param_name = _process_parameter_names(name=param_name) - if parallelize_plan is not None: - assert ( - param_name - in parallelize_plan.keys()), f"param name: {param_name} not in parallelize_plan :{parallelize_plan.keys()}" - placement = parallelize_plan[param_name] - local_loaded_weights = loaded_weights.redistribute(device_mesh=loaded_weights.device_mesh, - placements=placement).to_local() - else: - local_loaded_weights = loaded_weights.full_tensor() - return local_loaded_weights - - -def _process_parameter_names(name): - # Remove '.weight' if it exists at the end of the string - if name.endswith(".weight"): - name = name[:-7] - - # Remove 'model.layers.x.' or 'model.' prefix - if "model.layers" in name: - parts = name.split(".") - # Reconstruct the string without 'model.layers.x.' - name = ".".join(parts[3:]) # parts[0] is 'model', parts[1] is 'layers', parts[2] is 'x' - elif name.startswith("model."): - name = name[6:] # Remove 'model.' - - return name - - -__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__ = { - "GPT2LMHeadModel": gpt2_dtensor_weight_loader, - "LlamaForCausalLM": llama_dtensor_weight_loader, - "LLaMAForCausalLM": llama_dtensor_weight_loader, - "MistralForCausalLM": llama_dtensor_weight_loader, # mistral is the same as llama in vLLM - "InternLMForCausalLM": llama_dtensor_weight_loader, - "AquilaModel": llama_dtensor_weight_loader, - "AquilaForCausalLM": llama_dtensor_weight_loader, - "Phi3ForCausalLM": llama_dtensor_weight_loader, - "GemmaForCausalLM": gemma_dtensor_weight_loader, - "Gemma2ForCausalLM": gemma_dtensor_weight_loader, - "GPTBigCodeForCausalLM": gptbigcode_dtensor_load_weights, - "Starcoder2ForCausalLM": starcoder2_dtensor_load_weights, - "Qwen2ForCausalLM": qwen2_dtensor_weight_loader, - "DeepseekV2ForCausalLM": deepseekv2_dtensor_weight_loader, - "Qwen2VLForConditionalGeneration": qwen2vl_dtensor_weight_loader, -} - - -# the actor model is .state_dict() -# Load dtensor weights -def load_dtensor_weights(actor_weights: Dict, vllm_model: nn.Module): - weight_loader = _get_model_weight_loader(vllm_model.__class__.__name__) - weight_loader(actor_weights, vllm_model) - # NOTE(sgm) to reduce peak memory usage, we offload vllm model to cpu - # after init, and we need this after sync model weights for in first iter. - vllm_model = vllm_model.cuda() - - -def _get_model_weight_loader(arch: str): - if arch in __MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__: - return __MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__[arch] - raise ValueError(f"Model architectures {arch} are not supported for now. " - f"Supported architectures: {__MODEL_DTENSOR_WEIGHT_LOADER_REGISTRY__.keys()}") - - -# NOTE(sgm): we use per-parameter weight loader in each vllm sub -def update_dtensor_weight_loader(): - pass \ No newline at end of file diff --git a/verl/third_party/vllm/vllm_v_0_6_3/hf_weight_loader.py b/verl/third_party/vllm/vllm_v_0_6_3/hf_weight_loader.py deleted file mode 100644 index a3e5b22b2f..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/hf_weight_loader.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/model_loader - -from typing import Dict - -import torch.nn as nn -from vllm.model_executor.model_loader.utils import set_default_torch_dtype - - -def update_hf_weight_loader(): - print("no hf weight loader need to be updated") - return - - -def load_hf_weights(actor_weights: Dict, vllm_model: nn.Module): - assert isinstance(actor_weights, Dict) - with set_default_torch_dtype(next(vllm_model.parameters()).dtype): # TODO - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in actor_weights.keys(): - del actor_weights["lm_head.weight"] - vllm_model.load_weights(actor_weights.items()) - for _, module in vllm_model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - vllm_model = vllm_model.cuda() diff --git a/verl/third_party/vllm/vllm_v_0_6_3/llm.py b/verl/third_party/vllm/vllm_v_0_6_3/llm.py deleted file mode 100644 index cd3d646db4..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/llm.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/llm.py - -from typing import Dict, List, Optional, Tuple, Union - -import torch -import torch.nn as nn -from torch.nn.utils.rnn import pad_sequence -from transformers import PretrainedConfig, PreTrainedTokenizer, PreTrainedTokenizerFast -from verl.workers.rollout.tokenizer import HybridEngineBaseTokenizer -from vllm import LLM -from vllm.outputs import EmbeddingRequestOutput, RequestOutput -from vllm.utils import Counter - -from .arg_utils import EngineArgs -from .llm_engine_sp import LLMEngine - - -class LLM(LLM): - """An LLM for generating texts from given prompts and sampling parameters. - - This class includes a tokenizer, a language model (possibly distributed - across multiple GPUs), and GPU memory space allocated for intermediate - states (aka KV cache). Given a batch of prompts and sampling parameters, - this class generates texts from the model, using an intelligent batching - mechanism and efficient memory management. - - NOTE: This class is intended to be used for offline inference. For online - serving, use the `AsyncLLMEngine` class instead. - NOTE: For the comprehensive list of arguments, see `EngineArgs`. - - Args: - model: A HuggingFace Transformers model instance. - tokenizer: A HuggingFace Transformers tokenizer instance. - tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer - if available, and "slow" will always use the slow tokenizer. - trust_remote_code: Trust remote code (e.g., from HuggingFace) when - downloading the model and tokenizer. - tensor_parallel_size: The number of GPUs to use for distributed - execution with tensor parallelism. - dtype: The data type for the model weights and activations. Currently, - we support `float32`, `float16`, and `bfloat16`. If `auto`, we use - the `torch_dtype` attribute specified in the model config file. - However, if the `torch_dtype` in the config is `float32`, we will - use `float16` instead. - quantization: The method used to quantize the model weights. Currently, - we support "awq". If None, we assume the model weights are not - quantized and use `dtype` to determine the data type of the weights. - revision: The specific model version to use. It can be a branch name, - a tag name, or a commit id. - tokenizer_revision: The specific tokenizer version to use. It can be a - branch name, a tag name, or a commit id. - seed: The seed to initialize the random number generator for sampling. - gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to - reserve for the model weights, activations, and KV cache. Higher - values will increase the KV cache size and thus improve the model's - throughput. However, if the value is too high, it may cause out-of- - memory (OOM) errors. - swap_space: The size (GiB) of CPU memory per GPU to use as swap space. - This can be used for temporarily storing the states of the requests - when their `best_of` sampling parameters are larger than 1. If all - requests will have `best_of=1`, you can safely set this to 0. - Otherwise, too small values may cause out-of-memory (OOM) errors. - enforce_eager: Whether to enforce eager execution. If True, we will - disable CUDA graph and always execute the model in eager mode. - If False, we will use CUDA graph and eager execution in hybrid. - max_context_len_to_capture: Maximum context len covered by CUDA graphs. - When a sequence has context length larger than this, we fall back - to eager mode. - disable_custom_all_reduce: See ParallelConfig - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer], - model_hf_config: PretrainedConfig, - tokenizer_mode: str = "auto", - trust_remote_code: bool = False, - skip_tokenizer_init: bool = False, - tensor_parallel_size: int = 1, - dtype: str = "auto", - quantization: Optional[str] = None, - revision: Optional[str] = None, - tokenizer_revision: Optional[str] = None, - seed: int = 0, - gpu_memory_utilization: float = 0.9, - swap_space: int = 4, - cpu_offload_gb: float = 0, - enforce_eager: bool = False, - max_context_len_to_capture: Optional[int] = None, - max_seq_len_to_capture: int = 8192, - disable_custom_all_reduce: bool = False, - load_format="auto", - **kwargs, - ) -> None: - if "disable_log_stats" not in kwargs: - kwargs["disable_log_stats"] = True - removed_vision_keys = ("image_token_id", "image_feature_size", "image_input_shape", "image_input_type") - if any(k in kwargs for k in removed_vision_keys): - raise TypeError("There is no need to pass vision-related arguments anymore.") - engine_args = EngineArgs( - model_hf_config=model_hf_config, - # tokenizer=tokenizer, - tokenizer_mode=tokenizer_mode, - skip_tokenizer_init=skip_tokenizer_init, - trust_remote_code=trust_remote_code, - tensor_parallel_size=tensor_parallel_size, - dtype=dtype, - quantization=quantization, - revision=revision, - tokenizer_revision=tokenizer_revision, - seed=seed, - gpu_memory_utilization=gpu_memory_utilization, - swap_space=swap_space, - cpu_offload_gb=cpu_offload_gb, - enforce_eager=enforce_eager, - max_context_len_to_capture=max_context_len_to_capture, - max_seq_len_to_capture=max_seq_len_to_capture, - disable_custom_all_reduce=disable_custom_all_reduce, - load_format=load_format, - **kwargs, - ) - tokenizer_cls = (PreTrainedTokenizer, PreTrainedTokenizerFast, HybridEngineBaseTokenizer) - if not isinstance(tokenizer, tokenizer_cls): - raise ValueError( - f"Unexpected tokenizer type: {type(tokenizer)}. Must be" - "one of the following: PreTrainedTokenizer, PreTrainedTokenizerFast, verl.workers.rollout.HybridEngineBaseTokenizer" - ) - self.llm_engine = LLMEngine.from_engine_args(model, tokenizer, engine_args) # TODO: check usagecontext - self.request_counter = Counter() - - def init_cache_engine(self): - self.llm_engine.init_cache_engine() - - def free_cache_engine(self): - self.llm_engine.free_cache_engine() - - def get_tokenizer(self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: - return self.llm_engine.tokenizer - - def set_tokenizer( - self, - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], - ) -> None: - self.llm_engine.tokenizer = tokenizer - - def _run_engine(self, *, use_tqdm: bool) -> List[Union[RequestOutput, EmbeddingRequestOutput]]: - outputs = super()._run_engine(use_tqdm=use_tqdm) - return self._post_process_outputs(outputs) - - # # NOTE(shengguangming): add for verl - # # TODO(sgm): we can optimize it by making the dataloader yield List[int] without padding. - # def _pre_process_inputs(self, prompt_token_ids: torch.Tensor) -> List[int]: - # # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - # non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - # token_ids = prompt_token_ids[non_pad_index:].tolist() - # return token_ids - - # NOTE(shengguangming): add for verl - def _post_process_outputs(self, request_outputs: List[RequestOutput]) -> Tuple[torch.Tensor, torch.Tensor]: - output_token_ids = [] - logprobs = [] - for request_output in request_outputs: # List[RequestOutput] - outputs = request_output.outputs - for output in outputs: # List[CompletionOutput], usually len == 1 - output_token_ids.append(torch.tensor(output.token_ids)) - # TODO(shengguangming): can be optimzied by rewrite the Sampler._get_logprobs() logits - logprobs_dicts = output.logprobs - if logprobs_dicts is not None: - logprob = [] - for logprobs_dict, id in zip(logprobs_dicts, output.token_ids): - logprob.append(logprobs_dict[id].logprob) - logprobs.append(torch.tensor(logprob)) - - pad_token_id = (self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None - else self.llm_engine.tokenizer.eos_token_id) - output_token_ids = pad_sequence(output_token_ids, batch_first=True, padding_value=pad_token_id) - if len(logprobs) > 0: - logprobs = pad_sequence(logprobs, batch_first=True, padding_value=pad_token_id) - return output_token_ids, logprobs - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.llm_engine.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - def offload_model_weights(self) -> None: - self.llm_engine.offload_model_weights() diff --git a/verl/third_party/vllm/vllm_v_0_6_3/llm_engine_sp.py b/verl/third_party/vllm/vllm_v_0_6_3/llm_engine_sp.py deleted file mode 100644 index 10b112b259..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/llm_engine_sp.py +++ /dev/null @@ -1,408 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/engine/llm_engine.py - -from functools import partial -from typing import Callable, Dict, Optional, Type, Union - -import torch -import torch.nn as nn -from vllm.config import ( - CacheConfig, - DecodingConfig, - DeviceConfig, - EngineConfig, - LoadConfig, - LoRAConfig, - ModelConfig, - ObservabilityConfig, - ParallelConfig, - PromptAdapterConfig, - SchedulerConfig, - SpeculativeConfig, -) -from vllm.core.scheduler import Scheduler -from vllm.engine.arg_utils import EngineArgs -from vllm.engine.llm_engine import LLMEngine, SchedulerContext, SchedulerOutputState, _load_generation_config_dict -from vllm.engine.metrics_types import StatLoggerBase -from vllm.engine.output_processor.interfaces import SequenceGroupOutputProcessor -from vllm.engine.output_processor.stop_checker import StopChecker -from vllm.executor.executor_base import ExecutorBase -from vllm.inputs import INPUT_REGISTRY, InputRegistry -from vllm.inputs.preprocess import InputPreprocessor -from vllm.logger import init_logger -from vllm.sequence import Sequence -from vllm.tracing import init_tracer -from vllm.transformers_utils.detokenizer import Detokenizer -from vllm.transformers_utils.tokenizer import AnyTokenizer -from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message -from vllm.utils import Counter, weak_bind -from vllm.version import __version__ as VLLM_VERSION - -from .arg_utils import EngineArgs -from .config import LoadConfig, ModelConfig -from .tokenizer import TokenizerGroup - -logger = init_logger(__name__) -_LOCAL_LOGGING_INTERVAL_SEC = 5 - - -class LLMEngine(LLMEngine): - """An LLM engine that receives requests and generates texts. - - This is the main class for the vLLM engine. It receives requests - from clients and generates texts from the LLM. It includes a tokenizer, a - language model (possibly distributed across multiple GPUs), and GPU memory - space allocated for intermediate states (aka KV cache). This class utilizes - iteration-level scheduling and efficient memory management to maximize the - serving throughput. - - The :class:`~vllm.LLM` class wraps this class for offline batched inference - and the :class:`AsyncLLMEngine` class wraps this class for online serving. - - The config arguments are derived from :class:`~vllm.EngineArgs`. (See - :ref:`engine_args`) - - Args: - model_config: The configuration related to the LLM model. - cache_config: The configuration related to the KV cache memory - management. - parallel_config: The configuration related to distributed execution. - scheduler_config: The configuration related to the request scheduler. - device_config: The configuration related to the device. - lora_config (Optional): The configuration related to serving multi-LoRA. - speculative_config (Optional): The configuration related to speculative - decoding. - executor_class: The model executor class for managing distributed - execution. - prompt_adapter_config (Optional): The configuration related to serving - prompt adapters. - log_stats: Whether to log statistics. - usage_context: Specified entry point, used for usage info collection. - """ - - def __init__( - self, - # NOTE(sgm): first two arguments are added for verl - model: Union[nn.Module, Dict], # model itself or its parameter dict - tokenizer: nn.Module, - # NOTE(sgm): vllm original arguments - model_config: ModelConfig, - cache_config: CacheConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - speculative_config: Optional[SpeculativeConfig], - decoding_config: Optional[DecodingConfig], - observability_config: Optional[ObservabilityConfig], - prompt_adapter_config: Optional[PromptAdapterConfig], - executor_class: Type[ExecutorBase], - log_stats: bool, - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None, - input_registry: InputRegistry = INPUT_REGISTRY, - use_cached_outputs: bool = False, - ) -> None: - logger.info( - "Initializing an LLM engine (v%s) with config: " - "model=%r, speculative_config=%r, tokenizer=%r, " - "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, " - "override_neuron_config=%s, " - "rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, " - "trust_remote_code=%s, dtype=%s, max_seq_len=%d, " - "download_dir=%r, load_format=%s, tensor_parallel_size=%d, " - "pipeline_parallel_size=%d, " - "disable_custom_all_reduce=%s, quantization=%s, " - "enforce_eager=%s, kv_cache_dtype=%s, " - "quantization_param_path=%s, device_config=%s, " - "decoding_config=%r, observability_config=%r, " - "seed=%d, served_model_name=%s, use_v2_block_manager=%s, " - "num_scheduler_steps=%d, chunked_prefill_enabled=%s " - "multi_step_stream_outputs=%s, enable_prefix_caching=%s, " - "use_async_output_proc=%s, use_cached_outputs=%s, " - "mm_processor_kwargs=%s)", - VLLM_VERSION, - model_config.model, - speculative_config, - model_config.tokenizer, - model_config.skip_tokenizer_init, - model_config.tokenizer_mode, - model_config.revision, - model_config.override_neuron_config, - model_config.rope_scaling, - model_config.rope_theta, - model_config.tokenizer_revision, - model_config.trust_remote_code, - model_config.dtype, - model_config.max_model_len, - load_config.download_dir, - load_config.load_format, - parallel_config.tensor_parallel_size, - parallel_config.pipeline_parallel_size, - parallel_config.disable_custom_all_reduce, - model_config.quantization, - model_config.enforce_eager, - cache_config.cache_dtype, - model_config.quantization_param_path, - device_config.device, - decoding_config, - observability_config, - model_config.seed, - model_config.served_model_name, - scheduler_config.use_v2_block_manager, - scheduler_config.num_scheduler_steps, - scheduler_config.chunked_prefill_enabled, - scheduler_config.multi_step_stream_outputs, - cache_config.enable_prefix_caching, - model_config.use_async_output_proc, - use_cached_outputs, - model_config.mm_processor_kwargs, - ) - # TODO(woosuk): Print more configs in debug mode. - self.model_config = model_config - self.cache_config = cache_config - self.lora_config = lora_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.speculative_config = speculative_config - self.load_config = load_config - self.decoding_config = decoding_config or DecodingConfig() - self.prompt_adapter_config = prompt_adapter_config - self.observability_config = observability_config or ObservabilityConfig() - self.log_stats = log_stats - self.use_cached_outputs = use_cached_outputs - - if not self.model_config.skip_tokenizer_init: - self.tokenizer = self._init_tokenizer(tokenizer) - self.detokenizer = Detokenizer(self.tokenizer) - tokenizer_group = self.get_tokenizer_group() - else: - self.tokenizer = None - self.detokenizer = None - tokenizer_group = None - - # Ensure that the function doesn't contain a reference to self, - # to avoid engine GC issues - def get_tokenizer_for_seq(sequence: Sequence) -> AnyTokenizer: - assert tokenizer_group, "tokenizer_group cannot be None, " "make sure skip_tokenizer_init is False" - return tokenizer_group.get_lora_tokenizer(sequence.lora_request) - - self.seq_counter = Counter() - self.generation_config_fields = _load_generation_config_dict(model_config) - - self.input_preprocessor = InputPreprocessor(model_config, self.tokenizer) - - self.input_registry = input_registry - self.input_processor = input_registry.create_input_processor(model_config) - - self.model_executor = executor_class( - model=model, # add for spmd_gpu_executor - model_config=model_config, - cache_config=cache_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - device_config=device_config, - lora_config=lora_config, - speculative_config=speculative_config, - load_config=load_config, - prompt_adapter_config=prompt_adapter_config, - observability_config=self.observability_config, - ) - - if not self.model_config.embedding_mode: - self._initialize_kv_caches() - - # If usage stat is enabled, collect relevant info. - if is_usage_stats_enabled(): - from vllm.model_executor.model_loader import get_architecture_class_name - - usage_message.report_usage( - get_architecture_class_name(model_config), - usage_context, - extra_kvs={ - # Common configuration - "dtype": str(model_config.dtype), - "tensor_parallel_size": parallel_config.tensor_parallel_size, - "block_size": cache_config.block_size, - "gpu_memory_utilization": cache_config.gpu_memory_utilization, - # Quantization - "quantization": model_config.quantization, - "kv_cache_dtype": str(cache_config.cache_dtype), - # Feature flags - "enable_lora": bool(lora_config), - "enable_prompt_adapter": bool(prompt_adapter_config), - "enable_prefix_caching": cache_config.enable_prefix_caching, - "enforce_eager": model_config.enforce_eager, - "disable_custom_all_reduce": parallel_config.disable_custom_all_reduce, - }, - ) - - if self.tokenizer: - # Ping the tokenizer to ensure liveness if it runs in a - # different process. - self.tokenizer.ping() - - self.cached_scheduler_outputs = [ - SchedulerOutputState() for _ in range(self.parallel_config.pipeline_parallel_size) - ] - - self.scheduler_contexts = [ - SchedulerContext(multi_step_stream_outputs=self.scheduler_config.multi_step_stream_outputs) - for _ in range(self.parallel_config.pipeline_parallel_size) - ] - - if model_config.use_async_output_proc: - process_model_outputs = weak_bind(self._process_model_outputs) - - self.async_callbacks = [ - partial(process_model_outputs, ctx=self.scheduler_contexts[v_id]) - for v_id in range(self.parallel_config.pipeline_parallel_size) - ] - else: - self.async_callbacks = [] - - # Currently used by AsyncLLMEngine to ensure quick append - # of request outputs to asyncio queues - self.process_request_outputs_callback: Optional[Callable] = None - - # Create the scheduler. - # NOTE: the cache_config here have been updated with the numbers of - # GPU and CPU blocks, which are profiled in the distributed executor. - self.scheduler = [ - Scheduler( - scheduler_config, - cache_config, - lora_config, - parallel_config.pipeline_parallel_size, - self.async_callbacks[v_id] if model_config.use_async_output_proc else None, - ) for v_id in range(parallel_config.pipeline_parallel_size) - ] - - # Metric Logging. - if self.log_stats: - if stat_loggers is not None: - self.stat_loggers = stat_loggers - else: - # Lazy import for prometheus multiprocessing. - # We need to set PROMETHEUS_MULTIPROC_DIR environment variable - # before prometheus_client is imported. - # See https://prometheus.github.io/client_python/multiprocess/ - from vllm.engine.metrics import LoggingStatLogger, PrometheusStatLogger - - self.stat_loggers = { - "logging": - LoggingStatLogger(local_interval=_LOCAL_LOGGING_INTERVAL_SEC), - "prometheus": - PrometheusStatLogger( - local_interval=_LOCAL_LOGGING_INTERVAL_SEC, - labels=dict(model_name=model_config.served_model_name), - max_model_len=self.model_config.max_model_len, - ), - } - self.stat_loggers["prometheus"].info("cache_config", self.cache_config) - - self.tracer = None - if self.observability_config.otlp_traces_endpoint: - self.tracer = init_tracer("vllm.llm_engine", self.observability_config.otlp_traces_endpoint) - - # Create sequence output processor, e.g. for beam search or - # speculative decoding. - self.output_processor = SequenceGroupOutputProcessor.create_output_processor( - self.scheduler_config, - self.detokenizer, - self.scheduler, - self.seq_counter, - get_tokenizer_for_seq, - stop_checker=StopChecker( - self.scheduler_config.max_model_len, - get_tokenizer_for_seq, - ), - ) - - # TODO(sgm): add for verl but we may not tokenizer in Rollout - def _init_tokenizer(self, tokenizer, **tokenizer_init_kwargs): - init_kwargs = dict(enable_lora=bool(self.lora_config), - max_num_seqs=self.scheduler_config.max_num_seqs, - max_input_length=None) - init_kwargs.update(tokenizer_init_kwargs) - return TokenizerGroup(tokenizer, **init_kwargs) - - def init_cache_engine(self): - # TODO: check whether we should rebuild the CUDAGraph every iter when offload/load KVCache - # Re-capture CUDAGraph would be time-consuming - self.model_executor.init_cache_engine() - - def free_cache_engine(self): - self.model_executor.free_cache_engine() - - # NOTE(sgm): currently, we only support GPU executor - # The GPUExecutor remove the Ray dependency - @classmethod - def _get_executor_cls(cls, engine_config: EngineConfig) -> Type[ExecutorBase]: - distributed_executor_backend = engine_config.parallel_config.distributed_executor_backend - # Initialize the cluster and specify the executor class.] - assert (engine_config.device_config.device_type == "cuda" - ), "Currently, the vllm in verl only support running on GPU" - - # print('Waiting for debugger'); import os,debugpy; debugpy.listen(('localhost', 5678 + int(os.getenv('RANK', '0')))); debugpy.wait_for_client() - if engine_config.parallel_config.world_size == 1: - engine_config.load_config.load_format = "dummy_hf" - - from .spmd_gpu_executor import SPMDGPUExecutor - - executor_class = SPMDGPUExecutor - - return executor_class - - @classmethod - def from_engine_args( - cls, - model, - tokenizer, - engine_args: EngineArgs, - usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, - stat_loggers: Optional[Dict[str, StatLoggerBase]] = None, - ) -> "LLMEngine": - """Creates an LLM engine from the engine arguments.""" - # Create the engine configs. - engine_config = engine_args.create_engine_config() - executor_class = cls._get_executor_cls(engine_config) - # Initialize the cluster and specify the executor class. - assert (engine_config.device_config.device_type == "cuda" - ), "Currently, the vllm in verl only support running on GPU" - - from .spmd_gpu_executor import SPMDGPUExecutor - - executor_class = SPMDGPUExecutor - - # Create the LLM engine. - engine = cls( - model, - tokenizer, - **engine_config.to_dict(), - executor_class=executor_class, - log_stats=not engine_args.disable_log_stats, - usage_context=usage_context, - stat_loggers=stat_loggers, - ) - return engine - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.model_executor.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - def offload_model_weights(self) -> None: - self.model_executor.offload_model_weights() diff --git a/verl/third_party/vllm/vllm_v_0_6_3/megatron_weight_loaders.py b/verl/third_party/vllm/vllm_v_0_6_3/megatron_weight_loaders.py deleted file mode 100644 index 8f6b7133ac..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/megatron_weight_loaders.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/model_loader - -from typing import Dict - -import torch -import torch.nn as nn -from vllm.model_executor.layers.linear import * -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead, VocabParallelEmbedding -from vllm.model_executor.models import ModelRegistry - - -# NOTE(shengguangming): replace the origin weight loader function in the class -def parallel_weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Parallel Linear weight loader.""" - assert (param.size() == loaded_weight.size( - )), "the parameter size is not align with the loaded weight size, param size: {}, loaded_weight size: {}".format( - param.size(), loaded_weight.size()) - assert (param.data.dtype == loaded_weight.data.dtype - ), "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: - """Default weight loader.""" - assert param.size() == loaded_weight.size() - assert (param.data.dtype == loaded_weight.data.dtype - ), "if we want to shared weights, the data type should also be the same" - - param.data = loaded_weight.data - - -def gpt2_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_dict = dict(vllm_model.named_parameters(remove_duplicate=False)) - for name, loaded_weight in actor_weights.items(): - if "lm_head.weight" in name: - # GPT-2 ties the weights of the embedding layer and the final - # linear layer. - continue - if ".attn.bias" in name or ".attn.masked_bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - if not name.startswith("transformer."): - name = "transformer." + name - param = params_dict[name] - # The HF's GPT-2 implementation uses Conv1D instead of Linear. - # Because of this, we need to transpose the weights. - # Note(zhuohan): the logic below might break quantized models. - for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]: - if conv1d_weight_name not in name: - continue - if not name.endswith(".weight"): - continue - # TODO: check megatron - loaded_weight = loaded_weight.t() - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def qwen2_megatron_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_te_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv.layer_norm_weight", "input_layernorm.weight"), - ("self_attention.linear_qkv.layer_norm_bias", "input_layernorm.bias"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", "self_attn.o_proj"), - ("pre_mlp_layernorm", "post_attention_layernorm"), - ("mlp.linear_fc1.layer_norm_weight", "post_attention_layernorm.weight"), - ("mlp.linear_fc1.layer_norm_bias", "post_attention_layernorm.bias"), - ("mlp.linear_fc1", "mlp.gate_up_proj"), - ("mlp.linear_fc2", "mlp.down_proj"), - ("decoder.final_layernorm", "model.norm"), - ("output_layer", "lm_head"), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith(".bias") and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def llama_megatron_core_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", "self_attn.o_proj"), - ( - "input_layernorm", - "input_layernorm", - ), - ("pre_mlp_layernorm", "post_attention_layernorm"), - ("mlp.linear_fc1", "mlp.gate_up_proj"), - ("mlp.linear_fc2", "mlp.down_proj"), - ("decoder.final_layernorm", "model.norm"), - ("output_layer", "lm_head"), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - name = _replace_name(name, params_mapping) - if name.endswith(".bias") and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def _replace_name(megatron_name, name_mapping): - for m_name, v_name in name_mapping: - if m_name not in megatron_name: - continue - if "layers" in megatron_name: # deal with decoder layers - megatron_name = megatron_name.replace("decoder", "model") - megatron_name_list = megatron_name.split(".") - if "layer_norm_weight" in megatron_name_list or "layer_norm_bias" in megatron_name_list: - param_name_list = megatron_name_list[:3] - param_name_list.append(v_name) - param_name = ".".join(param_name_list) - else: - param_name_list = megatron_name_list[:3] - weight_or_bias = megatron_name_list[-1] - param_name_list.append(v_name) - param_name_list.append(weight_or_bias) - param_name = ".".join(param_name_list) - return param_name - else: - param_name = megatron_name.replace(m_name, v_name) - return param_name - - -def _replace_name(megatron_name, name_mapping): - for m_name, v_name in name_mapping: - if m_name not in megatron_name: - continue - if "layers" in megatron_name: # deal with decoder layers - megatron_name = megatron_name.replace("decoder", "model") - megatron_name_list = megatron_name.split(".") - if "layer_norm_weight" in megatron_name_list or "layer_norm_bias" in megatron_name_list: - param_name_list = megatron_name_list[:3] - param_name_list.append(v_name) - param_name = ".".join(param_name_list) - else: - param_name_list = megatron_name_list[:3] - weight_or_bias = megatron_name_list[-1] - param_name_list.append(v_name) - param_name_list.append(weight_or_bias) - param_name = ".".join(param_name_list) - return param_name - else: - param_name = megatron_name.replace(m_name, v_name) - return param_name - - -def mistral_megatron_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - # TODO: need to implement a general way to deal with prefix - params_dict = dict(vllm_model.named_parameters()) - for name, loaded_weight in actor_weights.items(): - if "rotary_emb.inv_freq" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -def megatron_core_te_weight_loader(actor_weights: Dict, vllm_model: nn.Module) -> nn.Module: - params_mapping = [ - # (megatron core gpt model name, vllm model name) - ("self_attention.linear_qkv.layer_norm_weight", "input_layernorm.weight"), - ("self_attention.linear_qkv.layer_norm_bias", "input_layernorm.bias"), - ("embedding.word_embeddings", "model.embed_tokens"), - ("self_attention.linear_qkv", "self_attn.qkv_proj"), - ("self_attention.linear_proj", "self_attn.o_proj"), - ("pre_mlp_layernorm", "post_attention_layernorm"), - ("mlp.linear_fc1.layer_norm_weight", "post_attention_layernorm.weight"), - ("mlp.linear_fc1.layer_norm_bias", "post_attention_layernorm.bias"), - ("mlp.linear_fc1", "mlp.gate_up_proj"), - ("mlp.linear_fc2", "mlp.down_proj"), - ("decoder.final_layernorm", "model.norm"), - ("output_layer", "lm_head"), - ] - # NOTE(shengguangming): the megatron llama may have this prefix - params_dict = dict(vllm_model.named_parameters()) - for original_name, loaded_weight in actor_weights.items(): - name = _replace_name(original_name, params_mapping) - if not name or name.endswith(".bias") and name not in params_dict: - continue - if "rotary_emb.inv_freq" in name: - continue - if vllm_model.config.tie_word_embeddings and "lm_head.weight" in name: - continue - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - -__LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__ = { - ColumnParallelLinear: parallel_weight_loader, - MergedColumnParallelLinear: parallel_weight_loader, - QKVParallelLinear: parallel_weight_loader, - RowParallelLinear: parallel_weight_loader, - VocabParallelEmbedding: parallel_weight_loader, - ParallelLMHead: parallel_weight_loader, - # "ScaledActivation.weight_loader": ScaledActivation, # TODO(shengguangming): latest commit in vllm fix awq for this function and add load_weights - # "default_weight_loader": default_weight_loader -} - -# for layer_class, weight_loader in __LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__.items(): -# # setattr(layer_class, 'megatron_weight_loader', weight_loader) -# layer_class.weight_loader = weight_loader - -__MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__ = { - "GPT2LMHeadModel": gpt2_weight_loader, - "LlamaForCausalLM": megatron_core_te_weight_loader, # use te backend for open-source megatron - "LLaMAForCausalLM": megatron_core_te_weight_loader, - "MistralForCausalLM": mistral_megatron_weight_loader, - 'Qwen2ForCausalLM': megatron_core_te_weight_loader, -} - - -# the actor model is .state_dict() -# Load megatron weights -def load_megatron_weights(actor_weights: Dict, vllm_model: nn.Module): - weight_loader = _get_model_weight_loader(vllm_model.__class__.__name__) - weight_loader(actor_weights, vllm_model) - # NOTE(sgm) to reduce peak memory usage, we offload vllm model to cpu - # after init, and we need this after sync model weights for in first iter. - vllm_model = vllm_model.cuda() - - -def _get_model_weight_loader(arch: str): - if arch in __MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__: - return __MODEL_MEGATRON_WEIGHT_LOADER_REGISTRY__[arch] - raise ValueError(f"Model architectures {arch} are not supported for now. " - f"Supported architectures: {ModelRegistry.get_supported_archs()}") - - -def update_megatron_weight_loader(): - for layer_class, weight_loader in __LAYER_WEIGHT_MEGATRON_LOADER_REGISTRY__.items(): - layer_class.weight_loader = weight_loader diff --git a/verl/third_party/vllm/vllm_v_0_6_3/model_loader.py b/verl/third_party/vllm/vllm_v_0_6_3/model_loader.py deleted file mode 100644 index f146a0eae2..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/model_loader.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models -"""Utilities for selecting and loading models.""" -from typing import Dict, Optional, Union - -import torch -import torch.nn as nn -from transformers import PreTrainedModel -from vllm.config import CacheConfig, DeviceConfig, LoadConfig, LoRAConfig, ModelConfig, ParallelConfig, SchedulerConfig -from vllm.distributed.communication_op import tensor_model_parallel_all_gather -from vllm.model_executor.model_loader import BaseModelLoader -from vllm.model_executor.model_loader.loader import _initialize_model -from vllm.model_executor.model_loader.utils import set_default_torch_dtype - -from .config import LoadConfig, LoadFormat, ModelConfig -from .dtensor_weight_loaders import load_dtensor_weights, update_dtensor_weight_loader -from .hf_weight_loader import update_hf_weight_loader -from .megatron_weight_loaders import load_megatron_weights, update_megatron_weight_loader - - -def get_model( - actor_model: Union[PreTrainedModel, Dict], - model_config: ModelConfig, - load_config: LoadConfig, - device_config: DeviceConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - lora_config: Optional[LoRAConfig], - cache_config: CacheConfig = None, -) -> nn.Module: - loader = get_model_loader(load_config) - if load_config.load_format.startswith("dummy"): - return loader.load_model( - model_config=model_config, - device_config=device_config, - lora_config=lora_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - cache_config=cache_config, - ) - else: - return loader.load_model( - actor_model=actor_model, - model_config=model_config, - device_config=device_config, - lora_config=lora_config, - parallel_config=parallel_config, - scheduler_config=scheduler_config, - cache_config=cache_config, - ) - - -def get_model_loader(load_config: LoadConfig) -> BaseModelLoader: - """Get a model loader based on the load format.""" - - if isinstance(load_config.load_format, type): - return load_config.load_format(load_config) - - if load_config.load_format == LoadFormat.AUTO: - update_megatron_weight_loader() - return MegatronLoader(load_config) - - # NOTE(sgm): change the weight_loader function in runtime - if load_config.load_format == LoadFormat.MEGATRON: - update_megatron_weight_loader() - return MegatronLoader(load_config) - - if load_config.load_format == LoadFormat.HF: - update_hf_weight_loader() - return HFLoader(load_config) - - if load_config.load_format == LoadFormat.DTENSOR: - update_dtensor_weight_loader() - return DTensorLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_HF: - update_hf_weight_loader() - return DummyModelLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_MEGATRON: - update_megatron_weight_loader() - return DummyModelLoader(load_config) - - if load_config.load_format == LoadFormat.DUMMY_DTENSOR: - update_dtensor_weight_loader() - return DummyModelLoader(load_config) - - raise ValueError("load format not supported in verl: {}, only support {} and {}".format( - load_config.load_format, LoadFormat.MEGATRON, LoadFormat.HF)) - - -class DummyModelLoader(BaseModelLoader): - """Model loader that will set model weights to random values.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def download_model(self, model_config: ModelConfig) -> None: - pass - - def load_model( - self, - *, - model_config: ModelConfig, - device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - cache_config: CacheConfig, - ) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, cache_config, scheduler_config) - # NOTE(woosuk): For accurate performance evaluation, we assign - # random values to the weights. - # initialize_dummy_weights(model) - return model.eval() - - -class MegatronLoader(BaseModelLoader): - """Model loader that can load the model weights from partitioned megatron model.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def download_model(self, model_config: ModelConfig) -> None: - pass # Nothing to download - - def _get_weights_iterator(actor_model: Union[PreTrainedModel, Dict]): - # NOTE(shengguangming) Load the weights from the actor model - pass - # if isinstance(actor_model, nn.Module): - # load_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), vllm_model=model) - # else: - # load_weights(actor_weights=actor_model, vllm_model=model) - # return actor_model - - def load_model( - self, - actor_model: Union[PreTrainedModel, Dict], - model_config: ModelConfig, - device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - cache_config: CacheConfig, - ) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, cache_config, scheduler_config) - - # TODO(sgm): This is a hack, we need to register the load_weight() func for each model in vllm - if isinstance(actor_model, nn.Module): - load_megatron_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), - vllm_model=model) - else: - load_megatron_weights(actor_weights=actor_model, vllm_model=model) - - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -class HFLoader(BaseModelLoader): - """Model loader that can load the model weights from model's full params.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def download_model(self, model_config: ModelConfig) -> None: - pass # Nothing to download - - def _get_weights_iterator(self, actor_model: Union[PreTrainedModel, Dict]): - if isinstance(actor_model, Dict): - return actor_model.items() - elif isinstance(actor_model, nn.Module): - return dict(actor_model.named_parameters()).items() - else: - raise ValueError(f"actor model should be Dict or nn.Module, but get {type(actor_model)}") - - def load_model( - self, - actor_model: Union[PreTrainedModel, Dict], - model_config: ModelConfig, - device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - cache_config: CacheConfig, - ) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - # with torch.device(device_config.device): - # NOTE(sgm): init the model in cpu - model = _initialize_model(model_config, self.load_config, lora_config, cache_config, scheduler_config) - model.load_weights(self._get_weights_iterator(actor_model)) - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -class DTensorLoader(BaseModelLoader): - """Model loader that can load the model weights from partitioned megatron model.""" - - def __init__(self, load_config: LoadConfig): - super().__init__(load_config) - if load_config.model_loader_extra_config: - raise ValueError(f"Model loader extra config is not supported for " - f"load format {load_config.load_format}") - - def download_model(self, model_config: ModelConfig) -> None: - pass # Nothing to download - - def _get_weights_iterator(actor_model: Union[PreTrainedModel, Dict]): - # NOTE(shengguangming) Load the weights from the actor model - pass - # if isinstance(actor_model, nn.Module): - # load_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), vllm_model=model) - # else: - # load_weights(actor_weights=actor_model, vllm_model=model) - # return actor_model - - def load_model( - self, - actor_model: Union[PreTrainedModel, Dict], - model_config: ModelConfig, - device_config: DeviceConfig, - lora_config: Optional[LoRAConfig], - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - cache_config: CacheConfig, - ) -> nn.Module: - with set_default_torch_dtype(model_config.dtype): - with torch.device(device_config.device): - model = _initialize_model(model_config, self.load_config, lora_config, cache_config, scheduler_config) - - # TODO(sgm): This is a hack, we need to register the load_weight() func for each model in vllm - if isinstance(actor_model, nn.Module): - load_dtensor_weights(actor_weights=dict(actor_model.named_parameters(remove_duplicate=False)), - vllm_model=model) - else: - load_dtensor_weights(actor_weights=actor_model, vllm_model=model) - - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - quant_method.process_weights_after_loading(module) - # FIXME: Remove this after Mixtral is updated - # to use quant_method. - if hasattr(module, "process_weights_after_loading"): - module.process_weights_after_loading() - # NOTE(sgm) Some weights are point to gpu, but still need this. - model = model.cuda() # NOTE (zhangchi.usc1992) We need this for vllm to profile memory usage - return model.eval() - - -# FIXME(sgm): hack the _get_logits function in vllm v0.4.2 -# as they use ray, the _get_logits result will only need to return to the driver node, -# therefore gather is enough. However, we use SPMD instead of a central scheduler, -# all_gather is required (aligned with v0.2.6) -def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor, - embedding_bias: Optional[torch.Tensor]) -> torch.Tensor: - # Get the logits for the next tokens. - logits = torch.matmul(hidden_states, embedding.t()) - if embedding_bias is not None: - logits += embedding_bias - logits = tensor_model_parallel_all_gather(logits) - # Remove paddings in vocab (if any). - if logits is not None: - logits = logits[:, :self.org_vocab_size] - return logits - - -from vllm.model_executor.layers.logits_processor import LogitsProcessor - - -def logitsprocessor_init( - self, - vocab_size: int, - org_vocab_size: Optional[int] = None, - scale: float = 1.0, - logits_as_input: bool = False, - soft_cap: Optional[float] = None, -) -> None: - """ - Args: - scale: A scaling factor to apply to the logits. - """ - super(LogitsProcessor, self).__init__() - self.scale = scale - self.vocab_size = vocab_size - # Whether the input is logits (default is hidden states). - self.logits_as_input = logits_as_input - # original vocabulary size (without LoRA). - self.org_vocab_size = org_vocab_size or vocab_size - # Soft cap the logits. Used in Gemma 2. - self.soft_cap = soft_cap - # Whether to use gather or all-gather to gather the logits. - self.use_gather = False - - -LogitsProcessor.__init__ = logitsprocessor_init # use all_gather diff --git a/verl/third_party/vllm/vllm_v_0_6_3/model_runner.py b/verl/third_party/vllm/vllm_v_0_6_3/model_runner.py deleted file mode 100644 index b0cceffb52..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/model_runner.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/model_runner.py - -import warnings -from enum import IntEnum -from typing import Dict, Optional, Union - -import torch -import torch.nn as nn -import vllm.envs as envs -from vllm.compilation.levels import CompilationLevel -from vllm.config import ( - CacheConfig, - DeviceConfig, - LoadConfig, - LoRAConfig, - ModelConfig, - ObservabilityConfig, - ParallelConfig, - PromptAdapterConfig, - SchedulerConfig, -) -from vllm.inputs import INPUT_REGISTRY, InputRegistry -from vllm.logger import init_logger -from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager -from vllm.model_executor.models.interfaces import supports_lora -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry -from vllm.prompt_adapter.worker_manager import LRUCacheWorkerPromptAdapterManager -from vllm.utils import DeviceMemoryProfiler, is_hip, supports_dynamo -from vllm.worker.model_runner import ModelRunner - -from .config import LoadConfig, ModelConfig -from .model_loader import get_model - -logger = init_logger(__name__) - - -# How batches are constructed. -class BatchType(IntEnum): - # Every batch is prefill. - PREFILL = 0 - # Every batch is decode. - DECODE = 1 - # Batch is a mixture of prefill and decode. - MIXED = 2 - - -class ModelRunner(ModelRunner): - - def __init__( - self, - model: Union[nn.Module, Dict], # [verl] model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - cache_config: CacheConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - kv_cache_dtype: Optional[str] = "auto", - is_driver_worker: bool = False, - prompt_adapter_config: Optional[PromptAdapterConfig] = None, - return_hidden_states: bool = False, - observability_config: Optional[ObservabilityConfig] = None, - input_registry: InputRegistry = INPUT_REGISTRY, - mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, - ): - - super().__init__( - model_config, - parallel_config, - scheduler_config, - device_config, - cache_config, - load_config, - lora_config, - kv_cache_dtype, - is_driver_worker=True, # a hack - prompt_adapter_config=prompt_adapter_config, - return_hidden_states=return_hidden_states, - observability_config=observability_config, - input_registry=input_registry, - mm_registry=mm_registry, - ) - - # NOTE(sgm): add for verl - self.model = model # this will be replaced by get_model() - - def load_model(self) -> None: - logger.info("Starting to load model %s...", self.model_config.model) - with DeviceMemoryProfiler() as m: - self.model = get_model( - self.model, - model_config=self.model_config, - device_config=self.device_config, - load_config=self.load_config, - lora_config=self.lora_config, - parallel_config=self.parallel_config, - scheduler_config=self.scheduler_config, - cache_config=self.cache_config, - ) - - self.model_memory_usage = m.consumed_memory - logger.info("Loading model weights took %.4f GB", self.model_memory_usage / float(2**30)) - - if self.lora_config: - assert supports_lora(self.model), f"{self.model.__class__.__name__} does not support LoRA yet." - - if supports_multimodal(self.model): - logger.warning("Regarding multimodal models, vLLM currently " - "only supports adding LoRA to language model.") - # It's necessary to distinguish between the max_position_embeddings - # of VLMs and LLMs. - if hasattr(self.model.config, "max_position_embeddings"): - max_pos_embeddings = self.model.config.max_position_embeddings - else: - max_pos_embeddings = self.model.config.text_config.max_position_embeddings - - self.lora_manager = LRUCacheWorkerLoRAManager( - self.scheduler_config.max_num_seqs, - self.scheduler_config.max_num_batched_tokens, - self.vocab_size, - self.lora_config, - self.device, - self.model.embedding_modules, - self.model.embedding_padding_modules, - max_position_embeddings=max_pos_embeddings, - ) - self.model = self.lora_manager.create_lora_manager(self.model) - - if self.prompt_adapter_config: - self.prompt_adapter_manager = LRUCacheWorkerPromptAdapterManager( - self.scheduler_config.max_num_seqs, - self.scheduler_config.max_num_batched_tokens, - self.device, - self.prompt_adapter_config, - ) - self.model = self.prompt_adapter_manager.create_prompt_adapter_manager(self.model) - - if self.kv_cache_dtype == "fp8" and is_hip(): - # Currently only ROCm accepts kv-cache scaling factors - # via quantization_param_path and this will be deprecated - # in the future. - if self.model_config.quantization_param_path is not None: - if callable(getattr(self.model, "load_kv_cache_scales", None)): - warnings.warn( - "Loading kv cache scaling factor from JSON is " - "deprecated and will be removed. Please include " - "kv cache scaling factors in the model checkpoint.", - FutureWarning, - stacklevel=2, - ) - self.model.load_kv_cache_scales(self.model_config.quantization_param_path) - logger.info("Loaded KV cache scaling factors from %s", self.model_config.quantization_param_path) - else: - raise RuntimeError( - "Using FP8 KV cache and scaling factors provided but " - "model %s does not support loading scaling factors.", - self.model.__class__, - ) - else: - logger.warning("Using FP8 KV cache but no scaling factors " - "provided. Defaulting to scaling factors of 1.0. " - "This may lead to less accurate results!") - - if envs.VLLM_TORCH_COMPILE_LEVEL == CompilationLevel.DYNAMO_AS_IS and supports_dynamo(): - from vllm.plugins import get_torch_compile_backend - - backend = get_torch_compile_backend() or "eager" - self.model = torch.compile(self.model, fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE, backend=backend) diff --git a/verl/third_party/vllm/vllm_v_0_6_3/parallel_state.py b/verl/third_party/vllm/vllm_v_0_6_3/parallel_state.py deleted file mode 100644 index 0150c1c678..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/parallel_state.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Adapted from -# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -"""Model and data parallel groups.""" -import os -from typing import Optional - -import torch -import torch.distributed -import vllm.distributed.parallel_state as ps -from vllm.distributed.parallel_state import ( - get_pp_group, - get_world_group, - init_distributed_environment, - init_model_parallel_group, -) -from vllm.logger import init_logger - -logger = init_logger(__name__) -""" -This version is strongly tied with Megatron to implement HybridEngine and weight sharing between vllm and Megatron. -- We assume the Megatron tp+dp+pp world is already established before calling this function. - -""" - -# Device mesh for using DTensor -_DEVICE_MESH = None - -# Tensor model parallel group that the current rank belongs to. -_TP = None -# Pipeline model parallel group that the current rank belongs to. -_PP = None - - -# This method is for initializing the ParallelGroup when using HybridEngine -def initialize_parallel_state( - distributed_init_method: str = "env://", - backend: str = "nccl", - tensor_model_parallel_size: int = 1, - num_tp_per_train_tp: int = 1, - pipeline_model_parallel_size: int = 1, -): - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. - rank = int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - init_distributed_environment(world_size, rank, distributed_init_method, local_rank, backend) - if torch.distributed.get_world_size() > 1: - # NOTE: build a sepearate inference group with infer tp & micro dp - initialize_model_parallel_for_vllm( - tensor_model_parallel_size=tensor_model_parallel_size, - num_tensor_model_parallel_groups_per_train_tp=num_tp_per_train_tp, - ) - else: - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - - -def ensure_model_parallel_initialized( - tensor_model_parallel_size: int, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """Helper to initialize model parallel groups if they are not initialized, - or ensure tensor-parallel and pipeline-parallel sizes are equal to expected - values if the model parallel groups are initialized. - """ - # get the backend of _DEVICE_WORLD_GROUP - backend = backend or torch.distributed.get_backend(get_world_group().device_group) - if not model_parallel_is_initialized(): - initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend) - return - - assert get_tensor_model_parallel_world_size() == tensor_model_parallel_size, ( - "tensor parallel group already initialized, but of unexpected size: " - f"{get_tensor_model_parallel_world_size()=} vs. " - f"{tensor_model_parallel_size=}") - pp_world_size = get_pp_group().world_size - assert pp_world_size == pipeline_model_parallel_size, ( - "pipeline parallel group already initialized, but of unexpected size: " - f"{pp_world_size=} vs. " - f"{pipeline_model_parallel_size=}") - - -# TODO(sgm): deviate from the v0.5.4, not pp now -def model_parallel_is_initialized(): - """Check if tensor and pipeline parallel groups are initialized.""" - return ps._TP is not None - # and _PIPELINE_MODEL_PARALLEL_GROUP is not None) - - -def initialize_model_parallel_for_vllm( - tensor_model_parallel_size: int, - num_tensor_model_parallel_groups_per_train_tp: int = 1, - pipeline_model_parallel_size: int = 1, -) -> None: - pass - - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - - assert isinstance(tensor_model_parallel_size, int) - - # assert num_tensor_model_parallel_groups_per_train_tp == 1 and not different_tp_group - # assert num_tensor_model_parallel_groups_per_train_tp > 1 and different_tp_group - - # Build the tensor model-parallel groups. - assert ps._TP is None, "tensor model parallel group is already initialized" - - global _TP - - world_size: int = torch.distributed.get_world_size() - - rank = torch.distributed.get_rank() - - backend = torch.distributed.get_backend() - - num_tensor_model_parallel_groups = world_size // tensor_model_parallel_size - - if num_tensor_model_parallel_groups_per_train_tp == 1: - # if tensor_model_parallel_size == train_tensor_parallel_size: - # using the same tp group as Megatron/vllm - assert _TP is None, "tensor model parallel group is already initialized" - group_ranks = [] - for i in range(num_tensor_model_parallel_groups): - ranks = range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) - group_ranks.append(ranks) - _TP = init_model_parallel_group( - group_ranks=group_ranks, - local_rank=get_world_group().local_rank, - backend=backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True, - ) - ps._TP = _TP - # _MICRO_DATA_PARALLEL_GROUP is move to hybrid engine - else: - # initialize a micro_dp group and a tp group - # assume training tp=4, infer tp=2, then, weight is partitioned as - # [1], [2], [3], [4] for training and [1,2], [1,2], [3,4], [3,4] for inference - - # Build the inference tp groups - # train_tp = train_tensor_parallel_size - train_tp = num_tensor_model_parallel_groups_per_train_tp * tensor_model_parallel_size - # num_tensor_model_parallel_groups_per_train_tp = train_tp // tensor_model_parallel_size - assert _TP is None, "tensor model parallel group is already initialized" - group_ranks = [] - for i in range(num_tensor_model_parallel_groups // num_tensor_model_parallel_groups_per_train_tp): - start = train_tp * i - end = train_tp * (i + 1) - for j in range(num_tensor_model_parallel_groups_per_train_tp): - ranks = list(range(start, end, num_tensor_model_parallel_groups_per_train_tp)) - for i in range(len(ranks)): - ranks[i] += j - group_ranks.append(ranks) - _TP = init_model_parallel_group( - group_ranks=group_ranks, - local_rank=get_world_group().local_rank, - backend=backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True, - ) - ps._TP = _TP - - # Build the pipeline model-parallel groups. - # global _PIPELINE_MODEL_PARALLEL_GROUP - # global _PIPELINE_GLOBAL_RANKS - # assert ps._PIPELINE_MODEL_PARALLEL_GROUP is None, ("pipeline model parallel group is already initialized") - - # ps._PIPELINE_MODEL_PARALLEL_GROUP = mpu.get_pipeline_model_parallel_group() - # ps._PIPELINE_GLOBAL_RANKS = mpu.get_pipeline_model_parallel_ranks() - - # TODO: init using device mesh (not support hybrid engine now) - # Build the pipeline model-parallel groups. - num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size - global _PP - assert _PP is None, "pipeline model parallel group is already initialized" - group_ranks = [] - for i in range(num_pipeline_model_parallel_groups): - ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) - group_ranks.append(ranks) - # pipeline parallel does not need custom allreduce - _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) - ps._PP = _PP # for verl - - -def initialize_model_parallel( - tensor_model_parallel_size: int = 1, - pipeline_model_parallel_size: int = 1, - backend: Optional[str] = None, -) -> None: - """ - NOTE: This method is a hack from the open-sourced version without - asertion of world_size = tp * pp - - Initialize model parallel groups. - - Arguments: - tensor_model_parallel_size: number of GPUs used for tensor model - parallelism. - pipeline_model_parallel_size: number of GPUs used for pipeline model - parallelism. - - Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we - use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize - the model pipeline. The present function will - create 4 tensor model-parallel groups and 2 pipeline model-parallel groups: - 4 tensor model-parallel groups: - [g0, g1], [g2, g3], [g4, g5], [g6, g7] - 2 pipeline model-parallel groups: - [g0, g2, g4, g6], [g1, g3, g5, g7] - Note that for efficiency, the caller should make sure adjacent ranks - are on the same DGX box. For example if we are using 2 DGX-1 boxes - with a total of 16 GPUs, rank 0 to 7 belong to the first box and - ranks 8 to 15 belong to the second box. - """ - # Get world size and rank. Ensure some consistencies. - assert torch.distributed.is_initialized() - world_size: int = torch.distributed.get_world_size() - backend = backend or torch.distributed.get_backend(ps.get_world_group().device_group) - - # NOTE(sgm) we don't assert world_size == tp * pp - # DP is not managed by vllm but by the VeRL WorkerGroup - # if (world_size != - # tensor_model_parallel_size * pipeline_model_parallel_size): - # raise RuntimeError( - # f"world_size ({world_size}) is not equal to " - # f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " - # f"pipeline_model_parallel_size ({pipeline_model_parallel_size})") - - num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size - rank = torch.distributed.get_rank() - global _TP - assert _TP is None, "tensor model parallel group is already initialized" - group_ranks = [] - for i in range(num_tensor_model_parallel_groups): - ranks = list(range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size)) - group_ranks.append(ranks) - - # message queue broadcaster is only used in tensor model parallel group - _TP = init_model_parallel_group( - group_ranks, - get_world_group().local_rank, - backend, - use_custom_allreduce=False, # TODO: check why True is not work in Ray trainer - use_message_queue_broadcaster=True, - ) - ps._TP = _TP - - # TODO: init using device mesh (not support hybrid engine now) - # Build the pipeline model-parallel groups. - num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size - global _PP - assert _PP is None, "pipeline model parallel group is already initialized" - group_ranks = [] - for i in range(num_pipeline_model_parallel_groups): - ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) - group_ranks.append(ranks) - # pipeline parallel does not need custom allreduce - _PP = init_model_parallel_group(group_ranks, get_world_group().local_rank, backend, use_custom_allreduce=False) - ps._PP = _PP # for verl - - -""" -Device mesh utilities -""" - - -def get_device_mesh(): - assert _DEVICE_MESH is not None, "device mesh is not initialized" - return _DEVICE_MESH - - -""" -Tensor model parallel utilities -""" - - -def get_tensor_model_parallel_group(): - """Get the tensor model parallel group the caller rank belongs to.""" - assert _TP is not None, "tensor model parallel group is not initialized" - return _TP.device_group - - -def get_tensor_model_parallel_world_size(): - """Return world size for the tensor model parallel group.""" - return torch.distributed.get_world_size(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_rank(): - """Return my rank for the tensor model parallel group.""" - return torch.distributed.get_rank(group=get_tensor_model_parallel_group()) - - -def get_tensor_model_parallel_src_rank(): - """Calculate the global rank corresponding to the first local rank - in the tensor model parallel group.""" - global_rank = torch.distributed.get_rank() - local_world_size = get_tensor_model_parallel_world_size() - return (global_rank // local_world_size) * local_world_size diff --git a/verl/third_party/vllm/vllm_v_0_6_3/spmd_gpu_executor.py b/verl/third_party/vllm/vllm_v_0_6_3/spmd_gpu_executor.py deleted file mode 100644 index 229a424c84..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/spmd_gpu_executor.py +++ /dev/null @@ -1,256 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/executor/gpu_executor.py - -import os -import socket -from typing import Dict, List, Optional, Set, Tuple - -import torch -from vllm.config import ( - CacheConfig, - DeviceConfig, - LoRAConfig, - ObservabilityConfig, - ParallelConfig, - PromptAdapterConfig, - SchedulerConfig, - SpeculativeConfig, -) -from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase -from vllm.logger import init_logger -from vllm.lora.request import LoRARequest -from vllm.model_executor.layers.sampler import SamplerOutput -from vllm.sequence import ExecuteModelRequest - -from .config import LoadConfig, ModelConfig - -logger = init_logger(__name__) - - -class SPMDGPUExecutor(ExecutorBase): - """SPMD-based multi-GPU executor implementations.""" - - def __init__( - self, - model, # pytorch model itself or its parameter dict - model_config: ModelConfig, - cache_config: CacheConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - load_config: LoadConfig, - lora_config: Optional[LoRAConfig], - speculative_config: Optional[SpeculativeConfig], - prompt_adapter_config: Optional[PromptAdapterConfig], - observability_config: Optional[ObservabilityConfig], - ) -> None: - self.model_config = model_config - self.cache_config = cache_config - self.lora_config = lora_config - self.load_config = load_config - self.parallel_config = parallel_config - self.scheduler_config = scheduler_config - self.device_config = device_config - self.speculative_config = speculative_config - self.prompt_adapter_config = prompt_adapter_config - self.observability_config = observability_config - - distributed_init_method = initialize_cluster(parallel_config) - self._init_executor(model, distributed_init_method) - - # TODO(sgm): verl not support speculative decode now - def _init_executor(self, model, distributed_init_method) -> None: - assert not self.speculative_config, "Speculative decoding not yet supported for multi-GPU backend." - - # Create the parallel worker for each GPU. - self._init_workers_sp(model, distributed_init_method) - - def _init_workers_sp(self, model, distributed_init_method: str): - # Lazy import the Worker to avoid importing torch.cuda/xformers - # before CUDA_VISIBLE_DEVICES is set in the Worker - from .worker import Worker # pylint: disable=import-outside-toplevel - - rank = int(os.getenv("RANK")) - local_rank = int(os.getenv("LOCAL_RANK")) - print(f"local rank {local_rank}") - - # see https://github.com/NVIDIA/nccl/issues/1234 - os.environ["NCCL_CUMEM_ENABLE"] = "0" - - self.worker = Worker( - model, - self.model_config, - self.parallel_config, - self.scheduler_config, - self.device_config, - self.cache_config, - self.load_config, - local_rank, - rank, - distributed_init_method, - lora_config=self.lora_config, - speculative_config=None, - prompt_adapter_config=self.speculative_config, - is_driver_worker=True, - model_runner_cls=None, # use the default one - ) - - # NOTE(shengguangming): torch.distributed.init_process_group will be called inside the init_model() - self.worker.init_device() - self.worker.load_model() - - def determine_num_available_blocks(self) -> Tuple[int, int]: - """Determine the number of available KV blocks. - - This invokes `determine_num_available_blocks` on each worker and takes - the min of the results, guaranteeing that the selected cache sizes are - compatible with all workers. - - Returns: - - tuple[num_gpu_blocks, num_cpu_blocks] - """ - # Get the maximum number of blocks that can be allocated on GPU and CPU. - num_blocks = self.worker.determine_num_available_blocks() - - # NOTE(shengguangming): Now we don't use a shared centralized controler but each process will - # have its own scheduler - num_gpu_blocks = num_blocks[0] - num_cpu_blocks = num_blocks[1] - - return num_gpu_blocks, num_cpu_blocks - - def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks: int) -> None: - """Initialize the KV cache in all workers.""" - - # NOTE: We log here to avoid multiple logs when number of workers is - # greater than one. We could log in the engine, but not all executors - # have GPUs. - logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks, num_cpu_blocks) - - self.cache_config.num_gpu_blocks = num_gpu_blocks - self.cache_config.num_cpu_blocks = num_cpu_blocks - - if torch.distributed.get_rank() == 0: - print( - f"before init cache memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB" - ) - self.worker.initialize_cache(num_gpu_blocks=num_gpu_blocks, num_cpu_blocks=num_cpu_blocks) - if torch.distributed.get_rank() == 0: - print( - f"after init cache memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB" - ) - - # NOTE(sgm): This will not profile & capture the model(CUDAGraph) when rebuilding KVCache - def init_cache_engine(self) -> None: - self.worker._init_cache_engine() - - def free_cache_engine(self) -> None: - self.worker.free_cache_engine() - - def execute_model(self, execute_model_req) -> List[SamplerOutput]: - all_outputs = self.worker.execute_model(execute_model_req=execute_model_req) - - # NOTE(sgm): - # Each GPU in vllm under verl has its own spmd_gpu_executor, therefore all GPUs should return the outputs - # In vllm with ray, only the driver worker returns the sampling results. - return all_outputs - - def add_lora(self, lora_request: LoRARequest) -> bool: - assert lora_request.lora_int_id > 0, "lora_id must be greater than 0." - return self.worker.add_lora(lora_request=lora_request) - - def remove_lora(self, lora_id: int) -> bool: - assert lora_id > 0, "lora_id must be greater than 0." - return self.worker.remove_lora(lora_id=lora_id) - - def list_loras(self) -> Set[int]: - return self.worker.list_loras() - - def check_health(self) -> None: - # SPMDExecutor will always be healthy as long as - # it's running. - return - - # NOTE(sgm) add for verl to pass the abstract class test, not used - from vllm.prompt_adapter.request import PromptAdapterRequest - - def add_prompt_adapter(self, prompt_adapter_request: PromptAdapterRequest) -> bool: - assert prompt_adapter_request.prompt_adapter_id > 0, "prompt_adapter_id must be greater than 0." - return self.worker.add_prompt_adapter(prompt_adapter_request) - - def list_prompt_adapters(self) -> Set[int]: - return self.worker.list_prompt_adapters() - - def pin_lora(self, lora_id: int) -> bool: - assert lora_id > 0, "lora_id must be greater than 0." - return self.worker.pin_lora(lora_id) - - def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool: - assert prompt_adapter_id > 0, "prompt_adapter_id must be greater than 0." - return self.worker.pin_prompt_adapter(prompt_adapter_id) - - def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool: - assert prompt_adapter_id > 0, "prompt_adapter_id must be greater than 0." - return self.worker.remove_prompt_adapter(prompt_adapter_id) - - # NOTE(sgm): add for verl - def offload_model_weights(self) -> None: - self.worker.offload_model_weights() - - def sync_model_weights(self, actor_weights: Dict[str, torch.Tensor], load_format: str) -> None: - self.worker.sync_model_weights(actor_weights=actor_weights, load_format=load_format) - - -def initialize_cluster( - parallel_config: ParallelConfig, - engine_use_ray: bool = False, - ray_address: Optional[str] = None, -) -> Tuple[str, Optional[None]]: - """Initialize the distributed cluster probably with Ray. - - Args: - parallel_config: The configurations for parallel execution. - - Returns: - The `distributed_init_method` is the address for initializing the - distributed backend. - """ - - # Initialize cluster locally. - port = get_open_port() - # We need to setup the distributed init method to make sure - # the distributed megatron code (e.g., get world size) works correctly. - # distributed_init_method = f"tcp://localhost:{port}" - distributed_init_method = "env://" - return distributed_init_method - - -def get_open_port(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - return s.getsockname()[1] - - -# TODO(sgm): not implemented async executor yet -class SPMDGPUExecutorAsync(SPMDGPUExecutor, ExecutorAsyncBase): - - async def execute_model_async(self, execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]: - """Executes one model step on the given sequences.""" - raise NotImplementedError - - async def check_health_async(self) -> None: - """Checks if the executor is healthy. If not, it should raise an - exception.""" - self.check_health() diff --git a/verl/third_party/vllm/vllm_v_0_6_3/tokenizer.py b/verl/third_party/vllm/vllm_v_0_6_3/tokenizer.py deleted file mode 100644 index b0b4d0e27c..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/tokenizer.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/tokenizer_group/tokenizer_group.py - -from typing import Optional - -from transformers import PreTrainedTokenizer -from vllm.transformers_utils.tokenizer_group import TokenizerGroup -from vllm.utils import LRUCache - - -class TokenizerGroup(TokenizerGroup): - """A group of tokenizers that can be used for LoRA adapters.""" - - def __init__(self, tokenizer: PreTrainedTokenizer, enable_lora: bool, max_num_seqs: int, - max_input_length: Optional[int]): - self.enable_lora = enable_lora - self.max_input_length = max_input_length - self.tokenizer = tokenizer - self.lora_tokenizers = LRUCache[PreTrainedTokenizer](capacity=max_num_seqs) if enable_lora else None - - # FIXME(sgm): for simplicity, we assign the special token here - @property - def pad_token_id(self): - return self.tokenizer.pad_token_id - - @property - def eos_token_id(self): - return self.tokenizer.eos_token_id diff --git a/verl/third_party/vllm/vllm_v_0_6_3/worker.py b/verl/third_party/vllm/vllm_v_0_6_3/worker.py deleted file mode 100644 index cb1a7ab80c..0000000000 --- a/verl/third_party/vllm/vllm_v_0_6_3/worker.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2023 The vLLM team. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/worker/worker.py -"""A GPU worker class.""" -import gc -import os -from typing import Dict, List, Optional, Tuple, Type, Union - -import torch -import torch.distributed -import torch.nn as nn -from vllm.config import ( - CacheConfig, - DeviceConfig, - LoRAConfig, - ParallelConfig, - PromptAdapterConfig, - SchedulerConfig, - SpeculativeConfig, -) - -# TODO(sgm): check why vllm has similar file in vllm.model_executor.parallel_utils.parallel_state -from vllm.distributed import get_tensor_model_parallel_group, init_distributed_environment, set_custom_all_reduce -from vllm.model_executor import set_random_seed -from vllm.model_executor.layers.sampler import SamplerOutput -from vllm.sequence import ExecuteModelRequest, IntermediateTensors -from vllm.worker.cache_engine import CacheEngine -from vllm.worker.embedding_model_runner import EmbeddingModelRunner -from vllm.worker.model_runner import GPUModelRunnerBase -from vllm.worker.model_runner_base import ModelRunnerInputBase -from vllm.worker.worker import Worker, _check_if_gpu_supports_dtype -from vllm.worker.worker_base import WorkerInput - -from .config import LoadConfig, LoadFormat, ModelConfig -from .dtensor_weight_loaders import load_dtensor_weights -from .hf_weight_loader import load_hf_weights -from .megatron_weight_loaders import load_megatron_weights -from .model_runner import ModelRunner -from .parallel_state import ensure_model_parallel_initialized - - -class Worker(Worker): - """A worker class that executes (a partition of) the model on a GPU. - - Each worker is associated with a single GPU. The worker is responsible for - maintaining the KV cache and executing the model on the GPU. In case of - distributed inference, each worker is assigned a partition of the model. - """ - - def __init__( - self, - model: Union[nn.Module, Dict], # model itself or its parameter dict - model_config: ModelConfig, - parallel_config: ParallelConfig, - scheduler_config: SchedulerConfig, - device_config: DeviceConfig, - cache_config: CacheConfig, - load_config: LoadConfig, - local_rank: int, - rank: int, - distributed_init_method: str, - lora_config: Optional[LoRAConfig] = None, - speculative_config: Optional[SpeculativeConfig] = None, - prompt_adapter_config: Optional[PromptAdapterConfig] = None, - is_driver_worker: bool = False, - model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None, - ) -> None: - # self.model = model # will be replaced in the init_model - self.model_config = model_config - self.parallel_config = parallel_config - self.parallel_config.rank = rank - self.scheduler_config = scheduler_config - self.device_config = device_config - self.cache_config = cache_config - self.local_rank = local_rank - self.rank = rank - self.distributed_init_method = distributed_init_method - self.lora_config = lora_config - self.load_config = load_config - self.prompt_adapter_config = prompt_adapter_config - self.is_driver_worker = is_driver_worker # TODO: we don't need driver - # if parallel_config and is_driver_worker: - # assert rank % parallel_config.tensor_parallel_size == 0, \ - # "Driver worker should be rank 0 of tensor parallel group." - if self.model_config.trust_remote_code: - # note: lazy import to avoid importing torch before initializing - from vllm.utils import init_cached_hf_modules - - init_cached_hf_modules() - - # Return hidden states from target model if the draft model is an - # mlp_speculator - speculative_args = ( - {} if speculative_config is None or (speculative_config.draft_model_config.model == model_config.model) or - (speculative_config.draft_model_config.hf_config.model_type not in ["medusa", "mlp_speculator"]) else { - "return_hidden_states": True - }) - - # TODO(sgm): set correct model runner class - ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner - if model_runner_cls is not None: - ModelRunnerClass = model_runner_cls - elif self.model_config.embedding_mode: - ModelRunnerClass = EmbeddingModelRunner - self.model_runner: GPUModelRunnerBase = ModelRunnerClass( - model, # [VERL]: add for verl - model_config, - parallel_config, - scheduler_config, - device_config, - cache_config, - load_config=load_config, - lora_config=self.lora_config, - kv_cache_dtype=self.cache_config.cache_dtype, - is_driver_worker=is_driver_worker, - prompt_adapter_config=prompt_adapter_config, - **speculative_args, - ) - - # Uninitialized cache engine. Will be initialized by - # initialize_cache. - self.cache_engine: List[CacheEngine] = None - # Initialize gpu_cache as embedding models don't initialize kv_caches - self.gpu_cache: Optional[List[List[torch.Tensor]]] = None - - # NOTE(sgm): [VERL] For offloading inference engine params - self.cpu_model = None - - def init_device(self) -> None: - if self.device_config.device.type == "cuda": - # torch.distributed.all_reduce does not free the input tensor until - # the synchronization point. This causes the memory usage to grow - # as the number of all_reduce calls increases. This env var disables - # this behavior. - # Related issue: - # https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573 - os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1" - - # NOTE(sgm): Modify for verl, Env vars will be set by TORCHRUN. - self.rank = self.rank if self.rank is not None else int(os.getenv("RANK", "-1")) - local_rank = int(os.getenv("LOCAL_RANK", "0")) - self.device = torch.device(f"cuda:{local_rank}") - if self.rank < 0: - raise ValueError("Invalid or unspecified rank.") - torch.cuda.set_device(self.device) - - # Use the world_size set by TORCHRUN - world_size = int(os.getenv("WORLD_SIZE", "-1")) - assert world_size != -1, "The world_size is set to -1, not initialized by TORCHRUN" - self.parallel_config.world_size = world_size - - _check_if_gpu_supports_dtype(self.model_config.dtype) - torch.cuda.empty_cache() - self.init_gpu_memory = torch.cuda.mem_get_info()[0] - else: - raise RuntimeError(f"Not support device type: {self.device_config.device}") - - # Initialize the distributed environment. - init_worker_distributed_environment(self.parallel_config, self.rank, self.distributed_init_method, - self.local_rank) - # Set random seed. - set_random_seed(self.model_config.seed) - # self.model = get_model(actor_model=self.model, model_config=self.model_config) - - @torch.inference_mode() - def determine_num_available_blocks(self) -> Tuple[int, int]: - """Profiles the peak memory usage of the model to determine how many - KV blocks may be allocated without OOMs. - - The engine will first conduct a profiling of the existing memory usage. - Then, it calculate the maximum possible number of GPU and CPU blocks - that can be allocated with the remaining free memory. - - .. tip:: - You may limit the usage of GPU memory - by adjusting the `gpu_memory_utilization` parameter. - """ - # Profile the memory usage of the model and get the maximum number of - # cache blocks that can be allocated with the remaining free memory. - torch.cuda.empty_cache() - # torch.cuda.reset_peak_memory_stats() - - # Execute a forward pass with dummy inputs to profile the memory usage - # of the model. - self.model_runner.profile_run() - - # Calculate the number of blocks that can be allocated with the - # profiled peak memory. - torch.cuda.synchronize() - free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info() - peak_memory = total_gpu_memory - free_gpu_memory - - assert peak_memory > 0, ("Error in memory profiling. This happens when the GPU memory was " - "not properly cleaned up before initializing the vLLM instance.") - - cache_block_size = self.get_cache_block_size_bytes() - - # NOTE(sgm) [VERL] use the remaining memory - num_gpu_blocks = int((free_gpu_memory * self.cache_config.gpu_memory_utilization) // cache_block_size) - # num_gpu_blocks = int((total_gpu_memory * self.cache_config.gpu_memory_utilization - peak_memory) // cache_block_size) - - num_cpu_blocks = int(self.cache_config.swap_space_bytes // cache_block_size) - num_gpu_blocks = max(num_gpu_blocks, 0) - num_cpu_blocks = max(num_cpu_blocks, 0) - if self.model_runner.lora_manager: - self.model_runner.remove_all_loras() - - # NOTE(sgm): Add for [VERL], synchronize number of blocks with all the rank - num_gpu_blocks = torch.tensor([num_gpu_blocks], device="cuda") - num_cpu_blocks = torch.tensor([num_cpu_blocks], device="cuda") - - torch.distributed.all_reduce(num_gpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group().device_group) - torch.distributed.all_reduce(num_cpu_blocks, - op=torch.distributed.ReduceOp.MIN, - group=get_tensor_model_parallel_group().device_group) - num_gpu_blocks = num_gpu_blocks.item() - num_cpu_blocks = num_cpu_blocks.item() - gc.collect() - torch.cuda.empty_cache() - return num_gpu_blocks, num_cpu_blocks - - def _init_cache_engine(self): - if self.cache_engine is None and self.gpu_cache is None: - super()._init_cache_engine() - - def free_cache_engine(self): - # ensure `enforce_eager=True` - self.cache_engine = None - self.gpu_cache = None - - # NOTE(sgm): [VERL]: adapt from _execute_model_spmd() - def execute_model(self, - execute_model_req: ExecuteModelRequest, - intermediate_tensors: Optional[IntermediateTensors] = None) -> Optional[List[SamplerOutput]]: - """ - Execute model in Single Program Multiple Data (SPMD) fashion. - All workers take the same request, prepare the input and - execute the model. - """ - assert execute_model_req is not None, ("_execute_model_spmd() requires each worker to take in an " - "ExecuteModelRequest") - worker_input: WorkerInput = self.prepare_worker_input(execute_model_req=execute_model_req) - model_input: ModelRunnerInputBase = self.model_runner.prepare_model_input( - execute_model_req.seq_group_metadata_list) - - # verl.worker.workerbase.WorkerBase - # swap cache - super().execute_worker(worker_input) - - # If there is no input, we don't need to execute the model. - if worker_input.num_seq_groups == 0: - return [] - - return self.model_runner.execute_model( - model_input, - self.kv_cache[worker_input.virtual_engine] if self.kv_cache is not None else None, - intermediate_tensors, - ) - - # assume the input is .state_dict() - def sync_model_weights(self, actor_weights: Dict, load_format: str): - if load_format in [LoadFormat.MEGATRON, LoadFormat.AUTO]: - load_megatron_weights(actor_weights, self.model_runner.model) - elif load_format == LoadFormat.HF: - # full model state dict without no sharding - load_hf_weights(actor_weights, self.model_runner.model) - elif load_format == LoadFormat.DTENSOR: - load_dtensor_weights(actor_weights, self.model_runner.model) - - def offload_model_weights(self) -> None: - if self.cpu_model == None: - self.cpu_model = {} - for name, params in self.model_runner.model.named_parameters(): - self.cpu_model[name] = torch.empty_like(params, device="cpu") - params.data = self.cpu_model[name] - else: - for name, params in self.model_runner.model.named_parameters(): - params.data = self.cpu_model[name] - - -def init_worker_distributed_environment( - parallel_config: ParallelConfig, - rank: int, - distributed_init_method: Optional[str] = "env://", - local_rank: int = -1, -) -> None: - """Initialize the distributed environment.""" - set_custom_all_reduce(not parallel_config.disable_custom_all_reduce) - - # NOTE(sgm) use tcp://localhost:xxxx will hang in HF setting without megatron - init_distributed_environment(parallel_config.world_size, rank, distributed_init_method, local_rank) - - ensure_model_parallel_initialized( - tensor_model_parallel_size=parallel_config.tensor_parallel_size, - pipeline_model_parallel_size=parallel_config.pipeline_parallel_size, - ) - - # TODO(sgm): check whether need this - # if pynccl_utils.is_initialized(): - # pynccl_world_size = pynccl_utils.get_world_size() - # if pynccl_world_size != parallel_config.world_size: - # raise RuntimeError( - # "pynccl is already initialized but the pynccl world " - # "size does not match parallel_config.world_size " - # f"({pynccl_world_size} vs. {parallel_config.world_size}).") - # elif parallel_config.world_size > 1: - # # NOTE(woosuk): We don't initialize pynccl process group when world size - # # is 1. - # # NOTE(kaichao): By default, pynccl is initialized for tp group. - # pynccl_utils.init_process_group( - # group=get_tensor_model_parallel_cpu_group()) - - # # Initialize a custom fast all-reduce implementation. - # if not parallel_config.disable_custom_all_reduce: - # init_custom_ar() - - # A small all_reduce for warmup. - torch.distributed.all_reduce(torch.zeros(1).cuda()) - # if pynccl_utils.is_initialized(): - # pynccl_utils.all_reduce(torch.zeros(1).cuda()) diff --git a/verl/trainer/__init__.py b/verl/trainer/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/trainer/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/trainer/config/evaluation.yaml b/verl/trainer/config/evaluation.yaml deleted file mode 100644 index d0e2cbd93a..0000000000 --- a/verl/trainer/config/evaluation.yaml +++ /dev/null @@ -1,10 +0,0 @@ -data: - path: /tmp/math_Qwen2-7B-Instruct.parquet - prompt_key: prompt - response_key: responses - data_source_key: data_source - reward_model_key: reward_model - -custom_reward_function: - path: null - name: compute_score diff --git a/verl/trainer/config/generation.yaml b/verl/trainer/config/generation.yaml deleted file mode 100644 index 8a6e010b5d..0000000000 --- a/verl/trainer/config/generation.yaml +++ /dev/null @@ -1,50 +0,0 @@ -trainer: - nnodes: 1 - n_gpus_per_node: 8 - -data: - path: ~/data/rlhf/math/test.parquet - prompt_key: prompt - n_samples: 5 - output_path: /opt/tiger/math_Qwen2-7B-Instruct.parquet - batch_size: 128 - -model: - path: ~/models/Qwen2-7B-Instruct - external_lib: null -rollout: - name: vllm - temperature: 1.0 - top_k: 50 # 0 for hf rollout, -1 for vllm rollout - top_p: 0.7 - prompt_length: 1536 - response_length: 512 - # for vllm rollout - dtype: bfloat16 # should align with FSDP - gpu_memory_utilization: 0.5 - ignore_eos: False - enforce_eager: True - free_cache_engine: True - load_format: dummy_dtensor - tensor_model_parallel_size: 1 - max_num_batched_tokens: 8192 - max_model_len: null - max_num_seqs: 1024 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: 8 - # for fire vllm rollout - use_fire_sampling: False # enable FIRE https://arxiv.org/abs/2410.21236 - # for hf rollout - do_sample: True - disable_log_stats: True - enable_chunked_prefill: True - n: 1 - sampling_params: - temperature: 1.0 - top_p: 0.95 - top_k: 50 -actor: - strategy: fsdp # This is for backward-compatibility - ulysses_sequence_parallel_size: 1 # sp size - fsdp_config: - fsdp_size: -1 \ No newline at end of file diff --git a/verl/trainer/config/ppo_megatron_trainer.yaml b/verl/trainer/config/ppo_megatron_trainer.yaml deleted file mode 100644 index 89015a316e..0000000000 --- a/verl/trainer/config/ppo_megatron_trainer.yaml +++ /dev/null @@ -1,210 +0,0 @@ -data: - tokenizer: null - train_files: ~/data/rlhf/gsm8k/train.parquet - val_files: ~/data/rlhf/gsm8k/test.parquet - prompt_key: prompt - reward_fn_key: data_source - max_prompt_length: 512 - max_response_length: 512 - train_batch_size: 1024 - gen_batch_size: ${data.train_batch_size} - val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves - return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs - return_raw_chat: False - shuffle: True - filter_overlong_prompts: False # for large-scale dataset, filtering overlong prompts could be timeconsuming. You cat set the filter_overlong_prompts_workers to use multiprocessing to speed up. - filter_overlong_prompts_workers: 1 - truncation: error - -actor_rollout_ref: - hybrid_engine: True - model: - path: ~/models/deepseek-llm-7b-chat - external_lib: null - override_config: {} - enable_gradient_checkpointing: False - actor: - strategy: megatron # This is for backward-compatibility - ppo_mini_batch_size: 256 - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - use_dynamic_bsz: False - use_torch_compile: True # False to disable torch compile - # pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) - clip_ratio: 0.2 # default value if clip_ratio_low and clip_ratio_high are not specified - clip_ratio_low: 0.2 - clip_ratio_high: 0.2 - clip_ratio_c: 3.0 # lower bound of the value for Dual-clip PPO from https://arxiv.org/pdf/1912.09729 - loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean" - # NOTE: "token-mean" is the default behavior - entropy_coeff: 0.001 - use_kl_loss: False # True for GRPO - kl_loss_coef: 0.001 # for grpo - kl_loss_type: low_var_kl # for grpo - ppo_epochs: 1 - shuffle: True - optim: - lr: 1e-6 - clip_grad: 1.0 - lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - weight_decay: 0.01 - megatron: - tensor_model_parallel_size: 4 - pipeline_model_parallel_size: 1 - virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests - sequence_parallel: True - use_distributed_optimizer: True - seed: 1 - load_weight: True - checkpoint: - contents: ['model', 'optimizer', 'extra'] # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space - ref: - megatron: - tensor_model_parallel_size: 4 - pipeline_model_parallel_size: 1 - virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests - sequence_parallel: True - use_distributed_optimizer: True - seed: 1 - load_weight: True - param_offload: False - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - rollout: - name: vllm - temperature: 1.0 - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1 - prompt_length: ${data.max_prompt_length} # for xperf_gpt - response_length: ${data.max_response_length} - # for vllm rollout - dtype: bfloat16 # should align with FSDP - gpu_memory_utilization: 0.5 - ignore_eos: False - enforce_eager: True - free_cache_engine: True - load_format: dummy_megatron - tensor_model_parallel_size: 2 - max_num_batched_tokens: 8192 - max_model_len: null - max_num_seqs: 1024 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - disable_log_stats: True - enable_chunked_prefill: False # could get higher throughput - # for hf rollout - do_sample: True - layer_name_map: - qkv_layer_name: qkv - gate_proj_layer_name: gate_up - # number of responses (i.e. num sample times) - n: 1 - val_kwargs: - # sampling parameters for validation - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1.0 - temperature: 0 - n: 1 - do_sample: False # default eager for validation - -critic: - rollout_n: ${actor_rollout_ref.rollout.n} - strategy: megatron - optim: - lr: 1e-5 - clip_grad: 1.0 - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - weight_decay: 0.01 - model: - path: ~/models/deepseek-llm-7b-chat - tokenizer_path: ${actor_rollout_ref.model.path} - override_config: {} - external_lib: ${actor_rollout_ref.model.external_lib} - enable_gradient_checkpointing: False - megatron: - tensor_model_parallel_size: 4 - pipeline_model_parallel_size: 1 - virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests - sequence_parallel: True - use_distributed_optimizer: True - seed: 1 - load_weight: True - ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs} - shuffle: ${actor_rollout_ref.actor.shuffle} - cliprange_value: 0.5 - kl_ctrl: - type: fixed - kl_coef: 0.001 - checkpoint: - contents: ['model', 'optimizer', 'extra'] # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space - -reward_model: - enable: False - strategy: megatron - megatron: - tensor_model_parallel_size: 4 - pipeline_model_parallel_size: 1 - virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests - sequence_parallel: True - use_distributed_optimizer: True - seed: 1 - model: - input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical - path: ~/models/FsfairX-LLaMA3-RM-v0.1 - external_lib: ${actor_rollout_ref.model.external_lib} - load_weight: True - param_offload: False - micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu - micro_batch_size_per_gpu: null - use_dynamic_bsz: ${critic.use_dynamic_bsz} - max_length: null - -custom_reward_function: - path: null - name: compute_score - -algorithm: - gamma: 1.0 - lam: 1.0 - adv_estimator: gae - use_kl_in_reward: False - kl_penalty: kl # how to estimate kl divergence - kl_ctrl: - type: fixed - kl_coef: 0.001 - horizon: 10000 - target_kl: 0.1 - -trainer: - balance_batch: True - total_epochs: 30 - total_training_steps: null - project_name: verl_examples - experiment_name: gsm8k - logger: ['console', 'wandb'] - log_val_generations: 0 - nnodes: 1 - n_gpus_per_node: 8 - save_freq: -1 - # auto: find the last ckpt to resume. If can't find, start from scratch - resume_mode: auto # or disable or resume_path if resume_from_path is set - resume_from_path: null - del_local_ckpt_after_load: False - val_before_train: True - test_freq: 2 - critic_warmup: 0 - default_hdfs_dir: null - default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} - max_actor_ckpt_to_keep: null - max_critic_ckpt_to_keep: null diff --git a/verl/trainer/config/ppo_trainer.yaml b/verl/trainer/config/ppo_trainer.yaml deleted file mode 100644 index 7dffd0e07c..0000000000 --- a/verl/trainer/config/ppo_trainer.yaml +++ /dev/null @@ -1,240 +0,0 @@ -data: - tokenizer: null - train_files: ~/data/rlhf/gsm8k/train.parquet - val_files: ~/data/rlhf/gsm8k/test.parquet - prompt_key: prompt - reward_fn_key: data_source - max_prompt_length: 512 - max_response_length: 512 - train_batch_size: 1024 - val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves - return_raw_input_ids: False # This should be set to true when the tokenizer between policy and rm differs - return_raw_chat: False - shuffle: True - filter_overlong_prompts: False # for large-scale dataset, filtering overlong prompts could be timeconsuming. You cat set the filter_overlong_prompts_workers to use multiprocessing to speed up. - filter_overlong_prompts_workers: 1 - truncation: error - image_key: images - dataloader_num_workers: 8 - -verifier: - reward_coef: 1 - -actor_rollout_ref: - hybrid_engine: True - model: - path: ~/models/deepseek-llm-7b-chat - external_lib: null - override_config: { } - enable_gradient_checkpointing: True - use_remove_padding: False - actor: - masking: False - strategy: fsdp # This is for backward-compatibility - ppo_mini_batch_size: 256 - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - use_dynamic_bsz: False - ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length} - grad_clip: 1.0 - # pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high) - clip_ratio: 0.2 # default value if clip_ratio_low and clip_ratio_high are not specified - clip_ratio_low: 0.2 - clip_ratio_high: 0.2 - clip_ratio_c: 3.0 # lower bound of the value for Dual-clip PPO from https://arxiv.org/pdf/1912.09729 - loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean" - entropy_coeff: 0.001 - use_kl_loss: False # True for GRPO - use_torch_compile: True # False to disable torch compile - kl_loss_coef: 0.001 # for grpo - kl_loss_type: low_var_kl # for grpo - ppo_epochs: 1 - shuffle: False - ulysses_sequence_parallel_size: 1 # sp size - checkpoint: - contents: ['model', 'optimizer', 'extra'] # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space - optim: - lr: 1e-6 - lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio. - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - weight_decay: 0.01 - fsdp_config: - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - param_offload: False - optimizer_offload: False - fsdp_size: -1 - ref: - fsdp_config: - param_offload: False - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} # sp size - rollout: - name: async - task_type: 'swegym' - temperature: 1.0 - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1 - use_fire_sampling: False # https://arxiv.org/abs/2410.21236 - prompt_length: ${data.max_prompt_length} # not use for opensource - response_length: ${data.max_response_length} - # for vllm rollout - dtype: bfloat16 # should align with FSDP - gpu_memory_utilization: 0.5 - ignore_eos: False - enforce_eager: True - free_cache_engine: True - load_format: dummy_dtensor - tensor_model_parallel_size: 2 - max_num_batched_tokens: 8192 - max_model_len: null - max_num_seqs: 1024 - log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu - log_prob_micro_batch_size_per_gpu: null - log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu} - disable_log_stats: True - enable_chunked_prefill: True # may get higher throughput when set to True. When activated, Please increase max_num_batched_tokens or decrease max_model_len. - # for hf rollout - do_sample: True - # number of responses (i.e. num sample times) - n: 1 # > 1 for grpo - enable_memory_saver: False - sampling_params: - temperature: 1.0 - top_p: 0.95 - max_new_tokens: ${data.max_response_length} - top_k: -1 # -1 is disable - min_p: 0 - repetition_penalty: 1 - # for agentic workflow - max_parallel_agents: 1 # how many agents for rollout at the same time - max_eval_parallel_agents: -1 # how many agents for evaluation at the same time. If -1, will be same as `max_parallel_agents`. - log_messages_dir: null - n_trajectories: 1 # This is the n for agent training. - max_iterations: 1 # how many turn to generate for one agent - remove_think_tokens: False # whether to remove think tokens for policy model - qwen3_enable_thinking: True # qwen3: whether to enable thinking mode or not - val_kwargs: - # sampling parameters for validation - top_k: -1 # 0 for hf rollout, -1 for vllm rollout - top_p: 1.0 - temperature: 0 - n: 1 - do_sample: False # default eager for validation - exchange_size: -1 - - -critic: - rollout_n: ${actor_rollout_ref.rollout.n} - strategy: fsdp - optim: - lr: 1e-5 - lr_warmup_steps_ratio: 0. # the total steps will be injected during runtime - min_lr_ratio: null # only useful for warmup with cosine - warmup_style: constant # select from constant/cosine - total_training_steps: -1 # must be override by program - weight_decay: 0.01 - model: - path: ~/models/deepseek-llm-7b-chat - tokenizer_path: ${actor_rollout_ref.model.path} - override_config: { } - external_lib: ${actor_rollout_ref.model.external_lib} - enable_gradient_checkpointing: True - use_remove_padding: False - fsdp_config: - param_offload: False - optimizer_offload: False - wrap_policy: - # transformer_layer_cls_to_wrap: None - min_num_params: 0 - fsdp_size: -1 - ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size} - ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu - ppo_micro_batch_size_per_gpu: null - forward_micro_batch_size: ${critic.ppo_micro_batch_size} - forward_micro_batch_size_per_gpu: ${critic.ppo_micro_batch_size_per_gpu} - use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz} - ppo_max_token_len_per_gpu: 32768 # (${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}) * 2 - forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu} - ulysses_sequence_parallel_size: 1 # sp size - ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs} - shuffle: ${actor_rollout_ref.actor.shuffle} - grad_clip: 1.0 - cliprange_value: 0.5 - checkpoint: - contents: ['model', 'optimizer', 'extra'] # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space - -reward_model: - enable: False - strategy: fsdp - model: - input_tokenizer: ${actor_rollout_ref.model.path} # set this to null if the chat template is identical - path: ~/models/FsfairX-LLaMA3-RM-v0.1 - external_lib: ${actor_rollout_ref.model.external_lib} - use_remove_padding: False - fsdp_config: - wrap_policy: - min_num_params: 0 - param_offload: False - fsdp_size: -1 - micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu - micro_batch_size_per_gpu: null # set a number - max_length: null - ulysses_sequence_parallel_size: 1 # sp size - use_dynamic_bsz: ${critic.use_dynamic_bsz} - forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu} - reward_manager: naive - score_timeout: 300 - num_processes: 128 - -custom_reward_function: - path: null - name: compute_score - -algorithm: - gamma: 1.0 - lam: 1.0 - adv_estimator: gae - use_kl_in_reward: False - kl_penalty: kl # how to estimate kl divergence - kl_ctrl: - type: fixed - kl_coef: 0.001 - horizon: 10000 - target_kl: 0.1 - -trainer: - balance_batch: True - total_epochs: 30 - total_training_steps: null - project_name: verl_examples - experiment_name: gsm8k - logger: [ 'console', 'wandb' ] - log_val_generations: 0 - nnodes: 1 - n_gpus_per_node: 8 - save_freq: -1 - # auto: find the last ckpt to resume. If can't find, start from scratch - resume_mode: auto # or disable or resume_path if resume_from_path is set - resume_from_path: null - val_before_train: False - test_freq: -1 - critic_warmup: 0 - default_hdfs_dir: null - del_local_ckpt_after_load: False - default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} - max_actor_ckpt_to_keep: null - max_critic_ckpt_to_keep: null - - diff --git a/verl/trainer/config/sft_trainer.yaml b/verl/trainer/config/sft_trainer.yaml deleted file mode 100644 index a18082456a..0000000000 --- a/verl/trainer/config/sft_trainer.yaml +++ /dev/null @@ -1,48 +0,0 @@ -data: - train_batch_size: 256 - micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu - micro_batch_size_per_gpu: 4 # this is also val batch size - train_files: ~/data/gsm8k/train.parquet - val_files: ~/data/gsm8k/test.parquet - prompt_key: question - response_key: answer - max_length: 1024 - truncation: error - balance_dp_token: False - chat_template: null - custom_cls: - path: null - name: null -model: - partial_pretrain: ~/models/gemma-1.1-7b-it - fsdp_config: - wrap_policy: - min_num_params: 0 - cpu_offload: False - offload_params: False - external_lib: null - enable_gradient_checkpointing: False - trust_remote_code: False - lora_rank: 0 # Set to positive value to enable LoRA (e.g., 32) - lora_alpha: 16 # LoRA scaling factor - target_modules: all-linear # Target modules for LoRA adaptation - use_liger: False -optim: - lr: 1e-5 - betas: [0.9, 0.95] - weight_decay: 0.01 - warmup_steps_ratio: 0.1 - clip_grad: 1.0 -ulysses_sequence_parallel_size: 1 -use_remove_padding: False -trainer: - default_local_dir: /tmp/sft_model - default_hdfs_dir: hdfs://tmp/experiments/gsm8k/gemma-1.1-7b-it/ # change the hdfs path here - resume_path: null - project_name: gsm8k-sft - experiment_name: test - total_epochs: 4 - total_training_steps: null - logger: ['console'] - seed: 1 - diff --git a/verl/trainer/fsdp_sft_trainer.py b/verl/trainer/fsdp_sft_trainer.py deleted file mode 100644 index 54089202af..0000000000 --- a/verl/trainer/fsdp_sft_trainer.py +++ /dev/null @@ -1,532 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -A lightweight one-file FSDP SFT Trainer -TODO(zhangchi.usc1992) -- Add calculation of mfu -- Add validation -""" - -import os - -os.environ['NCCL_DEBUG'] = 'WARN' -os.environ['TOKENIZERS_PARALLELISM'] = 'true' - -import logging -import re -from contextlib import nullcontext -import torch -import torch.distributed -from torch import nn, optim -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, MixedPrecision, ShardingStrategy, CPUOffload -from tqdm import tqdm -from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedModel, AutoConfig -from verl.utils.torch_functional import get_cosine_schedule_with_warmup -from tensordict import TensorDict -from torch.utils.data import DataLoader, DistributedSampler -from flash_attn.bert_padding import pad_input, unpad_input, rearrange, index_first_axis - -from verl.utils.fsdp_utils import get_fsdp_wrap_policy, init_fn, get_init_weight_context_manager -from verl.utils.dataset import SFTDataset -from verl.utils.fs import copy_to_local -from verl.utils.tracking import Tracking -from verl.utils.ulysses import get_ulysses_sequence_parallel_world_size, set_ulysses_sequence_parallel_group -from torch.distributed.device_mesh import DeviceMesh - -import verl.utils.hdfs_io as hdfs_io -from verl.utils.debug import log_gpu_memory_usage -from peft import LoraConfig, TaskType, get_peft_model - -from verl.workers.sharding_manager import FSDPUlyssesShardingManager -from verl.utils.ulysses import ulysses_pad_and_slice_inputs, gather_outpus_and_unpad -from verl import DataProto - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_SFT_LOGGING_LEVEL', 'WARN')) - - -def extract_step(path): - match = re.search(r'global_step_(\d+)', path) - if match: - return int(match.group(1)) - return None - - -def convert_to_regular_types(obj): - """Convert Hydra configs and other special types to regular Python types.""" - from omegaconf import ListConfig, DictConfig - if isinstance(obj, (ListConfig, DictConfig)): - return {k: convert_to_regular_types(v) for k, v in obj.items()} if isinstance(obj, DictConfig) else list(obj) - elif isinstance(obj, (list, tuple)): - return [convert_to_regular_types(x) for x in obj] - elif isinstance(obj, dict): - return {k: convert_to_regular_types(v) for k, v in obj.items()} - return obj - - -class FSDPSFTTrainer(object): - - def __init__(self, config, device_mesh: DeviceMesh, ulysses_device_mesh: DeviceMesh): - self.config = config - self.device_mesh = device_mesh - self.ulysses_device_mesh = ulysses_device_mesh - self.sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) - # build tokenizer first - local_model_path = copy_to_local(src=self.config.model.partial_pretrain, verbose=True) - from verl.utils import hf_tokenizer - self.tokenizer = hf_tokenizer(local_model_path, trust_remote_code=self.config.model.trust_remote_code) - if self.config.data.chat_template is not None: - raise ValueError('Apply Chat template from config is not supported yet.') - - # normalize dp size - self._normalize_config_bsz() - - # Set sequence parallel size - self.config.ulysses_sequence_parallel_size = getattr(self.config, 'ulysses_sequence_parallel_size', 1) - self.use_remove_padding = getattr(self.config, 'use_remove_padding', False) - if self.device_mesh.get_rank() == 0: - print(f'Using sequence parallel size: {self.config.ulysses_sequence_parallel_size}') - print(f'Using remove padding: {self.use_remove_padding}') - - self._build_dataloader() - # build model - self._build_model_optimizer() - - # TODO: add checkpoint manager - if self.device_mesh.get_rank() == 0: - print(self.config) - - def _normalize_config_bsz(self): - dp_size = self.device_mesh.size(0) if not self.ulysses_device_mesh else self.ulysses_device_mesh.size(0) - if self.device_mesh.get_rank() == 0: - print(f'Normalize batch size by dp {dp_size}') - - assert self.config.data.train_batch_size % dp_size == 0, f"Global batch size {self.config.data.train_batch_size} is not divisible by dp size {dp_size}" - - self.config.data.train_batch_size //= dp_size - - assert self.config.data.train_batch_size % self.config.data.micro_batch_size_per_gpu == 0 - - def _build_dataloader(self): - config = self.config - # build dataset - from verl.utils.import_utils import load_extern_type - if config.data.custom_cls.get("path", None): - dataset_cls = load_extern_type(config.data.custom_cls.path, config.data.custom_cls.name) - else: - dataset_cls = SFTDataset - self.train_dataset = dataset_cls(parquet_files=config.data.train_files, - tokenizer=self.tokenizer, - config=config.data) - self.val_dataset = dataset_cls(parquet_files=config.data.val_files, - tokenizer=self.tokenizer, - config=config.data) - - # build dataloader - # Use data parallel rank and size instead of global rank and world size - - # If doing SP, we need to use the local rank and size - if self.config.ulysses_sequence_parallel_size > 1: - rank = self.ulysses_device_mesh.get_local_rank('dp') - world_size = self.ulysses_device_mesh.size(0) - if self.ulysses_device_mesh.get_rank() == 0: - print(f'Using SP rank {rank} and size {world_size} for data distribution') - print(f'Each SP rank gets different data, but the same data WITHIN the same rank') - else: - rank = self.device_mesh.get_rank() - world_size = self.device_mesh.size() - if self.device_mesh.get_rank() == 0: - print(f'Using FSDP rank {rank} and size {world_size} for data distribution') - - self.train_sampler = DistributedSampler(self.train_dataset, - shuffle=True, - num_replicas=world_size, - rank=rank, - drop_last=True) - self.train_dataloader = DataLoader(dataset=self.train_dataset, - batch_size=config.data.train_batch_size, - sampler=self.train_sampler, - num_workers=8, - pin_memory=True, - drop_last=True) - - self.val_sampler = DistributedSampler(self.val_dataset, - shuffle=False, - num_replicas=world_size, - rank=rank, - drop_last=True) - self.val_dataloader = DataLoader(dataset=self.val_dataset, - batch_size=config.data.micro_batch_size_per_gpu, - sampler=self.val_sampler, - num_workers=8, - pin_memory=True, - drop_last=True) - - def _build_model_optimizer(self): - # TODO (zhangchi.usc1992): - # 1. support pretrain from random weights - # 2. support init directly from sharded weights - local_model_path = copy_to_local(src=self.config.model.partial_pretrain, verbose=True) - - if self.config.model.get('external_lib', None) is not None: - # This is used to import external_lib into the huggingface systems - import importlib - importlib.import_module(self.config.model.external_lib) - - log_gpu_memory_usage('Before model allocation', logger=logger) - - trust_remote_code = self.config.model.trust_remote_code - # load config first - config = AutoConfig.from_pretrained(local_model_path, trust_remote_code=trust_remote_code) - if self.config.ulysses_sequence_parallel_size > 1: - assert self.use_remove_padding, "Sequence parallel is only supported when remove_padding is enabled" - - # This may be very large - init_context = get_init_weight_context_manager(use_meta_tensor=not config.tie_word_embeddings, - mesh=self.device_mesh) - - with init_context(): - self.model: PreTrainedModel = AutoModelForCausalLM.from_pretrained(local_model_path, - config=config, - torch_dtype=torch.float32, - attn_implementation='flash_attention_2', - trust_remote_code=trust_remote_code) - - if self.use_remove_padding or self.config.ulysses_sequence_parallel_size > 1: - from verl.models.transformers.monkey_patch import apply_monkey_patch - apply_monkey_patch(model=self.model, ulysses_sp_size=self.config.ulysses_sequence_parallel_size) - - # Apply Liger kernel if use_liger is enabled - if self.config.model.get('use_liger', False): - from liger_kernel.transformers.monkey_patch import _apply_liger_kernel_to_instance - _apply_liger_kernel_to_instance(model=self.model) - - if self.config.model.get('lora_rank', 0) > 0: - self.model.enable_input_require_grads() - # Convert config to regular Python types before creating PEFT model - lora_config = { - 'task_type': TaskType.CAUSAL_LM, - 'r': self.config.model.lora_rank, - 'lora_alpha': self.config.model.lora_alpha, - 'target_modules': convert_to_regular_types(self.config.model.target_modules), - 'bias': "none" - } - self.model = get_peft_model(self.model, LoraConfig(**lora_config)) - - if self.config.model.enable_gradient_checkpointing: - self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant': False}) - - log_gpu_memory_usage('After model allocation', logger=logger) - - mixed_precision = MixedPrecision(param_dtype=torch.bfloat16, - reduce_dtype=torch.float32, - buffer_dtype=torch.float32) - - auto_wrap_policy = get_fsdp_wrap_policy(self.model, - config=self.config.model.fsdp_config.wrap_policy, - is_lora=self.config.model.get('lora_rank', 0) > 0) - if self.device_mesh.get_rank() == 0: - print(auto_wrap_policy) - - if not self.config.model.fsdp_config.cpu_offload: - cpu_offload = None - else: - cpu_offload = CPUOffload(offload_params=self.config.model.fsdp_config.offload_params) - - self.fsdp_model = FSDP(module=self.model, - auto_wrap_policy=auto_wrap_policy, - param_init_fn=init_fn, - sharding_strategy=ShardingStrategy.FULL_SHARD, - mixed_precision=mixed_precision, - device_mesh=self.device_mesh, - sync_module_states=True, - device_id=torch.cuda.current_device(), - cpu_offload=cpu_offload, - use_orig_params=False) - - log_gpu_memory_usage('After FSDP wrapping', logger=logger) - - self.optimizer = optim.AdamW(self.fsdp_model.parameters(), - lr=self.config.optim.lr, - betas=self.config.optim.betas, - weight_decay=self.config.optim.weight_decay) - - log_gpu_memory_usage('After initialize optimizer', logger=logger) - - self.steps_per_epoch = len(self.train_dataloader) - self.total_steps = self.steps_per_epoch * self.config.trainer.total_epochs - - if self.device_mesh.get_rank() == 0: - print( - f'Number of steps/epoch {self.steps_per_epoch}, number of epochs {self.config.trainer.total_epochs}, total number of steps {self.total_steps}' - ) - - num_warmup_steps = int(self.total_steps * self.config.optim.warmup_steps_ratio) - - self.lr_scheduler = get_cosine_schedule_with_warmup(optimizer=self.optimizer, - num_warmup_steps=num_warmup_steps, - num_training_steps=self.total_steps) - - def _compute_loss_and_backward(self, batch, do_backward=True): - """Compute loss with optional sequence parallelism and remove padding features""" - use_sp = self.use_remove_padding and self.config.ulysses_sequence_parallel_size > 1 - - # Move inputs to GPU and prepare loss mask - input_ids = batch['input_ids'].cuda() - attention_mask = batch['attention_mask'].cuda() - position_ids = batch['position_ids'].cuda() - loss_mask = batch.pop('loss_mask')[:, :-1].reshape(-1).cuda() - loss_fct = nn.CrossEntropyLoss(reduction='none') - - # Context manager for sequence parallel if needed - context = self.sharding_manager if use_sp else nullcontext() - with context: - with torch.autocast(device_type='cuda', dtype=torch.bfloat16): - if not use_sp: - # Standard forward pass without sequence parallel - labels = input_ids[:, 1:].contiguous() - output = self.fsdp_model(input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - use_cache=False) - logits = output.logits - - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = labels.contiguous() - # Flatten the tokens - shift_logits = shift_logits.view(-1, self.model.config.vocab_size) - shift_labels = shift_labels.view(-1) - # Enable model parallelism - shift_labels = shift_labels.to(shift_logits.device) - loss = loss_fct(shift_logits, shift_labels) - loss = loss * loss_mask.to(loss.device) - else: - # IMPORTANT: We have a big assumption here, so we can shard the SAME sequence across SP ranks - # i.e., each GPU has <1 sequence, and each SP group has 1 sequence - # 1. All SP ranks will receive the *SAME* batch - # 2. Different SP groups will receive *DIFFERENT* batches - # This is implemented by the DistributedSampler - - batch_size, seqlen = input_ids.shape - # Remove padding - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - - # Unpad position_ids to align rotary - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # Pad and slice inputs for sequence parallelism - input_ids_rmpad_sliced, position_ids_rmpad_padded, pad_size = ulysses_pad_and_slice_inputs( - input_ids_rmpad, position_ids_rmpad, sp_size=get_ulysses_sequence_parallel_world_size()) - # For computing loss - input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) - input_ids_rmpad_rolled, _, _ = ulysses_pad_and_slice_inputs( - input_ids_rmpad_rolled, None, get_ulysses_sequence_parallel_world_size()) - input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) # ((total_nnz / sp) + pad) - - # Forward pass - output = self.fsdp_model( - input_ids=input_ids_rmpad_sliced, - attention_mask=None, # Not needed with flash attention varlen - position_ids=position_ids_rmpad_padded, - use_cache=False) - - # Compute loss locally then aggregate - logits_rmpad = output.logits.squeeze(0) - input_ids_rmpad_rolled = input_ids_rmpad_rolled.to(logits_rmpad.device) - loss = loss_fct(logits_rmpad, input_ids_rmpad_rolled) - # Gather and unpad for sequence parallelism - loss = gather_outpus_and_unpad(loss, gather_dim=0, unpad_dim=0, padding_size=pad_size) - - # This is the loss collected from all ulysses ranks - full_loss = pad_input(hidden_states=loss.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen) - full_loss = full_loss.squeeze(-1)[:, :-1] # Remove last token's loss - full_loss = full_loss.reshape(-1) - loss_mask = loss_mask.to(full_loss.device) - loss = full_loss * loss_mask - - valid_token_this_rank = torch.sum(loss_mask) - - if self.config.data.balance_dp_token: - torch.distributed.all_reduce(valid_token_this_rank) - dp_size = self.ulysses_device_mesh.size('dp') if use_sp else torch.distributed.get_world_size() - else: - dp_size = 1 - - loss = torch.sum(loss) / (valid_token_this_rank + 1e-8) * dp_size - - if do_backward: - loss.backward() - return loss - - def training_step(self, batch: TensorDict): - self.fsdp_model.train() - - log_gpu_memory_usage('Before optimizer zero_grad', logger=logger) - - self.optimizer.zero_grad() - - log_gpu_memory_usage('After optimizer zero_grad', logger=logger) - - micro_batches = batch.split(self.config.data.micro_batch_size_per_gpu) - n_micro_batches = len(micro_batches) - step_loss = 0 - for micro_batch in micro_batches: - loss = self._compute_loss_and_backward(batch=micro_batch) / n_micro_batches - step_loss += loss.item() - - grad_norm = self.fsdp_model.clip_grad_norm_(max_norm=self.config.optim.clip_grad) - - log_gpu_memory_usage('Before optimizer step', logger=logger) - - # if grad_norm is not finite, skip the update - if not torch.isfinite(grad_norm): - print(f"WARN: grad_norm is not finite: {grad_norm}") - self.optimizer.zero_grad() - else: - self.optimizer.step() - - log_gpu_memory_usage('After optimizer step', logger=logger) - - self.lr_scheduler.step() - - # reduce loss across dp ranks - lr = self.lr_scheduler.get_last_lr()[0] - - log_gpu_memory_usage('After offload weights', logger=logger) - - step_loss = torch.tensor(step_loss).cuda() - torch.distributed.all_reduce(step_loss, op=torch.distributed.ReduceOp.AVG) - return {'train/loss': step_loss.detach().item(), 'train/lr(1e-3)': lr * 1e3} - - def validation_step(self, batch: TensorDict): - self.fsdp_model.eval() - with torch.no_grad(): - loss = self._compute_loss_and_backward(batch, do_backward=False) - torch.distributed.all_reduce(loss, op=torch.distributed.ReduceOp.AVG) - return loss - - def save_checkpoint(self, step): - # save checkpoint - from torch.distributed.fsdp import FullStateDictConfig, StateDictType - cfg = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) - with FSDP.state_dict_type(self.fsdp_model, StateDictType.FULL_STATE_DICT, cfg): - state_dict = self.fsdp_model.state_dict() - - path = os.path.join(self.config.trainer.default_local_dir, f'global_step_{step}') - # save huggingface model - if self.device_mesh.get_rank() == 0: - os.makedirs(path, exist_ok=True) - self.model.save_pretrained(path, state_dict=state_dict) - self.tokenizer.save_pretrained(path) - if self.config.trainer.default_hdfs_dir: - hdfs_io.makedirs(self.config.trainer.default_hdfs_dir, exist_ok=True) - hdfs_io.copy(src=path, dst=self.config.trainer.default_hdfs_dir, dirs_exist_ok=True) - torch.distributed.barrier() - - def fit(self): - rank = self.device_mesh.get_rank() - - # TODO: add a unified tracking - if rank == 0: - tracking = Tracking(project_name=self.config.trainer.project_name, - experiment_name=self.config.trainer.experiment_name, - default_backend=self.config.trainer.logger) - - global_step = 0 - # compute the total training steps. - # the total training steps in SFT is mainly for early exit - total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs - - if self.config.trainer.total_training_steps is not None: - total_training_steps = self.config.trainer.total_training_steps - - self.total_training_steps = total_training_steps - print(f'Total training steps: {self.total_training_steps}') - - # TODO (zhangchi.usc1992) add back checkpoint manager. Currently, it blocks when uploading to hdfs. So very slow. - - for epoch in range(self.config.trainer.total_epochs): - self.train_sampler.set_epoch(epoch=epoch) - for data in tqdm(self.train_dataloader, - total=self.steps_per_epoch, - desc=f"Epoch {epoch+1}/{self.config.trainer.total_epochs}"): - global_step += 1 - data = TensorDict(data, batch_size=self.config.data.train_batch_size).cuda() - metric = self.training_step(data) - if rank == 0: - tracking.log(data=metric, step=global_step) - - # for early exit validation - if global_step >= self.total_training_steps: - # Perform final validation - val_losses = [] - for val_data in self.val_dataloader: - val_data = TensorDict(val_data, batch_size=self.config.data.micro_batch_size_per_gpu).cuda() - val_loss = self.validation_step(val_data) - val_losses.append(val_loss) - if rank == 0: - avg_val_loss = torch.mean(torch.stack(val_losses)) - metric = {'val/loss': avg_val_loss.detach().item()} - tracking.log(data=metric, step=global_step) - torch.distributed.barrier() - - # Save final checkpoint - self.save_checkpoint(step=global_step) - return - - # validation - val_losses = [] - for data in self.val_dataloader: - data = TensorDict(data, batch_size=self.config.data.micro_batch_size_per_gpu).cuda() - val_loss = self.validation_step(data) - val_losses.append(val_loss) - if rank == 0: - val_loss = torch.mean(torch.stack(val_losses)) - metric = {'val/loss': val_loss.detach().item()} - tracking.log(data=metric, step=global_step) - torch.distributed.barrier() - - # save checkpoint - self.save_checkpoint(step=global_step) - - -from verl.trainer.fsdp_sft_trainer import FSDPSFTTrainer -import hydra - -from torch.distributed.device_mesh import init_device_mesh - -from verl.utils.distributed import initialize_global_process_group - - -@hydra.main(config_path='config', config_name='sft_trainer', version_base=None) -def main(config): - local_rank, rank, world_size = initialize_global_process_group() - - device_mesh = init_device_mesh(device_type='cuda', mesh_shape=(world_size,), mesh_dim_names=('fsdp',)) - dp_size = world_size // config.ulysses_sequence_parallel_size - ulysses_device_mesh = init_device_mesh(device_type='cuda', - mesh_shape=(dp_size, config.ulysses_sequence_parallel_size), - mesh_dim_names=('dp', 'sp')) - trainer = FSDPSFTTrainer(config=config, device_mesh=device_mesh, ulysses_device_mesh=ulysses_device_mesh) - trainer.fit() - - -if __name__ == '__main__': - main() diff --git a/verl/trainer/main_eval.py b/verl/trainer/main_eval.py deleted file mode 100644 index 40e871cf86..0000000000 --- a/verl/trainer/main_eval.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Offline evaluate the performance of a generated file using reward model and ground truth verifier. -The input is a parquet file that contains N generated sequences and (optional) the ground truth. - -""" - -import hydra -from verl.utils.fs import copy_to_local -import pandas as pd -import numpy as np -from tqdm import tqdm -from collections import defaultdict -import ray - - -def get_custom_reward_fn(config): - import importlib.util, os - - reward_fn_config = config.get("custom_reward_function") or {} - file_path = reward_fn_config.get("path") - if not file_path: - return None - - if not os.path.exists(file_path): - raise FileNotFoundError(f"Reward function file '{file_path}' not found.") - - spec = importlib.util.spec_from_file_location("custom_module", file_path) - module = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(module) - except Exception as e: - raise RuntimeError(f"Error loading module from '{file_path}': {e}") - - function_name = reward_fn_config.get("name") - - if not hasattr(module, function_name): - raise AttributeError(f"Reward function '{function_name}' not found in '{file_path}'.") - - print(f"using customized reward function '{function_name}' from '{file_path}'") - - return getattr(module, function_name) - - -@ray.remote -def process_item(reward_fn, data_source, response_lst, reward_data): - ground_truth = reward_data['ground_truth'] - score_lst = [reward_fn(data_source, r, ground_truth) for r in response_lst] - return data_source, np.mean(score_lst) - - -@hydra.main(config_path='config', config_name='evaluation', version_base=None) -def main(config): - local_path = copy_to_local(config.data.path) - dataset = pd.read_parquet(local_path) - prompts = dataset[config.data.prompt_key] - responses = dataset[config.data.response_key] - data_sources = dataset[config.data.data_source_key] - reward_model_data = dataset[config.data.reward_model_key] - - total = len(dataset) - - # Initialize Ray - if not ray.is_initialized(): - ray.init() - - # evaluate test_score based on data source - data_source_reward = defaultdict(list) - compute_score = get_custom_reward_fn(config) - - # Create remote tasks - remote_tasks = [ - process_item.remote(compute_score, data_sources[i], responses[i], reward_model_data[i]) for i in range(total) - ] - - # Process results as they come in - with tqdm(total=total) as pbar: - while len(remote_tasks) > 0: - # Use ray.wait to get completed tasks - done_ids, remote_tasks = ray.wait(remote_tasks) - for result_id in done_ids: - data_source, score = ray.get(result_id) - data_source_reward[data_source].append(score) - pbar.update(1) - - metric_dict = {} - for data_source, rewards in data_source_reward.items(): - metric_dict[f'test_score/{data_source}'] = np.mean(rewards) - - print(metric_dict) - - -if __name__ == '__main__': - main() diff --git a/verl/trainer/main_generation.py b/verl/trainer/main_generation.py deleted file mode 100644 index 75d986c83c..0000000000 --- a/verl/trainer/main_generation.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Generate responses given a dataset of prompts -""" -import ray -import numpy as np -import hydra -import os - -os.environ['NCCL_DEBUG'] = 'WARN' -os.environ['TOKENIZERS_PARALLELISM'] = 'true' -# os.environ['TORCH_COMPILE_DISABLE'] = '1' - -from verl.utils.model import compute_position_id_with_mask - -import pandas as pd - -from transformers import AutoTokenizer - -from verl import DataProto -from verl.utils.fs import copy_to_local -from verl.workers.fsdp_workers import ActorRolloutRefWorker -from verl.utils.hdfs_io import makedirs -from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup - - -@hydra.main(config_path='config', config_name='generation', version_base=None) -def main(config): - run_generation(config) - - -def run_generation(config) -> None: - - if not ray.is_initialized(): - # this is for local ray cluster - ray.init(runtime_env={'env_vars': {'TOKENIZERS_PARALLELISM': 'true', 'NCCL_DEBUG': 'WARN'}}) - - ray.get(main_task.remote(config)) - - -@ray.remote(num_cpus=1) -def main_task(config): - from pprint import pprint - from omegaconf import OmegaConf - pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values - OmegaConf.resolve(config) - local_path = copy_to_local(config.model.path) - from verl.utils import hf_tokenizer - trust_remote_code = config.data.get('trust_remote_code', False) - tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) - - if config.rollout.temperature == 0.: - assert config.data.n_samples == 1, 'When temperature=0, n_samples must be 1.' - - # read dataset. Note that the dataset should directly contain chat template format (e.g., a list of dictionary) - dataset = pd.read_parquet(config.data.path) - chat_lst = dataset[config.data.prompt_key].tolist() - - chat_lst = [chat.tolist() for chat in chat_lst] - - tokenizer.padding_side = 'left' - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - - ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(ActorRolloutRefWorker), config=config, role='rollout') - resource_pool = RayResourcePool(process_on_nodes=[config.trainer.n_gpus_per_node] * config.trainer.nnodes) - wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) - wg.init_model() - - total_samples = len(dataset) - # real_batch_size = data.batch['input_ids'].shape[0] - config_batch_size = config.data.batch_size - dispatch_dp_size = wg.world_size - num_batch = -(-total_samples // config_batch_size) - output_lst = [[] for _ in range(config.data.n_samples)] - - for batch_idx in range(num_batch): - print(f'[{batch_idx+1}/{num_batch}] Start to process.') - batch_chat_lst = chat_lst[batch_idx * config_batch_size:(batch_idx + 1) * config_batch_size] - inputs = tokenizer.apply_chat_template(batch_chat_lst, - add_generation_prompt=True, - padding=True, - truncation=True, - max_length=config.rollout.prompt_length, - return_tensors='pt', - return_dict=True, - tokenize=True) - input_ids = inputs['input_ids'] - attention_mask = inputs['attention_mask'] - position_ids = compute_position_id_with_mask(attention_mask) - - batch_dict = {'input_ids': input_ids, 'attention_mask': attention_mask, 'position_ids': position_ids} - - data = DataProto.from_dict(batch_dict) - real_batch_size = data.batch['input_ids'].shape[0] - if real_batch_size % dispatch_dp_size != 0: - dummy_data_size = dispatch_dp_size - real_batch_size % dispatch_dp_size - if dummy_data_size <= real_batch_size: - dummy_data = data[:dummy_data_size] - else: - dummy_data = data.repeat(-(-dummy_data_size // real_batch_size))[:dummy_data_size] - data = DataProto.concat([data, dummy_data]) - print( - f'real_batch_size {real_batch_size} is not divisible by dispatch_dp_size {dispatch_dp_size}, add {dummy_data_size} dummy data' - ) - - batch_size = data.batch['input_ids'].shape[0] - assert batch_size % dispatch_dp_size == 0, f'batch_size {batch_size} is not divisible by dispatch_dp_size {dispatch_dp_size}' - - print(f'[{batch_idx+1}/{num_batch}] Start to generate.') - # START TO GENERATE FOR n_samples TIMES - for i in range(config.data.n_samples): - output = wg.generate_sequences(data) - # remove dummy data - output = output[:real_batch_size] - output_text = tokenizer.batch_decode(output.batch['input_ids'][:, -config.rollout.response_length:], - skip_special_tokens=False) - - # remove the padding - pad_token = tokenizer.pad_token - output_text_unpad = [] - for text in output_text: - output_text_unpad.append(text.replace(pad_token, '')) - - output_lst[i].extend(output_text_unpad) - - # convert output_lst from (n_samples, n_data) to (n_data, n_sampels) - output_lst = np.array(output_lst, dtype=object) - output_lst = np.transpose(output_lst, axes=(1, 0)).tolist() - - # add to the data frame - dataset[f'responses'] = output_lst - - # write to a new parquet - output_dir = os.path.dirname(config.data.output_path) - makedirs(output_dir, exist_ok=True) - dataset.to_parquet(config.data.output_path) - - return output_text - - -if __name__ == '__main__': - main() diff --git a/verl/trainer/main_ppo.py b/verl/trainer/main_ppo.py deleted file mode 100644 index 5062bce668..0000000000 --- a/verl/trainer/main_ppo.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Note that we don't combine the main with ray_trainer as ray_trainer is used by other main. -""" -import os - -from verl.trainer.ppo.ray_trainer import RayPPOTrainer - -import ray -import hydra - - -def get_custom_reward_fn(config): - import importlib.util - - reward_fn_config = config.get("custom_reward_function") or {} - file_path = reward_fn_config.get("path") - if not file_path: - return None - - if not os.path.exists(file_path): - raise FileNotFoundError(f"Reward function file '{file_path}' not found.") - - spec = importlib.util.spec_from_file_location("custom_module", file_path) - module = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(module) - except Exception as e: - raise RuntimeError(f"Error loading module from '{file_path}': {e}") - - function_name = reward_fn_config.get("name") - - if not hasattr(module, function_name): - raise AttributeError(f"Reward function '{function_name}' not found in '{file_path}'.") - - print(f"using customized reward function '{function_name}' from '{file_path}'") - - return getattr(module, function_name) - - -@hydra.main(config_path='config', config_name='ppo_trainer', version_base=None) -def main(config): - run_ppo(config) - - -def run_ppo(config) -> None: - # TODO(linjunrong.ocss884): this ENV is left for resolving SGLang conflict with ray devices - # isolation, will solve in the future - os.environ["ENSURE_CUDA_VISIBLE_DEVICES"] = os.environ.get('CUDA_VISIBLE_DEVICES', '') - if not ray.is_initialized(): - # this is for local ray cluster - ray.init( - runtime_env={ - 'env_vars': { - 'TOKENIZERS_PARALLELISM': 'true', - 'NCCL_DEBUG': 'WARN', - 'VLLM_LOGGING_LEVEL': 'WARN', - **dict(os.environ) - } - }) - - runner = TaskRunner.remote() - ray.get(runner.run.remote(config)) - - -@ray.remote(num_cpus=1) # please make sure main_task is not scheduled on head -class TaskRunner: - - def run(self, config): - from verl.utils.fs import copy_to_local - # print initial config - from pprint import pprint - from omegaconf import OmegaConf - pprint(OmegaConf.to_container(config, resolve=True)) # resolve=True will eval symbol values - OmegaConf.resolve(config) - - # download the checkpoint from hdfs - local_path = copy_to_local(config.actor_rollout_ref.model.path) - # instantiate tokenizer - from verl.utils import hf_tokenizer, hf_processor - trust_remote_code = config.data.get('trust_remote_code', True) - tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) - processor = hf_processor(local_path, use_fast=True) # used for multimodal LLM, could be none - - # define worker classes - if config.actor_rollout_ref.actor.strategy == 'fsdp': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray import RayWorkerGroup - ray_worker_group_cls = RayWorkerGroup - - elif config.actor_rollout_ref.actor.strategy == 'megatron': - assert config.actor_rollout_ref.actor.strategy == config.critic.strategy - from verl.workers.megatron_workers import ActorRolloutRefWorker, CriticWorker - from verl.single_controller.ray.megatron import NVMegatronRayWorkerGroup - ray_worker_group_cls = NVMegatronRayWorkerGroup - - else: - raise NotImplementedError - - from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role - - if config.trainer.get("hybrid_engine", True): - role_worker_mapping = { - Role.ActorRollout: ray.remote(ActorRolloutRefWorker), - Role.Critic: ray.remote(CriticWorker), - Role.RefPolicy: ray.remote(ActorRolloutRefWorker) - } - - global_pool_id = 'global_pool' - resource_pool_spec = { - global_pool_id: [config.trainer.n_gpus_per_node] * config.trainer.nnodes, - } - mapping = { - Role.ActorRollout: global_pool_id, - Role.Critic: global_pool_id, - Role.RefPolicy: global_pool_id, - } - else: - role_worker_mapping = { - Role.Actor: ray.remote(ActorRolloutRefWorker), - Role.Critic: ray.remote(CriticWorker), - Role.RefPolicy: ray.remote(ActorRolloutRefWorker), - Role.Rollout: ray.remote(ActorRolloutRefWorker), - } - - placement = config.trainer.placement - resource_pool_spec = { - "actor_pool": [placement["actor"]] * config.trainer.nnodes, - "ref_pool": [placement["ref"]] * config.trainer.nnodes, - "critic_pool": [placement.get("critic", 0)] * config.trainer.nnodes, - "rollout_pool": [placement["rollout"]] * config.trainer.nnodes, - } - mapping = { - Role.Actor: "actor_pool", - Role.Critic: "critic_pool", - Role.RefPolicy: "ref_pool", - Role.Rollout: "rollout_pool", - } - - # we should adopt a multi-source reward function here - # - for rule-based rm, we directly call a reward score - # - for model-based rm, we call a model - # - for code related prompt, we send to a sandbox if there are test cases - # - finally, we combine all the rewards together - # - The reward type depends on the tag of the data - if config.reward_model.enable: - if config.reward_model.strategy == 'fsdp': - from verl.workers.fsdp_workers import RewardModelWorker - elif config.reward_model.strategy == 'megatron': - from verl.workers.megatron_workers import RewardModelWorker - else: - raise NotImplementedError - role_worker_mapping[Role.RewardModel] = ray.remote(RewardModelWorker) - mapping[Role.RewardModel] = global_pool_id - - #use reference model - if config.algorithm.use_kl_in_reward or config.actor_rollout_ref.actor.use_kl_loss: - role_worker_mapping[Role.RefPolicy] = ray.remote(ActorRolloutRefWorker) - mapping[Role.RefPolicy] = global_pool_id - - reward_manager_name = config.reward_model.get("reward_manager", "naive") - if reward_manager_name == 'naive': - from verl.workers.reward_manager import NaiveRewardManager - reward_manager_cls = NaiveRewardManager - elif reward_manager_name == 'prime': - from verl.workers.reward_manager import PrimeRewardManager - reward_manager_cls = PrimeRewardManager - elif reward_manager_name == 'dapo': - from verl.workers.reward_manager import DAPORewardManager - reward_manager_cls = DAPORewardManager - elif reward_manager_name == 'swebench': - from verl.workers.reward_manager import SWEBenchRewardManager - reward_manager_cls = SWEBenchRewardManager - else: - raise NotImplementedError - - compute_score = get_custom_reward_fn(config) - reward_fn = reward_manager_cls(tokenizer=tokenizer, - num_examine=0, - config=config, - compute_score=compute_score) - - # Note that we always use function-based RM for validation - val_reward_fn = reward_manager_cls(tokenizer=tokenizer, - num_examine=1, - config=config, - compute_score=compute_score) - resource_pool_manager = ResourcePoolManager(resource_pool_spec=resource_pool_spec, mapping=mapping) - - trainer = RayPPOTrainer(config=config, - tokenizer=tokenizer, - processor=processor, - role_worker_mapping=role_worker_mapping, - resource_pool_manager=resource_pool_manager, - ray_worker_group_cls=ray_worker_group_cls, - reward_fn=reward_fn, - val_reward_fn=val_reward_fn) - trainer.init_workers() - trainer.fit() - - -if __name__ == '__main__': - main() diff --git a/verl/trainer/ppo/__init__.py b/verl/trainer/ppo/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/trainer/ppo/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/trainer/ppo/core_algos.py b/verl/trainer/ppo/core_algos.py deleted file mode 100644 index df87f6a4d7..0000000000 --- a/verl/trainer/ppo/core_algos.py +++ /dev/null @@ -1,440 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Core functions to implement PPO algorithms. -The function implemented in this file should be used by trainer with different distributed strategies to -implement PPO -""" - -import numpy as np -import torch -from collections import defaultdict - -import verl.utils.torch_functional as verl_F - - -class AdaptiveKLController: - """ - Adaptive KL controller described in the paper: - https://arxiv.org/pdf/1909.08593.pdf - """ - - def __init__(self, init_kl_coef, target_kl, horizon): - self.value = init_kl_coef - self.target = target_kl - self.horizon = horizon - - def update(self, current_kl, n_steps): - target = self.target - proportional_error = np.clip(current_kl / target - 1, -0.2, 0.2) - mult = 1 + proportional_error * n_steps / self.horizon - self.value *= mult - - -class FixedKLController: - """Fixed KL controller.""" - - def __init__(self, kl_coef): - self.value = kl_coef - - def update(self, current_kl, n_steps): - pass - - -def get_kl_controller(kl_ctrl): - if kl_ctrl.type == 'fixed': - return FixedKLController(kl_coef=kl_ctrl.kl_coef) - elif kl_ctrl.type == 'adaptive': - assert kl_ctrl.horizon > 0, f'horizon must be larger than 0. Got {kl_ctrl.horizon}' - return AdaptiveKLController(init_kl_coef=kl_ctrl.kl_coef, target_kl=kl_ctrl.target_kl, horizon=kl_ctrl.horizon) - else: - raise NotImplementedError - - -def compute_gae_advantage_return(token_level_rewards: torch.Tensor, values: torch.Tensor, response_mask: torch.Tensor, - gamma: torch.Tensor, lam: torch.Tensor): - """Adapted from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py - - Args: - token_level_rewards: `(torch.Tensor)` - shape: (bs, response_length) - values: `(torch.Tensor)` - shape: (bs, response_length) - response_mask: `(torch.Tensor)` - shape: (bs, response_length). [EOS] mask. The token after [EOS] have mask zero. - gamma: `(float)` - discounted factor used in RL - lam: `(float)` - lambda value when computing Generalized Advantage Estimation (https://arxiv.org/abs/1506.02438) - - Returns: - advantages: `(torch.Tensor)` - shape: (bs, response_length) - Returns: `(torch.Tensor)` - shape: (bs, response_length) - - """ - with torch.no_grad(): - lastgaelam = 0 - advantages_reversed = [] - gen_len = token_level_rewards.shape[-1] - - for t in reversed(range(gen_len)): - nextvalues = values[:, t + 1] if t < gen_len - 1 else 0.0 - delta = token_level_rewards[:, t] + gamma * nextvalues - values[:, t] - lastgaelam = delta + gamma * lam * lastgaelam - advantages_reversed.append(lastgaelam) - advantages = torch.stack(advantages_reversed[::-1], dim=1) - - returns = advantages + values - advantages = verl_F.masked_whiten(advantages, response_mask) - return advantages, returns - - -# NOTE(sgm): this implementation only consider outcome supervision, where the reward is a scalar. -def compute_grpo_outcome_advantage(token_level_rewards: torch.Tensor, - response_mask: torch.Tensor, - index: np.ndarray, - epsilon: float = 1e-6): - """ - Compute advantage for GRPO, operating only on Outcome reward - (with only one scalar reward for each response). - Args: - token_level_rewards: `(torch.Tensor)` - shape: (bs, response_length) - response_mask: `(torch.Tensor)` - shape: (bs, response_length) - - Returns: - advantages: `(torch.Tensor)` - shape: (bs, response_length) - Returns: `(torch.Tensor)` - shape: (bs, response_length) - """ - scores = token_level_rewards.sum(dim=-1) - - id2score = defaultdict(list) - id2mean = {} - id2std = {} - - with torch.no_grad(): - bsz = scores.shape[0] - for i in range(bsz): - id2score[index[i]].append(scores[i]) - for idx in id2score: - if len(id2score[idx]) == 1: - id2mean[idx] = torch.tensor(0.0) - id2std[idx] = torch.tensor(1.0) - elif len(id2score[idx]) > 1: - id2mean[idx] = torch.mean(torch.tensor(id2score[idx])) - id2std[idx] = torch.std(torch.tensor([id2score[idx]])) - else: - raise ValueError(f"no score in prompt index: {idx}") - for i in range(bsz): - scores[i] = (scores[i] - id2mean[index[i]]) / (id2std[index[i]] + epsilon) - scores = scores.unsqueeze(-1) * response_mask - - return scores, scores - - -def compute_rloo_outcome_advantage(token_level_rewards: torch.Tensor, - response_mask: torch.Tensor, - index: np.ndarray, - epsilon: float = 1e-6): - """ - Compute advantage for RLOO based on https://arxiv.org/abs/2402.14740 - Args: - token_level_rewards: `(torch.Tensor)` - shape: (bs, response_length) - response_mask: `(torch.Tensor)` - shape: (bs, response_length) - - Returns: - advantages: `(torch.Tensor)` - shape: (bs, response_length) - Returns: `(torch.Tensor)` - shape: (bs, response_length) - """ - scores = token_level_rewards.sum(dim=-1) - - id2score = defaultdict(list) - id2mean = {} - - with torch.no_grad(): - bsz = scores.shape[0] - for i in range(bsz): - id2score[index[i]].append(scores[i]) - for idx in id2score: - if len(id2score[idx]) == 1: - id2mean[idx] = torch.tensor(0.0) - elif len(id2score[idx]) > 1: - id2mean[idx] = torch.mean(torch.tensor(id2score[idx])) - else: - raise ValueError(f"no score in prompt index: {idx}") - for i in range(bsz): - response_num = len(id2score[index[i]]) - if response_num > 1: - scores[i] = scores[i] * response_num / (response_num - - 1) - id2mean[index[i]] * response_num / (response_num - 1) - scores = scores.unsqueeze(-1) * response_mask - - return scores, scores - - -def compute_reinforce_plus_plus_outcome_advantage(token_level_rewards: torch.Tensor, response_mask: torch.Tensor, - gamma: torch.Tensor): - """ - Compute advantage for REINFORCE++. - This implementation is based on the paper: https://arxiv.org/abs/2501.03262 - Args: - token_level_rewards: `(torch.Tensor)` - shape: (bs, response_length) - response_mask: `(torch.Tensor)` - shape: (bs, response_length) - - Returns: - advantages: `(torch.Tensor)` - shape: (bs, response_length) - Returns: `(torch.Tensor)` - shape: (bs, response_length) - """ - - with torch.no_grad(): - returns = torch.zeros_like(token_level_rewards) - running_return = 0 - - for t in reversed(range(token_level_rewards.shape[1])): - running_return = token_level_rewards[:, t] + gamma * running_return - returns[:, t] = running_return - # Reset after EOS - running_return = running_return * response_mask[:, t] - - advantages = verl_F.masked_whiten(returns, response_mask) - advantages = advantages * response_mask - - return advantages, returns - - -def compute_remax_outcome_advantage(token_level_rewards: torch.Tensor, reward_baselines: torch.Tensor, - response_mask: torch.Tensor): - """ - Compute advantage for ReMax, operating only on Outcome reward - This implementation is based on the paper: https://arxiv.org/abs/2310.10505 - - (with only one scalar reward for each response). - Args: - token_level_rewards: `(torch.Tensor)` - shape: (bs, response_length) - reward_baselines: `(torch.Tensor)` - shape: (bs,) - response_mask: `(torch.Tensor)` - shape: (bs, response_length) - - Returns: - advantages: `(torch.Tensor)` - shape: (bs, response_length) - Returns: `(torch.Tensor)` - shape: (bs, response_length) - """ - - with torch.no_grad(): - returns = (token_level_rewards * response_mask).flip(dims=[-1]).cumsum(dim=-1).flip(dims=[-1]) - advantages = returns - reward_baselines.unsqueeze(-1) * response_mask - - return advantages, returns - - -def compute_rewards(token_level_scores, old_log_prob, ref_log_prob, kl_ratio): - kl = old_log_prob - ref_log_prob - return token_level_scores - kl * kl_ratio - - -def agg_loss(loss_mat: torch.Tensor, loss_mask: torch.Tensor, loss_agg_mode: str, preserve_grad: bool = False): - """ - Aggregate the loss matrix into a scalar. - Args: - loss_mat: `(torch.Tensor)` - shape: (bs, response_length) - loss_mask: `(torch.Tensor)` - shape: (bs, response_length) - loss_agg_mode: (str) choices: "token-mean" / "seq-mean-token-sum" / "seq-mean-token-mean" - "token-mean" is the default behavior - Returns: - loss: `a scalar torch.Tensor` - aggregated loss - """ - if loss_agg_mode == "token-mean": - loss = verl_F.masked_mean(loss_mat, loss_mask, preserve_grad=preserve_grad) - elif loss_agg_mode == "seq-mean-token-sum": - seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) / torch.sum(loss_mask, dim=-1) - loss = torch.mean(seq_losses) - elif loss_agg_mode == "seq-mean-token-mean": - seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) / torch.sum(loss_mask, dim=-1) - loss = torch.mean(seq_losses) - else: - raise ValueError(f"Invalid loss_agg_mode: {loss_agg_mode}") - - return loss - - -def compute_policy_loss(old_log_prob, - log_prob, - advantages, - response_mask, - cliprange=None, - cliprange_low=None, - cliprange_high=None, - clip_ratio_c=3.0, - loss_agg_mode="token-mean", - preserve_grad: bool = False): - """Adapted from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py#L1122 - Args: - old_log_prob: `(torch.Tensor)` - shape: (bs, response_length) - log_prob: `(torch.Tensor)` - shape: (bs, response_length) - advantages: `(torch.Tensor)` - shape: (bs, response_length) - response_mask: `(torch.Tensor)` - shape: (bs, response_length) - cliprange: (float) - The clip range used in PPO. See https://arxiv.org/abs/1707.06347 - cliprange_low: (float) - The lower clip range used in PPO. - cliprange_high: (float) - The higher clip range used in PPO. - clip_ratio_c: (float) default: 3.0 - The lower bound of the ratio for dual-clip PPO, See https://arxiv.org/pdf/1912.09729 - loss_agg_mode: (str) choices: "token-mean" / "seq-mean-token-sum" / "seq-mean-token-mean" - "token-mean" is the default behavior - - Returns: - pg_loss: `a scalar torch.Tensor` - policy gradient loss computed via PPO - pg_clipfrac: (float) - the fraction of policy gradient loss being clipped - ppo_kl: (float) - the estimated KL divergence between the latest updating policy and the old sampling policy - pg_clipfrac_lower: (float) - the fraction of policy gradient loss being clipped when the advantage is negative - """ - assert clip_ratio_c > 1.0, f"The lower bound of the clip_ratio_c for dual-clip PPO should be greater than 1.0, but get the value: {clip_ratio_c}." - negative_approx_kl = log_prob - old_log_prob - ratio = torch.exp(negative_approx_kl) - ppo_kl = verl_F.masked_mean(-negative_approx_kl, response_mask, preserve_grad=preserve_grad) - - pg_losses1 = -advantages * ratio - if cliprange_low is None: - cliprange_low = cliprange - if cliprange_high is None: - cliprange_high = cliprange - pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, - 1 + cliprange_high) # - clip(ratio, 1-cliprange, 1+cliprange) * A - clip_pg_losses1 = torch.maximum(pg_losses1, - pg_losses2) # max(-ratio * A, -clip(ratio, 1-cliprange, 1+cliprange) * A) - pg_clipfrac = verl_F.masked_mean(torch.gt(pg_losses2, pg_losses1).float(), response_mask, preserve_grad=preserve_grad) - - pg_losses3 = -advantages * clip_ratio_c - clip_pg_losses2 = torch.min(pg_losses3, clip_pg_losses1) - pg_clipfrac_lower = verl_F.masked_mean( - torch.gt(clip_pg_losses2, pg_losses3) * (advantages < 0).float(), response_mask, preserve_grad=preserve_grad) - - pg_losses = torch.where(advantages < 0, clip_pg_losses2, clip_pg_losses1) - pg_loss = agg_loss(loss_mat=pg_losses, loss_mask=response_mask, loss_agg_mode=loss_agg_mode, preserve_grad=preserve_grad) - - return pg_loss, pg_clipfrac, ppo_kl, pg_clipfrac_lower - - -def compute_entropy_loss(logits, response_mask): - """Compute Categorical entropy loss - - Args: - logits: `(torch.Tensor)` - shape: (bs, response_length, vocab_size) - response_mask: `(torch.Tensor)` - shape: (bs, response_length) - - Returns: - entropy: a scalar torch.Tensor - - """ - # compute entropy - entropy = verl_F.entropy_from_logits(logits) # (bs, response_len) - entropy_loss = verl_F.masked_mean(entropy, mask=response_mask) - return entropy_loss - - -def compute_value_loss(vpreds, returns, values, response_mask, cliprange_value): - """Compute the value loss. Copied from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py#L1151 - - Args: - vpreds (`torch.FloatTensor`): - Predicted values of the value head, shape (`batch_size`, `response_length`) - values (`torch.FloatTensor`): - Old values of value head, shape (`batch_size`, `response_length`) - returns: (`torch.FloatTensor`): - Ground truth returns, shape (`batch_size`, `response_length`) - - Returns: - vf_loss: a scalar (`torch.FloatTensor`): - value function loss - vf_clipfrac: a float - The ratio of vf being clipped - - """ - vpredclipped = verl_F.clip_by_value(vpreds, values - cliprange_value, values + cliprange_value) - vf_losses1 = (vpreds - returns)**2 - vf_losses2 = (vpredclipped - returns)**2 - vf_loss = 0.5 * verl_F.masked_mean(torch.max(vf_losses1, vf_losses2), response_mask) - vf_clipfrac = verl_F.masked_mean(torch.gt(vf_losses2, vf_losses1).float(), response_mask) - return vf_loss, vf_clipfrac - - -def kl_penalty(logprob: torch.FloatTensor, ref_logprob: torch.FloatTensor, kl_penalty) -> torch.FloatTensor: - """Compute KL divergence given logprob and ref_logprob. - Copied from https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_trainer.py#L1104 - - Args: - logprob: - ref_logprob: - - Returns: - - """ - if kl_penalty == "kl": - return logprob - ref_logprob - - if kl_penalty == "abs": - return (logprob - ref_logprob).abs() - - if kl_penalty == "mse": - return 0.5 * (logprob - ref_logprob).square() - - print("logprob.shape: ", logprob.shape) - print("ref_logprob.shape: ", ref_logprob.shape) - - # J. Schulman. Approximating kl divergence, 2020. - # # URL http://joschu.net/blog/kl-approx.html. - if kl_penalty == 'low_var_kl': - kl = ref_logprob - logprob - ratio = torch.exp(kl) - kld = (ratio - kl - 1).contiguous() - return torch.clamp(kld, min=-10, max=10) - - if kl_penalty == "full": - # so, here logprob and ref_logprob should contain the logits for every token in vocabulary - raise NotImplementedError - - raise NotImplementedError diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py deleted file mode 100644 index a5b8e4e63b..0000000000 --- a/verl/trainer/ppo/metric_utils.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Metrics related to the PPO trainer. -""" - -import torch -from typing import Any, Dict, List, Callable -import numpy as np -from verl import DataProto -from collections import Counter, defaultdict -from functools import partial - - -def reduce_metrics(metrics: Dict[str, List[Any]]) -> Dict[str, Any]: - for key, val in metrics.items(): - metrics[key] = np.mean(val) - return metrics - - -def _compute_response_info(batch: DataProto) -> Dict[str, Any]: - response_length = batch.batch['responses'].shape[-1] - - prompt_mask = batch.batch['attention_mask'][:, :-response_length] - response_mask = batch.batch['attention_mask'][:, -response_length:] - - prompt_length = prompt_mask.sum(-1).float() - response_length = response_mask.sum(-1).float() # (batch_size,) - - return dict( - response_mask=response_mask, - prompt_length=prompt_length, - response_length=response_length, - ) - - -def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> Dict[str, Any]: - # TODO: add response length - sequence_score = batch.batch['token_level_scores'].sum(-1) - sequence_reward = batch.batch['token_level_rewards'].sum(-1) - - advantages = batch.batch['advantages'] - returns = batch.batch['returns'] - - max_response_length = batch.batch['responses'].shape[-1] - - prompt_mask = batch.batch['attention_mask'][:, :-max_response_length].bool() - response_mask = batch.batch['attention_mask'][:, -max_response_length:].bool() - - max_prompt_length = prompt_mask.size(-1) - - response_info = _compute_response_info(batch) - prompt_length = response_info['prompt_length'] - response_length = response_info['response_length'] - - valid_adv = torch.masked_select(advantages, response_mask) - valid_returns = torch.masked_select(returns, response_mask) - - if use_critic: - values = batch.batch['values'] - valid_values = torch.masked_select(values, response_mask) - return_diff_var = torch.var(valid_returns - valid_values) - return_var = torch.var(valid_returns) - - metrics = { - # score - 'critic/score/mean': - torch.mean(sequence_score).detach().item(), - 'critic/score/max': - torch.max(sequence_score).detach().item(), - 'critic/score/min': - torch.min(sequence_score).detach().item(), - # reward - 'critic/rewards/mean': - torch.mean(sequence_reward).detach().item(), - 'critic/rewards/max': - torch.max(sequence_reward).detach().item(), - 'critic/rewards/min': - torch.min(sequence_reward).detach().item(), - # adv - 'critic/advantages/mean': - torch.mean(valid_adv).detach().item(), - 'critic/advantages/max': - torch.max(valid_adv).detach().item(), - 'critic/advantages/min': - torch.min(valid_adv).detach().item(), - # returns - 'critic/returns/mean': - torch.mean(valid_returns).detach().item(), - 'critic/returns/max': - torch.max(valid_returns).detach().item(), - 'critic/returns/min': - torch.min(valid_returns).detach().item(), - **({ - # values - 'critic/values/mean': torch.mean(valid_values).detach().item(), - 'critic/values/max': torch.max(valid_values).detach().item(), - 'critic/values/min': torch.min(valid_values).detach().item(), - # vf explained var - 'critic/vf_explained_var': (1.0 - return_diff_var / (return_var + 1e-5)).detach().item(), - } if use_critic else {}), - - # response length - 'response_length/mean': - torch.mean(response_length).detach().item(), - 'response_length/max': - torch.max(response_length).detach().item(), - 'response_length/min': - torch.min(response_length).detach().item(), - 'response_length/clip_ratio': - torch.mean(torch.eq(response_length, max_response_length).float()).detach().item(), - # prompt length - 'prompt_length/mean': - torch.mean(prompt_length).detach().item(), - 'prompt_length/max': - torch.max(prompt_length).detach().item(), - 'prompt_length/min': - torch.min(prompt_length).detach().item(), - 'prompt_length/clip_ratio': - torch.mean(torch.eq(prompt_length, max_prompt_length).float()).detach().item(), - } - return metrics - - -def compute_timing_metrics(batch: DataProto, timing_raw: Dict[str, float]) -> Dict[str, Any]: - response_info = _compute_response_info(batch) - num_prompt_tokens = torch.sum(response_info['prompt_length']).item() - num_response_tokens = torch.sum(response_info['response_length']).item() - num_overall_tokens = num_prompt_tokens + num_response_tokens - - num_tokens_of_section = { - 'gen': num_response_tokens, - **{ - name: num_overall_tokens for name in ['ref', 'values', 'adv', 'update_critic', 'update_actor'] - }, - } - - return { - **{ - f'timing_s/{name}': value for name, value in timing_raw.items() - }, - **{ - f'timing_per_token_ms/{name}': timing_raw[name] * 1000 / num_tokens_of_section[name] for name in set(num_tokens_of_section.keys( - )) & set(timing_raw.keys()) - }, - } - - -def compute_throughout_metrics(batch: DataProto, timing_raw: Dict[str, float], n_gpus: int) -> Dict[str, Any]: - total_num_tokens = sum(batch.meta_info['global_token_num']) - time = timing_raw['step'] - # estimated_flops, promised_flops = flops_function.estimate_flops(num_tokens, time) - # f'Actual TFLOPs/s/GPU​': estimated_flops/(n_gpus), - # f'Theoretical TFLOPs/s/GPU​': promised_flops, - return { - 'perf/total_num_tokens': total_num_tokens, - 'perf/time_per_step': time, - 'perf/throughput': total_num_tokens / (time * n_gpus), - } - - -def bootstrap_metric(data: list[dict[str, Any]], - subset_size: int, - reduce_fns: list[Callable[[np.ndarray], float]], - n_bootstrap: int = 1000, - seed: int = 42) -> list[tuple[float, float]]: - np.random.seed(seed) - - bootstrap_metric_lsts = [[] for _ in range(len(reduce_fns))] - for _ in range(n_bootstrap): - bootstrap_idxs = np.random.choice(len(data), size=subset_size, replace=True) - bootstrap_data = [data[i] for i in bootstrap_idxs] - for i, reduce_fn in enumerate(reduce_fns): - bootstrap_metric_lsts[i].append(reduce_fn(bootstrap_data)) - return [(np.mean(lst), np.std(lst)) for lst in bootstrap_metric_lsts] - - -def calc_maj_val(data: list[dict[str, Any]], vote_key: str, val_key: str) -> float: - """ - Calculate the majority voting metric - """ - vote2vals = defaultdict(list) - for d in data: - vote2vals[d[vote_key]].append(d[val_key]) - - vote2cnt = {k: len(v) for k, v in vote2vals.items()} - maj_vote = max(vote2cnt, key=vote2cnt.get) - - maj_val = vote2vals[maj_vote][0] - - return maj_val - - -def process_validation_metrics(data_sources: list[str], sample_inputs: list[str], - infos_dict: dict[str, list[Any]]) -> dict[str, dict[str, dict[str, float]]]: - """Process validation metrics into a structured format. - - Args: - data_sources: Array of data source identifiers for each sample - sample_inputs: List of input prompts - infos_dict: variable name -> list of values for each sample - - Returns: - dict[str, dict[str, dict[str, float]]]: data source -> variable name -> metric value - """ - # Group metrics by data source, prompt and variable - data_src2prompt2var2vals = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - for sample_idx, data_source in enumerate(data_sources): - prompt = sample_inputs[sample_idx] - var2vals = data_src2prompt2var2vals[data_source][prompt] - for var_name, var_vals in infos_dict.items(): - var2vals[var_name].append(var_vals[sample_idx]) - - # Calculate metrics for each group - data_src2prompt2var2metric = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) - for data_source, prompt2var2vals in data_src2prompt2var2vals.items(): - for prompt, var2vals in prompt2var2vals.items(): - n_resps = len(var2vals["final_reward"]) - preds = var2vals["pred"] - for var_name, var_vals in var2vals.items(): - if var_name in ["pred", "final_reward"]: - continue - metric = {} - - metric[f"mean@{n_resps}"] = np.mean(var_vals) - metric[f"std@{n_resps}"] = np.std(var_vals) - - ns = [] - n = 2 - while n < n_resps: - ns.append(n) - n *= 2 - ns.append(n_resps) - - data = [{"val": val, "pred": pred} for val, pred in zip(var_vals, preds)] - for n in ns: - (bon_mean, bon_std), (won_mean, won_std), (maj_n_mean, maj_n_std) = bootstrap_metric( - data, - subset_size=n, - reduce_fns=[ - lambda arr: np.max([d["val"] for d in arr]), lambda arr: np.min([d["val"] for d in arr]), - partial(calc_maj_val, vote_key="pred", val_key="val") - ]) - metric[f"best@{n}/mean"], metric[f"best@{n}/std"] = bon_mean, bon_std - metric[f"worst@{n}/mean"], metric[f"worst@{n}/std"] = won_mean, won_std - metric[f"maj@{n}/mean"], metric[f"maj@{n}/std"] = maj_n_mean, maj_n_std - - data_src2prompt2var2metric[data_source][prompt][var_name] = metric - - # Aggregate metrics across prompts - data_src2var2metric2prompt_vals = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - for data_source, prompt2var2metric in data_src2prompt2var2metric.items(): - for prompt, var2metric in prompt2var2metric.items(): - for var_name, metric in var2metric.items(): - for metric_name, metric_val in metric.items(): - data_src2var2metric2prompt_vals[data_source][var_name][metric_name].append(metric_val) - - data_src2var2metric2val = defaultdict(lambda: defaultdict(lambda: defaultdict(float))) - for data_source, var2metric2prompt_vals in data_src2var2metric2prompt_vals.items(): - for var_name, metric2prompt_vals in var2metric2prompt_vals.items(): - for metric_name, prompt_vals in metric2prompt_vals.items(): - data_src2var2metric2val[data_source][var_name][metric_name] = np.mean(prompt_vals) - - return data_src2var2metric2val diff --git a/verl/trainer/ppo/ray_trainer.py b/verl/trainer/ppo/ray_trainer.py deleted file mode 100644 index c30563854d..0000000000 --- a/verl/trainer/ppo/ray_trainer.py +++ /dev/null @@ -1,1085 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -FSDP PPO Trainer with Ray-based single controller. -This trainer supports model-agonistic model initialization with huggingface -""" - -import os -import uuid -from contextlib import contextmanager -from dataclasses import dataclass, field -from enum import Enum -from pprint import pprint -from typing import Type, Dict -from copy import deepcopy -from collections import defaultdict -from functools import partial -from tqdm import tqdm - -import ray -import numpy as np -import ray -from codetiming import Timer -from omegaconf import OmegaConf, open_dict -from verl import DataProto -from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto -from verl.single_controller.base import Worker -from verl.single_controller.ray import RayResourcePool, RayWorkerGroup, RayClassWithInitArgs -from verl.single_controller.ray.base import create_colocated_worker_cls -from verl.trainer.ppo import core_algos -from verl.trainer.ppo.metric_utils import compute_data_metrics, compute_throughout_metrics, compute_timing_metrics, reduce_metrics, bootstrap_metric, calc_maj_val, process_validation_metrics -from verl.utils.seqlen_balancing import get_seqlen_balanced_partitions, log_seqlen_unbalance -from verl.utils.checkpoint.checkpoint_manager import find_latest_ckpt_path -from verl.utils.dataset.rl_dataset import RLHFDataset, collate_fn -from verl.utils.tracking import ValidationGenerationsLogger -from torch.utils.data import RandomSampler, SequentialSampler -from torchdata.stateful_dataloader import StatefulDataLoader - -WorkerType = Type[Worker] - - -class Role(Enum): - """ - To create more roles dynamically, you can subclass Role and add new members - """ - Actor = 0 - Rollout = 1 - ActorRollout = 2 - Critic = 3 - RefPolicy = 4 - RewardModel = 5 - ActorRolloutRef = 6 - - -class AdvantageEstimator(str, Enum): - """ - Using an enumeration class to avoid spelling errors in adv_estimator - """ - GAE = 'gae' - GRPO = 'grpo' - REINFORCE_PLUS_PLUS = 'reinforce_plus_plus' - REMAX = 'remax' - RLOO = 'rloo' - - -@dataclass -class ResourcePoolManager: - """ - Define a resource pool specification. Resource pool will be initialized first. - Mapping - """ - resource_pool_spec: dict[str, list[int]] - mapping: dict[Role, str] - resource_pool_dict: dict[str, RayResourcePool] = field(default_factory=dict) - - def create_resource_pool(self): - for resource_pool_name, process_on_nodes in self.resource_pool_spec.items(): - # max_colocate_count means the number of WorkerGroups (i.e. processes) in each RayResourcePool - # For FSDP backend, we recommend using max_colocate_count=1 that merge all WorkerGroups into one. - # For Megatron backend, we recommend using max_colocate_count>1 that can utilize different WorkerGroup for differnt models - resource_pool = RayResourcePool(process_on_nodes=process_on_nodes, - use_gpu=True, - max_colocate_count=1, - name_prefix=resource_pool_name) - self.resource_pool_dict[resource_pool_name] = resource_pool - - self._check_resource_available() - - def get_resource_pool(self, role: Role) -> RayResourcePool: - """Get the resource pool of the worker_cls""" - return self.resource_pool_dict[self.mapping[role]] - - def get_n_gpus(self) -> int: - """Get the number of gpus in this cluster.""" - return sum([n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes]) - - def _check_resource_available(self): - """Check if the resource pool can be satisfied in this ray cluster.""" - node_available_resources = ray.state.available_resources_per_node() - node_available_gpus = {node: node_info.get('GPU', 0) for node, node_info in node_available_resources.items()} - - # check total required gpus can be satisfied - total_available_gpus = sum(node_available_gpus.values()) - total_required_gpus = sum( - [n_gpus for process_on_nodes in self.resource_pool_spec.values() for n_gpus in process_on_nodes]) - if total_available_gpus < total_required_gpus: - raise ValueError( - f"Total available GPUs {total_available_gpus} is less than total desired GPUs {total_required_gpus}") - - # check each resource pool can be satisfied, O(#resource_pools * #nodes) - for resource_pool_name, process_on_nodes in self.resource_pool_spec.items(): - num_gpus, num_nodes = process_on_nodes[0], len(process_on_nodes) - for node, available_gpus in node_available_gpus.items(): - if available_gpus >= num_gpus: - node_available_gpus[node] -= num_gpus - num_nodes -= 1 - if num_nodes == 0: - break - if num_nodes > 0: - raise ValueError( - f"Resource pool {resource_pool_name}: {num_gpus}*{num_nodes} cannot be satisfied in this ray cluster" - ) - - -import torch -from verl.utils.torch_functional import masked_mean - - -def apply_kl_penalty(data: DataProto, kl_ctrl: core_algos.AdaptiveKLController, kl_penalty='kl'): - responses = data.batch['responses'] - response_length = responses.size(1) - token_level_scores = data.batch['token_level_scores'] - batch_size = data.batch.batch_size[0] - - attention_mask = data.batch['attention_mask'] - response_mask = attention_mask[:, -response_length:] - - # compute kl between ref_policy and current policy - # When apply_kl_penalty, algorithm.use_kl_in_reward=True, so the reference model has been enabled. - if 'ref_log_prob' in data.batch.keys(): - kld = core_algos.kl_penalty(data.batch['old_log_probs'], data.batch['ref_log_prob'], - kl_penalty=kl_penalty) # (batch_size, response_length) - kld = kld * response_mask - beta = kl_ctrl.value - else: - beta = 0 - kld = torch.zeros_like(response_mask, dtype=torch.float32) - - token_level_rewards = token_level_scores - beta * kld - - current_kl = masked_mean(kld, mask=response_mask, axis=-1) # average over sequence - current_kl = torch.mean(current_kl, dim=0).item() - - # according to https://github.com/huggingface/trl/blob/951ca1841f29114b969b57b26c7d3e80a39f75a0/trl/trainer/ppo_trainer.py#L837 - kl_ctrl.update(current_kl=current_kl, n_steps=batch_size) - data.batch['token_level_rewards'] = token_level_rewards - - metrics = {'actor/reward_kl_penalty': current_kl, 'actor/reward_kl_penalty_coeff': beta} - - return data, metrics - - -def compute_response_mask(data: DataProto): - responses = data.batch['responses'] - response_length = responses.size(1) - attention_mask = data.batch['attention_mask'] - return attention_mask[:, -response_length:] - - -def compute_advantage(data: DataProto, adv_estimator, gamma=1.0, lam=1.0, num_repeat=1): - # Back-compatible with trainers that do not compute response mask in fit - if "response_mask" not in data.batch.keys(): - data.batch['response_mask'] = compute_response_mask(data) - # prepare response group - # TODO: add other ways to estimate advantages - if adv_estimator == AdvantageEstimator.GAE: - values = data.batch['values'] - advantages, returns = core_algos.compute_gae_advantage_return( - token_level_rewards=data.batch['token_level_rewards'], - values=data.batch['values'], - response_mask=data.batch['response_mask'], - gamma=gamma, - lam=lam) - data.batch['advantages'] = advantages - data.batch['returns'] = returns - elif adv_estimator == AdvantageEstimator.GRPO: - advantages, returns = core_algos.compute_grpo_outcome_advantage( - token_level_rewards=data.batch['token_level_rewards'], - response_mask=data.batch['response_mask'], - index=data.non_tensor_batch['uid']) - data.batch['advantages'] = advantages - data.batch['returns'] = returns - elif adv_estimator == AdvantageEstimator.REINFORCE_PLUS_PLUS: - advantages, returns = core_algos.compute_reinforce_plus_plus_outcome_advantage( - token_level_rewards=data.batch['token_level_rewards'], - response_mask=data.batch['response_mask'], - gamma=gamma) - data.batch['advantages'] = advantages - data.batch['returns'] = returns - elif adv_estimator == AdvantageEstimator.REMAX: - advantages, returns = core_algos.compute_remax_outcome_advantage( - token_level_rewards=data.batch['token_level_rewards'], - reward_baselines=data.batch['reward_baselines'], - response_mask=data.batch['response_mask']) - - data.batch['advantages'] = advantages - data.batch['returns'] = returns - elif adv_estimator == AdvantageEstimator.RLOO: - advantages, returns = core_algos.compute_rloo_outcome_advantage( - token_level_rewards=data.batch['token_level_rewards'], - response_mask=data.batch['response_mask'], - index=data.non_tensor_batch['uid']) - data.batch['advantages'] = advantages - data.batch['returns'] = returns - else: - raise NotImplementedError - return data - - -@contextmanager -def _timer(name: str, timing_raw: Dict[str, float]): - with Timer(name=name, logger=None) as timer: - yield - timing_raw[name] = timer.last - - -class RayPPOTrainer(object): - """ - Note that this trainer runs on the driver process on a single CPU/GPU node. - """ - - # TODO: support each role have individual ray_worker_group_cls, - # i.e., support different backend of different role - def __init__(self, - config, - tokenizer, - role_worker_mapping: dict[Role, WorkerType], - resource_pool_manager: ResourcePoolManager, - ray_worker_group_cls: type[RayWorkerGroup] = RayWorkerGroup, - processor=None, - reward_fn=None, - val_reward_fn=None): - - # assert torch.cuda.is_available(), 'cuda must be available on driver' - - self.tokenizer = tokenizer - self.processor = processor - self.config = config - self.reward_fn = reward_fn - self.val_reward_fn = val_reward_fn - - self.hybrid_engine = config.actor_rollout_ref.hybrid_engine - assert self.hybrid_engine, 'Currently, only support hybrid engine' - - if self.hybrid_engine: - assert Role.ActorRollout in role_worker_mapping, f'{role_worker_mapping.keys()=}' - - self.role_worker_mapping = role_worker_mapping - self.resource_pool_manager = resource_pool_manager - self.use_reference_policy = Role.RefPolicy in role_worker_mapping - self.use_rm = Role.RewardModel in role_worker_mapping - self.ray_worker_group_cls = ray_worker_group_cls - self.validation_generations_logger = ValidationGenerationsLogger() - - # define in-reward KL control - # kl loss control currently not suppoorted - if config.algorithm.use_kl_in_reward: - self.kl_ctrl_in_reward = core_algos.get_kl_controller(config.algorithm.kl_ctrl) - - if self.config.algorithm.adv_estimator == AdvantageEstimator.GAE: - self.use_critic = True - elif self.config.algorithm.adv_estimator in [ - AdvantageEstimator.GRPO, AdvantageEstimator.REINFORCE_PLUS_PLUS, AdvantageEstimator.REMAX, - AdvantageEstimator.RLOO - ]: - self.use_critic = False - else: - raise NotImplementedError - - self._validate_config() - self._create_dataloader() - - - def _validate_config(self): - config = self.config - # number of GPUs total - n_gpus = config.trainer.n_gpus_per_node * config.trainer.nnodes - - # 1. Check total batch size for data correctness - real_train_batch_size = config.data.train_batch_size * config.actor_rollout_ref.rollout.n - assert real_train_batch_size % n_gpus == 0, \ - f"real_train_batch_size ({real_train_batch_size}) must be divisible by total n_gpus ({n_gpus})." - - # A helper function to check "micro_batch_size" vs "micro_batch_size_per_gpu" - # We throw an error if the user sets both. The new convention is "..._micro_batch_size_per_gpu". - def check_mutually_exclusive(mbs, mbs_per_gpu, name: str): - settings = { - "actor_rollout_ref.actor": "micro_batch_size", - "critic": "micro_batch_size", - "reward_model": "micro_batch_size", - "actor_rollout_ref.ref": "log_prob_micro_batch_size", - "actor_rollout_ref.rollout": "log_prob_micro_batch_size", - } - - if name in settings: - param = settings[name] - param_per_gpu = f"{param}_per_gpu" - - if mbs is None and mbs_per_gpu is None: - raise ValueError( - f"[{name}] Please set at least one of '{name}.{param}' or '{name}.{param_per_gpu}'.") - - if mbs is not None and mbs_per_gpu is not None: - raise ValueError( - f"[{name}] You have set both '{name}.{param}' AND '{name}.{param_per_gpu}'. " - f"Please remove '{name}.{param}' because only '*_{param_per_gpu}' is supported (the former is deprecated)." - ) - - if not config.actor_rollout_ref.actor.use_dynamic_bsz: - # actor: ppo_micro_batch_size vs. ppo_micro_batch_size_per_gpu - check_mutually_exclusive(config.actor_rollout_ref.actor.ppo_micro_batch_size, - config.actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu, - "actor_rollout_ref.actor") - - if self.use_reference_policy: - # reference: log_prob_micro_batch_size vs. log_prob_micro_batch_size_per_gpu - check_mutually_exclusive(config.actor_rollout_ref.ref.log_prob_micro_batch_size, - config.actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu, - "actor_rollout_ref.ref") - - # The rollout section also has log_prob_micro_batch_size vs. log_prob_micro_batch_size_per_gpu - check_mutually_exclusive(config.actor_rollout_ref.rollout.log_prob_micro_batch_size, - config.actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu, - "actor_rollout_ref.rollout") - - if self.use_critic and not config.critic.use_dynamic_bsz: - # Check for critic micro-batch size conflicts - check_mutually_exclusive(config.critic.ppo_micro_batch_size, config.critic.ppo_micro_batch_size_per_gpu, - "critic") - - # Check for reward model micro-batch size conflicts - if config.reward_model.enable and not config.reward_model.use_dynamic_bsz: - check_mutually_exclusive(config.reward_model.micro_batch_size, config.reward_model.micro_batch_size_per_gpu, - "reward_model") - - # Actor - # check if train_batch_size is larger than ppo_mini_batch_size - # if NOT dynamic_bsz, we must ensure: - # ppo_mini_batch_size is divisible by ppo_micro_batch_size - # ppo_micro_batch_size * sequence_parallel_size >= n_gpus - if not config.actor_rollout_ref.actor.use_dynamic_bsz: - assert config.data.train_batch_size >= config.actor_rollout_ref.actor.ppo_mini_batch_size - sp_size = config.actor_rollout_ref.actor.get('ulysses_sequence_parallel_size', 1) - if config.actor_rollout_ref.actor.ppo_micro_batch_size is not None: - assert config.actor_rollout_ref.actor.ppo_mini_batch_size % config.actor_rollout_ref.actor.ppo_micro_batch_size == 0 - assert config.actor_rollout_ref.actor.ppo_micro_batch_size * sp_size >= n_gpus - - if config.actor_rollout_ref.actor.get('loss_agg_mode', None) is not None: - assert config.actor_rollout_ref.actor.loss_agg_mode in [ - "token-mean", "seq-mean-token-sum", "seq-mean-token-mean" - ], f"Invalid loss_agg_mode: {config.actor_rollout_ref.actor.loss_agg_mode}" - - if config.algorithm.use_kl_in_reward and config.actor_rollout_ref.actor.use_kl_loss: - print(f"NOTICE: You have both enabled in-reward kl and kl loss.") - - # critic - if self.use_critic and not config.critic.use_dynamic_bsz: - assert config.data.train_batch_size >= config.critic.ppo_mini_batch_size - sp_size = config.critic.get('ulysses_sequence_parallel_size', 1) - if config.critic.ppo_micro_batch_size is not None: - assert config.critic.ppo_mini_batch_size % config.critic.ppo_micro_batch_size == 0 - assert config.critic.ppo_micro_batch_size * sp_size >= n_gpus - - # Check if use_remove_padding is enabled when using sequence parallelism for fsdp - if config.actor_rollout_ref.actor.strategy == 'fsdp': - if config.actor_rollout_ref.actor.get('ulysses_sequence_parallel_size', 1) > 1 or \ - config.actor_rollout_ref.ref.get('ulysses_sequence_parallel_size', 1) > 1: - assert config.actor_rollout_ref.model.use_remove_padding, \ - "When using sequence parallelism for actor/ref policy, you must enable `use_remove_padding`." - - if self.use_critic and config.critic.strategy == 'fsdp': - if config.critic.get('ulysses_sequence_parallel_size', 1) > 1: - assert config.critic.model.use_remove_padding, \ - "When using sequence parallelism for critic, you must enable `use_remove_padding`." - - if config.data.get('val_batch_size', None) is not None: - print( - f"WARNING: val_batch_size is deprecated. Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves." - ) - - # check eval config - if config.actor_rollout_ref.rollout.val_kwargs.do_sample: - assert config.actor_rollout_ref.rollout.temperature > 0, \ - "validation gen temperature should be greater than 0 when enabling do_sample" - - if config.actor_rollout_ref.rollout.max_eval_parallel_agents <= 0: - print(f"`max_eval_parallel_agents` has not been set. Setting it to `max_parallel_agents` i.e {config.actor_rollout_ref.rollout.max_parallel_agents}") - config.actor_rollout_ref.rollout.max_eval_parallel_agents = config.actor_rollout_ref.rollout.max_parallel_agents - - print("[validate_config] All configuration checks passed successfully!") - - def _create_dataloader(self): - - # TODO: we have to make sure the batch size is divisible by the dp size - self.train_dataset = RLHFDataset( - parquet_files=self.config.data.train_files, - tokenizer=self.tokenizer, - processor=self.processor, - prompt_key=self.config.data.prompt_key, - image_key=self.config.data.get('image_key', 'images'), - max_prompt_length=self.config.data.max_prompt_length, - filter_prompts=True, - return_raw_chat=self.config.data.get('return_raw_chat', False), - truncation=self.config.data.get('truncation', 'error'), - filter_overlong_prompts=self.config.data.get('filter_overlong_prompts', False), - num_workers=self.config.data.get('filter_overlong_prompts_workers', None), - ) - - assert self.train_dataset.truncation == self.config.data.get( - 'truncation', 'error' - ), f'dataset truncation {self.train_dataset.truncation} must be the same as config {self.config.data.get("truncation", "error")}' - - self.val_dataset = RLHFDataset( - parquet_files=self.config.data.val_files, - tokenizer=self.tokenizer, - processor=self.processor, - prompt_key=self.config.data.prompt_key, - image_key=self.config.data.get('image_key', 'images'), - max_prompt_length=self.config.data.max_prompt_length, - filter_prompts=True, - return_raw_chat=self.config.data.get('return_raw_chat', False), - truncation=self.config.data.get('truncation', 'error'), - filter_overlong_prompts=self.config.data.get('filter_overlong_prompts', False), - num_workers=self.config.data.get('filter_overlong_prompts_workers', None), - ) - # use sampler for better ckpt resume - if self.config.data.shuffle: - train_dataloader_generator = torch.Generator() - train_dataloader_generator.manual_seed(self.config.data.get('seed', 1)) - sampler = RandomSampler(data_source=self.train_dataset, generator=train_dataloader_generator) - else: - sampler = SequentialSampler(data_source=self.train_dataset) - - self.train_dataloader = StatefulDataLoader(dataset=self.train_dataset, - batch_size=self.config.data.train_batch_size, - num_workers=self.config.data.dataloader_num_workers, - drop_last=True, - collate_fn=collate_fn, - sampler=sampler) - - self.val_dataloader = StatefulDataLoader( - dataset=self.val_dataset, - # Validation datasets are sent to inference engines as a whole batch, - # which will schedule the memory themselves. - batch_size=len(self.val_dataset), - num_workers=self.config.data.dataloader_num_workers, - shuffle=False, - drop_last=False, - collate_fn=collate_fn) - - assert len(self.train_dataloader) >= 1 - assert len( - self.val_dataloader - ) == 1, "Validation dataloader must have a single batch, which inference engines will schedule the memory themselves." - - print(f'Size of train dataloader: {len(self.train_dataloader)}') - - # inject total_training_steps to actor/critic optim_config. This is hacky. - total_training_steps = len(self.train_dataloader) * self.config.trainer.total_epochs - - if self.config.trainer.total_training_steps is not None: - total_training_steps = self.config.trainer.total_training_steps - - self.total_training_steps = total_training_steps - print(f'Total training steps: {self.total_training_steps}') - - OmegaConf.set_struct(self.config, True) - with open_dict(self.config): - self.config.actor_rollout_ref.actor.optim.total_training_steps = total_training_steps - self.config.critic.optim.total_training_steps = total_training_steps - - def _maybe_log_val_generations(self, inputs, outputs, scores): - """Log a table of validation samples to the configured logger (wandb or swanlab)""" - - generations_to_log = self.config.trainer.log_val_generations - - if generations_to_log == 0: - return - - import numpy as np - - # Create tuples of (input, output, score) and sort by input text - samples = list(zip(inputs, outputs, scores)) - samples.sort(key=lambda x: x[0]) # Sort by input text - - # Use fixed random seed for deterministic shuffling - rng = np.random.RandomState(42) - rng.shuffle(samples) - - # Take first N samples after shuffling - samples = samples[:generations_to_log] - - # Log to each configured logger - self.validation_generations_logger.log(self.config.trainer.logger, samples, self.global_steps) - - def _validate(self): - reward_tensor_lst = [] - data_source_lst = [] - reward_extra_infos_dict: dict[str, list] = defaultdict(list) - - # Lists to collect samples for the table - sample_inputs = [] - sample_outputs = [] - sample_scores = [] - - for test_data in self.val_dataloader: - test_batch = DataProto.from_single_dict(test_data) - - # repeat test batch - test_batch = test_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.val_kwargs.n, - interleave=True) - - # we only do validation on rule-based rm - if self.config.reward_model.enable and test_batch[0].non_tensor_batch['reward_model']['style'] == 'model': - return {} - - # Store original inputs - input_ids = test_batch.batch['input_ids'] - # TODO: Can we keep special tokens except for padding tokens? - input_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in input_ids] - sample_inputs.extend(input_texts) - - if self.config.actor_rollout_ref.rollout.task_type == "swegym": - non_tensor_batch_keys = ["instance"] - else: - non_tensor_batch_keys = ["raw_prompt_ids"] - - if 'multi_modal_inputs' in test_batch.non_tensor_batch.keys(): - test_gen_batch = test_batch.pop( - batch_keys=['input_ids', 'attention_mask', 'position_ids'], - non_tensor_batch_keys= non_tensor_batch_keys + ['multi_modal_data', 'multi_modal_inputs'], - ) - else: - test_gen_batch = test_batch.pop( - batch_keys=['input_ids', 'attention_mask', 'position_ids'], - non_tensor_batch_keys=non_tensor_batch_keys, - ) - - test_gen_batch.meta_info = { - 'eos_token_id': self.tokenizer.eos_token_id, - 'pad_token_id': self.tokenizer.pad_token_id, - 'recompute_log_prob': False, - 'do_sample': self.config.actor_rollout_ref.rollout.val_kwargs.do_sample, - 'validate': True, - } - print(f'test_gen_batch meta info: {test_gen_batch.meta_info}') - - # pad to be divisible by dp_size - test_gen_batch_padded, pad_size = pad_dataproto_to_divisor(test_gen_batch, self.actor_rollout_wg.world_size) - test_output_gen_batch_padded = self.actor_rollout_wg.generate_sequences(test_gen_batch_padded) - - # unpad - test_output_gen_batch = unpad_dataproto(test_output_gen_batch_padded, pad_size=pad_size) - - print('validation generation end') - - # Store generated outputs - output_ids = test_output_gen_batch.batch['responses'] - output_texts = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in output_ids] - sample_outputs.extend(output_texts) - - # repeat to align with repeated responses in rollout - test_batch = test_batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n_trajectories, interleave=True) - test_batch = test_batch.union(test_output_gen_batch) - - result = self.val_reward_fn(test_batch, return_dict=True) - reward_tensor = result["reward_tensor"] - scores = reward_tensor.sum(-1).cpu().tolist() - sample_scores.extend(scores) - if "reward_extra_info" in result: - reward_extra_infos_dict["final_reward"].extend(scores) - for key, lst in result["reward_extra_info"].items(): - reward_extra_infos_dict[key].extend(lst) - - data_source_lst.append(test_batch.non_tensor_batch.get('data_source', ['unknown'] * reward_tensor.shape[0])) - - self._maybe_log_val_generations(inputs=sample_inputs, outputs=sample_outputs, scores=sample_scores) - - for lst in reward_extra_infos_dict.values(): - assert len(lst) == 0 or len(lst) == len(sample_scores) - - data_sources = np.concatenate(data_source_lst, axis=0) - - data_src2var2metric2val = process_validation_metrics(data_sources, sample_inputs, reward_extra_infos_dict) - - metric_dict = {} - for data_source, var2metric2val in data_src2var2metric2val.items(): - core_var = "acc" if "acc" in var2metric2val else "final_reward" - for var_name, metric2val in var2metric2val.items(): - n_max = max([int(name.split("@")[-1].split("/")[0]) for name in metric2val.keys()]) - for metric_name, metric_val in metric2val.items(): - if var_name == core_var and any( - metric_name.startswith(pfx) - for pfx in ["mean", "std", "maj", "best"]) and f"@{n_max}/" in metric_name: - metric_sec = "val-core" - else: - metric_sec = "val-aux" - pfx = f"{metric_sec}/{data_source}/{var_name}/{metric_name}" - metric_dict[pfx] = metric_val - - return metric_dict - - def init_workers(self): - """Init resource pool and worker group""" - self.resource_pool_manager.create_resource_pool() - - self.resource_pool_to_cls = {pool: {} for pool in self.resource_pool_manager.resource_pool_dict.values()} - - # create actor and rollout - if self.hybrid_engine: - resource_pool = self.resource_pool_manager.get_resource_pool(Role.ActorRollout) - actor_rollout_cls = RayClassWithInitArgs(cls=self.role_worker_mapping[Role.ActorRollout], - config=self.config.actor_rollout_ref, - role='actor_rollout') - self.resource_pool_to_cls[resource_pool]['actor_rollout'] = actor_rollout_cls - else: - actor_pool = self.resource_pool_manager.get_resource_pool(Role.Actor) - actor_cls = RayClassWithInitArgs( - cls=self.role_worker_mapping[Role.Actor], - config=self.config.actor_rollout_ref, - role='actor', - ) - self.resource_pool_to_cls[actor_pool]['actor'] = actor_cls - - rollout_pool = self.resource_pool_manager.get_resource_pool(Role.Rollout) - rollout_cls = RayClassWithInitArgs( - cls=self.role_worker_mapping[Role.Rollout], - config=self.config.actor_rollout_ref, - role='rollout', - ) - self.resource_pool_to_cls[rollout_pool]['rollout'] = rollout_cls - - # create critic - if self.use_critic: - resource_pool = self.resource_pool_manager.get_resource_pool(Role.Critic) - critic_cls = RayClassWithInitArgs(cls=self.role_worker_mapping[Role.Critic], config=self.config.critic) - self.resource_pool_to_cls[resource_pool]['critic'] = critic_cls - - # create reference policy if needed - if self.use_reference_policy: - resource_pool = self.resource_pool_manager.get_resource_pool(Role.RefPolicy) - ref_policy_cls = RayClassWithInitArgs(self.role_worker_mapping[Role.RefPolicy], - config=self.config.actor_rollout_ref, - role='ref') - self.resource_pool_to_cls[resource_pool]['ref'] = ref_policy_cls - - # create a reward model if reward_fn is None - if self.use_rm: - # we create a RM here - resource_pool = self.resource_pool_manager.get_resource_pool(Role.RewardModel) - rm_cls = RayClassWithInitArgs(self.role_worker_mapping[Role.RewardModel], config=self.config.reward_model) - self.resource_pool_to_cls[resource_pool]['rm'] = rm_cls - - # initialize WorkerGroup - # NOTE: if you want to use a different resource pool for each role, which can support different parallel size, - # you should not use `create_colocated_worker_cls`. Instead, directly pass different resource pool to different worker groups. - # See https://github.com/volcengine/verl/blob/master/examples/ray/tutorial.ipynb for more information. - if self.hybrid_engine: - all_wg: dict[str, RayWorkerGroup] = {} - self.wg_dicts = [] - for resource_pool, class_dict in self.resource_pool_to_cls.items(): - worker_dict_cls = create_colocated_worker_cls(class_dict=class_dict) - wg_dict = self.ray_worker_group_cls(resource_pool=resource_pool, ray_cls_with_init=worker_dict_cls) - spawn_wg = wg_dict.spawn(prefix_set=class_dict.keys()) - all_wg.update(spawn_wg) - # keep the referece of WorkerDict to support ray >= 2.31. Ref: https://github.com/ray-project/ray/pull/45699 - self.wg_dicts.append(wg_dict) - - if self.use_critic: - self.critic_wg = all_wg['critic'] - self.critic_wg.init_model() - - if self.use_reference_policy: - self.ref_policy_wg = all_wg['ref'] - self.ref_policy_wg.init_model() - - if self.use_rm: - self.rm_wg = all_wg['rm'] - self.rm_wg.init_model() - - # we should create rollout at the end so that vllm can have a better estimation of kv cache memory - self.actor_rollout_wg = all_wg['actor_rollout'] - self.actor_rollout_wg.init_model() - self.actor_wg = self.rollout_wg = self.actor_rollout_wg - else: - if self.use_critic: - self.critic_wg = self.ray_worker_group_cls( - resource_pool=self.resource_pool_manager.get_resource_pool(Role.Critic), - ray_cls_with_init=critic_cls, - ) - self.critic_wg.init_model() - - if self.use_reference_policy: - self.ref_policy_wg = self.ray_worker_group_cls( - resource_pool=self.resource_pool_manager.get_resource_pool(Role.RefPolicy), - ray_cls_with_init=ref_policy_cls, - ) - self.ref_policy_wg.init_model() - - if self.use_rm: - self.rm_wg = self.ray_worker_group_cls( - resource_pool=self.resource_pool_manager.get_resource_pool(Role.RewardModel), - ray_cls_with_init=rm_cls, - ) - self.rm_wg.init_model() - - self.actor_wg = self.ray_worker_group_cls( - resource_pool=self.resource_pool_manager.get_resource_pool(Role.Actor), - ray_cls_with_init=actor_cls, - ) - o1 = self.actor_wg.execute_all_async("init_model") - - self.rollout_wg = self.ray_worker_group_cls( - resource_pool=self.resource_pool_manager.get_resource_pool(Role.Rollout), - ray_cls_with_init=rollout_cls, - ) - o2 = self.rollout_wg.execute_all_async("init_model") - - ray.get(o1) - ray.get(o2) - - - def _save_checkpoint(self, save_hf=False): - # path: given_path + `/global_step_{global_steps}` + `/actor` - - if save_hf: - # TODO：is not ready yet - from torch.distributed.fsdp import FullStateDictConfig, StateDictType - from torch.distributed.fsdp import FullyShardedDataParallel as FSDP - import verl.utils.hdfs_io as hdfs_io - - cfg = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) - with FSDP.state_dict_type(self.actor_wg.fsdp_model, StateDictType.FULL_STATE_DICT, cfg): - state_dict = self.actor_wg.fsdp_model.state_dict() - - local_global_step_folder = os.path.join(self.config.trainer.default_local_dir, - f'global_step_{self.global_steps}') - actor_local_path = os.path.join(local_global_step_folder, 'actor') - - # Save HuggingFace model on rank 0 - if self.actor_wg.device_mesh.get_rank() == 0: - os.makedirs(actor_local_path, exist_ok=True) - self.actor_wg.model.save_pretrained(actor_local_path, state_dict=state_dict) - self.actor_wg.tokenizer.save_pretrained(actor_local_path) - - # Copy to HDFS if configured - if self.config.trainer.default_hdfs_dir: - hdfs_io.makedirs(self.config.trainer.default_hdfs_dir, exist_ok=True) - hdfs_io.copy(src=actor_local_path, - dst=os.path.join(self.config.trainer.default_hdfs_dir, - f'global_step_{self.global_steps}', 'actor'), - dirs_exist_ok=True) - torch.distributed.barrier() - else: - local_global_step_folder = os.path.join(self.config.trainer.default_local_dir, - f'global_step_{self.global_steps}') - actor_local_path = os.path.join(local_global_step_folder, 'actor') - - actor_remote_path = None if self.config.trainer.default_hdfs_dir is None else os.path.join( - self.config.trainer.default_hdfs_dir, f'global_step_{self.global_steps}', 'actor') - - remove_previous_ckpt_in_save = self.config.trainer.get('remove_previous_ckpt_in_save', False) - if remove_previous_ckpt_in_save: - print( - 'Warning: remove_previous_ckpt_in_save is deprecated, set max_actor_ckpt_to_keep=1 and max_critic_ckpt_to_keep=1 instead' - ) - max_actor_ckpt_to_keep = self.config.trainer.get('max_actor_ckpt_to_keep', - None) if not remove_previous_ckpt_in_save else 1 - max_critic_ckpt_to_keep = self.config.trainer.get('max_critic_ckpt_to_keep', - None) if not remove_previous_ckpt_in_save else 1 - - self.actor_rollout_wg.save_checkpoint(actor_local_path, - actor_remote_path, - self.global_steps, - max_ckpt_to_keep=max_actor_ckpt_to_keep) - - if self.use_critic: - critic_local_path = os.path.join(local_global_step_folder, 'critic') - critic_remote_path = None if self.config.trainer.default_hdfs_dir is None else os.path.join( - self.config.trainer.default_hdfs_dir, f'global_step_{self.global_steps}', 'critic') - self.critic_wg.save_checkpoint(critic_local_path, - critic_remote_path, - self.global_steps, - max_ckpt_to_keep=max_critic_ckpt_to_keep) - - # save dataloader - dataloader_local_path = os.path.join(local_global_step_folder, 'data.pt') - dataloader_state_dict = self.train_dataloader.state_dict() - torch.save(dataloader_state_dict, dataloader_local_path) - - # latest checkpointed iteration tracker (for atomic usage) - local_latest_checkpointed_iteration = os.path.join(self.config.trainer.default_local_dir, - 'latest_checkpointed_iteration.txt') - with open(local_latest_checkpointed_iteration, 'w') as f: - f.write(str(self.global_steps)) - - def _load_checkpoint(self): - if self.config.trainer.resume_mode == 'disable': - return 0 - - # load from hdfs - if self.config.trainer.default_hdfs_dir is not None: - raise NotImplementedError('load from hdfs is not implemented yet') - else: - checkpoint_folder = self.config.trainer.default_local_dir # TODO: check path - if not os.path.isabs(checkpoint_folder): - working_dir = os.getcwd() - checkpoint_folder = os.path.join(working_dir, checkpoint_folder) - global_step_folder = find_latest_ckpt_path(checkpoint_folder) # None if no latest - - # find global_step_folder - if self.config.trainer.resume_mode == 'auto': - if global_step_folder is None: - print('Training from scratch') - return 0 - else: - if self.config.trainer.resume_mode == "resume_path": - assert isinstance(self.config.trainer.resume_from_path, str), "resume ckpt must be str type" - assert 'global_step_' in self.config.trainer.resume_from_path, "resume ckpt must specify the global_steps" - global_step_folder = self.config.trainer.resume_from_path - if not os.path.isabs(global_step_folder): - working_dir = os.getcwd() - global_step_folder = os.path.join(working_dir, global_step_folder) - print(f'Load from checkpoint folder: {global_step_folder}') - # set global step - self.global_steps = int(global_step_folder.split('global_step_')[-1]) - - print(f'Setting global step to {self.global_steps}') - print(f'Resuming from {global_step_folder}') - - actor_path = os.path.join(global_step_folder, 'actor') - critic_path = os.path.join(global_step_folder, 'critic') - # load actor - self.actor_wg.load_checkpoint(actor_path, del_local_after_load=self.config.trainer.del_local_ckpt_after_load) - # load critic - if self.use_critic: - self.critic_wg.load_checkpoint(critic_path, - del_local_after_load=self.config.trainer.del_local_ckpt_after_load) - - # load dataloader, - # TODO: from remote not implemented yet - dataloader_local_path = os.path.join(global_step_folder, 'data.pt') - if os.path.exists(dataloader_local_path): - dataloader_state_dict = torch.load(dataloader_local_path, weights_only=False) - self.train_dataloader.load_state_dict(dataloader_state_dict) - else: - print(f"Warning: No dataloader state found at {dataloader_local_path}, will start from scratch") - - def _balance_batch(self, batch: DataProto, metrics, logging_prefix='global_seqlen'): - """Reorder the data on single controller such that each dp rank gets similar total tokens""" - attention_mask = batch.batch['attention_mask'] - batch_size = attention_mask.shape[0] - global_seqlen_lst = batch.batch['attention_mask'].view(batch_size, -1).sum(-1).tolist() # (train_batch_size,) - world_size = self.actor_wg.world_size - global_partition_lst = get_seqlen_balanced_partitions(global_seqlen_lst, - k_partitions=world_size, - equal_size=True) - # reorder based on index. The data will be automatically equally partitioned by dispatch function - global_idx = torch.tensor([j for partition in global_partition_lst for j in partition]) - batch.reorder(global_idx) - global_balance_stats = log_seqlen_unbalance(seqlen_list=global_seqlen_lst, - partitions=global_partition_lst, - prefix=logging_prefix) - metrics.update(global_balance_stats) - - def fit(self): - """ - The training loop of PPO. - The driver process only need to call the compute functions of the worker group through RPC to construct the PPO dataflow. - The light-weight advantage computation is done on the driver process. - """ - from verl.utils.tracking import Tracking - from omegaconf import OmegaConf - - logger = Tracking(project_name=self.config.trainer.project_name, - experiment_name=self.config.trainer.experiment_name, - default_backend=self.config.trainer.logger, - config=OmegaConf.to_container(self.config, resolve=True)) - - self.global_steps = 0 - - # load checkpoint before doing anything - self._load_checkpoint() - - # perform validation before training - # currently, we only support validation using the reward_function. - if self.val_reward_fn is not None and self.config.trainer.get('val_before_train', True): - val_metrics = self._validate() - pprint(f'Initial validation metrics: {val_metrics}') - logger.log(data=val_metrics, step=self.global_steps) - if self.config.trainer.get('val_only', False): - return - - # add tqdm - progress_bar = tqdm(total=self.total_training_steps, initial=self.global_steps, desc="Training Progress") - - # we start from step 1 - self.global_steps += 1 - last_val_metrics = None - - for epoch in range(self.config.trainer.total_epochs): - for batch_dict in self.train_dataloader: - metrics = {} - timing_raw = {} - - batch: DataProto = DataProto.from_single_dict(batch_dict) - - # pop those keys for generation - if 'multi_modal_inputs' in batch.non_tensor_batch.keys(): - gen_batch = batch.pop( - batch_keys=['input_ids', 'attention_mask', 'position_ids'], - non_tensor_batch_keys=['raw_prompt_ids', 'multi_modal_data', 'multi_modal_inputs'], - ) - else: - batch_keys = ['input_ids', 'attention_mask', 'position_ids'] - - if self.config.actor_rollout_ref.rollout.task_type == "swegym": # TODO(haoran): pass arg list here - gen_batch = batch.pop(batch_keys=batch_keys, - non_tensor_batch_keys=['instance']) - else: - gen_batch = batch.pop(batch_keys=batch_keys, - non_tensor_batch_keys=['raw_prompt_ids']) - - is_last_step = self.global_steps >= self.total_training_steps - - with _timer('step', timing_raw): - # generate a batch - import inspect - with _timer('gen', timing_raw): - if self.actor_wg is not self.rollout_wg: - # for syncing params - self.actor_wg.execute_all_async("generate_sequences", gen_batch) - gen_batch_output = self.rollout_wg.generate_sequences(gen_batch) - - if self.config.algorithm.adv_estimator == AdvantageEstimator.REMAX: - with _timer('gen_max', timing_raw): - gen_baseline_batch = deepcopy(gen_batch) - gen_baseline_batch.meta_info['do_sample'] = False - gen_baseline_output = self.actor_wg.generate_sequences(gen_baseline_batch) - - batch = batch.union(gen_baseline_output) - reward_baseline_tensor = self.reward_fn(batch) - reward_baseline_tensor = reward_baseline_tensor.sum(dim=-1) - - batch.pop(batch_keys=list(gen_baseline_output.batch.keys())) - - batch.batch['reward_baselines'] = reward_baseline_tensor - - del gen_baseline_batch, gen_baseline_output - - batch.non_tensor_batch['uid'] = np.array([str(uuid.uuid4()) for _ in range(len(batch.batch))], - dtype=object) - # repeat to align with repeated responses in rollout - batch = batch.repeat(repeat_times=self.config.actor_rollout_ref.rollout.n_trajectories, interleave=True) - batch = batch.union(gen_batch_output) - - # batch.batch['response_mask'] = compute_response_mask(batch) - # balance the number of valid tokens on each dp rank. - # Note that this breaks the order of data inside the batch. - # Please take care when you implement group based adv computation such as GRPO and rloo - if self.config.trainer.get('balance_batch', False): - self._balance_batch(batch, metrics=metrics) - - # compute global_valid tokens - batch.meta_info['global_token_num'] = torch.sum(batch.batch['attention_mask'], dim=-1).tolist() - - # recompute old_log_probs - with _timer('old_log_prob', timing_raw): - old_log_prob = self.actor_wg.compute_log_prob(batch) - batch = batch.union(old_log_prob) - - if self.use_reference_policy: - # compute reference log_prob - with _timer('ref', timing_raw): - ref_log_prob = self.ref_policy_wg.compute_ref_log_prob(batch) - batch = batch.union(ref_log_prob) - - # compute values - if self.use_critic: - with _timer('values', timing_raw): - values = self.critic_wg.compute_values(batch) - batch = batch.union(values) - - with _timer('adv', timing_raw): - # compute scores. Support both model and function-based. - # We first compute the scores using reward model. Then, we call reward_fn to combine - # the results from reward model and rule-based results. - if self.use_rm: - # we first compute reward model score - reward_tensor = self.rm_wg.compute_rm_score(batch) - batch = batch.union(reward_tensor) - - # we combine with rule-based rm - # we combine with rule-based rm - reward_tensor_dict, reward_metrics = self.reward_fn(batch) - batch.batch['token_level_scores'] = reward_tensor_dict['all'] - for k, v in reward_metrics.items(): - metrics['train_reward/' + k] = v - # decomposed rewards: - for k, v in reward_tensor_dict.items(): - batch.batch[k]=v - - # compute rewards. apply_kl_penalty if available - if self.config.algorithm.use_kl_in_reward: - batch, kl_metrics = apply_kl_penalty(batch, - kl_ctrl=self.kl_ctrl, - kl_penalty=self.config.algorithm.kl_penalty) - metrics.update(kl_metrics) - else: - batch.batch['token_level_rewards'] = batch.batch['token_level_scores'] - - # compute advantages, executed on the driver process - batch = compute_advantage(batch, - adv_estimator=self.config.algorithm.adv_estimator, - gamma=self.config.algorithm.gamma, - lam=self.config.algorithm.lam, - num_repeat=self.config.actor_rollout_ref.rollout.n_trajectories) - - # update critic - if self.use_critic: - with _timer('update_critic', timing_raw): - critic_output = self.critic_wg.update_critic(batch) - critic_output_metrics = reduce_metrics(critic_output.meta_info['metrics']) - metrics.update(critic_output_metrics) - - # implement critic warmup - if self.config.trainer.critic_warmup <= self.global_steps: - # update actor - with _timer('update_actor', timing_raw): - actor_output = self.actor_wg.update_actor(batch) - actor_output_metrics = reduce_metrics(actor_output.meta_info['metrics']) - metrics.update(actor_output_metrics) - - # validate - if self.val_reward_fn is not None and self.config.trainer.test_freq > 0 and \ - (is_last_step or self.global_steps % self.config.trainer.test_freq == 0): - with _timer('testing', timing_raw): - val_metrics: dict = self._validate() - if is_last_step: - last_val_metrics = val_metrics - metrics.update(val_metrics) - - if self.config.trainer.save_freq > 0 and ( is_last_step or \ - self.global_steps % self.config.trainer.save_freq == 0): - with _timer('save_checkpoint', timing_raw): - self._save_checkpoint() - - # collect metrics - metrics.update(compute_data_metrics(batch=batch, use_critic=self.use_critic)) - metrics.update(compute_timing_metrics(batch=batch, timing_raw=timing_raw)) - # TODO: implement actual tflpo and theoretical tflpo - n_gpus = self.resource_pool_manager.get_n_gpus() - metrics.update(compute_throughout_metrics(batch=batch, timing_raw=timing_raw, n_gpus=n_gpus)) - - # TODO: make a canonical logger that supports various backend - logger.log(data=metrics, step=self.global_steps) - - if is_last_step: - pprint(f'Final validation metrics: {last_val_metrics}') - progress_bar.close() - return - - progress_bar.update(1) - self.global_steps += 1 diff --git a/verl/trainer/runtime_env.yaml b/verl/trainer/runtime_env.yaml deleted file mode 100644 index 87bd05a9aa..0000000000 --- a/verl/trainer/runtime_env.yaml +++ /dev/null @@ -1,5 +0,0 @@ -working_dir: ./ -excludes: ["/.git/"] -env_vars: - TORCH_NCCL_AVOID_RECORD_STREAMS: "1" - VLLM_ATTENTION_BACKEND: "XFORMERS" \ No newline at end of file diff --git a/verl/utils/__init__.py b/verl/utils/__init__.py deleted file mode 100644 index bc781029d0..0000000000 --- a/verl/utils/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import tokenizer -from .tokenizer import hf_tokenizer, hf_processor - -__all__ = tokenizer.__all__ \ No newline at end of file diff --git a/verl/utils/checkpoint/__init__.py b/verl/utils/checkpoint/__init__.py deleted file mode 100644 index 7a7aadbc9d..0000000000 --- a/verl/utils/checkpoint/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/verl/utils/checkpoint/checkpoint_manager.py b/verl/utils/checkpoint/checkpoint_manager.py deleted file mode 100644 index af32e85d97..0000000000 --- a/verl/utils/checkpoint/checkpoint_manager.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import shutil -from filelock import FileLock -import tempfile -from typing import Union -import torch -import torch.distributed -from transformers import PreTrainedTokenizer, ProcessorMixin -import numpy as np -import random -import re - - -class BaseCheckpointManager: - """ - A checkpoint manager that saves and loads - - model - - optimizer - - lr_scheduler - - extra_states - in a SPMD way. - - We save - - sharded model states and optimizer states - - full lr_scheduler states - - huggingface tokenizer and config for ckpt merge - """ - - def __init__(self, - model, - optimizer: torch.optim.Optimizer, - lr_scheduler: torch.optim.lr_scheduler.LRScheduler = None, - processing_class: Union[PreTrainedTokenizer, ProcessorMixin] = None, - checkpoint_contents: list = ['model', 'optimizer', 'extra']): - self.previous_global_step = None - self.previous_saved_paths = [] - - self.model = model - self.optimizer = optimizer - self.lr_scheduler = lr_scheduler - self.processing_class = processing_class - self.checkpoint_contents = checkpoint_contents - - self.rank = torch.distributed.get_rank() - self.world_size = torch.distributed.get_world_size() - - def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_after_load: bool = False): - raise NotImplementedError - - def save_checkpoint(self, - local_path: str, - hdfs_path: str = None, - global_step: int = 0, - max_ckpt_to_keep: int = None): - raise NotImplementedError - - @staticmethod - def checkpath(local_path: str, hdfs_path: str): - assert local_path is not None or hdfs_path is not None, "local_path and hdfs_path cannot be both None" - return True if local_path is not None else False, local_path if local_path is not None else hdfs_path - - def remove_previous_save_local_path(self, path): - if isinstance(path, str): - path = [path] - for p in path: - abs_path = os.path.abspath(p) - print(f'Checkpoint manager remove previous save local path: {abs_path}') - if not os.path.exists(abs_path): - continue - shutil.rmtree(abs_path, ignore_errors=True) - - @staticmethod - def local_mkdir(path): - if not os.path.isabs(path): - working_dir = os.getcwd() - path = os.path.join(working_dir, path) - - # Using hash value of path as lock file name to avoid long file name - lock_filename = f"ckpt_{hash(path) & 0xFFFFFFFF:08x}.lock" - lock_path = os.path.join(tempfile.gettempdir(), lock_filename) - - try: - with FileLock(lock_path, timeout=60): # Add timeout - # make a new dir - os.makedirs(path, exist_ok=True) - except Exception as e: - print(f"Warning: Failed to acquire lock for {path}: {e}") - # Even if the lock is not acquired, try to create the directory - os.makedirs(path, exist_ok=True) - - return path - - @staticmethod - def get_rng_state(): - rng_state = { - 'cpu': torch.get_rng_state(), - 'cuda': torch.cuda.get_rng_state(), - 'numpy': np.random.get_state(), - 'random': random.getstate(), - } - return rng_state - - @staticmethod - def load_rng_state(rng_state): - torch.set_rng_state(rng_state['cpu']) - torch.cuda.set_rng_state(rng_state['cuda']) - np.random.set_state(rng_state['numpy']) - random.setstate(rng_state['random']) - - -def find_latest_ckpt_path(path, directory_format="global_step_{}"): - if path is None: - return None - - tracker_file = get_checkpoint_tracker_filename(path) - if not os.path.exists(tracker_file): - print("Checkpoint tracker file does not exist: %s", tracker_file) - return None - - with open(tracker_file, "rb") as f: - iteration = int(f.read().decode()) - ckpt_path = os.path.join(path, directory_format.format(iteration)) - if not os.path.exists(ckpt_path): - print("Checkpoint does not exist: %s", ckpt_path) - return None - - print("Found checkpoint: %s", ckpt_path) - return ckpt_path - - -def get_checkpoint_tracker_filename(root_path: str): - """ - Tracker file rescords the latest chckpoint during training to restart from. - """ - return os.path.join(root_path, "latest_checkpointed_iteration.txt") diff --git a/verl/utils/checkpoint/fsdp_checkpoint_manager.py b/verl/utils/checkpoint/fsdp_checkpoint_manager.py deleted file mode 100644 index c59f844dfe..0000000000 --- a/verl/utils/checkpoint/fsdp_checkpoint_manager.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ray -import os - -import warnings -from typing import Union -import torch -import torch.distributed -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, StateDictType -from torch.distributed.fsdp import ShardedStateDictConfig, ShardedOptimStateDictConfig - -from verl.utils.fs import copy_to_local, is_non_local - -from transformers import PreTrainedTokenizer, ProcessorMixin - -from .checkpoint_manager import BaseCheckpointManager - - -class FSDPCheckpointManager(BaseCheckpointManager): - """ - A checkpoint manager that saves and loads - - model - - optimizer - - lr_scheduler - - extra_states - in a SPMD way. - - We save - - sharded model states and optimizer states - - full lr_scheduler states - - huggingface tokenizer/processor and config for ckpt merge - """ - - def __init__(self, - model: FSDP, - optimizer: torch.optim.Optimizer, - lr_scheduler: torch.optim.lr_scheduler.LRScheduler, - processing_class: Union[PreTrainedTokenizer, ProcessorMixin] = None, - checkpoint_contents: list = ['model', 'optimizer', 'extra'], - **kwargs): - - if processing_class is None: - assert "tokenizer" in kwargs, "tokenizer or processor must be provided" - warnings.warn("`tokenizer` is deprecated. use `processing_class` instead.", DeprecationWarning) - processing_class = kwargs.pop("tokenizer") - assert "model" in checkpoint_contents and "optimizer" in checkpoint_contents and "extra" in checkpoint_contents, f"FSDPCheckpointManager must include ['model', 'optimizer', 'extra'], got {checkpoint_contents}" - - super().__init__(model, - optimizer, - lr_scheduler=lr_scheduler, - processing_class=processing_class, - checkpoint_contents=checkpoint_contents) - - def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_after_load=False): - if local_path is None: - return - - # every rank download its own checkpoint - remote_model_path = os.path.join(local_path, f'model_world_size_{self.world_size}_rank_{self.rank}.pt') - remote_optim_path = os.path.join(local_path, f'optim_world_size_{self.world_size}_rank_{self.rank}.pt') - remote_extra_state_path = os.path.join(local_path, - f'extra_state_world_size_{self.world_size}_rank_{self.rank}.pt') - print( - f'[rank-{self.rank}]: Loading from {remote_model_path} and {remote_optim_path} and {remote_extra_state_path}' - ) - local_model_path = copy_to_local(remote_model_path) - local_optim_path = copy_to_local(remote_optim_path) - local_extra_state_path = copy_to_local(remote_extra_state_path) - - model_state_dict = torch.load(local_model_path, weights_only=False) - optimizer_state_dict = torch.load(local_optim_path, weights_only=False) - extra_state_dict = torch.load(local_extra_state_path, weights_only=False) - - if del_local_after_load: - try: - os.remove(local_model_path) if is_non_local(local_model_path) else None - os.remove(local_optim_path) if is_non_local(local_optim_path) else None - os.remove(local_extra_state_path) if is_non_local(local_extra_state_path) else None - except Exception as e: - print( - f'[rank-{self.rank}]: remove local resume ckpt file after loading failed, exception {e} will be ignored' - ) - - lr_scheduler_state_dict = extra_state_dict['lr_scheduler'] - - state_dict_cfg = ShardedStateDictConfig(offload_to_cpu=True) - optim_cfg = ShardedOptimStateDictConfig(offload_to_cpu=True) - with FSDP.state_dict_type(self.model, StateDictType.SHARDED_STATE_DICT, state_dict_cfg, optim_cfg): - self.model.load_state_dict(model_state_dict) - if self.optimizer is not None: - self.optimizer.load_state_dict(optimizer_state_dict) - # recover random state - if 'rng' in extra_state_dict: - # 'rng' may not exist for backward compatibility - self.load_rng_state(extra_state_dict['rng']) - - if self.lr_scheduler is not None: - self.lr_scheduler.load_state_dict(lr_scheduler_state_dict) - - def save_checkpoint(self, local_path: str, hdfs_path: str = None, global_step: int = 0, max_ckpt_to_keep=None): - if local_path is None: - return - - # record the previous global step - self.previous_global_step = global_step - - # remove previous local_path - if max_ckpt_to_keep and isinstance(max_ckpt_to_keep, int) and max_ckpt_to_keep > 0 and len( - self.previous_saved_paths) >= max_ckpt_to_keep: - keep_start = len(self.previous_saved_paths) - max_ckpt_to_keep + 1 - self.remove_previous_save_local_path(self.previous_saved_paths[:keep_start]) - self.previous_saved_paths = self.previous_saved_paths[keep_start:] - - local_path = self.local_mkdir(local_path) - torch.distributed.barrier() - - # every rank will save its own model and optim shard - state_dict_cfg = ShardedStateDictConfig(offload_to_cpu=True) - optim_cfg = ShardedOptimStateDictConfig(offload_to_cpu=True) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - with FSDP.state_dict_type(self.model, StateDictType.SHARDED_STATE_DICT, state_dict_cfg, optim_cfg): - model_state_dict = self.model.state_dict() - if self.optimizer is not None: - optimizer_state_dict = self.optimizer.state_dict() - else: - optimizer_state_dict = None - if self.lr_scheduler is not None: - lr_scheduler_state_dict = self.lr_scheduler.state_dict() - else: - lr_scheduler_state_dict = None - - extra_state_dict = { - 'lr_scheduler': lr_scheduler_state_dict, - 'rng': self.get_rng_state(), - } - model_path = os.path.join(local_path, f'model_world_size_{self.world_size}_rank_{self.rank}.pt') - optim_path = os.path.join(local_path, f'optim_world_size_{self.world_size}_rank_{self.rank}.pt') - extra_path = os.path.join(local_path, f'extra_state_world_size_{self.world_size}_rank_{self.rank}.pt') - - print(f'[rank-{self.rank}]: Saving model to {os.path.abspath(model_path)}') - print(f'[rank-{self.rank}]: Saving checkpoint to {os.path.abspath(model_path)}') - print(f'[rank-{self.rank}]: Saving extra_state to {os.path.abspath(extra_path)}') - torch.save(model_state_dict, model_path) - torch.save(optimizer_state_dict, optim_path) # TODO: address optimizer is None - torch.save(extra_state_dict, extra_path) - - if "hf_model" in self.checkpoint_contents: - # wait for everyone to dump to local - torch.distributed.barrier() - - if self.rank == 0: - hf_local_path = os.path.join(local_path, 'huggingface') - os.makedirs(hf_local_path, exist_ok=True) - self.model._fsdp_wrapped_module.config.save_pretrained(hf_local_path) - self.processing_class.save_pretrained(hf_local_path) - - torch.distributed.barrier() - - self.previous_saved_paths.append(local_path) diff --git a/verl/utils/checkpoint/megatron_checkpoint_manager.py b/verl/utils/checkpoint/megatron_checkpoint_manager.py deleted file mode 100644 index c001e4e2ef..0000000000 --- a/verl/utils/checkpoint/megatron_checkpoint_manager.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ray -import os -import random -import numpy as np - -import warnings -from typing import Union -import torch -import torch.distributed -from torch.nn.parallel import DistributedDataParallel as torchDDP - -from verl.utils.fs import copy_to_local, is_non_local -from verl.models.weight_loader_registry import get_weight_saver -from verl.models.weight_loader_registry import get_weight_loader -from verl.utils.model import load_megatron_model_weights -from verl.utils.megatron_utils import TransformerConfig, get_model_checkpoint_path, get_hf_model_checkpoint_path, get_optimizer_checkpoint_path, get_rng_states_checkpoint_path, unwrap_model - -from .checkpoint_manager import BaseCheckpointManager -from transformers import AutoModelForCausalLM - -from megatron.core import mpu, tensor_parallel -from megatron.core.dist_checkpointing.mapping import ShardedObject -from megatron.core.transformer.module import Float16Module -from megatron.core.distributed import DistributedDataParallel as LocalDDP - - -class MegatronCheckpointManager(BaseCheckpointManager): - """ - A checkpoint manager that saves and loads - - model - - optimizer - - lr_scheduler - - extra_states - in a SPMD way. - - We save - - sharded model states and optimizer states - - full lr_scheduler states - - huggingface tokenizer/processor and config for ckpt merge - """ - - def __init__(self, - config, - model_config, - role, - model: torch.nn.ModuleList, - arch: str, - hf_config, - param_dtype: torch.dtype, - share_embeddings_and_output_weights: bool, - tokenizer, - optimizer, - use_distributed_optimizer: bool, - checkpoint_contents: list = ['model', 'optimizer', 'extra'], - **kwargs): - - super().__init__(model, - optimizer=optimizer, - lr_scheduler=None, - processing_class=tokenizer, - checkpoint_contents=checkpoint_contents) - self.arch = arch - self.config = config - self.role = role - self.is_value_model = False - if self.role in ["reward", "critic"]: - self.is_value_model = True - self.model_config = model_config - self.hf_config = hf_config - self.param_dtype = param_dtype - self.share_embeddings_and_output_weights = share_embeddings_and_output_weights - self.model_path = self.config.model.path - self.use_distributed_optimizer = use_distributed_optimizer - - self.rank = torch.distributed.get_rank() - - self.weight_saver = get_weight_saver(self.arch) - - def get_rng_state(self, use_dist_ckpt: bool = False, data_parallel_random_init: bool = False): - """ collect rng state across data parallel ranks """ - rng_state = { - 'random_rng_state': random.getstate(), - 'np_rng_state': np.random.get_state(), - 'torch_rng_state': torch.get_rng_state(), - 'cuda_rng_state': torch.cuda.get_rng_state(), - 'rng_tracker_states': tensor_parallel.get_cuda_rng_tracker().get_states() - } - - rng_state_list = None - if torch.distributed.is_initialized() and \ - mpu.get_data_parallel_world_size() > 1 and data_parallel_random_init: - rng_state_list = \ - [None for i in range(mpu.get_data_parallel_world_size())] - torch.distributed.all_gather_object(rng_state_list, rng_state, group=mpu.get_data_parallel_group()) - else: - rng_state_list = [rng_state] - - if use_dist_ckpt: - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() - rng_state_list = ShardedObject('rng_state', - rng_state_list, (pp_size, tp_size), (pp_rank, tp_rank), - replica_id=mpu.get_data_parallel_rank(with_context_parallel=True)) - - return rng_state_list - - def get_checkpoint_name(self, - checkpoints_path, - pipeline_parallel=None, - tensor_rank=None, - pipeline_rank=None, - expert_parallel=None, - expert_rank=None, - return_base_dir=True, - basename="model.pt"): - """Determine the directory name for this rank's checkpoint.""" - # Use both the tensor and pipeline MP rank. - if pipeline_parallel is None: - pipeline_parallel = (mpu.get_pipeline_model_parallel_world_size() > 1) - if tensor_rank is None: - tensor_rank = mpu.get_tensor_model_parallel_rank() - if pipeline_rank is None: - pipeline_rank = mpu.get_pipeline_model_parallel_rank() - if expert_parallel is None: - expert_parallel = (mpu.get_expert_model_parallel_world_size() > 1) - if expert_rank is None: - expert_rank = mpu.get_expert_model_parallel_rank() - - # Use both the tensor and pipeline MP rank. If using the distributed - # optimizer, then the optimizer's path must additionally include the - # data parallel rank. - if not pipeline_parallel: - common_path = os.path.join(checkpoints_path, f'mp_rank_{tensor_rank:02d}') - else: - common_path = os.path.join(checkpoints_path, f'mp_rank_{tensor_rank:02d}_{pipeline_rank:03d}') - - if expert_parallel: - common_path = common_path + f'_{expert_rank:03d}' - - os.makedirs(common_path, exist_ok=True) - - if return_base_dir: - return common_path - return os.path.join(common_path, basename) - - def load_optimizer(self, ckpt_path): - # TODO: Check Optimizer format and distributed optimizer - optimizer_path = get_optimizer_checkpoint_path(ckpt_path) - print(f"Loading optimizer from {optimizer_path}") - self.optimizer.load_parameter_state(optimizer_path) - - def load_rng_states(self, ckpt_path, data_parallel_random_init=False, use_dist_ckpt=False): - rng_state_path = get_rng_states_checkpoint_path(ckpt_path) - print(f"Loading rng states from {rng_state_path}") - rng_state = torch.load(rng_state_path) - # access rng_state for data parallel rank - if not use_dist_ckpt: - if data_parallel_random_init: - rng_state = rng_state[mpu.get_data_parallel_rank()] - else: - rng_state = rng_state[0] - random.setstate(rng_state['random_rng_state']) - np.random.set_state(rng_state['np_rng_state']) - torch.set_rng_state(rng_state['torch_rng_state']) - torch.cuda.set_rng_state(rng_state['cuda_rng_state']) - # Check for empty states array - if not rng_state['rng_tracker_states']: - raise KeyError - tensor_parallel.get_cuda_rng_tracker().set_states(rng_state['rng_tracker_states']) - - def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_after_load=False): - if local_path is None: - return - - if 'model' in self.checkpoint_contents: - model_path = get_model_checkpoint_path(local_path) - ckpt_name = self.get_checkpoint_name(model_path, return_base_dir=False) - state_dicts = torch.load(os.path.join(ckpt_name)) - assert len(state_dicts) == len( - self.model), f'state_dicts length: {len(state_dicts)} mismatch with model length: {len(self.model)}' - for vpp_rank, (state_dict, model) in enumerate(zip(state_dicts, self.model)): - model.load_state_dict(state_dict) - print(f'Loaded sharded model checkpoint from {model_path}') - - if 'optimizer' in self.checkpoint_contents: - self.load_optimizer(local_path) - - if 'extra' in self.checkpoint_contents: - self.load_rng_states(local_path) - - if del_local_after_load: - try: - os.remove(local_path) if is_non_local(local_path) else None - except Exception as e: - print( - f'[rank-{self.rank}]: remove local resume ckpt file after loading failed, exception {e} will be ignored' - ) - - def save_checkpoint(self, local_path: str, hdfs_path: str = None, global_step: int = 0, max_ckpt_to_keep=None): - # record the previous global step - self.previous_global_step = global_step - - # remove previous local_path - if max_ckpt_to_keep and isinstance(max_ckpt_to_keep, int) and max_ckpt_to_keep > 0 and len( - self.previous_saved_paths) >= max_ckpt_to_keep: - keep_start = len(self.previous_saved_paths) - max_ckpt_to_keep + 1 - self.remove_previous_save_local_path(self.previous_saved_paths[:keep_start]) - self.previous_saved_paths = self.previous_saved_paths[keep_start:] - - local_path = self.local_mkdir(local_path) - - # Save Model - if 'model' in self.checkpoint_contents and mpu.get_data_parallel_rank() == 0: - state_dicts = [] - - for vpp_rank, model in enumerate(self.model): - state_dict = model.state_dict() - state_dicts.append(state_dict) - - print(f'Saving sharded model checkpoint to {local_path}') - model_ckpt_path = get_model_checkpoint_path(local_path) - hf_model_ckpt_path = get_hf_model_checkpoint_path(local_path) - ckpt_name = self.get_checkpoint_name(model_ckpt_path, return_base_dir=False) - torch.save(state_dicts, os.path.join(ckpt_name)) - self.processing_class.save_pretrained(hf_model_ckpt_path) # tokenizer will be saved to hf_model_ckpt_path - print(f'Saved checkpoint to {model_ckpt_path}') - if hdfs_path is not None: - print(f'Uploading checkpoint to {hdfs_path}') - from verl.utils import hdfs_io - hdfs_io.makedirs(hdfs_path, exist_ok=True) - hdfs_io.copy(src=model_ckpt_path, dst=hdfs_path, dirs_exist_ok=True) - - if 'hf_model' in self.checkpoint_contents: - # wait for everyone to dump to local - state_dict = self.weight_saver(self.model, - self.hf_config, - dtype=self.param_dtype, - is_value_model=self.is_value_model, - tie_word_embeddings=self.share_embeddings_and_output_weights) - - torch.distributed.barrier() - print(f'self.param_dtype: {self.param_dtype}') - for key in state_dict.keys(): - print(f'state_dict[key].dtype: {key} {state_dict[key].dtype}') - torch.distributed.barrier() - if self.rank == 0: - hf_model_ckpt_path = get_hf_model_checkpoint_path(local_path) - from accelerate import init_empty_weights - import warnings - with init_empty_weights(), warnings.catch_warnings(): - warnings.simplefilter("ignore") - if 'mistral7b-rm' in self.config.model.path: - from transformers import MistralForSequenceClassification - model = MistralForSequenceClassification.from_pretrained( - self.config.model.path) # use score head instead of lm_head - state_dict['score.weight'] = state_dict['score.weight'] - else: - from transformers import AutoModelForCausalLM - model = AutoModelForCausalLM.from_pretrained(self.config.model.path, torch_dtype="auto") - model.save_pretrained(hf_model_ckpt_path, state_dict=state_dict) - if hdfs_path is not None: - print(f'Uploading checkpoint to {hdfs_path}') - from verl.utils import hdfs_io - hdfs_io.makedirs(hdfs_path, exist_ok=True) - hdfs_io.copy(src=hf_model_ckpt_path, dst=hdfs_path, dirs_exist_ok=True) - - # Save Optimizer - if 'optimizer' in self.checkpoint_contents: - torch.distributed.barrier() - - optimizer_path = get_optimizer_checkpoint_path(local_path) - self.optimizer.save_parameter_state(optimizer_path) - if self.rank == 0: - print(f"saving optimizer state to {optimizer_path}") - - # Save RNG States - if 'extra' in self.checkpoint_contents: - torch.distributed.barrier() - - rng_state_path = get_rng_states_checkpoint_path(local_path) - rng_state = self.get_rng_state() - torch.save(rng_state, rng_state_path) - if self.rank == 0: - print(f"saving rng states to {rng_state_path}") - - self.previous_saved_paths.append(local_path) diff --git a/verl/utils/config.py b/verl/utils/config.py deleted file mode 100644 index 5c9298c42a..0000000000 --- a/verl/utils/config.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict - -from omegaconf import DictConfig - - -def update_dict_with_config(dictionary: Dict, config: DictConfig): - for key in dictionary: - if hasattr(config, key): - dictionary[key] = getattr(config, key) diff --git a/verl/utils/dataset/README.md b/verl/utils/dataset/README.md deleted file mode 100644 index f886a70aab..0000000000 --- a/verl/utils/dataset/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Dataset Format -## RLHF dataset -We combine all the data sources into a single parquet files. We directly organize the prompt into the chat format so that multi-turn chats can be easily incorporated. In the prompt, we may add instruction following texts to guide the model output the answers in a particular format so that we can extract the answers. - -Math problems -```json -{ - "data_source": "openai/gsm8k", - "prompt": [{"role": "user", "content": "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May? Let's think step by step and output the final answer after \"####\""}], - "ability": "math", - "reward_model": { - "style": "rule", - "ground_truth": ["72"] - }, -} -``` diff --git a/verl/utils/dataset/__init__.py b/verl/utils/dataset/__init__.py deleted file mode 100644 index b833e441e7..0000000000 --- a/verl/utils/dataset/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .rl_dataset import RLHFDataset -from .rm_dataset import RMDataset -from .sft_dataset import SFTDataset diff --git a/verl/utils/dataset/rl_dataset.py b/verl/utils/dataset/rl_dataset.py deleted file mode 100644 index d22d3e4dae..0000000000 --- a/verl/utils/dataset/rl_dataset.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from omegaconf import ListConfig -import os -from typing import List, Union, Optional, Callable -import copy -import datasets -from collections import defaultdict - -import torch -import numpy as np -from torch.utils.data import Dataset -from transformers import PreTrainedTokenizer, ProcessorMixin -from verl.utils.model import compute_position_id_with_mask -import verl.utils.torch_functional as verl_F - - -def collate_fn(data_list: list[dict]) -> dict: - tensors = defaultdict(list) - non_tensors = defaultdict(list) - - for data in data_list: - for key, val in data.items(): - if isinstance(val, torch.Tensor): - tensors[key].append(val) - else: - non_tensors[key].append(val) - - for key, val in tensors.items(): - tensors[key] = torch.stack(val, dim=0) - - for key, val in non_tensors.items(): - non_tensors[key] = np.array(val, dtype=object) - - return {**tensors, **non_tensors} - - -def process_image(image: dict, max_pixels: int = 2048 * 2048, min_pixels: int = 512 * 512): - import math - from io import BytesIO - from PIL import Image - - if isinstance(image, dict): - image = Image.open(BytesIO(image['bytes'])) - - if (image.width * image.height) > max_pixels: - resize_factor = math.sqrt(max_pixels / (image.width * image.height)) - width, height = int(image.width * resize_factor), int(image.height * resize_factor) - image = image.resize((width, height)) - - if (image.width * image.height) < min_pixels: - resize_factor = math.sqrt(min_pixels / (image.width * image.height)) - width, height = int(image.width * resize_factor), int(image.height * resize_factor) - image = image.resize((width, height)) - - if image.mode != 'RGB': - image = image.convert('RGB') - - return image - - -class RLHFDataset(Dataset): - """ - We assume the dataset contains a column that contains prompts and other information - """ - - def __init__(self, - parquet_files: Union[str, List[str]], - tokenizer: PreTrainedTokenizer, - processor: Optional[ProcessorMixin] = None, - prompt_key: str = 'prompt', - image_key: str = 'images', - max_prompt_length: int = 1024, - filter_prompts=True, - cache_dir: str = '~/.cache/verl/rlhf', - chat_template_func: Optional[Callable] = None, - return_raw_chat: bool = False, - truncation: str = 'error', - filter_overlong_prompts: bool = False, - num_workers: Optional[int] = None): - if not isinstance(parquet_files, (List, ListConfig)): - parquet_files = [parquet_files] - - self.parquet_files = copy.deepcopy(parquet_files) - self.original_parquet_files = copy.deepcopy(parquet_files) # use for resume - self.cache_dir = os.path.expanduser(cache_dir) - self.tokenizer = tokenizer - self.processor = processor - - self.prompt_key = prompt_key - self.image_key = image_key - self.max_prompt_length = max_prompt_length - self.filter_prompts = filter_prompts - - self.return_raw_chat = return_raw_chat - self.chat_template_func = chat_template_func - self.truncation = truncation - self.filter_overlong_prompts = filter_overlong_prompts - if num_workers is None: - self.num_workers = max(1, os.cpu_count() // 4) - else: - self.num_workers = min(num_workers, os.cpu_count()) - - # whether to store the dataset in state_dict() - # default not store - self.serialize_dataset = False - self._download() - self._read_files_and_tokenize() - - def _download(self, use_origin_parquet=False): - from verl.utils.fs import copy_to_local - parquet_files = self.parquet_files if not use_origin_parquet else self.original_parquet_files - for i, parquet_file in enumerate(parquet_files): - self.parquet_files[i] = copy_to_local(src=parquet_file, cache_dir=self.cache_dir) - - def _read_files_and_tokenize(self): - dataframes = [] - for parquet_file in self.parquet_files: - # read parquet files and cache - dataframe = datasets.load_dataset("parquet", data_files=parquet_file)["train"] - dataframes.append(dataframe) - self.dataframe: datasets.Dataset = datasets.concatenate_datasets(dataframes) - - print(f'dataset len: {len(self.dataframe)}') - - # filter out too long prompts - if self.filter_overlong_prompts: - tokenizer = self.tokenizer - prompt_key = self.prompt_key - self.dataframe = self.dataframe.filter( - lambda doc: len(tokenizer.apply_chat_template(doc[prompt_key], add_generation_prompt=True) - ) <= self.max_prompt_length, - num_proc=self.num_workers, - desc=f"Filtering prompts longer than {self.max_prompt_length} tokens") - - print(f'filter dataset len: {len(self.dataframe)}') - - def resume_dataset_state(self): - self.serialize_dataset = False if hasattr(self, 'original_parquet_files') else True - # resume dataframe if not it's serialized in data.pt - if not self.serialize_dataset: - self._download(use_origin_parquet=True) # download and resume from original parquet files - self._read_files_and_tokenize() - else: - print(r'old dataloader ckpt file is used, please train from scratch for better ckpt performance') - - def __len__(self): - return len(self.dataframe) - - def __getitem__(self, item): - """ - Note that we also return the raw_input_ids so that it can be combined with other chat template - """ - row_dict: dict = self.dataframe[item] - chat = row_dict.pop(self.prompt_key) - - prompt_with_chat_template = self.tokenizer.apply_chat_template(chat, add_generation_prompt=True, tokenize=False) - - is_multi_modal = self.image_key in row_dict - if is_multi_modal: # expand image token - raw_prompt = prompt_with_chat_template.replace('', '<|vision_start|><|image_pad|><|vision_end|>') - row_dict['multi_modal_data'] = {'image': [process_image(image) for image in row_dict.pop(self.image_key)]} - image_inputs = self.processor.image_processor(row_dict['multi_modal_data']['image'], return_tensors='pt') - image_grid_thw = image_inputs['image_grid_thw'] - row_dict['multi_modal_inputs'] = {key: val for key, val in image_inputs.items()} - - if image_grid_thw is not None: - merge_length = self.processor.image_processor.merge_size**2 - index = 0 - while '' in prompt_with_chat_template: - prompt_with_chat_template = prompt_with_chat_template.replace( - '', - '<|vision_start|>' + '<|placeholder|>' * (image_grid_thw[index].prod() // merge_length) + - '<|vision_end|>', - 1, - ) - index += 1 - - prompt_with_chat_template = prompt_with_chat_template.replace('<|placeholder|>', - self.processor.image_token) - else: - raw_prompt = prompt_with_chat_template - - input_ids, attention_mask = verl_F.tokenize_and_postprocess_data(prompt=prompt_with_chat_template, - tokenizer=self.tokenizer, - max_length=self.max_prompt_length, - pad_token_id=self.tokenizer.pad_token_id, - left_pad=True, - truncation=self.truncation) - - if is_multi_modal: - from verl.models.transformers.qwen2_vl import get_rope_index - - position_ids = get_rope_index( - self.processor, - input_ids=input_ids[0], - image_grid_thw=image_grid_thw, - attention_mask=attention_mask[0], - ) # (3, seq_len) - else: - position_ids = compute_position_id_with_mask(attention_mask) - - row_dict['input_ids'] = input_ids[0] - row_dict['attention_mask'] = attention_mask[0] - row_dict['position_ids'] = position_ids[0] - row_dict['raw_prompt_ids'] = self.tokenizer.encode(raw_prompt, add_special_tokens=False) - - - - # encode prompts without chat template - if self.return_raw_chat: - row_dict['raw_prompt'] = chat - - # add index for each prompt - index = row_dict.get("extra_info", {}).get("index", 0) - row_dict["index"] = index - - return row_dict - - def __getstate__(self): - if not self.serialize_dataset: - state = self.__dict__.copy() - - if 'dataframe' in state: - del state['dataframe'] - return state - return self.__dict__.copy() diff --git a/verl/utils/dataset/rm_dataset.py b/verl/utils/dataset/rm_dataset.py deleted file mode 100644 index be137895b4..0000000000 --- a/verl/utils/dataset/rm_dataset.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from typing import List, Union - -import pandas as pd - -import torch -from torch.utils.data import Dataset -from transformers import AutoTokenizer - -from verl.utils import hf_tokenizer - - -def download_files_distributed(download_fn): - import torch.distributed - if torch.distributed.is_initialized(): - if torch.distributed.get_rank() == 0: - # download files - download_fn() - - torch.distributed.barrier() - else: - # download anyway - download_fn() - - -class RMDataset(Dataset): - - def __init__(self, - parquet_files: Union[str, List[str]], - tokenizer, - prompt_key='prompt', - chosen_key='chosen', - rejected_key='rejected', - max_length=1024, - add_eos=True, - cache_dir='~/.cache/verl/rm'): - if not isinstance(parquet_files, List): - parquet_files = [parquet_files] - - self.parquet_files = parquet_files - self.cache_dir = os.path.expanduser(cache_dir) - if isinstance(tokenizer, str): - tokenizer = hf_tokenizer(tokenizer) - self.tokenizer = tokenizer - - self.prompt_key = prompt_key - self.chosen_key = chosen_key - self.rejected_key = rejected_key - - self.add_eos = add_eos - self.max_length = max_length - - self._download() - self._read_files_and_tokenize() - - def _download(self): - - def _download_files(): - from verl.utils.fs import copy, is_non_local - os.makedirs(self.cache_dir, exist_ok=True) - assert os.path.exists(self.cache_dir) - for i, parquet_file in enumerate(self.parquet_files): - if is_non_local(parquet_file): - dst = os.path.join(self.cache_dir, os.path.basename(parquet_file)) - if not os.path.exists(dst): - copy(src=parquet_file, dst=dst) - self.parquet_files[i] = dst - - download_files_distributed(_download_files) - - def _read_files_and_tokenize(self): - dataframes = [] - for parquet_file in self.parquet_files: - # read parquet files and cache - dataframe = pd.read_parquet(parquet_file) - dataframes.append(dataframe) - self.dataframe = pd.concat(dataframes) - self.prompts = self.dataframe[self.prompt_key].tolist() - self.chosen_responses = self.dataframe[self.chosen_key].tolist() - self.rejected_responses = self.dataframe[self.rejected_key].tolist() - - def __len__(self): - return len(self.prompts) - - def _pad_to_length(self, input_ids, attention_mask): - curr_length = input_ids.shape[-1] - - if curr_length < self.max_length: - input_ids = torch.cat( - (input_ids, torch.zeros(size=(self.max_length - curr_length,), dtype=input_ids.dtype)), dim=-1) - attention_mask = torch.cat( - (attention_mask, torch.zeros(size=(self.max_length - curr_length,), dtype=attention_mask.dtype)), - dim=-1) - elif curr_length > self.max_length: - input_ids = input_ids[:self.max_length] - attention_mask = attention_mask[:self.max_length] - - return input_ids, attention_mask - - def __getitem__(self, item): - prompt = self.prompts[item] - chosen_response = self.chosen_responses[item] - rejected_response = self.rejected_responses[item] - - prompt_ids = self.tokenizer(prompt, return_tensors='pt')['input_ids'][0] - chosen_response_ids = self.tokenizer(chosen_response, return_tensors='pt')['input_ids'][0] - rejected_response_ids = self.tokenizer(rejected_response, return_tensors='pt')['input_ids'][0] - - if self.add_eos: - chosen_response_ids = torch.cat((chosen_response_ids, torch.tensor([self.tokenizer.eos_token_id])), dim=-1) - rejected_response_ids = torch.cat((rejected_response_ids, torch.tensor([self.tokenizer.eos_token_id])), - dim=-1) - - chosen_input_ids = torch.cat((prompt_ids, chosen_response_ids), dim=-1) - chosen_attention_mask = torch.ones_like(chosen_input_ids) - - rejected_input_ids = torch.cat((prompt_ids, rejected_response_ids), dim=-1) - rejected_attention_mask = torch.ones_like(rejected_input_ids) - - chosen_input_ids, chosen_attention_mask = self._pad_to_length(chosen_input_ids, chosen_attention_mask) - rejected_input_ids, rejected_attention_mask = self._pad_to_length(rejected_input_ids, rejected_attention_mask) - - input_ids = torch.stack((chosen_input_ids, rejected_input_ids), dim=0) - attention_mask = torch.stack((rejected_input_ids, rejected_attention_mask), dim=0) - - return { - 'input_ids': input_ids, - 'attention_mask': attention_mask, - } \ No newline at end of file diff --git a/verl/utils/dataset/sft_dataset.py b/verl/utils/dataset/sft_dataset.py deleted file mode 100644 index 7407239dfc..0000000000 --- a/verl/utils/dataset/sft_dataset.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -SFT dataset -- We assume user pass a single parquet file. -- We load all the data into the memory. -Each parquet file contains -""" - -from typing import List, Union - -import pandas as pd - -import torch -from torch.utils.data import Dataset -from transformers import AutoTokenizer, PreTrainedTokenizer - -from verl.utils.fs import copy_to_local -from verl.utils.model import compute_position_id_with_mask -from verl.utils import hf_tokenizer - - -class SFTDataset(Dataset): - """ - This is an in-memory SFTDataset - - Arguments: - config (OmegaConf): the data config - """ - - def __init__(self, parquet_files: Union[str, List[str]], tokenizer, config): - - prompt_key = config.get('prompt_key', 'prompt') - prompt_dict_keys = config.get('prompt_dict_keys', None) - response_key = config.get('response_key', 'response') - response_dict_keys = config.get('response_dict_keys', None) - max_length = config.get('max_length', 1024) - truncation = config.get('truncation', 'error') - - assert truncation in ['error', 'left', 'right'] - self.truncation = truncation - - if not isinstance(parquet_files, List): - parquet_files = [parquet_files] - - self.parquet_files = parquet_files - if isinstance(tokenizer, str): - tokenizer = hf_tokenizer(tokenizer) - self.tokenizer: PreTrainedTokenizer = tokenizer - - self.prompt_key = prompt_key if isinstance(prompt_key, (tuple, list)) else [prompt_key] - self.response_key = response_key if isinstance(response_key, (tuple, list)) else [response_key] - self.prompt_dict_keys = [] if not prompt_dict_keys else prompt_dict_keys - self.response_dict_keys = [] if not response_dict_keys else response_dict_keys - - self.max_length = max_length - - self._download() - self._read_files_and_tokenize() - - def _download(self): - for i, parquet_file in enumerate(self.parquet_files): - self.parquet_files[i] = copy_to_local(parquet_file, verbose=True) - - def _read_files_and_tokenize(self): - - def series_to_item(ls): - import pandas, numpy - while isinstance(ls, (pandas.core.series.Series, numpy.ndarray)) and len(ls) == 1: - ls = ls[0] - return ls - - dataframes = [] - for parquet_file in self.parquet_files: - # read parquet files and cache - dataframe = pd.read_parquet(parquet_file) - dataframes.append(dataframe) - self.dataframe = pd.concat(dataframes) - self.prompts = self.dataframe[self.prompt_key] - for key in self.prompt_dict_keys: - # type(x): pandas.core.series.Series - # type(x[0]): numpy.ndarray - # type(x[0][0]): dict - try: - self.prompts = self.prompts.apply(lambda x: series_to_item(x)[key], axis=1) - except Exception: - print(f'self.prompts={self.prompts}') - raise - self.prompts = self.prompts.tolist() - self.responses = self.dataframe[self.response_key] - for key in self.response_dict_keys: - try: - self.responses = self.responses.apply(lambda x: series_to_item(x)[key], axis=1) - except Exception: - print(f'self.responses={self.responses}') - raise - self.responses = self.responses.tolist() - - def __len__(self): - return len(self.prompts) - - def __getitem__(self, item): - tokenizer = self.tokenizer - - prompt = self.prompts[item] - response = self.responses[item] - - # apply chat template - prompt_chat = [{'role': 'user', 'content': prompt}] - - # string - prompt_chat_str = tokenizer.apply_chat_template(prompt_chat, add_generation_prompt=True, tokenize=False) - response_chat_str = response + tokenizer.eos_token - - # tokenize - prompt_ids_output = tokenizer(prompt_chat_str, return_tensors='pt', add_special_tokens=False) - prompt_ids = prompt_ids_output['input_ids'][0] - prompt_attention_mask = prompt_ids_output['attention_mask'][0] - - response_ids_output = tokenizer(response_chat_str, return_tensors='pt', add_special_tokens=False) - response_ids = response_ids_output['input_ids'][0] - response_attention_mask = response_ids_output['attention_mask'][0] - - prompt_length = prompt_ids.shape[0] - response_length = response_ids.shape[0] - - input_ids = torch.cat((prompt_ids, response_ids), dim=-1) - attention_mask = torch.cat((prompt_attention_mask, response_attention_mask), dim=-1) - - # padding to max length - sequence_length = input_ids.shape[0] - if sequence_length < self.max_length: - padded_input_ids = torch.ones(size=(self.max_length - sequence_length,), - dtype=input_ids.dtype) * self.tokenizer.pad_token_id - padded_attention_mask = torch.zeros(size=(self.max_length - sequence_length,), dtype=attention_mask.dtype) - - input_ids = torch.cat((input_ids, padded_input_ids)) - attention_mask = torch.cat((attention_mask, padded_attention_mask)) - elif sequence_length > self.max_length: - if self.truncation == 'left': - # actually, left truncation may not be reasonable - input_ids = input_ids[-self.max_length:] - attention_mask = attention_mask[-self.max_length:] - elif self.truncation == 'right': - input_ids = input_ids[:self.max_length] - attention_mask = attention_mask[:self.max_length] - elif self.truncation == 'error': - raise NotImplementedError(f'{sequence_length=} is larger than {self.max_length=}') - else: - raise NotImplementedError(f'Unknown truncation method {self.truncation}') - - position_ids = compute_position_id_with_mask(attention_mask) - - loss_mask = attention_mask.clone() - if prompt_length > 1: - # mask out prompt for SFT. - loss_mask[:min(prompt_length, loss_mask.size(0)) - 1] = 0 - # mask out the last token in response - loss_mask[min(prompt_length + response_length, loss_mask.size(0)) - 1] = 0 - - return { - 'input_ids': input_ids, - 'attention_mask': attention_mask, - 'position_ids': position_ids, - 'loss_mask': loss_mask - } diff --git a/verl/utils/debug/__init__.py b/verl/utils/debug/__init__.py deleted file mode 100644 index 0d0b3432eb..0000000000 --- a/verl/utils/debug/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .performance import log_gpu_memory_usage \ No newline at end of file diff --git a/verl/utils/debug/performance.py b/verl/utils/debug/performance.py deleted file mode 100644 index 096a13c5ae..0000000000 --- a/verl/utils/debug/performance.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.distributed as dist -import logging - - -def log_gpu_memory_usage(head: str, logger: logging.Logger = None, level=logging.INFO, rank: int = 0): - if (not dist.is_initialized()) or (rank is None) or (dist.get_rank() == rank): - memory_allocated = torch.cuda.memory_allocated() / 1024**3 - memory_reserved = torch.cuda.memory_reserved() / 1024**3 - free, total = torch.cuda.mem_get_info() - - message = f'{head}, current proc memory allocated (GB): {memory_allocated}, memory reserved (GB): {memory_reserved}; global memory free (GB): {free / 1024**3}, total (GB): {total / 1024**3}' - - if logger is None: - print(message) - else: - logger.log(msg=message, level=level) diff --git a/verl/utils/debug/trajectory_tracker.py b/verl/utils/debug/trajectory_tracker.py deleted file mode 100644 index 33b2546852..0000000000 --- a/verl/utils/debug/trajectory_tracker.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Trajectory tracker can be inserted into code to save the intermediate results. -The results will be dump to hdfs for offline comparison. -Each process will have a client that first move all the tensors to CPU -""" - -from verl.utils.hdfs_io import makedirs, copy -import torch -import os -import ray -import io -import tempfile - -from collections import deque - -remote_copy = ray.remote(copy) - - -@ray.remote -def save_to_hdfs(data: io.BytesIO, name, hdfs_dir, verbose): - filename = name + '.pth' - with tempfile.TemporaryDirectory() as tmpdirname: - local_filepath = os.path.join(tmpdirname, filename) - with open(local_filepath, 'wb') as f: - f.write(data.getbuffer()) - # upload to hdfs - - if verbose: - print(f'Saving {local_filepath} to {hdfs_dir}') - try: - copy(local_filepath, hdfs_dir) - except Exception as e: - print(e) - - -@ray.remote -class TrajectoryTracker(): - - def __init__(self, hdfs_dir, verbose) -> None: - self.hdfs_dir = hdfs_dir - makedirs(hdfs_dir) - self.verbose = verbose - - self.handle = deque() - - def dump(self, data: io.BytesIO, name): - # get a temp file and write to it - self.handle.append(save_to_hdfs.remote(data, name, self.hdfs_dir, self.verbose)) - - def wait_for_hdfs(self): - while len(self.handle) != 0: - future = self.handle.popleft() - ray.get(future) - - -def dump_data(data, name): - enable = os.getenv('VERL_ENABLE_TRACKER', '0') == '1' - if not enable: - return - buffer = io.BytesIO() - torch.save(data, buffer) - tracker = get_trajectory_tracker() - ray.get(tracker.dump.remote(buffer, name)) - - -def get_trajectory_tracker(): - hdfs_dir = os.getenv('VERL_TRACKER_HDFS_DIR', default=None) - verbose = os.getenv('VERL_TRACKER_VERBOSE', default='0') == '1' - assert hdfs_dir is not None - tracker = TrajectoryTracker.options(name="global_tracker", get_if_exists=True, - lifetime="detached").remote(hdfs_dir, verbose) - return tracker - - -if __name__ == '__main__': - # testing - os.environ['VERL_ENABLE_TRACKER'] = '1' - os.environ['VERL_TRACKER_HDFS_DIR'] = '~/debug/test' - - @ray.remote - def process(iter): - data = {'obs': torch.randn(10, 20)} - dump_data(data, f'process_{iter}_obs') - - ray.init() - - output_lst = [] - - for i in range(10): - output_lst.append(process.remote(i)) - - out = ray.get(output_lst) - - tracker = get_trajectory_tracker() - ray.get(tracker.wait_for_hdfs.remote()) diff --git a/verl/utils/distributed.py b/verl/utils/distributed.py deleted file mode 100644 index 6fea5a29cd..0000000000 --- a/verl/utils/distributed.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for distributed training.""" -import os - - -def initialize_global_process_group(timeout_second=36000): - import torch.distributed - from datetime import timedelta - torch.distributed.init_process_group('nccl', timeout=timedelta(seconds=timeout_second)) - local_rank = int(os.environ["LOCAL_RANK"]) - rank = int(os.environ["RANK"]) - world_size = int(os.environ["WORLD_SIZE"]) - - if torch.distributed.is_initialized(): - torch.cuda.set_device(local_rank) - return local_rank, rank, world_size diff --git a/verl/utils/flops_counter.py b/verl/utils/flops_counter.py deleted file mode 100644 index 9bcebc8518..0000000000 --- a/verl/utils/flops_counter.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -from transformers import PretrainedConfig - -VALID_CONFIG_TYPE = {"llama", "qwen2", "qwen2_vl", "qwen2_5_vl", "deepseek_v3"} - - -def get_device_flops(unit="T"): - - def unit_convert(number, level): - units = ["B", "K", "M", "G", "T", "P"] - if number <= 0: - return number - ptr = 0 - while ptr < len(units) and units[ptr] != level: - number /= 1000 - ptr += 1 - return number - - device_name = torch.cuda.get_device_name() - flops = float("inf") # INF flops for unkown gpu type - - if "MI300X" in device_name: - flops = 1336e12 - elif "H100" in device_name or "H800" in device_name: - flops = 989e12 - elif "A100" in device_name or "A800" in device_name: - flops = 312e12 - elif "L40" in device_name: - flops = 181.05e12 - elif "L20" in device_name: - flops = 119.5e12 - elif "H20" in device_name: - flops = 148e12 - elif "910B" in device_name: - flops = 354e12 - flops_unit = unit_convert(flops, unit) - return flops_unit - - -class FlopsCounter: - """ - Used to count mfu during training loop - - Example: - flops_counter = FlopsCounter(config) - flops_achieved, flops_promised = flops_counter.estimate_flops(tokens_list, delta_time) - - """ - - def __init__(self, config: PretrainedConfig): - if not config.model_type in VALID_CONFIG_TYPE: - print(f"Only support config type of {VALID_CONFIG_TYPE}, but got {config.model_type}. " - f"MFU will always be zero.") - - self.estimate_func = { - 'qwen2': self._estimate_qwen2_flops, - 'llama': self._estimate_qwen2_flops, - 'qwen2_vl': self._estimate_qwen2_flops, - 'qwen2_5_vl': self._estimate_qwen2_flops, - 'deepseek_v3': self._estimate_deepseek_v3_flops, - } - self.config = config - - def _estimate_unknown_flops(self, tokens_sum, batch_seqlens, delta_time): - return 0 - - def _estimate_qwen2_flops(self, tokens_sum, batch_seqlens, delta_time): - hidden_size = self.config.hidden_size - vocab_size = self.config.vocab_size - num_hidden_layers = self.config.num_hidden_layers - num_key_value_heads = self.config.num_key_value_heads - num_attention_heads = self.config.num_attention_heads - intermediate_size = self.config.intermediate_size - - head_dim = hidden_size // num_attention_heads - q_size = num_attention_heads * head_dim - k_size = num_key_value_heads * head_dim - v_size = num_key_value_heads * head_dim - - # non-attn per layer parm - # Qwen2/LLama use SwiGelu, gate, having up and down linear layer in mlp - mlp_N = hidden_size * intermediate_size * 3 - attn_linear_N = hidden_size * (q_size + k_size + v_size + num_attention_heads * head_dim) - emd_and_lm_head_N = vocab_size * hidden_size * 2 - # non-attn all_layer parm - dense_N = (mlp_N + attn_linear_N) * num_hidden_layers + emd_and_lm_head_N - # non-attn all_layer & all_token fwd & bwd flops - dense_N_flops = 6 * dense_N * tokens_sum - - # attn all_layer & all_token fwd & bwd flops - seqlen_square_sum = 0 - for seqlen in batch_seqlens: - seqlen_square_sum += seqlen * seqlen - attn_qkv_flops = 12 * seqlen_square_sum * head_dim * num_attention_heads * num_hidden_layers - - # all_layer & all_token fwd & bwd flops - flops_all_token = dense_N_flops + attn_qkv_flops - flops_achieved = flops_all_token * (1.0 / delta_time) / 1e12 - return flops_achieved - - def _estimate_deepseek_v3_flops(self, tokens_sum, batch_seqlens, delta_time): - hidden_size = self.config.hidden_size - vocab_size = self.config.vocab_size - moe_intermediate_size = self.config.moe_intermediate_size - num_hidden_layers = self.config.num_hidden_layers - first_k_dense_replace = self.config.first_k_dense_replace - num_query_heads = self.config.num_attention_heads - moe_num_expert = self.config.n_routed_experts - - moe_topk = self.config.num_experts_per_tok - share_expert_num = self.config.n_shared_experts - - # non-attn per layer parm - moe_gata_N = hidden_size * moe_num_expert - # moe has fc1_1, fc1_2 and fc2 using SwiGLU in ExpertMlp layer & shared experts - moe_expertmlp_N = hidden_size * moe_intermediate_size * (moe_topk + share_expert_num) * 3 - # MLA attn - attn_linear_N = 0 - q_head_dim = self.config.qk_nope_head_dim + self.config.qk_rope_head_dim - if self.config.q_lora_rank is None: - attn_linear_N += hidden_size * num_query_heads * q_head_dim - else: - attn_linear_N += hidden_size * self.config.q_lora_rank - attn_linear_N += num_query_heads * q_head_dim * self.config.q_lora_rank - - attn_linear_N += hidden_size * (self.config.kv_lora_rank + self.config.qk_rope_head_dim) - attn_linear_N += (num_query_heads * (q_head_dim - self.config.qk_rope_head_dim + self.config.v_head_dim) * - self.config.kv_lora_rank) - attn_linear_N += num_query_heads * self.config.v_head_dim * hidden_size - emd_and_lm_head_N = vocab_size * hidden_size * 2 - # non-attn all_layer parm - moe_N = ((moe_gata_N + moe_expertmlp_N + attn_linear_N) * (num_hidden_layers - first_k_dense_replace) + - (hidden_size * self.config.intermediate_size * 3 + attn_linear_N) * first_k_dense_replace + - emd_and_lm_head_N) - # non-attn all_layer & all_token fwd & bwd flops - dense_N_flops = 6 * moe_N * tokens_sum - - # attn all_layer & all_token fwd & bwd flops - seqlen_square_sum = 0 - for seqlen in batch_seqlens: - seqlen_square_sum += seqlen * seqlen * num_hidden_layers - - attn_qkv_flops = 12 * seqlen_square_sum * q_head_dim * num_query_heads - # all_layer & all_token fwd & bwk flops - flops_all_token = dense_N_flops + attn_qkv_flops - flops_achieved = flops_all_token * (1.0 / delta_time) / 1e12 - - return flops_achieved - - def estimate_flops(self, batch_seqlens, delta_time): - """ - Estimate the FLOPS based on the number of valid tokens in the current batch and the time taken. - - Args: - batch_seqlens (List[int]): A list where each element represents the number of valid tokens in the current batch. - delta_time (float): The time taken to process the batch, in seconds. - - Returns: - estimated_flops (float): The estimated FLOPS based on the input tokens and time. - promised_flops (float): The expected FLOPS of the current device. - """ - tokens_sum = sum(batch_seqlens) - func = self.estimate_func.get(self.config.model_type, self._estimate_unknown_flops) - estimated_flops = func(tokens_sum, batch_seqlens, delta_time) - promised_flops = get_device_flops() - return estimated_flops, promised_flops diff --git a/verl/utils/fs.py b/verl/utils/fs.py deleted file mode 100644 index 769b7e48e0..0000000000 --- a/verl/utils/fs.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -*- coding: utf-8 -*- -"""File-system agnostic IO APIs""" -import os -import tempfile -import hashlib - -try: - from hdfs_io import copy, makedirs, exists # for internal use only -except ImportError: - from .hdfs_io import copy, makedirs, exists - -__all__ = ["copy", "exists", "makedirs"] - -_HDFS_PREFIX = "hdfs://" - - -def is_non_local(path): - return path.startswith(_HDFS_PREFIX) - - -def md5_encode(path: str) -> str: - return hashlib.md5(path.encode()).hexdigest() - - -def get_local_temp_path(hdfs_path: str, cache_dir: str) -> str: - """Return a local temp path that joins cache_dir and basename of hdfs_path - - Args: - hdfs_path: - cache_dir: - - Returns: - - """ - # make a base64 encoding of hdfs_path to avoid directory conflict - encoded_hdfs_path = md5_encode(hdfs_path) - temp_dir = os.path.join(cache_dir, encoded_hdfs_path) - os.makedirs(temp_dir, exist_ok=True) - dst = os.path.join(temp_dir, os.path.basename(hdfs_path)) - return dst - - -def copy_to_local(src: str, cache_dir=None, filelock='.file.lock', verbose=False) -> str: - """Copy src from hdfs to local if src is on hdfs or directly return src. - If cache_dir is None, we will use the default cache dir of the system. Note that this may cause conflicts if - the src name is the same between calls - - Args: - src (str): a HDFS path of a local path - - Returns: - a local path of the copied file - """ - return copy_local_path_from_hdfs(src, cache_dir, filelock, verbose) - - -def copy_local_path_from_hdfs(src: str, cache_dir=None, filelock='.file.lock', verbose=False) -> str: - """Deprecated. Please use copy_to_local instead.""" - from filelock import FileLock - - assert src[-1] != '/', f'Make sure the last char in src is not / because it will cause error. Got {src}' - - if is_non_local(src): - # download from hdfs to local - if cache_dir is None: - # get a temp folder - cache_dir = tempfile.gettempdir() - os.makedirs(cache_dir, exist_ok=True) - assert os.path.exists(cache_dir) - local_path = get_local_temp_path(src, cache_dir) - # get a specific lock - filelock = md5_encode(src) + '.lock' - lock_file = os.path.join(cache_dir, filelock) - with FileLock(lock_file=lock_file): - if not os.path.exists(local_path): - if verbose: - print(f'Copy from {src} to {local_path}') - copy(src, local_path) - return local_path - else: - return src diff --git a/verl/utils/fsdp_utils.py b/verl/utils/fsdp_utils.py deleted file mode 100644 index b3f5b73534..0000000000 --- a/verl/utils/fsdp_utils.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict -import functools -import json -import math -import itertools -import os -from contextlib import contextmanager -from torch.distributed import DeviceMesh -from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy, transformer_auto_wrap_policy -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP -from torch.distributed.fsdp._runtime_utils import _lazy_init -from transformers.trainer_pt_utils import get_module_class_from_name -import torch -import torch.nn as nn -import torch.distributed as dist - - -def init_fn(x: torch.nn.Module): - if not torch.distributed.get_rank() == 0: - x = x.to_empty(device=torch.cuda.current_device(), recurse=False) - torch.cuda.empty_cache() - return x - - -def get_init_weight_context_manager(use_meta_tensor=True, mesh: DeviceMesh = None): - from accelerate import init_empty_weights - cpu_init_weights = lambda: torch.device('cpu') - if use_meta_tensor: - if mesh is None: - init_context = init_empty_weights if torch.distributed.get_rank() != 0 else cpu_init_weights - else: - init_context = init_empty_weights if mesh.get_coordinate()[-1] != 0 else cpu_init_weights - else: - init_context = cpu_init_weights - return init_context - - -# Copyright 2020-present the HuggingFace Inc. team. -# Adapted from https://github.com/huggingface/transformers/src/transformers/trainer.py -def get_fsdp_wrap_policy(module, config=None, is_lora=False): - """Get FSDP wrap policy for the module. - - Args: - module: The module to get wrap policy for - config: Configuration for wrap policy - is_lora: Whether to enable lambda policy for LoRA modules - """ - if config is None: - config = {} - - if config.get('disable', False): - return None - - default_transformer_cls_names_to_wrap = getattr(module, "_no_split_modules", None) - fsdp_transformer_layer_cls_to_wrap = config.get("transformer_layer_cls_to_wrap", - default_transformer_cls_names_to_wrap) - min_num_params = config.get('min_num_params', 0) - auto_wrap_policy = None - - policies = [] - - from torch.distributed.fsdp.wrap import _or_policy, lambda_auto_wrap_policy, transformer_auto_wrap_policy - - # Add lambda policy for LoRA modules if is_lora is True - if is_lora: - - def lambda_policy_fn(module): - if (len(list(module.named_children())) == 0 and getattr(module, "weight", None) is not None and - module.weight.requires_grad): - return True - return False - - lambda_policy = functools.partial(lambda_auto_wrap_policy, lambda_fn=lambda_policy_fn) - policies.append(lambda_policy) - - if min_num_params > 0: - size_policy = functools.partial(size_based_auto_wrap_policy, min_num_params=min_num_params) - policies.append(size_policy) - elif fsdp_transformer_layer_cls_to_wrap is not None: - transformer_cls_to_wrap = set() - for layer_class in fsdp_transformer_layer_cls_to_wrap: - transformer_cls = get_module_class_from_name(module, layer_class) - if transformer_cls is None: - raise Exception("Could not find the transformer layer class to wrap in the model.") - else: - transformer_cls_to_wrap.add(transformer_cls) - - transformer_policy = functools.partial( - transformer_auto_wrap_policy, - transformer_layer_cls=transformer_cls_to_wrap, - ) - policies.append(transformer_policy) - - if len(policies) > 0: - auto_wrap_policy = functools.partial(_or_policy, policies=policies) - - return auto_wrap_policy - - -@torch.no_grad() -def offload_fsdp_model_to_cpu(model: FSDP, empty_cache: bool = True): - assert isinstance(model, FSDP) - # lazy init FSDP model - _lazy_init(model, model) - assert model._is_root, f"Only support root model offloading to CPU" - for handle in model._all_handles: - if handle._offload_params: - continue - flat_param = handle.flat_param - assert flat_param.data.data_ptr() == flat_param._local_shard.data_ptr() and \ - id(flat_param.data) != id(flat_param._local_shard) and \ - flat_param.data.size() == flat_param._local_shard.size() - handle.flat_param_to(torch.device("cpu"), non_blocking=True) - # the following still keeps id(._local_shard) != id(.data) - flat_param._local_shard = flat_param.data - assert id(flat_param._local_shard) != id(flat_param.data) - if empty_cache: - torch.cuda.empty_cache() - - -@torch.no_grad() -def load_fsdp_model_to_gpu(model: FSDP): - assert isinstance(model, FSDP) - # lazy init FSDP model - _lazy_init(model, model) - assert model._is_root, f"Only support root model loading to GPU" - device_id = torch.cuda.current_device() - for handle in model._all_handles: - if handle._offload_params: - continue - flat_param = handle.flat_param - handle.flat_param_to(torch.device(f"cuda:{device_id}"), non_blocking=True) - # the following still keeps id(._local_shard) != id(.data) - flat_param._local_shard = flat_param.data - - -@torch.no_grad() -def offload_fsdp_optimizer(optimizer): - if not optimizer.state: - return - for param_group in optimizer.param_groups: - for param in param_group['params']: - state = optimizer.state[param] - for key, value in state.items(): - if isinstance(value, torch.Tensor): - state[key] = value.to("cpu", non_blocking=True) - - -@torch.no_grad() -def load_fsdp_optimizer(optimizer, device_id): - if not optimizer.state: - return - for param_group in optimizer.param_groups: - for param in param_group['params']: - state = optimizer.state[param] - for key, value in state.items(): - if isinstance(value, torch.Tensor): - state[key] = value.to(device_id, non_blocking=True) - - -@contextmanager -def meta_device_init(): - """ - Create model parameters with meta device. - - Note buffers in model will still be initialized in default device (e.g., CPU), - since the buffers can be non-persistent and filled with expected values that can - NOT be captured in meta device. - """ - device = torch.device("meta") - old_register_parameter = nn.Module.register_parameter - registered = set() - - def register_empty_parameter(module, name, param): - old_register_parameter(module, name, param) - # we will skip register shared parameters as it - # is already registered previously - if param is not None and param not in registered: - param_cls = type(module._parameters[name]) - kwargs = module._parameters[name].__dict__ - kwargs["requires_grad"] = param.requires_grad - module._parameters[name] = param_cls(module._parameters[name].to(device), **kwargs) - registered.add(module._parameters[name]) - - try: - nn.Module.register_parameter = register_empty_parameter - yield - finally: - registered.clear() - nn.Module.register_parameter = old_register_parameter - - -def parallel_load_safetensors(filepath): - """ - Parallel load safetensors from huggingface checkpoint - - Huggingface checkpoint contains: - - - config.json: a json file for model configuration - - model.safetensor.index.json: a json file for safetensors (parameters & buffers) index - - model-000x-of-ooxx.safetensors: a binary file for safetensors (parameters & buffers) chunks - - Or (when model is small), - - - model.safetensors: a binary file for all parameters and buffers - - Each rank will own a part of model chunks and load them directly into GPU memory. - """ - from safetensors.torch import load_file - - safetensors2param = {} - - index_file = os.path.join(filepath, "model.safetensors.index.json") - if os.path.exists(index_file): - index = json.load(open(index_file, "rb")) - for param_name, filename in index["weight_map"].items(): - safetensors2param.setdefault(filename, []).append(param_name) - else: - # in this case, the model is small and we can load it all at once - param_file = os.path.join(filepath, "model.safetensors") - assert os.path.exists(param_file), f"Cannot find {param_file}" - states = load_file(param_file) - for param_name in states: - safetensors2param.setdefault("model.safetensors", []).append(param_name) - del states - - total_files = len(safetensors2param) - ckpt_chunks = sorted(safetensors2param.keys()) - world_size = dist.get_world_size() - size = int(math.ceil(total_files / world_size)) - ckpt_chunks = [ckpt_chunks[rank * size:rank * size + size] for rank in range(world_size)] - - shard_states = {} - device = torch.cuda.current_device() - for rank, files in enumerate(ckpt_chunks): - if rank == dist.get_rank(): - for file in files: - file = os.path.join(filepath, file) - states = load_file(file, device=device) - # print(f"rank {rank} loading {file}...") - shard_states.update(states) - else: - for file in files: - for param_name in safetensors2param[file]: - shard_states[param_name] = rank - return shard_states - - -def parallel_init_module_fn(module: torch.nn.Module, shard_states: Dict[str, torch.nn.Parameter]): - """ - Generate a function to initialize sub-modules in the `module` with `shard_states` - from huggingface checkpoint. - - Args: - module (torch.nn.Module): the global module to be initialized - shard_states (Dict[str, torch.nn.Parameter]): the shard states from huggingface checkpoint - - Returns: - init_fn (Callable): a function to initialize sub-modules in the `module` with `shard_states` - """ - - state2fqn = {} - for name, state in itertools.chain(module.named_parameters(remove_duplicate=False), - module.named_buffers(remove_duplicate=False)): - state2fqn.setdefault(state, []).append(name) - # remove standalone parameters and buffers - shared = {s for s, names in state2fqn.items() if len(names) > 1} - materialized_states = {} - - @torch.no_grad() - def create_and_sync_state(param_name, state, is_param): - assert param_name in shard_states, f"{param_name} not loaded" - device = torch.cuda.current_device() - if is_param: - param = torch.nn.Parameter(torch.empty_like(state.data, device=device), requires_grad=state.requires_grad) - else: # buffer - param = torch.empty_like(state.data, device=device) - loaded = shard_states[param_name] - if isinstance(loaded, (torch.nn.Parameter, torch.Tensor)): - # NOTE: loaded.dtype can be different with param.dtype - param.data.copy_(loaded.data) - dist.broadcast(param.data, src=dist.get_rank()) - else: - assert isinstance(loaded, int) # the rank that holds the state - dist.broadcast(param.data, src=loaded) - shard_states.pop(param_name) - del loaded - return param - - def init_fn(sub_mod: torch.nn.Module, recurse: bool = True): - param_and_buffers = tuple(sub_mod.named_parameters(recurse=False)) + tuple(sub_mod.named_buffers(recurse=False)) - # param_and_buffers = sorted(sub_mod.named_parameters(recurse=False), key=lambda x: x[0]) - for name, state in param_and_buffers: - if not state.is_meta: - continue - is_param = name in sub_mod._parameters - fqn = state2fqn[state].pop(0) - # non-persistent buffers will not be saved in state dict, we can safely skip it - if (not is_param) and fqn not in shard_states: - if state.is_meta: - raise RuntimeError( - f"find a non-persistent buffer ({fqn}) initiated with device meta. " - "Such buffer is not saved in checkpoint and user should guarantee to init in CPU / GPU device.") - continue - # for shared parameter, we get it from the first time it is created - if state in shared: - if state not in materialized_states: - materialized_states[state] = create_and_sync_state(fqn, state, is_param) - else: - if fqn in shard_states: - shard_states.pop(fqn) - materialize_state = materialized_states[state] - # for not shared parameter, we create it directly - else: - materialize_state = create_and_sync_state(fqn, state, is_param) - if is_param: - sub_mod._parameters[name] = materialize_state - else: - sub_mod._buffers[name] = materialize_state - if recurse: - for module in sub_mod.children(): - init_fn(module, recurse=True) - - # for debug - # if len(shard_states) == 0: print("clear") - return sub_mod - - return init_fn diff --git a/verl/utils/hdfs_io.py b/verl/utils/hdfs_io.py deleted file mode 100644 index 08c4ecb9a5..0000000000 --- a/verl/utils/hdfs_io.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import shutil -import logging - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_SFT_LOGGING_LEVEL', 'WARN')) - -_HDFS_PREFIX = "hdfs://" - -_HDFS_BIN_PATH = shutil.which('hdfs') - - -def exists(path: str, **kwargs) -> bool: - r"""Works like os.path.exists() but supports hdfs. - - Test whether a path exists. Returns False for broken symbolic links. - - Args: - path (str): path to test - - Returns: - bool: True if the path exists, False otherwise - """ - if _is_non_local(path): - return _exists(path, **kwargs) - return os.path.exists(path) - - -def _exists(file_path: str): - """ hdfs capable to check whether a file_path is exists """ - if file_path.startswith("hdfs"): - return _run_cmd(_hdfs_cmd(f"-test -e {file_path}")) == 0 - return os.path.exists(file_path) - - -def makedirs(name, mode=0o777, exist_ok=False, **kwargs) -> None: - r"""Works like os.makedirs() but supports hdfs. - - Super-mkdir; create a leaf directory and all intermediate ones. Works like - mkdir, except that any intermediate path segment (not just the rightmost) - will be created if it does not exist. If the target directory already - exists, raise an OSError if exist_ok is False. Otherwise no exception is - raised. This is recursive. - - Args: - name (str): directory to create - mode (int): file mode bits - exist_ok (bool): if True, do not raise an exception if the directory already exists - kwargs: keyword arguments for hdfs - - """ - if _is_non_local(name): - # TODO(haibin.lin): - # - handle OSError for hdfs(?) - # - support exist_ok for hdfs(?) - _mkdir(name, **kwargs) - else: - os.makedirs(name, mode=mode, exist_ok=exist_ok) - - -def _mkdir(file_path: str) -> bool: - """hdfs mkdir""" - if file_path.startswith("hdfs"): - _run_cmd(_hdfs_cmd(f"-mkdir -p {file_path}")) - else: - os.makedirs(file_path, exist_ok=True) - return True - - -def copy(src: str, dst: str, **kwargs) -> bool: - r"""Works like shutil.copy() for file, and shutil.copytree for dir, and supports hdfs. - - Copy data and mode bits ("cp src dst"). Return the file's destination. - The destination may be a directory. - If source and destination are the same file, a SameFileError will be - raised. - - Arg: - src (str): source file path - dst (str): destination file path - kwargs: keyword arguments for hdfs copy - - Returns: - str: destination file path - - """ - if _is_non_local(src) or _is_non_local(dst): - # TODO(haibin.lin): - # - handle SameFileError for hdfs files(?) - # - return file destination for hdfs files - return _copy(src, dst) - else: - if os.path.isdir(src): - return shutil.copytree(src, dst, **kwargs) - else: - return shutil.copy(src, dst, **kwargs) - - -def _copy(from_path: str, to_path: str, timeout: int = None) -> bool: - if to_path.startswith("hdfs"): - if from_path.startswith("hdfs"): - returncode = _run_cmd(_hdfs_cmd(f"-cp -f {from_path} {to_path}"), timeout=timeout) - else: - returncode = _run_cmd(_hdfs_cmd(f"-put -f {from_path} {to_path}"), timeout=timeout) - else: - if from_path.startswith("hdfs"): - returncode = _run_cmd(_hdfs_cmd(f"-get \ - {from_path} {to_path}"), timeout=timeout) - else: - try: - shutil.copy(from_path, to_path) - returncode = 0 - except shutil.SameFileError: - returncode = 0 - except Exception as e: - logger.warning(f"copy {from_path} {to_path} failed: {e}") - returncode = -1 - return returncode == 0 - - -def _run_cmd(cmd: str, timeout=None): - return os.system(cmd) - - -def _hdfs_cmd(cmd: str) -> str: - return f"{_HDFS_BIN_PATH} dfs {cmd}" - - -def _is_non_local(path: str): - return path.startswith(_HDFS_PREFIX) diff --git a/verl/utils/import_utils.py b/verl/utils/import_utils.py deleted file mode 100644 index bd82e64e66..0000000000 --- a/verl/utils/import_utils.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Utilities to check if packages are available. -We assume package availability won't change during runtime. -""" - -from functools import cache -from typing import List, Optional - - -@cache -def is_megatron_core_available(): - try: - from megatron.core import parallel_state as mpu - return True - except ImportError: - return False - - -@cache -def is_vllm_available(): - try: - import vllm - from verl.third_party.vllm import parallel_state as vllm_ps - return True - except (ImportError, ValueError): - return False - - -@cache -def is_sglang_available(): - try: - import sglang - return True - except ImportError: - return False - - -def import_external_libs(external_libs=None): - if external_libs is None: - return - if not isinstance(external_libs, List): - external_libs = [external_libs] - import importlib - for external_lib in external_libs: - importlib.import_module(external_lib) - - -def load_extern_type(file_path: Optional[str], type_name: Optional[str]): - """Load a external data type based on the file path and type name""" - import importlib.util, os - - if not file_path: - return None - - if not os.path.exists(file_path): - raise FileNotFoundError(f"Custom type file '{file_path}' not found.") - - spec = importlib.util.spec_from_file_location("custom_module", file_path) - module = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(module) - except Exception as e: - raise RuntimeError(f"Error loading module from '{file_path}': {e}") - - if not hasattr(module, type_name): - raise AttributeError(f"Custom type '{type_name}' not found in '{file_path}'.") - - return getattr(module, type_name) \ No newline at end of file diff --git a/verl/utils/logger/__init__.py b/verl/utils/logger/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/utils/logger/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/utils/logger/aggregate_logger.py b/verl/utils/logger/aggregate_logger.py deleted file mode 100644 index b7712c598f..0000000000 --- a/verl/utils/logger/aggregate_logger.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -A Ray logger will receive logging info from different processes. -""" -import numbers -from typing import Dict - - -def concat_dict_to_str(dict: Dict, step): - output = [f'step:{step}'] - for k, v in dict.items(): - if isinstance(v, numbers.Number): - output.append(f'{k}:{v:.6f}') - output_str = ' - '.join(output) - return output_str - - -class LocalLogger: - - def __init__(self, remote_logger=None, enable_wandb=False, print_to_console=False): - self.print_to_console = print_to_console - if print_to_console: - print('Using LocalLogger is deprecated. The constructor API will change ') - - def flush(self): - pass - - def log(self, data, step): - if self.print_to_console: - print(concat_dict_to_str(data, step=step), flush=True) \ No newline at end of file diff --git a/verl/utils/logging_utils.py b/verl/utils/logging_utils.py deleted file mode 100644 index 3298353472..0000000000 --- a/verl/utils/logging_utils.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import torch - - -def set_basic_config(level): - """ - This function sets the global logging format and level. It will be called when import verl - """ - logging.basicConfig(format='%(levelname)s:%(asctime)s:%(message)s', level=level) - - -def log_to_file(string): - print(string) - if os.path.isdir('logs'): - with open(f'logs/log_{torch.distributed.get_rank()}', 'a+') as f: - f.write(string + '\n') diff --git a/verl/utils/megatron/__init__.py b/verl/utils/megatron/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/utils/megatron/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/utils/megatron/memory.py b/verl/utils/megatron/memory.py deleted file mode 100644 index 5e8570ed49..0000000000 --- a/verl/utils/megatron/memory.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch - - -class MemoryBuffer: - - def __init__(self, numel, numel_padded, dtype): - self.numel = numel - self.numel_padded = numel_padded - self.dtype = dtype - self.data = torch.zeros(self.numel_padded, - dtype=self.dtype, - device=torch.cuda.current_device(), - requires_grad=False) - - def zero(self): - """Reset the buffer to zero.""" - self.data.zero_() - - def get(self, shape, start_index): - """Return a tensor with the input `shape` as a view into the - 1-D data starting at `start_index`.""" - end_index = start_index + shape.numel() - assert end_index <= self.numel, \ - 'requested tensor is out of the buffer range.' - buffer_tensor = self.data[start_index:end_index] - buffer_tensor = buffer_tensor.view(shape) - return buffer_tensor diff --git a/verl/utils/megatron/optimizer.py b/verl/utils/megatron/optimizer.py deleted file mode 100644 index 1d936d9ed5..0000000000 --- a/verl/utils/megatron/optimizer.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -from packaging.version import Version - -from apex.optimizers import FusedAdam as Adam -from apex.optimizers import FusedSGD as SGD - -from megatron.core.optimizer import OptimizerConfig - -from megatron.core.optimizer import get_megatron_optimizer as get_megatron_optimizer_native - - -def get_megatron_optimizer( - model, - config: OptimizerConfig, - no_weight_decay_cond=None, - scale_lr_cond=None, - lr_mult=1.0, - check_for_nan_in_loss_and_grad=False, - overlap_param_gather=False # add for verl -): - # Base optimizer. - return get_megatron_optimizer_native(config=config, - model_chunks=model, - no_weight_decay_cond=no_weight_decay_cond, - scale_lr_cond=scale_lr_cond, - lr_mult=lr_mult) diff --git a/verl/utils/megatron/pipeline_parallel.py b/verl/utils/megatron/pipeline_parallel.py deleted file mode 100644 index 3a3790bb1a..0000000000 --- a/verl/utils/megatron/pipeline_parallel.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -from megatron.core import parallel_state as mpu - -from .sequence_parallel import pad_to_sequence_parallel - - -def compute_transformers_input_shapes(batches, meta_info): - from flash_attn.bert_padding import unpad_input # flash 2 is a must for Megatron - # pre-compute input shapes for each micro-batch at each pp stage - input_shapes = [] - for model_inputs in batches: - input_ids = model_inputs['input_ids'] - attention_mask = model_inputs['attention_mask'] - input_ids_rmpad = unpad_input(input_ids.unsqueeze(dim=-1), attention_mask)[0] # (total_nnz, 1) - if meta_info['sequence_parallel']: - input_ids_rmpad = pad_to_sequence_parallel(input_ids_rmpad) - # compute shapes for model_inputs - input_shapes.append( - torch.Size([ - input_ids_rmpad.shape[0] // mpu.get_tensor_model_parallel_world_size(), 1, meta_info['hidden_size'] - ])) - else: - # compute shapes for model_inputs - input_shapes.append(torch.Size([input_ids_rmpad.shape[0], 1, meta_info['hidden_size']])) - return input_shapes - - -def make_batch_generator(batches, vpp_size): - if vpp_size > 1: - # has vpp - batch_generator = [batches] * vpp_size # number of vpp chunks - batch_generator = [iter(b) for b in batch_generator] - else: - # no vpp - batch_generator = iter(batches) - return batch_generator diff --git a/verl/utils/megatron/sequence_parallel.py b/verl/utils/megatron/sequence_parallel.py deleted file mode 100644 index 4b76cb295e..0000000000 --- a/verl/utils/megatron/sequence_parallel.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import torch.nn.functional as F -from megatron.core import parallel_state as mpu - - -def mark_parameter_as_sequence_parallel(parameter): - setattr(parameter, 'sequence_parallel', True) - - -def is_sequence_parallel_param(param): - return hasattr(param, 'sequence_parallel') and param.sequence_parallel - - -def pad_to_sequence_parallel(unpad_tokens: torch.Tensor): - """pad the tokens such that the total length is a multiple of sp world size - - Args: - unpad_tokens: (total_nnz, ...). Tokens after removing padding - - Returns: - - """ - total_nnz = unpad_tokens.shape[0] - sp_world_size = mpu.get_tensor_model_parallel_world_size() - - if total_nnz % sp_world_size == 0: - pad_size = 0 - else: - pad_size = sp_world_size - total_nnz % sp_world_size - - if pad_size > 0: - if unpad_tokens.ndim == 1: - unpad_tokens = F.pad(unpad_tokens, (0, pad_size)) - elif unpad_tokens.ndim == 2: - unpad_tokens = F.pad(unpad_tokens, (0, 0, 0, pad_size)) - else: - raise NotImplementedError(f'Padding dim {unpad_tokens.ndim()} is not supported') - - return unpad_tokens diff --git a/verl/utils/megatron/tensor_parallel.py b/verl/utils/megatron/tensor_parallel.py deleted file mode 100644 index 522d5008f4..0000000000 --- a/verl/utils/megatron/tensor_parallel.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Utilities for using tensor_parallel in megatron -""" -from typing import Dict -import torch -from torch.nn import init -import torch.distributed as dist -from megatron.core import ModelParallelConfig -from megatron.core import parallel_state as mpu, tensor_parallel -import verl.utils.torch_functional as verl_F - - -def update_kwargs_with_config(dictionary: Dict, config: ModelParallelConfig): - dictionary['config'] = config - return dictionary - - -def get_default_kwargs_for_model_parallel_config(): - model_parallel_config_kwargs = { - 'params_dtype': torch.float32, - 'use_cpu_initialization': False, - 'perform_initialization': True, - 'gradient_accumulation_fusion': False, - 'sequence_parallel': False, - } - return model_parallel_config_kwargs - - -def get_default_model_parallel_config(): - return ModelParallelConfig(**get_default_kwargs_for_model_parallel_config()) - - -def get_common_default_kwargs_for_parallel_linear(): - default_model_parallel_config = get_default_model_parallel_config() - common_default_kwargs = { - 'init_method': init.xavier_normal_, - 'stride': 1, - 'keep_master_weight_for_test': False, - 'config': default_model_parallel_config, - } - return common_default_kwargs - - -def get_default_kwargs_for_column_parallel_linear(): - model_parallel_config_kwargs = get_default_kwargs_for_model_parallel_config() - column_parallel_config_kwargs = { - 'async_tensor_model_parallel_allreduce': False, - } - model_parallel_config_kwargs.update(column_parallel_config_kwargs) - column_default_kwargs = { - 'config': ModelParallelConfig(**model_parallel_config_kwargs), - } - common_default_kwargs = get_common_default_kwargs_for_parallel_linear() - common_default_kwargs.update(column_default_kwargs) - return common_default_kwargs - - -def get_default_kwargs_for_row_parallel_linear(): - common_default_kwargs = get_common_default_kwargs_for_parallel_linear() - return common_default_kwargs - - -def get_default_kwargs_for_parallel_embedding(): - model_parallel_config_kwargs = get_default_kwargs_for_model_parallel_config() - embedding_default_kwargs = { - 'init_method': init.xavier_normal_, - 'config': ModelParallelConfig(**model_parallel_config_kwargs), - } - return embedding_default_kwargs - - -def is_tensor_parallel_param(param): - return (hasattr(param, 'tensor_model_parallel') and param.tensor_model_parallel) - - -def get_tensor_parallel_partition_dim(param): - assert is_tensor_parallel_param(param) - return param.partition_dim - - -def get_tensor_parallel_partition_stride(param): - assert is_tensor_parallel_param(param) - return param.partition_stride - - -class _VocabParallelEntropy(torch.autograd.Function): - - @staticmethod - def forward(ctx, vocab_parallel_logits: torch.Tensor) -> torch.Tensor: - logits_max = vocab_parallel_logits.max(dim=-1, keepdim=True).values - dist.all_reduce(logits_max, op=dist.ReduceOp.MAX, group=mpu.get_tensor_model_parallel_group()) - normalized_vocab_parallel_logits = vocab_parallel_logits - logits_max - normalized_exp_logits = normalized_vocab_parallel_logits.exp() - normalized_sum_exp_logits = normalized_exp_logits.sum(dim=-1, keepdim=True) - dist.all_reduce(normalized_sum_exp_logits, group=mpu.get_tensor_model_parallel_group()) - softmax_logits = normalized_exp_logits / normalized_sum_exp_logits - # This consume too much VRAM, causing OOM, try optimize - # sum_softmax_times_logits = (softmax_logits * vocab_parallel_logits).sum(dim=-1, keepdim=True) - original_shape = softmax_logits.shape - sum_softmax_times_logits = torch.bmm(softmax_logits.view(-1, 1, original_shape[-1]), - vocab_parallel_logits.view(-1, original_shape[-1], - 1)).view(original_shape[:-1] + (1,)) - dist.all_reduce(sum_softmax_times_logits, group=mpu.get_tensor_model_parallel_group()) - entropy = logits_max + normalized_sum_exp_logits.log() - sum_softmax_times_logits - ctx.save_for_backward(vocab_parallel_logits, softmax_logits, sum_softmax_times_logits) - return entropy.squeeze(dim=-1) - - @staticmethod - def backward(ctx, grad_output: torch.Tensor) -> torch.Tensor: - vocab_parallel_logits, softmax_logits, sum_softmax_times_logits = ctx.saved_tensors - grad_input = grad_output.unsqueeze(dim=-1) * softmax_logits * (sum_softmax_times_logits - vocab_parallel_logits) - return grad_input - - -def vocab_parallel_entropy(vocab_parallel_logits: torch.Tensor) -> torch.Tensor: - """Compute entropy when the logits are sharded in tp ranks - - Args: - vocab_parallel_logits: (total_nnz, vocab_size // tp_size) - - Returns: (total_nnz,) - - """ - return _VocabParallelEntropy.apply(vocab_parallel_logits) - - -def vocab_parallel_log_probs_from_logits(logits, labels): - """TODO(zhangchi.usc1992): We may change the implementation later""" - return -tensor_parallel.vocab_parallel_cross_entropy(vocab_parallel_logits=logits, target=labels) - - -def vocab_parallel_log_probs_from_logits_response_rmpad(input_ids, attention_mask, logits_rmpad, response_length): - """Similar to log_probs_from_logits_response_rmpad, but the logits_rmpad is now spliited across tensor parallel region. - This will further reduce the peak memory usage during training - - Args: - input_ids: [batch_size, seqlen] - attention_mask: [batch_size, seqlen] - logits_rmpad: [total_nnz, vocab_size // tp_size] - response_length: int - - """ - from flash_attn.bert_padding import pad_input, unpad_input - - batch_size, seqlen = input_ids.shape - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), attention_mask=attention_mask) - input_ids_rmpad = input_ids_rmpad.squeeze(-1) - input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=0) - full_log_probs_rmpad = vocab_parallel_log_probs_from_logits(logits=logits_rmpad, - labels=input_ids_rmpad_rolled) # (total_nnz,) - full_output = pad_input(hidden_states=full_log_probs_rmpad.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen) - output = full_output.squeeze(-1)[:, -response_length - 1:-1] # [batch_size, response_length] - return output diff --git a/verl/utils/megatron_utils.py b/verl/utils/megatron_utils.py deleted file mode 100644 index 9812639b5f..0000000000 --- a/verl/utils/megatron_utils.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Pretrain utilities.""" -import os -from typing import Dict - -import torch -import torch.nn as nn -import torch.nn.functional as F -from megatron.core import ModelParallelConfig -from megatron.core import mpu, tensor_parallel -from megatron.core.distributed import DistributedDataParallel as DDP -from megatron.core.distributed import DistributedDataParallelConfig -from megatron.core.enums import ModelType -from megatron.core.optimizer import OptimizerConfig -from megatron.core.transformer import TransformerConfig -from megatron.core.transformer.module import Float16Module -from megatron.core.utils import get_attr_wrapped_model -from omegaconf import DictConfig - -from verl.utils.memory_buffer import build_memory_reference_from_module -from verl.utils.torch_dtypes import PrecisionType - - -def get_model_config(model): - return get_attr_wrapped_model(model, 'config', allow_none=False) - - -def get_model(model_provider_func, - model_type=ModelType.encoder_or_decoder, - wrap_with_ddp=True, - use_distributed_optimizer=True): - """Build the model.""" - # Build model. - if mpu.get_pipeline_model_parallel_world_size() > 1 and \ - mpu.get_virtual_pipeline_model_parallel_world_size() is not None: - assert model_type != ModelType.encoder_and_decoder, \ - "Interleaved schedule not supported for model with both encoder and decoder" - model = [] - for i in range(mpu.get_virtual_pipeline_model_parallel_world_size()): - mpu.set_virtual_pipeline_model_parallel_rank(i) - # Set pre_process and post_process only after virtual rank is set. - pre_process = mpu.is_pipeline_first_stage() - post_process = mpu.is_pipeline_last_stage() - this_model = model_provider_func(pre_process=pre_process, post_process=post_process) - this_model.model_type = model_type - model.append(this_model) - else: - pre_process = mpu.is_pipeline_first_stage() - post_process = mpu.is_pipeline_last_stage() - add_encoder = True - add_decoder = True - if model_type == ModelType.encoder_and_decoder: - if mpu.get_pipeline_model_parallel_world_size() > 1: - assert mpu.get_pipeline_model_parallel_split_rank() is not None, \ - "Split rank needs to be specified for model with both encoder and decoder" - rank = mpu.get_pipeline_model_parallel_rank() - split_rank = mpu.get_pipeline_model_parallel_split_rank() - world_size = mpu.get_pipeline_model_parallel_world_size() - pre_process = rank == 0 or rank == split_rank - post_process = (rank == (split_rank - 1)) or (rank == (world_size - 1)) - add_encoder = mpu.is_pipeline_stage_before_split() - add_decoder = mpu.is_pipeline_stage_after_split() - model = model_provider_func(pre_process=pre_process, - post_process=post_process, - add_encoder=add_encoder, - add_decoder=add_decoder) - else: - model = model_provider_func(pre_process=pre_process, post_process=post_process) - model.model_type = model_type - - if not isinstance(model, list): - model = [model] - - # Set tensor model parallel attributes if not set. - # Only parameters that are already tensor model parallel have these - # attributes set for them. We should make sure the default attributes - # are set for all params so the optimizer can use them. - for model_module in model: - for param in model_module.parameters(): - tensor_parallel.set_defaults_if_not_set_tensor_model_parallel_attributes(param) - - # Print number of parameters. - if mpu.get_data_parallel_rank() == 0: - print(' > number of parameters on (tensor, pipeline) ' - 'model parallel rank ({}, {}): {}'.format( - mpu.get_tensor_model_parallel_rank(), mpu.get_pipeline_model_parallel_rank(), - sum([sum([p.nelement() for p in model_module.parameters()]) for model_module in model])), - flush=True) - - # GPU allocation. - for model_module in model: - model_module.cuda(torch.cuda.current_device()) - - # Fp16 conversion. - config: TransformerConfig = get_model_config(model[0]) - config.fp8 = None - tfconfig: TransformerConfig = model[0].config - if config.fp16 or config.bf16: # the ModelParallelConfig in GPTModel - model = [Float16Module(config, model_module) for model_module in model] - - if wrap_with_ddp: - ddp_models = [] - for model_chunk_idx, model_chunk in enumerate(model): - ddp_model = DDP( - config=tfconfig, - module=model_chunk, - disable_bucketing=(model_chunk_idx > 0), - ddp_config=DistributedDataParallelConfig( - overlap_grad_reduce=False, - use_distributed_optimizer=use_distributed_optimizer, - grad_reduce_in_fp32=True, # [old] accumulate_allreduce_grads_in_fp32=True, - )) - ddp_models.append(ddp_model) - model = ddp_models - # # Broadcast params from data parallel src rank to other data parallel ranks. - # # if args.data_parallel_random_init: - for model_module in model: - model_module.broadcast_params() - return model - - -ALL_MODULE_WRAPPER_CLASSNAMES = (DDP, Float16Module) - - -def unwrap_model(model, module_instances=ALL_MODULE_WRAPPER_CLASSNAMES): - return_list = True - if not isinstance(model, list): - model = [model] - return_list = False - unwrapped_model = [] - for model_module in model: - while isinstance(model_module, module_instances): - model_module = model_module.module - unwrapped_model.append(model_module) - if not return_list: - return unwrapped_model[0] - return unwrapped_model - - -from transformers import PretrainedConfig - - -def convert_config(hf_config: PretrainedConfig, megatron_config) -> TransformerConfig: - print(f'megatron config {megatron_config}') - dt = PrecisionType.to_dtype(megatron_config.params_dtype) - print(f'pipeline_dtype=megatron_config {dt}') - if "Qwen2ForCausalLM" in hf_config.architectures: - qkv_bias = True - else: - qkv_bias = getattr(hf_config, 'attention_bias', False) - overlap_p2p_comm = mpu.get_virtual_pipeline_model_parallel_world_size( - ) is not None and mpu.get_virtual_pipeline_model_parallel_world_size() > 1 - batch_p2p_comm = False - transformer_config = TransformerConfig( - num_layers=hf_config.num_hidden_layers, - hidden_size=hf_config.hidden_size, - num_attention_heads=hf_config.num_attention_heads, - num_query_groups=hf_config.num_key_value_heads, - ffn_hidden_size=hf_config.intermediate_size, - # max_position_embeddings=hf_config.max_position_embeddings, - activation_func=F.silu, - normalization='RMSNorm', - # rotary_percent=False, # default, - gated_linear_unit=True, # for llama - use_cpu_initialization=True, - apply_residual_connection_post_layernorm=False, # check what's this mean - add_bias_linear=False, - tensor_model_parallel_size=mpu.get_tensor_model_parallel_world_size(), - pipeline_model_parallel_size=mpu.get_pipeline_model_parallel_world_size(), - virtual_pipeline_model_parallel_size=mpu.get_virtual_pipeline_model_parallel_world_size(), - overlap_p2p_comm=overlap_p2p_comm, - batch_p2p_comm=batch_p2p_comm, - pipeline_dtype=dt, - params_dtype=dt, - sequence_parallel=True, - variable_seq_lengths=True, - masked_softmax_fusion=True, - moe_token_dispatcher_type="alltoall", - attention_dropout=hf_config.attention_dropout, - hidden_dropout=getattr(hf_config, 'hidden_dropout', 0.0), - add_qkv_bias=qkv_bias, - bf16=dt is torch.bfloat16) - - return transformer_config - - -def init_megatron_optim_config(optim_config: Dict) -> OptimizerConfig: - config = OptimizerConfig( - optimizer='adam', - lr=optim_config.get('lr'), - clip_grad=optim_config.get('clip_grad'), - weight_decay=1e-2, - bf16=True, - params_dtype=torch.bfloat16, - use_distributed_optimizer=True, - ) - return config - - -def init_model_parallel_config(config: DictConfig) -> ModelParallelConfig: - # TODO(sgm): check how to disable megatron timers - timers = None - return ModelParallelConfig(tensor_model_parallel_size=config.get('tensor_model_parallel_size'), - pipeline_model_parallel_size=config.get('pipeline_model_parallel_size'), - virtual_pipeline_model_parallel_size=config.get('virtual_pipeline_model_parallel_size'), - sequence_parallel=config.get('sequence_parallel'), - params_dtype=PrecisionType.to_dtype(config.get('param_dtype')), - pipeline_dtype=PrecisionType.to_dtype(config.get('param_dtype')), - bf16=True, - fp16=False, - timers=timers) - - -def offload_megatron_param_and_grad(module_list: nn.ModuleList, offload_grad=False, hybrid_engine=None): - if hybrid_engine is not None: - pp_rank = mpu.get_pipeline_model_parallel_rank() - for buffer in hybrid_engine.memory_buffers[pp_rank].values(): - buffer.data = buffer.data.to('cpu', non_blocking=True) - build_memory_reference_from_module(module_list, hybrid_engine.memory_buffers[pp_rank], maintain_weight=True) - else: - for module in module_list: - for _, param in module.named_parameters(): - param.data = param.data.to('cpu', non_blocking=True) - if offload_grad and param.grad is not None: - param.grad = param.grad.to("cpu", non_blocking=True) - torch.cuda.empty_cache() - - -def load_megatron_param_and_grad(module_list: nn.ModuleList, device_id, load_grad=False, hybrid_engine=None): - if hybrid_engine is not None: - pp_rank = mpu.get_pipeline_model_parallel_rank() - for buffer in hybrid_engine.memory_buffers[pp_rank].values(): - buffer.data = buffer.data.to(device_id, non_blocking=True) - build_memory_reference_from_module(module_list, hybrid_engine.memory_buffers[pp_rank], maintain_weight=True) - else: - for module in module_list: - for _, param in module.named_parameters(): - param.data = param.data.to(device_id, non_blocking=True) - if load_grad and param.grad is not None: - param.grad = param.grad.to(device_id, non_blocking=True) - torch.cuda.empty_cache() - - -def print_rank_0(message): - """If distributed is initialized, print only on rank 0.""" - if torch.distributed.is_initialized(): - if torch.distributed.get_rank() == 0: - print(message, flush=True) - else: - print(message, flush=True) - - -def get_model_checkpoint_path(checkpoint_path): - os.makedirs(checkpoint_path, exist_ok=True) - return os.path.join(checkpoint_path, "model") - - -def get_hf_model_checkpoint_path(checkpoint_path): - os.makedirs(checkpoint_path, exist_ok=True) - return os.path.join(checkpoint_path, "huggingface") - - -def get_optimizer_checkpoint_path(checkpoint_path, use_distributed_optimizer=True): - os.makedirs(os.path.join(checkpoint_path, "optim"), exist_ok=True) - if not use_distributed_optimizer: - return os.path.join(checkpoint_path, "optim", "optim.pt") - pp_rank = mpu.get_pipeline_model_parallel_rank() - tp_rank = mpu.get_tensor_model_parallel_rank() - #TODO: support ep - return os.path.join(checkpoint_path, f"optim", f"distrib_optim_pp{pp_rank}_tp{tp_rank}.pt") - - -def get_rng_states_checkpoint_path(checkpoint_path, data_parallel_random_init=False): - os.makedirs(os.path.join(checkpoint_path, "rng_states"), exist_ok=True) - if not data_parallel_random_init: - return os.path.join(checkpoint_path, f'rng_states', "rng_states.pt") - dp_rank = mpu.get_data_parallel_rank() - return os.path.join(checkpoint_path, f'rng_states', f"rng_states_{dp_rank}.pt") diff --git a/verl/utils/memory_buffer.py b/verl/utils/memory_buffer.py deleted file mode 100644 index 07d9f23f38..0000000000 --- a/verl/utils/memory_buffer.py +++ /dev/null @@ -1,217 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This file contains utilities to manipulate torch memory buffers -""" - -from typing import Dict, List, Optional - -import torch -from torch import nn - - -class MemoryBuffer: - """ - A memory buffer is a contiguous torch tensor that may combine multiple tensors sharing with the underlying - memory. It must have a unique type to support this behavior. - """ - - def __init__(self, numel: int, numel_padded: int, dtype: torch.dtype, source: Optional[torch.Tensor] = None): - self.numel = numel - self.numel_padded = numel_padded - self.dtype = dtype - if source is not None: - self.data = source - else: - self.data = torch.zeros(self.numel_padded, dtype=self.dtype, device='cuda', requires_grad=False) - - def zero(self): - """Reset the buffer to zero.""" - self.data.zero_() - - def get(self, shape, start_index): - """Return a tensor with the input `shape` as a view into the - 1-D data starting at `start_index`.""" - end_index = start_index + shape.numel() - assert end_index <= self.numel, \ - 'requested tensor is out of the buffer range.' - buffer_tensor = self.data[start_index:end_index] - buffer_tensor = buffer_tensor.view(shape) - return buffer_tensor - - -def calc_padded_numel(shape: torch.Size, dtype: torch.dtype): - """for cuda memory alignment, make sure alignment by 128-bits""" - align_numel = 128 // torch.finfo(dtype).bits - numel = shape.numel() - return (numel + align_numel - 1) // align_numel * align_numel - - -def get_weight_buffer_meta_from_module(module: nn.Module) -> Dict[str, Dict]: - """ - Return a dictionary containing name to a shape and dtype. - """ - weight_buffer_meta = {} - for name, param in sorted(module.named_parameters()): - weight_buffer_meta[name] = {'shape': param.shape, 'dtype': param.dtype} - return weight_buffer_meta - - -def build_memory_buffer(weight_buffer_meta: Dict[str, Dict]) -> Dict[torch.dtype, MemoryBuffer]: - """Build the memory buffer given weight_buffer_meta - - Args: - weight_buffer_meta: contains mapping from name to a dictionary containing shape and dtype of the tensors - - Returns: a large memory buffer for each dtype that can hold all the tensors - - """ - memory_buffers = {} - total_numel_map = {} # map from dtype to the total numel - for name, meta_info in sorted(weight_buffer_meta.items()): - shape = meta_info['shape'] - dtype = meta_info['dtype'] - - assert isinstance(shape, torch.Size) - assert isinstance(dtype, torch.dtype) - - if dtype not in total_numel_map: - total_numel_map[dtype] = 0 - - total_numel_map[dtype] += calc_padded_numel(shape, dtype) - - for dtype, total_numel in total_numel_map.items(): - memory_buffers[dtype] = MemoryBuffer(total_numel, total_numel, dtype) - - return memory_buffers - - -def build_memory_reference_from_module(module: torch.nn.Module, - memory_buffers: Dict[torch.dtype, MemoryBuffer], - maintain_weight=True): - start_index = {} - for dtype in memory_buffers.keys(): - start_index[dtype] = 0 - for name, param in sorted(module.named_parameters()): - memory_buffer = memory_buffers[param.dtype] - buffer = memory_buffer.get(shape=param.shape, start_index=start_index[param.dtype]) - # need to increment start_index - start_index[param.dtype] += calc_padded_numel(param.shape, dtype) - if maintain_weight: - buffer.copy_(param.data) - param.data = buffer - - -def build_memory_reference(weight_buffer_meta: Dict[str, Dict], memory_buffers: Dict[torch.dtype, MemoryBuffer]): - """Build the memory references. The memory buffers are built using the build_memory_buffer API. - This API will allocate a weight buffer pointer to the memory buffer according to the weight_buffer_meta. - - Args: - weight_buffer_meta: - memory_buffers: - - Returns: - - """ - start_idx = {} - weight_buffers = {} - for dtype in memory_buffers.keys(): - start_idx[dtype] = 0 - - for name, meta_info in sorted(weight_buffer_meta.items()): - shape = meta_info['shape'] - dtype = meta_info['dtype'] - - buffer = memory_buffers[dtype].get(shape, start_index=start_idx[dtype]) - start_idx[dtype] += calc_padded_numel(shape, dtype) - weight_buffers[name] = buffer - - return weight_buffers - - -class MemoryBufferModuleWrapper: - """ - Note that we do not design MemoryBufferModuleWrapper as an nn.Module due to - - It will change the checkpoint name - """ - - def __init__(self, module: nn.Module): - super().__init__() - self.module = module - self.weight_buffer_meta = get_weight_buffer_meta_from_module(self.module) - self.memory_buffers = build_memory_buffer(self.weight_buffer_meta) - build_memory_reference_from_module(self.module, self.memory_buffers) - - def get_memory_buffers(self): - return self.memory_buffers - - def get_weight_buffer_meta(self): - return self.weight_buffer_meta - - -class MegatronMemoryBufferForRollout(object): - """ - We assume that - - inference engine has tp + dp - - actor has tp + pp + dp - - the tp between inference engine and actor should be the same - - memory_buffers: contains a list of memory_buffers, each is a dict from dtype to MemoryBuffer - - weight_buffers: contains a list of weight_buffers, each is a dict from name to param - - named_parameters: a dict from name to parameter that normalizes the names from pp and vpp. Note that - the named_parameters may not be directly compatible with inference engine. User has to take care of - this part such as the layout mismatches. (e.g. qkv transpose) - - Note that weight_buffer, named_parameters and memory_buffers share the same underlying GPU memory. - - When doing weight sync, the data is transfer via memory buffers - """ - - def __init__(self, transform_memory_param_fn): - self._memory_buffers = [] - self._weight_buffers = [] - self._named_parameters = {} - self.transform_memory_param_fn = transform_memory_param_fn - - def initialize_weight_buffer(self, weight_buffer_meta_pp: List[Dict[str, Dict]]): - """ - Initialize the weight buffer. The weight buffer is obtained according to the actor. We will construct - a large buffer for each dtype in the weight_buffer. - - Args: - weight_buffer_meta: contains pp models, each pp models contains a dictionary of mapping from - - Returns: None - - """ - self.weight_buffer_meta_pp = weight_buffer_meta_pp - - for weight_buffer_meta in self.weight_buffer_meta_pp: - memory_buffer = build_memory_buffer(weight_buffer_meta) - self._memory_buffers.append(memory_buffer) - self._weight_buffers.append(None) - - def build_memory_reference(self): - for i, weight_buffer_meta in enumerate(self.weight_buffer_meta_pp): - self._weight_buffers[i] = build_memory_reference(weight_buffer_meta, self._memory_buffers[i]) - self._named_parameters = self.transform_memory_param_fn(self._weight_buffers) - - @property - def named_parameters(self): - return self._named_parameters - - @property - def weight_buffers(self): - return self._weight_buffers - - @property - def memory_buffers(self): - return self._memory_buffers diff --git a/verl/utils/model.py b/verl/utils/model.py deleted file mode 100644 index 6af6542ba8..0000000000 --- a/verl/utils/model.py +++ /dev/null @@ -1,445 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Utilities to create common models from huggingface -""" -import os -import warnings -from typing import Dict, Type, Optional - -import numpy as np -import torch -from torch import nn -from transformers import AutoConfig, AutoModelForCausalLM, PretrainedConfig, MistralForSequenceClassification, GenerationConfig -from verl.models.registry import ModelRegistry - - -class LambdaLayer(nn.Module): - - def __init__(self, fn): - super().__init__() - self.fn = fn - - def forward(self, *args, **kwargs): - return self.fn(*args, **kwargs) - - -def squeeze(x): - return torch.squeeze(x, dim=-1) - - -def update_model_config(module_config, override_config_kwargs): - for key, val in override_config_kwargs.items(): - setattr(module_config, key, val) - - -def get_huggingface_actor_config(model_name: str, override_config_kwargs=None, trust_remote_code=False) -> Dict: - if override_config_kwargs is None: - override_config_kwargs = {} - assert isinstance(override_config_kwargs, Dict), \ - f'override_config_kwargs must be a dict, got {type(override_config_kwargs)}' - module_config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code) - update_model_config(module_config, override_config_kwargs) - - return module_config - - -def get_generation_config( - model: str, - trust_remote_code: bool = False, -) -> Optional[GenerationConfig]: - try: - return GenerationConfig.from_pretrained(model) - except OSError: # Not found - try: - config = get_huggingface_actor_config( - model, - trust_remote_code=trust_remote_code, - ) - return GenerationConfig.from_model_config(config) - except OSError: # Not found - return None - - -def create_huggingface_actor(model_name: str, override_config_kwargs=None, automodel_kwargs=None) -> nn.Module: - """ - - Args: - model_name: - override_config_kwargs: - - Returns: - - """ - if override_config_kwargs is None: - override_config_kwargs = {} - if automodel_kwargs is None: - automodel_kwargs = {} - assert isinstance(override_config_kwargs, Dict), \ - f'override_config_kwargs must be a dict, got {type(override_config_kwargs)}' - module_config = get_huggingface_actor_config(model_name, - override_config_kwargs, - trust_remote_code=automodel_kwargs.get('trust_remote_code', False)) - module: nn.Module = AutoModelForCausalLM.from_config(module_config, **automodel_kwargs) - return module - - -def create_huggingface_critic(model_name: str, override_config_kwargs=None, automodel_kwargs=None) -> nn.Module: - """ - - Args: - model_name: - override_config_kwargs: - - Returns: - - """ - critic_module: nn.Module = create_huggingface_actor(model_name, - override_config_kwargs=override_config_kwargs, - automodel_kwargs=automodel_kwargs) - if automodel_kwargs is None: - automodel_kwargs = {} - torch_dtype = automodel_kwargs.get('torch_dtype', torch.float32) - critic_module.lm_head = nn.Sequential(nn.Linear(critic_module.config.hidden_size, 1, dtype=torch_dtype), - LambdaLayer(fn=squeeze)) - return critic_module - - -def get_model_size(model: nn.Module, scale='auto'): - n_params = sum(p.numel() for p in model.parameters()) - - if scale == 'auto': - if n_params > 1e9: - scale = 'B' - elif n_params > 1e6: - scale = 'M' - elif n_params > 1e3: - scale = 'K' - else: - scale = '' - - if scale == 'B': - n_params = n_params / 1e9 - elif scale == 'M': - n_params = n_params / 1e6 - elif scale == 'K': - n_params = n_params / 1e3 - elif scale == '': - pass - else: - raise NotImplemented(f'Unknown scale {scale}') - - return n_params, scale - - -def print_model_size(model: nn.Module, name: str = None): - n_params, scale = get_model_size(model, scale='auto') - if name is None: - name = model.__class__.__name__ - print(f'{name} contains {n_params:.2f}{scale} parameters') - - -def create_random_mask(input_ids: torch.Tensor, - max_ratio_of_valid_token: float, - max_ratio_of_left_padding: float, - min_ratio_of_valid_token: float = 0): - """Create a random mask given input_ids. Support left padding and right padding. - Process: - - Sample valid token length - - Sample left_padding length - - Generate padding - - Args: - input_ids: - shape (batch_size, seq_len) - - Returns: - - """ - assert max_ratio_of_valid_token > 0 and max_ratio_of_valid_token <= 1. - assert max_ratio_of_left_padding >= 0 and max_ratio_of_left_padding < 1. - assert min_ratio_of_valid_token <= max_ratio_of_valid_token - - batch_size, sequence_length = input_ids.shape - max_num_valid_tokens = int(sequence_length * max_ratio_of_valid_token) - min_num_valid_tokens = max(1, int(sequence_length * min_ratio_of_valid_token)) - max_left_padding = int(sequence_length * max_ratio_of_left_padding) - assert max_num_valid_tokens + max_left_padding <= sequence_length - assert max_num_valid_tokens > 0 and max_ratio_of_valid_token <= sequence_length - masks = torch.ones_like(input_ids, dtype=torch.int64) - # TODO: we can make this faster - for i in range(batch_size): - num_left_padding = np.random.randint(low=0, high=max_left_padding + 1, dtype=np.int64) - num_valid = np.random.randint(low=min_num_valid_tokens, high=max_num_valid_tokens + 1, dtype=np.int64) - - for index in range(num_left_padding): - masks[i, index] = 0 - - for index in range(num_left_padding + num_valid, sequence_length): - masks[i, index] = 0 - return masks - - -def compute_position_id_with_mask(mask): - return torch.clip(torch.cumsum(mask, dim=-1) - 1, min=0, max=None) - - -def normalize_pp_vpp_params(params, num_hidden_layers, layer_name='layers'): - """ - Normalize the pp vpp params into a complete named parameters. - This is useful when gather parameters from pp ranks and passed to a model without pp - - params: List[List[Dict[str, param]]] - params contains a list of pp, with a list of vpp named_parameters in each vpp chunk. - output: Dict[str, param] - - """ - - def normalize_model_name(name, pp_rank, vpp_rank, pp_size, vpp_size, num_layers): - """ - Transform the model name in each model_chunk in each pp stage into the name in inference engine - """ - if vpp_size > 1: - # print(f'try to bind vpp params to inference engine...') - layers_per_pp = num_layers // pp_size - layers_per_vpp = layers_per_pp // vpp_size - pp_offset = layers_per_vpp * pp_rank - vpp_offset = (layers_per_vpp * pp_size) * vpp_rank - layer_offset = pp_offset + vpp_offset - else: - layers_per_pp = num_layers // pp_size - layer_offset = layers_per_pp * pp_rank - - if layer_name in name: # belong to an intermediate layer - split_name = name.split('.') - # find the num next to split_name - for i, name in enumerate(split_name): - if name == layer_name: - break - layer_num_idx = i + 1 - # check the name - assert len(split_name) >= layer_num_idx + 1, f'split_name = {split_name}' - assert split_name[layer_num_idx].isdigit(), f'split_name = {split_name}' - # increment layer_num_idx by layer_offset - split_name[layer_num_idx] = str(int(split_name[layer_num_idx]) + layer_offset) - name = '.'.join(split_name) # weight name in inference_tp_model - return name - - pp_size = len(params) - normalized_name_to_param = {} - for pp_rank in range(len(params)): - vpp_size = len(params[pp_rank]) - for vpp_rank in range(vpp_size): - for name, param in params[pp_rank][vpp_rank].items(): - normalized_name = normalize_model_name(name, pp_rank, vpp_rank, pp_size, vpp_size, num_hidden_layers) - normalized_name_to_param[normalized_name] = param - - return normalized_name_to_param - - -def get_parallel_model_from_config(config, - megatron_config, - pre_process=None, - post_process=None, - share_embeddings_and_output_weights=False, - value=False): - from megatron.core import ModelParallelConfig - assert isinstance(megatron_config, ModelParallelConfig) - model_class = _get_parallel_model_architecture_from_config(config, value) - - model = model_class(config, - megatron_config, - pre_process=pre_process, - post_process=post_process, - share_embeddings_and_output_weights=share_embeddings_and_output_weights) - return model - - -def _get_parallel_model_architecture_from_config(config: PretrainedConfig, value=False) -> Type[nn.Module]: - architectures = getattr(config, "architectures", []) - for arch in architectures: - model_cls = ModelRegistry.load_model_cls(arch, value) - print(f'after load model cls') - if model_cls is not None: - return model_cls - raise ValueError(f"Model architectures {architectures} are not supported for now. " - f"Supported architectures: {ModelRegistry.get_supported_archs()}") - - -def load_megatron_model_weights(config, - model_config, - parallel_model, - params_dtype, - is_value_model=False, - local_cache_path='~/.cache/verl/rlhf'): - assert hasattr(model_config, "architectures"), "architectures cannot be empty when load weight!" - architectures = getattr(model_config, "architectures", []) - local_cache_path = os.path.expanduser(local_cache_path) - - if config.model.path.startswith("hdfs:"): - from verl.utils.fs import copy_to_local - print(f'start download from {config.model.path}') - local_model_path = copy_to_local(src=config.model.path, cache_dir=local_cache_path) - print('finish download') - else: - local_model_path = config.model.path - print(f"load from local dir {local_model_path}") - - # TODO: to find a better way to load mistral7b-rm lm_head - from verl.utils.fsdp_utils import get_init_weight_context_manager - init_context = get_init_weight_context_manager(use_meta_tensor=not model_config.tie_word_embeddings) - with init_context(), warnings.catch_warnings(): - warnings.simplefilter("ignore") - if 'mistral7b-rm' in config.model.path: - model = MistralForSequenceClassification.from_pretrained( - local_model_path, device_map="auto", low_cpu_mem_usage=True) # use score head instead of lm_head - state_dict = model.state_dict() - state_dict['lm_head.weight'] = state_dict['score.weight'] - state_dict['model.embed_tokens.weight'] = state_dict[ - 'model.embed_tokens.weight'][:32000] # workaround, 32001 -> 32000 - is_value_model = True - else: - model = AutoModelForCausalLM.from_pretrained(local_model_path, - torch_dtype="auto", - device_map="auto", - low_cpu_mem_usage=True) - state_dict = model.state_dict() - - from verl.models.weight_loader_registry import get_weight_loader - print(f'before weight loader: architectures = {architectures}...') - for arch in architectures: - print(f'call weight loader arch = {arch}, model config = {model.config}') - weight_loader = get_weight_loader(arch) - weight_loader(state_dict=state_dict, - wrapped_models=parallel_model, - config=model.config, - params_dtype=params_dtype, - is_value_model=is_value_model, - tie_word_embeddings=model_config.tie_word_embeddings) - return model.config - - -# pad input_ids_rmpad, cu_seqlens and max_seqlen_in_batch to be divisible by tp -def pad_packed_inputs(unpad_tokens: torch.Tensor, cu_seqlens, max_seqlen_in_batch, size): - """pad the tokens such that the total length is a multiple of size. - This function is useful when applying sequence parallel and context parallel - - Args: - unpad_tokens: (total_nnz, ...). Tokens after removing padding - cu_seqlens: (total_nnz + 1,) - max_seqlen_in_batch: int - - Returns: - - """ - F = nn.functional - - total_nnz = unpad_tokens.shape[0] - - if total_nnz % size == 0: - pad_size = 0 - else: - pad_size = size - total_nnz % size - - # we assume adding a new data in the batch with seqlen pad_size - if pad_size > 0: - if unpad_tokens.ndim == 1: - unpad_tokens = F.pad(unpad_tokens, (0, pad_size)) - elif unpad_tokens.ndim == 2: - unpad_tokens = F.pad(unpad_tokens, (0, 0, 0, pad_size)) - else: - raise NotImplementedError(f'Padding dim {unpad_tokens.ndim()} is not supported') - - cu_seqlens = F.pad(cu_seqlens, (0, 1), value=pad_size + cu_seqlens[-1]) - max_seqlen_in_batch = max(max_seqlen_in_batch, pad_size) - - return unpad_tokens, cu_seqlens, max_seqlen_in_batch - - -def load_megatron_gptmodel_weights(config, - model_config, - parallel_model, - params_dtype, - is_value_model=False, - local_cache_path='~/.cache/verl/rlhf'): - assert hasattr(model_config, "architectures"), "architectures cannot be empty when load weight!" - architectures = getattr(model_config, "architectures", []) - local_cache_path = os.path.expanduser(local_cache_path) - - if config.model.path.startswith("hdfs:"): - from verl.utils.fs import copy_to_local - print(f'start download from {config.model.path}') - local_model_path = copy_to_local(src=config.model.path, cache_dir=local_cache_path) - print('finish download') - else: - print(f"load from local dir {config.model.path}") - local_model_path = config.model.path - - # TODO: to find a better way to load mistral7b-rm lm_head - if 'mistral7b-rm' in config.model.path: - model = MistralForSequenceClassification.from_pretrained(local_model_path) # use score head instead of lm_head - state_dict = model.state_dict() - state_dict['lm_head.weight'] = state_dict['score.weight'] - state_dict['model.embed_tokens.weight'] = state_dict[ - 'model.embed_tokens.weight'][:32000] # workaround, 32001 -> 32000 - is_value_model = True - else: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - model = AutoModelForCausalLM.from_pretrained(local_model_path) - state_dict = model.state_dict() - - from verl.models.mcore.loader import load_state_dict_to_megatron_gptmodel - load_state_dict_to_megatron_gptmodel(state_dict=state_dict, - wrapped_models=parallel_model, - config=model.config, - params_dtype=params_dtype, - is_value_model=is_value_model) - del state_dict, model - - -def get_parallel_gptmodel_from_config(tfconfig, - hf_config, - pre_process=None, - post_process=None, - share_embeddings_and_output_weights=False, - value=False): - from megatron.core.models.gpt.gpt_model import GPTModel - from megatron.core.models.gpt.gpt_layer_specs import get_gpt_decoder_block_spec - from megatron.core import parallel_state as mpu - from megatron.core import tensor_parallel - use_te = True - assert tfconfig.normalization == "RMSNorm", 'only RMSNorm is supported for now' - transformer_layer_spec = get_gpt_decoder_block_spec(tfconfig, use_transformer_engine=use_te) - rope_scaling_args = {} - if hf_config.rope_scaling is not None: - assert hf_config.rope_scaling['type'] == 'linear', "only linear scaling is supported for now" - rope_scaling_args['seq_len_interpolation_factor'] = hf_config.rope_scaling['factor'] - parallel_model = GPTModel(config=tfconfig, - transformer_layer_spec=transformer_layer_spec, - vocab_size=hf_config.vocab_size, - max_sequence_length=hf_config.max_position_embeddings, - pre_process=pre_process, - post_process=post_process, - share_embeddings_and_output_weights=share_embeddings_and_output_weights, - position_embedding_type='rope', - rotary_base=hf_config.rope_theta, - **rope_scaling_args) - # # for layer in parallel_model.decoder.layers: layer.self_attention.core_attention.flash_attention.softmax_scale = None - if post_process and value: - from verl.models.llama.megatron.layers.parallel_linear import LinearForLastLayer - parallel_model.output_layer = LinearForLastLayer(input_size=tfconfig.hidden_size, - output_size=1, - config=tfconfig) - return parallel_model diff --git a/verl/utils/py_functional.py b/verl/utils/py_functional.py deleted file mode 100644 index 8f5a0e1767..0000000000 --- a/verl/utils/py_functional.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Contain small python utility functions -""" - -from typing import Dict -from types import SimpleNamespace - - -def union_two_dict(dict1: Dict, dict2: Dict): - """Union two dict. Will throw an error if there is an item not the same object with the same key. - - Args: - dict1: - dict2: - - Returns: - - """ - for key, val in dict2.items(): - if key in dict1: - assert dict2[key] == dict1[key], \ - f'{key} in meta_dict1 and meta_dict2 are not the same object' - dict1[key] = val - - return dict1 - - -def append_to_dict(data: Dict, new_data: Dict): - for key, val in new_data.items(): - if key not in data: - data[key] = [] - data[key].append(val) - - -class NestedNamespace(SimpleNamespace): - - def __init__(self, dictionary, **kwargs): - super().__init__(**kwargs) - for key, value in dictionary.items(): - if isinstance(value, dict): - self.__setattr__(key, NestedNamespace(value)) - else: - self.__setattr__(key, value) diff --git a/verl/utils/ray_utils.py b/verl/utils/ray_utils.py deleted file mode 100644 index 9a75df6c37..0000000000 --- a/verl/utils/ray_utils.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Contains commonly used utilities for ray -""" - -import ray - -import concurrent.futures - - -def parallel_put(data_list, max_workers=None): - - def put_data(index, data): - return index, ray.put(data) - - if max_workers is None: - max_workers = min(len(data_list), 16) - - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - data_list_f = [executor.submit(put_data, i, data) for i, data in enumerate(data_list)] - res_lst = [] - for future in concurrent.futures.as_completed(data_list_f): - res_lst.append(future.result()) - - # reorder based on index - output = [None for _ in range(len(data_list))] - for res in res_lst: - index, data_ref = res - output[index] = data_ref - - return output diff --git a/verl/utils/rendezvous/__init__.py b/verl/utils/rendezvous/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/utils/rendezvous/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/utils/rendezvous/ray_backend.py b/verl/utils/rendezvous/ray_backend.py deleted file mode 100644 index c0d2bd906f..0000000000 --- a/verl/utils/rendezvous/ray_backend.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import time - -from cupy.cuda.nccl import NcclCommunicator, get_unique_id - -import ray -from ray.util import list_named_actors - - -@ray.remote -class NCCLIDStore: - - def __init__(self, nccl_id): - self._nccl_id = nccl_id - - def get(self): - return self._nccl_id - - -def get_nccl_id_store_by_name(name): - all_actors = list_named_actors(all_namespaces=True) - matched_actors = [actor for actor in all_actors if actor.get("name", None) == name] - if len(matched_actors) == 1: - actor = matched_actors[0] - return ray.get_actor(**actor) - elif len(matched_actors) > 1: - logging.warning(f"multiple actors with same name found: {matched_actors}") - elif len(matched_actors) == 0: - logging.info(f"failed to get any actor named {name}") - return None - - -def create_nccl_communicator_in_ray(rank: int, - world_size: int, - group_name: str, - max_retries: int = 100, - interval_s: int = 5): - if rank == 0: - nccl_id = get_unique_id() - nccl_id_store = NCCLIDStore.options(name=group_name).remote(nccl_id) - - assert ray.get(nccl_id_store.get.remote()) == nccl_id - communicator = NcclCommunicator( - ndev=world_size, - commId=nccl_id, - rank=0, - ) - return communicator - else: - for i in range(max_retries): - nccl_id_store = get_nccl_id_store_by_name(group_name) - if nccl_id_store is not None: - logging.info(f"nccl_id_store {group_name} got") - nccl_id = ray.get(nccl_id_store.get.remote()) - logging.info(f"nccl id for {group_name} got: {nccl_id}") - communicator = NcclCommunicator( - ndev=world_size, - commId=nccl_id, - rank=rank, - ) - return communicator - logging.info(f"failed to get nccl_id for {i+1} time, sleep for {interval_s} seconds") - time.sleep(interval_s) diff --git a/verl/utils/reward_score/__init__.py b/verl/utils/reward_score/__init__.py deleted file mode 100644 index 1b6f6d75fb..0000000000 --- a/verl/utils/reward_score/__init__.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# from . import gsm8k, math, prime_math, prime_code - - -def _default_compute_score(data_source, solution_str, ground_truth, extra_info=None, question="", tokenizer=None): - if data_source == 'openai/gsm8k': - from . import gsm8k - res = gsm8k.compute_score(solution_str, ground_truth) - elif data_source in ['lighteval/MATH', 'DigitalLearningGmbH/MATH-lighteval']: - from . import math - res = math.compute_score(solution_str, ground_truth) - # [Optional] Math-Verify Integration - # For enhanced accuracy, consider utilizing Math-Verify (https://github.com/huggingface/Math-Verify). - # Note: Math-Verify needs to be manually installed via pip: `pip install math-verify`. - # To use it, override the `compute_score` function with the following implementation: - - # from . import math_verify - # res = math_verify.compute_score(solution_str, ground_truth) - elif data_source == 'math_dapo' or data_source.startswith("aime"): - from . import math_dapo - res = math_dapo.compute_score(solution_str, ground_truth) - elif data_source in [ - 'numina_aops_forum', 'numina_synthetic_math', 'numina_amc_aime', 'numina_synthetic_amc', 'numina_cn_k12', - 'numina_olympiads' - ]: - from . import prime_math - res = prime_math.compute_score(solution_str, ground_truth) - elif data_source in ['codecontests', 'apps', 'codeforces', 'taco']: - from . import prime_code - res = prime_code.compute_score(solution_str, ground_truth, continuous=True) - elif data_source in ['hiyouga/geometry3k']: - from . import geo3k - res = geo3k.compute_score(solution_str, ground_truth) - elif data_source in ["princeton-nlp/SWE-bench_Verified", "princeton-nlp/SWE-bench_Lite", "princeton-nlp/SWE-bench", "SWE-Gym/SWE-Gym"]: - from . import openhands_swebench - assert extra_info is not None and 'instance_id' in extra_info, "instance_id is required for openhands_swebench" - # assume that the instance id is in the extra_info - # print(f"Inside reward score init: calling into compute score") - res = openhands_swebench.compute_score(solution_str, ground_truth, extra_info['instance_id'], data_source) - else: - raise NotImplementedError(f"Reward function is not implemented for {data_source=}") - - if isinstance(res, dict): - return res - elif isinstance(res, (int, float, bool)): - return float(res) - else: - return float(res[0]) diff --git a/verl/utils/reward_score/geo3k.py b/verl/utils/reward_score/geo3k.py deleted file mode 100644 index 92c4554696..0000000000 --- a/verl/utils/reward_score/geo3k.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -from mathruler.grader import extract_boxed_content, grade_answer - - -def format_reward(predict_str: str) -> float: - pattern = re.compile(r'.*.*\\boxed\{.*\}.*', re.DOTALL) - match_result = re.fullmatch(pattern, predict_str) - return 1.0 if match_result else 0.0 - - -def acc_reward(predict_str: str, ground_truth: str) -> float: - answer = extract_boxed_content(predict_str) - return 1.0 if grade_answer(answer, ground_truth) else 0.0 - - -def compute_score(predict_str: str, ground_truth: str) -> float: - return 0.9 * acc_reward(predict_str, ground_truth) + 0.1 * format_reward(predict_str) diff --git a/verl/utils/reward_score/gsm8k.py b/verl/utils/reward_score/gsm8k.py deleted file mode 100644 index 7091037643..0000000000 --- a/verl/utils/reward_score/gsm8k.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - - -def extract_solution(solution_str, method='strict'): - assert method in ['strict', 'flexible'] - - if method == 'strict': - # this also tests the formatting of the model - solution = re.search("#### (\\-?[0-9\\.\\,]+)", solution_str) - if solution is None: - final_answer = None - else: - final_answer = solution.group(0) - final_answer = final_answer.split('#### ')[1].replace(',', '').replace('$', '') - elif method == 'flexible': - answer = re.findall("(\\-?[0-9\\.\\,]+)", solution_str) - final_answer = None - if len(answer) == 0: - # no reward is there is no answer - pass - else: - invalid_str = ['', '.'] - # find the last number that is not '.' - for final_answer in reversed(answer): - if final_answer not in invalid_str: - break - return final_answer - - -def compute_score(solution_str, ground_truth, method='strict', format_score=0., score=1.): - """The scoring function for GSM8k. - - Reference: Trung, Luong, et al. "Reft: Reasoning with reinforced fine-tuning." Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). 2024. - - Args: - solution_str: the solution text - ground_truth: the ground truth - method: the method to extract the solution, choices are 'strict' and 'flexible' - format_score: the score for the format - score: the score for the correct answer - """ - answer = extract_solution(solution_str=solution_str, method=method) - if answer is None: - return 0 - else: - if answer == ground_truth: - return score - else: - return format_score \ No newline at end of file diff --git a/verl/utils/reward_score/math.py b/verl/utils/reward_score/math.py deleted file mode 100644 index 50792aa6ed..0000000000 --- a/verl/utils/reward_score/math.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/hendrycks_math/utils.py - - -def compute_score(solution_str, ground_truth) -> float: - retval = 0. - try: - string_in_last_boxed = last_boxed_only_string(solution_str) - if string_in_last_boxed is not None: - answer = remove_boxed(string_in_last_boxed) - if is_equiv(answer, ground_truth): - retval = 1. - except Exception as e: - print(e) - - return retval - - -# string normalization from https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/hendrycks_math.py -def is_equiv(str1, str2, verbose=False): - if str1 is None and str2 is None: - print("WARNING: Both None") - return True - if str1 is None or str2 is None: - return False - - try: - ss1 = strip_string(str1) - ss2 = strip_string(str2) - if verbose: - print(ss1, ss2) - return ss1 == ss2 - except Exception: - return str1 == str2 - - -def remove_boxed(s): - if "\\boxed " in s: - left = "\\boxed " - assert s[:len(left)] == left - return s[len(left):] - - left = "\\boxed{" - - assert s[:len(left)] == left - assert s[-1] == "}" - - return s[len(left):-1] - - -def last_boxed_only_string(string): - idx = string.rfind("\\boxed") - if "\\boxed " in string: - return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] - if idx < 0: - idx = string.rfind("\\fbox") - if idx < 0: - return None - - i = idx - right_brace_idx = None - num_left_braces_open = 0 - while i < len(string): - if string[i] == "{": - num_left_braces_open += 1 - if string[i] == "}": - num_left_braces_open -= 1 - if num_left_braces_open == 0: - right_brace_idx = i - break - i += 1 - - if right_brace_idx is None: - retval = None - else: - retval = string[idx:right_brace_idx + 1] - - return retval - - -def fix_fracs(string): - substrs = string.split("\\frac") - new_str = substrs[0] - if len(substrs) > 1: - substrs = substrs[1:] - for substr in substrs: - new_str += "\\frac" - if substr[0] == "{": - new_str += substr - else: - try: - assert len(substr) >= 2 - except AssertionError: - return string - a = substr[0] - b = substr[1] - if b != "{": - if len(substr) > 2: - post_substr = substr[2:] - new_str += "{" + a + "}{" + b + "}" + post_substr - else: - new_str += "{" + a + "}{" + b + "}" - else: - if len(substr) > 2: - post_substr = substr[2:] - new_str += "{" + a + "}" + b + post_substr - else: - new_str += "{" + a + "}" + b - string = new_str - return string - - -def fix_a_slash_b(string): - if len(string.split("/")) != 2: - return string - a = string.split("/")[0] - b = string.split("/")[1] - try: - a = int(a) - b = int(b) - assert string == "{}/{}".format(a, b) - new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" - return new_string - except AssertionError: - return string - - -def remove_right_units(string): - # "\\text{ " only ever occurs (at least in the val set) when describing units - if "\\text{ " in string: - splits = string.split("\\text{ ") - assert len(splits) == 2 - return splits[0] - else: - return string - - -def fix_sqrt(string): - if "\\sqrt" not in string: - return string - splits = string.split("\\sqrt") - new_string = splits[0] - for split in splits[1:]: - if split[0] != "{": - a = split[0] - new_substr = "\\sqrt{" + a + "}" + split[1:] - else: - new_substr = "\\sqrt" + split - new_string += new_substr - return new_string - - -def strip_string(string): - # linebreaks - string = string.replace("\n", "") - - # remove inverse spaces - string = string.replace("\\!", "") - - # replace \\ with \ - string = string.replace("\\\\", "\\") - - # replace tfrac and dfrac with frac - string = string.replace("tfrac", "frac") - string = string.replace("dfrac", "frac") - - # remove \left and \right - string = string.replace("\\left", "") - string = string.replace("\\right", "") - - # Remove circ (degrees) - string = string.replace("^{\\circ}", "") - string = string.replace("^\\circ", "") - - # remove dollar signs - string = string.replace("\\$", "") - - # remove units (on the right) - string = remove_right_units(string) - - # remove percentage - string = string.replace("\\%", "") - string = string.replace("\%", "") # noqa: W605 - - # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string - string = string.replace(" .", " 0.") - string = string.replace("{.", "{0.") - # if empty, return empty string - if len(string) == 0: - return string - if string[0] == ".": - string = "0" + string - - # to consider: get rid of e.g. "k = " or "q = " at beginning - if len(string.split("=")) == 2: - if len(string.split("=")[0]) <= 2: - string = string.split("=")[1] - - # fix sqrt3 --> sqrt{3} - string = fix_sqrt(string) - - # remove spaces - string = string.replace(" ", "") - - # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} - string = fix_fracs(string) - - # manually change 0.5 --> \frac{1}{2} - if string == "0.5": - string = "\\frac{1}{2}" - - # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y - string = fix_a_slash_b(string) - - return string diff --git a/verl/utils/reward_score/math_dapo.py b/verl/utils/reward_score/math_dapo.py deleted file mode 100644 index 624651ad0d..0000000000 --- a/verl/utils/reward_score/math_dapo.py +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/hendrycks_math/utils.py - -import re -import signal -from typing import Optional - - -def last_boxed_only_string(string: str) -> Optional[str]: - """Extract the last LaTeX boxed expression from a string. - - Args: - string: Input string containing LaTeX code - - Returns: - The last boxed expression or None if not found - """ - idx = string.rfind("\\boxed{") - if idx < 0: - return None - - i = idx - right_brace_idx = None - num_left_braces_open = 0 - - while i < len(string): - if string[i] == "{": - num_left_braces_open += 1 - if string[i] == "}": - num_left_braces_open -= 1 - if num_left_braces_open == 0: - right_brace_idx = i - break - i += 1 - - return string[idx:right_brace_idx + 1] if right_brace_idx is not None else None - - -def remove_boxed(s: str) -> str: - """Remove the LaTeX boxed command from a string. - - Args: - s: String with format "\\boxed{content}" - - Returns: - The content inside the boxed command - """ - left = "\\boxed{" - assert s[:len(left)] == left, f"box error: {s}" - assert s[-1] == "}", f"box error: {s}" - return s[len(left):-1] - - -class timeout: - - def __init__(self, seconds=1, error_message="Timeout"): - self.seconds = seconds - self.error_message = error_message - - def handle_timeout(self, signum, frame): - raise TimeoutError(self.error_message) - - def __enter__(self): - signal.signal(signal.SIGALRM, self.handle_timeout) - signal.alarm(self.seconds) - - def __exit__(self, type, value, traceback): - signal.alarm(0) - - -# Constants for normalization -SUBSTITUTIONS = [ - ("an ", ""), - ("a ", ""), - (".$", "$"), - ("\\$", ""), - (r"\ ", ""), - (" ", ""), - ("mbox", "text"), - (",\\text{and}", ","), - ("\\text{and}", ","), - ("\\text{m}", "\\text{}"), -] - -REMOVED_EXPRESSIONS = [ - "square", - "ways", - "integers", - "dollars", - "mph", - "inches", - "hours", - "km", - "units", - "\\ldots", - "sue", - "points", - "feet", - "minutes", - "digits", - "cents", - "degrees", - "cm", - "gm", - "pounds", - "meters", - "meals", - "edges", - "students", - "childrentickets", - "multiples", - "\\text{s}", - "\\text{.}", - "\\text{\ns}", - "\\text{}^2", - "\\text{}^3", - "\\text{\n}", - "\\text{}", - r"\mathrm{th}", - r"^\circ", - r"^{\circ}", - r"\;", - r",\!", - "{,}", - '"', - "\\dots", -] - - -def normalize_final_answer(final_answer: str) -> str: - """Normalize a final answer to a quantitative reasoning question. - - Args: - final_answer: The answer string to normalize - - Returns: - Normalized answer string - """ - final_answer = final_answer.split("=")[-1] - - # Apply substitutions and removals - for before, after in SUBSTITUTIONS: - final_answer = final_answer.replace(before, after) - for expr in REMOVED_EXPRESSIONS: - final_answer = final_answer.replace(expr, "") - - # Extract and normalize LaTeX math - final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) - final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) - - # Normalize shorthand TeX: - # \fracab -> \frac{a}{b} - # \frac{abc}{bef} -> \frac{abc}{bef} - # \fracabc -> \frac{a}{b}c - # \sqrta -> \sqrt{a} - # \sqrtab -> sqrt{a}b - final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) - final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) - final_answer = final_answer.replace("$", "") - - # Normalize numbers - if final_answer.replace(",", "").isdigit(): - final_answer = final_answer.replace(",", "") - - return final_answer.strip() - - -def is_correct_minerva(solution_str: str, - gt: str, - gt_need_extract: bool = False, - answer_pattern: str = r"(?i)Answer\s*:\s*([^\n]+)") -> tuple[bool, str]: - """Check if the solution is correct according to Minerva criteria. - - Args: - solution_str: The solution string to check - gt: The ground truth answer - gt_need_extract: Whether the ground truth needs extraction - answer_pattern: Regex pattern to extract the answer - - Returns: - Tuple of (is_correct, normalized_prediction) - """ - # Extract answer from solution - match = re.findall(answer_pattern, solution_str) - extracted_answer = match[-1] if match else "[INVALID]" - pred = normalize_final_answer(extracted_answer) - - # Process ground truth - if gt_need_extract: - gt = normalize_final_answer(remove_boxed(last_boxed_only_string(gt))) - else: - gt = normalize_final_answer(gt) - - return (pred == gt), pred - - -def is_correct_strict_box(pred: str, - gt: str, - pause_tokens_index: Optional[list[int]] = None) -> tuple[int, Optional[str]]: - """Check if the prediction is correct using strict boxed answer criteria. - - Args: - pred: The prediction string - gt: The ground truth answer - pause_tokens_index: Indices of pause tokens - - Returns: - Tuple of (score, extracted_prediction) - """ - # Extract the relevant part of the prediction - if pause_tokens_index is not None: - assert len(pause_tokens_index) == 4 - pred = pred[pause_tokens_index[-1] - 100:] - else: - pred = pred[-100:] - - # Extract and check the boxed answer - boxed_pred = last_boxed_only_string(pred) - extracted_pred = remove_boxed(boxed_pred) if boxed_pred is not None else None - - return 1 if (extracted_pred == gt) else -1, extracted_pred - - -def verify(solution_str: str, - answer: str, - strict_box_verify: bool = False, - pause_tokens_index: Optional[list[int]] = None) -> bool: - """Verify if the solution is correct. - - Args: - solution_str: The solution string to verify - answer: The ground truth answer - strict_box_verify: Whether to use strict box verification - pause_tokens_index: Indices of pause tokens - - Returns: - True if the solution is correct, False otherwise - """ - if strict_box_verify: - correct, pred = is_correct_strict_box(solution_str, answer, pause_tokens_index) - return correct == 1, pred - - correct, pred = is_correct_minerva(solution_str, answer) - return correct, pred - - -def compute_score(solution_str: str, - ground_truth: str, - strict_box_verify: bool = False, - pause_tokens_index: Optional[list[int]] = None) -> float: - """Compute the reward score for a solution. - - Args: - solution_str: The solution string - ground_truth: The ground truth answer - config: Configuration object containing reward model settings - pause_tokens_index: Indices of pause tokens - - Returns: - Reward score (1.0 for correct, -1.0 for incorrect) - """ - # Limit solution length for efficiency - solution_str = solution_str[-300:] # The longest answer in MATH-500 has 159 characters - - # Verify the solution - correct, pred = verify(solution_str, ground_truth, strict_box_verify, pause_tokens_index) - - reward = 1.0 if correct else -1.0 - acc = correct - - return { - "score": reward, - "acc": acc, - "pred": pred, - } diff --git a/verl/utils/reward_score/math_verify.py b/verl/utils/reward_score/math_verify.py deleted file mode 100644 index 83a83bb046..0000000000 --- a/verl/utils/reward_score/math_verify.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - from math_verify.metric import math_metric - from math_verify.parser import LatexExtractionConfig, ExprExtractionConfig -except ImportError: - print("To use Math-Verify, please install it first by running `pip install math-verify`.") - - -def compute_score(model_output: str, ground_truth: str) -> bool: - verify_func = math_metric( - gold_extraction_target=(LatexExtractionConfig(),), - pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig()), - ) - ret_score = 0. - - # Wrap the ground truth in \boxed{} format for verification - ground_truth_boxed = "\\boxed{" + ground_truth + "}" - try: - ret_score, _ = verify_func([ground_truth_boxed], [model_output]) - except Exception as e: - pass - - return ret_score diff --git a/verl/utils/reward_score/openhands_swebench/__init__.py b/verl/utils/reward_score/openhands_swebench/__init__.py deleted file mode 100644 index b1b58d438d..0000000000 --- a/verl/utils/reward_score/openhands_swebench/__init__.py +++ /dev/null @@ -1,388 +0,0 @@ -# Modified from https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks/swe_bench/eval_infer.py -# commit around: 1a7003a7056f5b459207c2e2d2c6db4646967c53 -# (DL): Deleted logic, in addition to dataset processing: -# 1. test spec processing, assume ground truth is already provided (reference: prime_code) -# 2. -import json -import os -import tempfile -import time - -from pathlib import Path -from typing import cast - -import pandas as pd -from datasets import Dataset, load_dataset, load_from_disk -from swegym.harness.grading import get_eval_report -from swegym.harness.run_evaluation import ( - APPLY_PATCH_FAIL, - APPLY_PATCH_PASS, -) - -from openhands.core.config import ( - AppConfig, - SandboxConfig -) -from openhands.core.logger import openhands_logger as logger -from openhands.core.main import create_runtime -from openhands.events.action import CmdRunAction -from openhands.events.observation import CmdOutputObservation -from openhands.utils.async_utils import call_async_from_sync - - -from swebench.harness.constants import ( - SWEbenchInstance, - KEY_INSTANCE_ID -) - - -# TODO: migrate all swe-bench docker to ghcr.io/openhands -DOCKER_IMAGE_PREFIX = os.environ.get('EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/xingyaoww/') -logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}') - -"""Mapping instance_id to resource_factor. -Different instances may have different resource requirements. -e.g., some instances may require more memory/CPU to run inference. -This file tracks the resource requirements of different instances. -""" - -import json -import os - -from openhands.core.logger import openhands_logger as logger - -# OpenHands evaluation/benchmarks/swe_bench/resource/mapping.py -CUR_DIR = os.path.dirname(os.path.abspath(__file__)) -DEFAULT_RUNTIME_RESOURCE_FACTOR = int( - os.environ.get('DEFAULT_RUNTIME_RESOURCE_FACTOR', 1) -) - -""" -# Adapt from https://github.com/SWE-bench/SWE-bench/blob/main/swebench/harness/utils.py -def load_swebench_dataset( - name="princeton-nlp/SWE-bench", split="test", instance_ids=None -) -> list[SWEbenchInstance]: - """""" - Load SWE-bench dataset from Hugging Face Datasets or local .json/.jsonl file - """""" - # check that all instance IDs are in the dataset - if instance_ids: - instance_ids = set(instance_ids) - # Load from local .json/.jsonl file - if name.endswith(".json") or name.endswith(".jsonl"): - dataset = json.loads(Path(name).read_text()) - dataset_ids = {instance[KEY_INSTANCE_ID] for instance in dataset} - else: - # Load from Hugging Face Datasets - if name.lower() in {"swe-bench", "swebench", "swe_bench"}: - name = "princeton-nlp/SWE-bench" - elif name.lower() in { - "swe-bench-lite", - "swebench-lite", - "swe_bench_lite", - "swe-bench_lite", - "lite", - }: - name = "princeton-nlp/SWE-bench_Lite" - elif "swe-gym" in name.lower(): - name = "SWE-Gym/SWE-Gym" - if (Path(name) / split / "dataset_info.json").exists(): - dataset = cast(Dataset, load_from_disk(Path(name) / split)) - else: - dataset = cast(Dataset, load_dataset(name, split=split)) - dataset_ids = {instance[KEY_INSTANCE_ID] for instance in dataset} - if instance_ids: - if instance_ids - dataset_ids: - raise ValueError( - ( - "Some instance IDs not found in dataset!" - f"\nMissing IDs:\n{' '.join(instance_ids - dataset_ids)}" - ) - ) - dataset = [ - instance - for instance in dataset - if instance[KEY_INSTANCE_ID] in instance_ids - ] - return [cast(SWEbenchInstance, instance) for instance in dataset] -""" - -# dataset to resource mapping -_global_resource_mapping: dict[str, dict[str, float]] = {} -def get_resource_mapping(dataset_name: str) -> dict[str, float]: - if dataset_name not in _global_resource_mapping: - file_path = os.path.join(CUR_DIR, f'{dataset_name}.json') - if not os.path.exists(file_path): - logger.warning(f'Resource mapping for {dataset_name} not found.') - return None - - with open(file_path, 'r') as f: - _global_resource_mapping[dataset_name] = json.load(f) - logger.info(f'Loaded resource mapping for {dataset_name}') - return _global_resource_mapping[dataset_name] - -def get_instance_resource_factor(dataset_name: str, instance_id: str) -> int: - resource_mapping = get_resource_mapping(dataset_name) - if resource_mapping is None: - return DEFAULT_RUNTIME_RESOURCE_FACTOR - return int(resource_mapping.get(instance_id, DEFAULT_RUNTIME_RESOURCE_FACTOR)) - -def get_default_sandbox_config_for_eval() -> SandboxConfig: - return SandboxConfig( - use_host_network=False, - # large enough timeout, since some testcases take very long to run - timeout=300, - api_key=os.environ.get('ALLHANDS_API_KEY', None), - remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'), - keep_runtime_alive=False, - remote_runtime_init_timeout=3600, - remote_runtime_api_timeout=120, - remote_runtime_enable_retries=True, - remote_runtime_class='sysbox', - ) - -def get_instance_docker_image(instance_id: str) -> str: - image_name = 'sweb.eval.x86_64.' + instance_id - image_name = image_name.replace( - '__', '_s_' - ) # to comply with docker image naming convention - return (DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + image_name).lower() - -def process_git_patch(patch): - if not isinstance(patch, str): - return '' - - if not patch.strip(): - print(f'Skipping empty patch....') - # skip empty patches - return '' - - patch = patch.replace('\r\n', '\n') - # There might be some weird characters at the beginning of the patch - # due to some OpenHands inference command outputs - - # FOR EXAMPLE: - # git diff --no-color --cached 895f28f9cbed817c00ab68770433170d83132d90 - # 0 - # diff --git a/django/db/models/sql/.backup.query.py b/django/db/models/sql/.backup.query.py - # new file mode 100644 - # index 0000000000..fc13db5948 - - # We "find" the first line that starts with "diff" and then we remove lines before it - lines = patch.split('\n') - for i, line in enumerate(lines): - if line.startswith('diff --git'): - patch = '\n'.join(lines[i:]) - break - - patch = patch.rstrip() + '\n' # Make sure the last line ends with a newline - return patch - - -def get_config(data_source, instance_id) -> AppConfig: - # We use a different instance image for the each instance of swe-bench eval - base_container_image = get_instance_docker_image(instance_id) - logger.info( - f'Using instance container image: {base_container_image}. ' - f'Please make sure this image exists. ' - f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.' - ) - sandbox_config = get_default_sandbox_config_for_eval() - sandbox_config.base_container_image = base_container_image - sandbox_config.remote_runtime_resource_factor = get_instance_resource_factor( - dataset_name=data_source, - instance_id=instance_id, - ) - config = AppConfig( - run_as_openhands=False, - runtime="remote", - sandbox=sandbox_config, - # do not mount workspace - workspace_base=None, - workspace_mount_path=None, - ) - return config - -# modified from the process_instance in eval_infer.py -def compute_score( - model_patch, - test_spec, - instance_id, - data_source, - ): - model_patch = process_git_patch(model_patch) - config = get_config(data_source, instance_id) - instance = {"instance_id": instance_id, "model_patch": model_patch, "test_spec": test_spec, "test_result": {}} - - instance['test_result']['report'] = { - 'empty_generation': False, - 'resolved': False, - 'failed_apply_patch': False, - 'error_eval': False, - 'test_timeout': False, - } - - # print(f"(DL Debug in openhands swebench init) Inside compute score with model_patch: {model_patch} {model_patch == ""}") - if model_patch == '': - # print(f"(DL Debug in openhands swebench init) Inside compute score empty branch with instance {instance}") - instance['test_result']['report']['empty_generation'] = True - return instance['test_result']['report']['resolved'] # (DL): This is 0, but for compatibility, we return the resolved value - - try: - runtime = create_runtime(config) - call_async_from_sync(runtime.connect) - # Get patch and save it to /tmp/patch.diff - with tempfile.TemporaryDirectory() as temp_dir: - # Patch file - patch_file_path = os.path.join(temp_dir, 'patch.diff') - with open(patch_file_path, 'w') as f: - f.write(model_patch) - runtime.copy_to(patch_file_path, '/tmp') - # Eval script - eval_script_path = os.path.join(temp_dir, 'eval.sh') - with open(eval_script_path, 'w') as f: - f.write(test_spec.eval_script) - runtime.copy_to(eval_script_path, '/tmp') - - # Set +x - action = CmdRunAction(command='chmod +x /tmp/eval.sh') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert obs.exit_code == 0 - - print(f"(DL Debug in openhands swebench init) Inside compute score L253") - # Apply patch - exec_command = ( - 'cd /testbed && ' - "(git apply -v /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " - "(echo 'Failed to apply patch with git apply, trying with patch command...' && " - "(patch --batch --fuzz=5 -p1 -i /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " - "echo 'APPLY_PATCH_FAIL')))" - ) - action = CmdRunAction(command=exec_command) - action.set_hard_timeout(600) - obs = runtime.run_action(action) - assert isinstance(obs, CmdOutputObservation) - apply_patch_output = obs.content - assert isinstance(apply_patch_output, str) - instance['test_result']['apply_patch_output'] = apply_patch_output - - # print(f"(DL Debug in openhands swebench init) Inside compute score L269") - if 'APPLY_PATCH_FAIL' in apply_patch_output: - logger.info(f'[{instance_id}] {APPLY_PATCH_FAIL}:\n{apply_patch_output}') - instance['test_result']['report']['failed_apply_patch'] = True - - return instance['test_result']['report']['resolved'] - elif 'APPLY_PATCH_PASS' in apply_patch_output: - logger.info(f'[{instance_id}] {APPLY_PATCH_PASS}:\n{apply_patch_output}') - - # Run eval script in background and save output to log file - log_file = '/tmp/eval_output.log' - action = CmdRunAction(command=f'/tmp/eval.sh > {log_file} 2>&1 & echo $!') - action.set_hard_timeout(300) # Short timeout just to get the process ID - obs = runtime.run_action(action) - - # print(f"(DL Debug in openhands swebench init) Inside compute score L284") - if isinstance(obs, CmdOutputObservation) and obs.exit_code == 0: - pid = obs.content.split()[-1].strip() - logger.info( - f'[{instance_id}] Evaluation process started with PID: {pid}' - ) - - # Poll for completion - start_time = time.time() - timeout = 1800 # 30 minutes - while True: - seconds_elapsed = time.time() - start_time - if seconds_elapsed > timeout: - logger.info( - f'[{instance_id}] Evaluation timed out after {timeout} seconds' - ) - instance['test_result']['report']['test_timeout'] = True - break - check_action = CmdRunAction( - command=f'ps -p {pid} > /dev/null; echo $?' - ) - check_action.set_hard_timeout(300) - check_obs = runtime.run_action(check_action) - if ( - isinstance(check_obs, CmdOutputObservation) - and check_obs.content.split()[-1].strip() == '1' - ): - logger.info( - f'[{instance_id}] Evaluation process completed after {seconds_elapsed} seconds' - ) - break - logger.info( - f'[{instance_id}] [{seconds_elapsed:.0f}s] Evaluation still running, waiting...' - ) - time.sleep(30) # Wait for 30 seconds before checking again - # print(f"(DL Debug in openhands swebench init) Inside compute score L320") - - # Read the log file - cat_action = CmdRunAction(command=f'cat {log_file}') - cat_action.set_hard_timeout(300) - cat_obs = runtime.run_action(cat_action) - - # Grade answer - if isinstance(cat_obs, CmdOutputObservation) and cat_obs.exit_code == 0: - test_output = cat_obs.content - assert isinstance(test_output, str) - instance['test_result']['test_output'] = test_output - - # Get report from test output - logger.info(f'[{instance_id}] Grading answer...') - # print(f"(DL Debug in openhands swebench init) Inside compute score L335") - with tempfile.TemporaryDirectory() as temp_dir: - # Create a directory structure that matches the expected format - # NOTE: this is a hack to make the eval report format consistent - # with the original SWE-Bench eval script - log_dir = os.path.join(temp_dir, 'logs', instance_id.lower()) - os.makedirs(log_dir, exist_ok=True) - test_output_path = os.path.join(log_dir, 'test_output.txt') - with open(test_output_path, 'w') as f: - f.write(test_output) - try: - _report = get_eval_report( - test_spec=test_spec, - prediction={ - 'model_patch': model_patch, - 'instance_id': instance_id, - }, - log_path=test_output_path, - include_tests_status=True, - ) - report = _report[instance_id] - logger.info( - f"[{instance_id}] report: {report}\nResult for {instance_id}: resolved: {report['resolved']}" - ) - instance['test_result']['report']['resolved'] = report[ - 'resolved' - ] - except Exception as e: - logger.error( - f'[{instance_id}] Error when getting eval report: {e}' - ) - instance['test_result']['report']['resolved'] = False - instance['test_result']['report']['error_eval'] = True - # print(f"(DL Debug in openhands swebench init) Inside compute score L367") - else: - logger.info(f'[{instance_id}] Error when starting eval:\n{obs.content}') - instance['test_result']['report']['error_eval'] = True - - return instance['test_result']['report']['resolved'] - else: - logger.info( - f'[{instance_id}] Unexpected output when applying patch:\n{apply_patch_output}' - ) - return instance['test_result']['report']['resolved'] - # (DL): This is not raised, we should always return a number for training - # raise RuntimeError( - # instance_id, - # f'Unexpected output when applying patch:\n{apply_patch_output}', - # logger, - # ) - finally: - runtime.close() diff --git a/verl/utils/reward_score/prime_code/__init__.py b/verl/utils/reward_score/prime_code/__init__.py deleted file mode 100644 index 49b0d30f57..0000000000 --- a/verl/utils/reward_score/prime_code/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .utils import check_correctness as apps_check_correctness -import json -import re -import traceback - - -def compute_score(completion, test_cases, continuous=False): - # try to get code solution from completion. if the completion is pure code, this will not take effect. - solution = completion.split('```python')[-1].split('```')[0] - try: - try: - if not isinstance(test_cases, dict): - test_cases = json.loads(test_cases) - except Exception as e: - print(f"Error:{e}") - - # Complete check on all in-out pairs first. If there is no failure, per-sample test can be skipped. - try: - res, metadata = apps_check_correctness(in_outs=test_cases, generation=solution, timeout=5, debug=False) - metadata = dict(enumerate(metadata))[0] - success = all(map(lambda x: x == True, res)) - if success: - return success, metadata - except Exception as e: - pass - - test_cases_list = [] - inputs = test_cases["inputs"] - outputs = test_cases["outputs"] - for i in range(len(inputs)): - test_cases_list.append({"inputs": [inputs[i]], "outputs": [outputs[i]]}) - - if continuous: - # per sample test: if continuous score is needed, test first 10 samples regardless of failures - # do not test all samples cuz some problems have enormous test cases - metadata_list = [] - res_list = [] - for test_case_id, test_case in enumerate(test_cases_list): - res, metadata = apps_check_correctness(in_outs=test_case, generation=solution, timeout=5, debug=False) - try: - metadata = dict(enumerate(metadata))[0] # metadata can be empty occasionally - except Exception as e: - metadata = {} - metadata["test_case"] = {} - metadata["test_case"]["input"] = str(test_case["inputs"][0]) - metadata["test_case"]["output"] = str(test_case["outputs"][0]) - metadata["test_case"]["res"] = str(res) - metadata_list.append(metadata) - res_list.extend(res) - - if test_case_id >= 9: - break - res_count = len(res_list) if len(res_list) > 0 else 1 - success = sum(map(lambda x: x == True, res_list)) / res_count - except Exception as e: - traceback.print_exc(10) - success = False - metadata_list = None - return success, metadata_list diff --git a/verl/utils/reward_score/prime_code/testing_util.py b/verl/utils/reward_score/prime_code/testing_util.py deleted file mode 100644 index 34845d07ef..0000000000 --- a/verl/utils/reward_score/prime_code/testing_util.py +++ /dev/null @@ -1,721 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import ast -import json -import sys -import faulthandler -import platform - -# used for debugging to time steps -from datetime import datetime - -# to run the solution files we're using a timing based approach -import signal - -import numpy as np - -# for capturing the stdout -from io import StringIO - -# used for testing the code that reads from input -from unittest.mock import patch, mock_open - -from pyext import RuntimeModule - -from enum import Enum - -import traceback - - -def truncatefn(s, length=300): - assert isinstance(s, str) - if len(s) <= length: - return s - - return s[:length // 2] + "...(truncated) ..." + s[-length // 2:] - - -class CODE_TYPE(Enum): - call_based = 0 - standard_input = 1 - - -# stuff for setting up signal timer -class TimeoutException(Exception): - pass - - -def timeout_handler(signum, frame): - print("alarm went off") - return - # raise TimeoutException # this is an unhandled exception. just return None is OK - - -signal.signal(signal.SIGALRM, timeout_handler) - -# timeout = 6 # seconds - - -# used to capture stdout as a list -# from https://stackoverflow.com/a/16571630/6416660 -# alternative use redirect_stdout() from contextlib -class Capturing(list): - - def __enter__(self): - self._stdout = sys.stdout - sys.stdout = self._stringio = StringIO() - # Make closing the StringIO a no-op - self._stringio.close = lambda x: 1 - return self - - def __exit__(self, *args): - self.append(self._stringio.getvalue()) - del self._stringio # free up some memory - sys.stdout = self._stdout - - -def only_int_check(val): - return isinstance(val, int) - - -def string_int_check(val): - return isinstance(val, str) and val.isdigit() - - -def combined_int_check(val): - return only_int_check(val) or string_int_check(val) - - -def clean_traceback(error_traceback): - file_start = error_traceback.find('File \"\"') - # print(file_start) - error_traceback = "Traceback (most recent call last):\n " + error_traceback[file_start:] - return error_traceback - - -def run_test(in_outs, test=None, debug=False, timeout=15): - """ - if test(generated_code) is not None it'll try to run the code. - otherwise it'll just return an input and output pair. - """ - # Disable functionalities that can make destructive changes to the test. - reliability_guard() - - if debug: - print(f"start = {datetime.now().time()}") - - if in_outs: - if in_outs.get("fn_name") is None: - which_type = CODE_TYPE.standard_input # Standard input - method_name = None - else: - which_type = CODE_TYPE.call_based # Call-based - method_name = in_outs["fn_name"] - - if debug: - print(f"loaded input_output = {datetime.now().time()}") - - if test is None: - assert False, "should not happen: test code is none" - return in_outs, {"error": "no test code provided"} - elif test is not None: - results = [] - sol = "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(6*10**5)\n" - if debug: - print(f"loading test code = {datetime.now().time()}") - - if which_type == CODE_TYPE.call_based: - - sol += test - if debug: - print(f"sol = {sol}") - signal.alarm(timeout) - try: - tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol) - if "class Solution" not in test: - tmp = tmp_sol - else: - tmp = tmp_sol.Solution() - signal.alarm(0) - except Exception as e: - signal.alarm(0) - error_traceback = traceback.format_exc() - if debug: - print(f"type 0 compilation error = {e}") - results.append(-2) - return results, { - "error": repr(e), - # "error_code": -1, - # "error_message": "Compilation Error", - "traceback": clean_traceback(error_traceback), - } - signal.alarm(0) - - elif which_type == CODE_TYPE.standard_input: - # sol - # if code has if __name__ == "__main__": then remove it - try: - astree = ast.parse(test) - last_block = astree.body[-1] - if isinstance(last_block, ast.If): - condition = last_block.test - if ast.unparse(condition).strip() == "__name__ == '__main__'": - test = (ast.unparse(astree.body[:-1]) + "\n" + ast.unparse(last_block.body)) - except: - pass - - tmp_test = test.split("\n") - - new_test = [] - for x in tmp_test: - if (not x.startswith("from ")) and (not x.startswith("import ")): - new_test.append("\t" + x + "\n") - else: - new_test.append(x + "\n") - tmp_test = new_test - - new_test = "" - started = False - for i in tmp_test: - if i.startswith("\t") and not started: - new_test += "stdin = sys.stdin\nstdout = sys.stdout\n" - new_test += "def code():\n" - new_test += i - started = True - elif started and ((i.startswith("from ")) or (i.startswith("import "))): - new_test += "\t" + i - else: - new_test += i - tmp_test = new_test - - sol += tmp_test - if debug: - print(f"sol = {sol}") - method_name = "code" - signal.alarm(timeout) - try: - tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol) - tmp = tmp_sol - signal.alarm(0) - except Exception as e: - signal.alarm(0) - error_traceback = traceback.format_exc() - if debug: - print(f"type 1 compilation error = {e}") - results.append(-2) - return results, { - "error": repr(e), - # "error_code": -1, - # "error_message": "Compilation Error", - "traceback": clean_traceback(error_traceback), - } - signal.alarm(0) - if debug: - print(f"get method = {datetime.now().time()}") - - try: - method = getattr(tmp, method_name) # get_attr second arg must be str - except: - signal.alarm(0) - error_traceback = traceback.format_exc() - e = sys.exc_info() - print(f"unable to get function error = {e}") - results.append(-2) - return results, { - "error": repr(e), - # "error_code": -1, - # "error_message": "Unable to extract code", - "traceback": clean_traceback(error_traceback), - } - - for index, inputs in enumerate(in_outs["inputs"]): - raw_inputs = inputs - raw_outputs = in_outs["outputs"][index] - if which_type == CODE_TYPE.call_based: - inputs = [json.loads(line) for line in inputs.split("\n")] - in_outs["outputs"][index] = json.loads(in_outs["outputs"][index]) - - truncate_line_size = 300 // (raw_inputs.count("\n") + 1) - raw_inputs = "\n".join( - [truncatefn(line, truncate_line_size) for line in raw_inputs.strip().split("\n")]) - raw_outputs = truncatefn(raw_outputs, 200) - else: - raw_inputs = truncatefn(raw_inputs) - raw_outputs = truncatefn(raw_outputs, 200) - # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list) - try: - if isinstance(inputs[0], dict): - inputs = [{int(k): v for k, v in inputs[0].items()}] - except: - True - try: - if isinstance(in_outs["outputs"][index], dict): - in_outs["outputs"][index] = [{int(k): v for k, v in in_outs["outputs"][index].items()}] - except: - True - try: - if isinstance(in_outs["outputs"][index][0], dict): - in_outs["outputs"][index] = [{int(k): v for k, v in in_outs["outputs"][index][0].items()}] - except: - True - - if debug: - print( - f"time: {datetime.now().time()} testing index = {index} inputs = {inputs}, {type(inputs)}. type = {which_type}" - ) - if which_type == CODE_TYPE.call_based: # Call-based - signal.alarm(timeout) - faulthandler.enable() - try: - output = method(*inputs) - raw_true_output = output - - raw_true_output_copy = json.dumps(output) - raw_true_output_copy = truncatefn(raw_true_output_copy, 200) - - # ground truth sequences are not tuples - if isinstance(output, tuple): - output = list(output) - - tmp_result = output == in_outs["outputs"][index] - if (isinstance(in_outs["outputs"][index], list) and in_outs["outputs"][index]): - tmp_result = tmp_result or (output == in_outs["outputs"][index][0]) - - # ground truth sequences are not tuples - try: - if isinstance(output[0], tuple): - tmp_result = tmp_result or ([list(x) for x in output] == in_outs["outputs"][index][0]) - except: - True - results.append(tmp_result) - if tmp_result != True: - return results, { - "output": raw_true_output_copy, - "expected": raw_outputs, - "inputs": raw_inputs, - # "error_code": -2, - "error_message": "Wrong Answer", - } - # reset the alarm - signal.alarm(0) - except Exception as e: - signal.alarm(0) - error_traceback = traceback.format_exc() - faulthandler.disable() - if debug: - print(f"Standard input runtime error or time limit exceeded error = {e}") - results.append(-1) - if "timeoutexception" in repr(e).lower(): - return results, { - "error": repr(e), - # "error_code": -3, - # "error_message": "Time Limit Exceeded", - # "inputs": raw_inputs, - # "expected": raw_outputs, - "traceback": clean_traceback(error_traceback), - } - else: - return results, { - "error": repr(e), - # "error_code": -4, - # "error_message": "Runtime Error", - # "inputs": raw_inputs, - # "expected": raw_outputs, - "traceback": clean_traceback(error_traceback), - } - faulthandler.disable() - signal.alarm(0) - if debug: - print( - f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" - ) - elif which_type == CODE_TYPE.standard_input: # Standard input - faulthandler.enable() - passed = False - - if isinstance(inputs, list): - inputs = "\n".join(inputs) - if isinstance(in_outs["outputs"][index], list): - in_outs["outputs"][index] = "\n".join(in_outs["outputs"][index]) - - signal.alarm(timeout) - with Capturing() as output: - try: - call_method(method, inputs) - # reset the alarm - signal.alarm(0) - passed = True - except Exception as e: - # runtime error or took too long - signal.alarm(0) - error_traceback = traceback.format_exc() - print(f"Call-based runtime error or time limit exceeded error = {repr(e)}{e}") - results.append(-1) - if "timeoutexception" in repr(e).lower(): - return results, { - "error": repr(e), - # "error_code": -3, - # "error_message": "Time Limit Exceeded", - # "inputs": raw_inputs, - # "expected": raw_outputs, - "traceback": clean_traceback(error_traceback), - } - else: - return results, { - "error": repr(e), - # "error_code": -4, - # "error_message": "Runtime Error", - # "inputs": raw_inputs, - # "expected": raw_outputs, - "traceback": clean_traceback(error_traceback), - } - signal.alarm(0) - raw_true_output = output[0] - raw_true_output_copy = truncatefn(raw_true_output, 200) - output = raw_true_output.splitlines() - if not passed: - if debug: - nl = "\n" - if not isinstance(inputs, list): - print( - f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" - ) - else: - print( - f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" - ) - continue - - if passed and debug: - print(f"==> output = {output}, test outputs = {in_outs['outputs'][index]}") - - if custom_compare_(output, in_outs["outputs"][index]): - tmp_result = True - results.append(tmp_result) - continue - - # ground truth sequences are expressed as lists not tuples - if isinstance(output, tuple): - output = list(output) - - tmp_result = False - try: - tmp_result = output == [in_outs["outputs"][index]] - if isinstance(in_outs["outputs"][index], list): - tmp_result = tmp_result or (output == in_outs["outputs"][index]) - if isinstance(output[0], str): - tmp_result = tmp_result or ([e.strip() for e in output] == in_outs["outputs"][index]) - except Exception as e: - if debug: - print(f"Failed check1 exception = {e}") - pass - - if tmp_result == True: - results.append(tmp_result) - continue - - # try one more time without \n - if isinstance(in_outs["outputs"][index], list): - for tmp_index, i in enumerate(in_outs["outputs"][index]): - in_outs["outputs"][index][tmp_index] = i.split("\n") - in_outs["outputs"][index][tmp_index] = [ - x.strip() for x in in_outs["outputs"][index][tmp_index] if x - ] - else: - in_outs["outputs"][index] = in_outs["outputs"][index].split("\n") - in_outs["outputs"][index] = list(filter(len, in_outs["outputs"][index])) - in_outs["outputs"][index] = list(map(lambda x: x.strip(), in_outs["outputs"][index])) - - try: - tmp_result = output == [in_outs["outputs"][index]] - if isinstance(in_outs["outputs"][index], list): - tmp_result = tmp_result or (output == in_outs["outputs"][index]) - except Exception as e: - if debug: - print(f"Failed check2 exception = {e}") - pass - - if tmp_result == True: - results.append(tmp_result) - continue - - # try by converting the output into a split up list too - if isinstance(output, list): - output = list(filter(len, output)) - - if debug: - nl = "\n" - if not isinstance(inputs, list): - print( - f"@1 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]} {tmp_result=}" - ) - else: - print( - f"@1 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]} {tmp_result=}" - ) - - if tmp_result == True: - results.append(tmp_result) - continue - - if debug: - print(f"{tmp_result=} @a") - - try: - tmp_result = output == [in_outs["outputs"][index]] - if isinstance(in_outs["outputs"][index], list): - tmp_result = tmp_result or (output == in_outs["outputs"][index]) - except Exception as e: - if debug: - print(f"Failed check3 exception = {e}") - pass - - if debug: - print(f"{tmp_result=} @b") - - try: - all_ints = all( - combined_int_check(e1) and combined_int_check(e2) - for e1, e2 in zip(output, in_outs["outputs"][index])) - if not all_ints: - if debug: - print([ - combined_int_check(e1) and combined_int_check(e2) - for e1, e2 in zip(output, in_outs["outputs"][index]) - ]) - output_float = [float(e) for e in output] - gt_float = [float(e) for e in in_outs["outputs"][index]] - tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and - np.allclose(output_float, gt_float)) - except Exception as e: - pass - - if debug: - print(f"{tmp_result=} @c") - - try: - if isinstance(output[0], list): - all_ints = all( - combined_int_check(e1) and combined_int_check(e2) - for e1, e2 in zip(output[0], in_outs["outputs"][index])) - if not all_ints: - output_float = [float(e) for e in output[0]] - gt_float = [float(e) for e in in_outs["outputs"][index][0]] - tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and - np.allclose(output_float, gt_float)) - except Exception as e: - pass - - if tmp_result == True: - results.append(tmp_result) - continue - - if debug: - print(f"{tmp_result=} @d") - # try by converting the stuff into split up list - if isinstance(in_outs["outputs"][index], list): - for tmp_index, i in enumerate(in_outs["outputs"][index]): - in_outs["outputs"][index][tmp_index] = set(i.split()) - else: - in_outs["outputs"][index] = set(in_outs["outputs"][index].split()) - - if debug: - print(f"{tmp_result=} @e") - - try: - tmp_result = output == in_outs["outputs"][index] - except Exception as e: - if debug: - print(f"Failed check4 exception = {e}") - continue - - if tmp_result == True: - results.append(tmp_result) - continue - - if debug: - print(f"{tmp_result=} @f") - - # try by converting the output into a split up list too - if isinstance(output, list): - for tmp_index, i in enumerate(output): - output[tmp_index] = i.split() - output = list(filter(len, output)) - for tmp_index, i in enumerate(output): - output[tmp_index] = set(i) - else: - output = output.split() - output = list(filter(len, output)) - output = set(output) - - if debug: - print(f"{tmp_result=} @g") - - if tmp_result == True and debug: - print("PASSED") - - results.append(tmp_result) - if tmp_result != True: - return results, { - "output": raw_true_output_copy, - "expected": raw_outputs, - "inputs": raw_inputs, - # "error_code": -2, - "error_message": "Wrong Answer", - } - - if debug: - nl = "\n" - if not isinstance(inputs, list): - print( - f"@2 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" - ) - else: - print( - f"@2 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" - ) - - print(f"results = {results}") - - return results, {} - - -def custom_compare_(output, ground_truth): - - if isinstance(output, list): - output_1 = "\n".join(output) - if stripped_string_compare(output_1, ground_truth): - return True - - if isinstance(output, list): - output_2 = [o.lstrip().rstrip() for o in output] - output_2 = "\n".join(output_2) - if stripped_string_compare(output_2, ground_truth): - return True - - return False - - -def stripped_string_compare(s1, s2): - s1 = s1.lstrip().rstrip() - s2 = s2.lstrip().rstrip() - return s1 == s2 - - -def call_method(method, inputs): - - if isinstance(inputs, list): - inputs = "\n".join(inputs) - - inputs_line_iterator = iter(inputs.split("\n")) - - # sys.setrecursionlimit(10000) - - # @patch('builtins.input', side_effect=inputs.split("\n")) - @patch("builtins.open", mock_open(read_data=inputs)) - @patch("sys.stdin", StringIO(inputs)) - @patch("sys.stdin.readline", lambda *args: next(inputs_line_iterator)) - @patch("sys.stdin.readlines", lambda *args: inputs.split("\n")) - @patch("sys.stdin.read", lambda *args: inputs) - # @patch('sys.stdout.write', print) - def _inner_call_method(_method): - try: - return _method() - except SystemExit as e: - pass - finally: - pass - - return _inner_call_method(method) - - -def reliability_guard(maximum_memory_bytes=None): - """ - This disables various destructive functions and prevents the generated code - from interfering with the test (e.g. fork bomb, killing other processes, - removing filesystem files, etc.) - WARNING - This function is NOT a security sandbox. Untrusted code, including, model- - generated code, should not be blindly executed outside of one. See the - Codex paper for more information about OpenAI's code sandbox, and proceed - with caution. - """ - - if maximum_memory_bytes is not None: - import resource - - resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) - resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) - if not platform.uname().system == "Darwin": - resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) - - faulthandler.disable() - - import builtins - - builtins.exit = None - builtins.quit = None - - import os - - os.environ["OMP_NUM_THREADS"] = "1" - - os.kill = None - os.system = None # 防止干扰repl评测 - os.putenv = None - os.remove = None - os.removedirs = None - os.rmdir = None - os.fchdir = None - os.setuid = None - os.fork = None - os.forkpty = None - os.killpg = None - os.rename = None - os.renames = None - os.truncate = None - os.replace = None - os.unlink = None - os.fchmod = None - os.fchown = None - os.chmod = None - os.chown = None - os.chroot = None - os.fchdir = None - os.lchflags = None - os.lchmod = None - os.lchown = None - os.getcwd = None - os.chdir = None - - import shutil - - shutil.rmtree = None - shutil.move = None - shutil.chown = None - - import subprocess - - subprocess.Popen = None # type: ignore - - __builtins__["help"] = None - - import sys - - sys.modules["ipdb"] = None - sys.modules["joblib"] = None - sys.modules["resource"] = None - sys.modules["psutil"] = None - sys.modules["tkinter"] = None diff --git a/verl/utils/reward_score/prime_code/utils.py b/verl/utils/reward_score/prime_code/utils.py deleted file mode 100644 index 07f3db88e0..0000000000 --- a/verl/utils/reward_score/prime_code/utils.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Borrowed from: https://huggingface.co/spaces/codeparrot/apps_metric/blob/main/utils.py - -import multiprocessing -from typing import Dict, Optional -from datasets import load_dataset -from .testing_util import run_test -import traceback -import os, sys - - -def _temp_run(sample, generation, debug, result, metadata_list, timeout): - with open(os.devnull, 'w') as devnull: - sys.stdout = devnull - sys.stderr = devnull - try: - res, metadata = run_test(in_outs=sample, test=generation, debug=debug, timeout=timeout) - result.append(res) - metadata_list.append(metadata) - except Exception as e: - # print(e) # some tracebacks are extremely long. - traceback.print_exc(10) - result.append([-1 for i in range(len(sample['inputs']))]) - metadata_list.append({}) - - -def check_correctness(in_outs: Optional[dict], generation, timeout=10, debug=True): - """Check correctness of code generation with a global timeout. - The global timeout is to catch some extreme/rare cases not handled by the timeouts - inside `run_test`""" - - manager = multiprocessing.Manager() - result = manager.list() - metadata_list = manager.list() - p = multiprocessing.Process(target=_temp_run, args=(in_outs, generation, debug, result, metadata_list, timeout)) - p.start() - p.join(timeout=timeout + 1) - if p.is_alive(): - p.kill() - # p.terminate() - if not result: - # consider that all tests failed - result = [[-1 for i in range(len(in_outs["inputs"]))]] - if debug: - print(f"global timeout") - return result[0], metadata_list diff --git a/verl/utils/reward_score/prime_math/__init__.py b/verl/utils/reward_score/prime_math/__init__.py deleted file mode 100644 index 49c6acbd57..0000000000 --- a/verl/utils/reward_score/prime_math/__init__.py +++ /dev/null @@ -1,430 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Answer checker API that uses sympy to simplify expressions and check for equality. - -Call grade_answer(given_answer: str, ground_truth: str). - -FROM: https://github.com/openai/prm800k/blob/main/prm800k/grading/grader.py -""" -import re -import sympy -from pylatexenc import latex2text -from sympy.parsing import sympy_parser -import os -from . import math_normalize -from .grader import math_equal - -# import math_normalize -# from grader import math_equal - -# sympy might hang -- we don't care about trying to be lenient in these cases -BAD_SUBSTRINGS = ["^{", "^("] -BAD_REGEXES = ["\^[0-9]+\^", "\^[0-9][0-9]+"] -TUPLE_CHARS = "()[]" - - -def timeout(timeout_seconds: int = 8): - if os.name == "posix": - import signal - - def decorator(func): - - def handler(signum, frame): - raise TimeoutError("Operation timed out!") - - def wrapper(*args, **kwargs): - old_handler = signal.getsignal(signal.SIGALRM) - signal.signal(signal.SIGALRM, handler) - signal.alarm(timeout_seconds) - - try: - return func(*args, **kwargs) - finally: - signal.alarm(0) - signal.signal(signal.SIGALRM, old_handler) - - return wrapper - - return decorator - else: - raise NotImplementedError(f"Unsupported OS: {os.name}") - - -def _sympy_parse(expr: str): - """Parses an expression with sympy.""" - py_expr = expr.replace("^", "**") - return sympy_parser.parse_expr( - py_expr, - transformations=(sympy_parser.standard_transformations + (sympy_parser.implicit_multiplication_application,)), - ) - - -def _parse_latex(expr: str) -> str: - """Attempts to parse latex to an expression sympy can read.""" - expr = expr.replace("\\tfrac", "\\frac") - expr = expr.replace("\\dfrac", "\\frac") - expr = expr.replace("\\frac", " \\frac") # Play nice with mixed numbers. - expr = latex2text.LatexNodes2Text().latex_to_text(expr) - - # Replace the specific characters that this parser uses. - expr = expr.replace("√", "sqrt") - expr = expr.replace("π", "pi") - expr = expr.replace("∞", "inf") - expr = expr.replace("∪", "U") - expr = expr.replace("·", "*") - expr = expr.replace("×", "*") - - return expr.strip() - - -def _is_float(num: str) -> bool: - try: - float(num) - return True - except ValueError: - return False - - -def _is_int(x: float) -> bool: - try: - return abs(x - int(round(x))) <= 1e-7 - except: - return False - - -def _is_frac(expr: str) -> bool: - return bool(re.search(r"^-?[0-9]+.?/0*[1-9][0-9]*.?$", expr)) - - -def _str_is_int(x: str) -> bool: - try: - x = _strip_properly_formatted_commas(x) - x = float(x) - return abs(x - int(round(x))) <= 1e-7 - except: - return False - - -def _str_to_int(x: str) -> bool: - x = x.replace(",", "") - x = float(x) - return int(x) - - -def _inject_implicit_mixed_number(step: str): - """ - Automatically make a mixed number evalable - e.g. 7 3/4 => 7+3/4 - """ - p1 = re.compile("([0-9]) +([0-9])") - step = p1.sub("\\1+\\2", step) ## implicit mults - return step - - -def _strip_properly_formatted_commas(expr: str): - # We want to be careful because we don't want to strip tuple commas - p1 = re.compile("(\d)(,)(\d\d\d)($|\D)") - while True: - next_expr = p1.sub("\\1\\3\\4", expr) - if next_expr == expr: - break - expr = next_expr - return next_expr - - -def _normalize(expr: str) -> str: - """Normalize answer expressions.""" - if expr is None: - return None - - # Remove enclosing `\text{}`. - m = re.search("^\\\\text\{(?P.+?)\}$", expr) - if m is not None: - expr = m.group("text") - - expr = expr.replace("\\%", "%") - expr = expr.replace("\\$", "$") - expr = expr.replace("$", "") - expr = expr.replace("%", "") - expr = expr.replace(" or ", " , ") - expr = expr.replace(" and ", " , ") - - expr = expr.replace("million", "*10^6") - expr = expr.replace("billion", "*10^9") - expr = expr.replace("trillion", "*10^12") - - for unit in [ - "degree", - "cm", - "centimeter", - "meter", - "mile", - "second", - "minute", - "hour", - "day", - "week", - "month", - "year", - "foot", - "feet", - "inch", - "yard", - "liter", - ]: - expr = re.sub(f"{unit}(es)?(s)? *(\^[0-9]+)?", "", expr) - expr = re.sub(f"\^ *\\\\circ", "", expr) - - if len(expr) > 0 and expr[0] == "{" and expr[-1] == "}": - expr = expr[1:-1] - - expr = re.sub(",\\\\! *", "", expr) - if _is_float(expr) and _is_int(float(expr)): - expr = str(int(round(float(expr)))) - if "\\" in expr: - try: - expr = _parse_latex(expr) - except: - pass - - # edge case with mixed numbers and negative signs - expr = re.sub("- *", "-", expr) - - expr = _inject_implicit_mixed_number(expr) - - # don't be case sensitive for text answers - expr = expr.lower() - - if _str_is_int(expr): - expr = str(_str_to_int(expr)) - - return expr - - -def count_unknown_letters_in_expr(expr: str): - expr = expr.replace("sqrt", "") - expr = expr.replace("frac", "") - letters_in_expr = set([x for x in expr if x.isalpha()]) - return len(letters_in_expr) - - -def should_allow_eval(expr: str): - # we don't want to try parsing unknown text or functions of more than two variables - if count_unknown_letters_in_expr(expr) > 2: - return False - - for bad_string in BAD_SUBSTRINGS: - if bad_string in expr: - return False - - for bad_regex in BAD_REGEXES: - if re.search(bad_regex, expr) is not None: - return False - - return True - - -@timeout(timeout_seconds=10) -def are_equal_under_sympy(ground_truth_normalized: str, given_normalized: str): - are_equal = False - try: - expr = f"({ground_truth_normalized})-({given_normalized})" - if should_allow_eval(expr): - sympy_diff = _sympy_parse(expr) - simplified = sympy.simplify(sympy_diff) - if simplified == 0: - are_equal = True - except: - pass - return are_equal - - -def split_tuple(expr: str): - """ - Split the elements in a tuple/interval, while handling well-formatted commas in large numbers - """ - expr = _strip_properly_formatted_commas(expr) - if len(expr) == 0: - return [] - if (len(expr) > 2 and expr[0] in TUPLE_CHARS and expr[-1] in TUPLE_CHARS and - all([ch not in expr[1:-1] for ch in TUPLE_CHARS])): - elems = [elem.strip() for elem in expr[1:-1].split(",")] - else: - elems = [expr] - return elems - - -def grade_answer(given_answer: str, ground_truth: str) -> bool: - """ - The answer will be considered correct if: - (a) it normalizes to the same string as the ground truth answer - OR - (b) sympy can simplify the difference between the expressions to 0 - """ - if given_answer is None: - return False - - ground_truth_normalized_mathd = math_normalize.normalize_answer(ground_truth) - given_answer_normalized_mathd = math_normalize.normalize_answer(given_answer) - - # be at least as lenient as mathd - if ground_truth_normalized_mathd == given_answer_normalized_mathd: - return True - - ground_truth_normalized = _normalize(ground_truth) - given_normalized = _normalize(given_answer) - - if ground_truth_normalized is None: - return False - - if ground_truth_normalized == given_normalized: - return True - - if len(given_normalized) == 0: - return False - - ground_truth_elems = split_tuple(ground_truth_normalized) - given_elems = split_tuple(given_normalized) - - if len(ground_truth_elems) > 1 and (ground_truth_normalized[0] != given_normalized[0] or - ground_truth_normalized[-1] != given_normalized[-1]): - is_correct = False - elif len(ground_truth_elems) != len(given_elems): - is_correct = False - else: - for ground_truth_elem, given_elem in zip(ground_truth_elems, given_elems): - if _is_frac(ground_truth_elem) and _is_frac(given_elem): - # if fractions aren't reduced, then shouldn't be marked as correct - # so, we don't want to allow sympy.simplify in this case - is_correct = ground_truth_elem == given_elem - elif _str_is_int(ground_truth_elem) != _str_is_int(given_elem): - # if the ground truth answer is an integer, we require the given answer to be a strict match (no sympy.simplify) - is_correct = False - else: - is_correct = are_equal_under_sympy(ground_truth_elem, given_elem) - if not is_correct: - break - - return is_correct - - -def remove_boxed(s): - left = "\\boxed{" - try: - assert s[:len(left)] == left - assert s[-1] == "}" - return s[len(left):-1] - except: - return None - - -def _last_boxed_only_string(string): - idx = string.rfind("\\boxed") - if idx < 0: - idx = string.rfind("\\fbox") - if idx < 0: - return None - - i = idx - left_brace_idx = None - right_brace_idx = None - num_left_braces_open = 0 - while i < len(string): - if string[i] == "{": - num_left_braces_open += 1 - if left_brace_idx is None: - left_brace_idx = i - elif string[i] == "}": - num_left_braces_open -= 1 - if num_left_braces_open == 0: - right_brace_idx = i - break - - i += 1 - - if left_brace_idx is None or right_brace_idx is None: - return None - - return string[left_brace_idx + 1:right_brace_idx].strip() - - -def match_answer(response): - is_matched = False - for ans_marker in ['answer:', "answer is", "answers are"]: - ans_idx = response.lower().rfind(ans_marker) - if ans_idx != -1: - is_matched = True - response = response[ans_idx + len(ans_marker):].strip() - if response.endswith("\n"): - response = response[:-2] - - for ans_marker in ["is answer", "is the answer", "are answers", "are the answers"]: - ans_idx = response.lower().rfind(ans_marker) - if ans_idx != -1: - is_matched = True - response = response[:ans_idx].strip() - if response.endswith("\n"): - response = response[:-2] - - # Find boxed - ans_boxed = _last_boxed_only_string(response) - if ans_boxed: - is_matched = True - response = ans_boxed - - if ". " in response: - dot_idx = response.lower().rfind(". ") - if dot_idx != -1: - response = response[:dot_idx].strip() - - for ans_marker in ['be ', "is ", "are ", "=", ": ", "get ", 'be\n', "is\n", "are\n", ":\n", "get\n"]: - ans_idx = response.lower().rfind(ans_marker) - if ans_idx != -1: - is_matched = True - response = response[ans_idx + len(ans_marker):].strip() - if response.endswith("\n"): - response = response[:-2] - - is_matched = is_matched if any([c.isdigit() for c in response]) else False # answer must have a digit - # Grade - return is_matched, response - - -import math - - -def compute_score(model_output: str, ground_truth: str) -> bool: - model_output = str(model_output) - ground_truth = str(ground_truth) - - is_matched, extracted_model_output = match_answer(model_output) - format_correctness = "Step 2:" in model_output and "\\box" in model_output - - # grade simple algebra questions. if succeeded, return; otherwise, proceed to more complex grading - if grade_answer(extracted_model_output, ground_truth): - return True, True, extracted_model_output - - try: - if "\pi" in extracted_model_output or "\pi" in ground_truth: - equivs = [] - for pi in [math.pi, 3.14]: - equivs.append(math_equal(extracted_model_output, ground_truth, timeout=True, pi=pi)) - is_correct = any(equivs) - else: - is_correct = math_equal(extracted_model_output, ground_truth, timeout=True) - except: - is_correct = False - - return is_correct, format_correctness, extracted_model_output diff --git a/verl/utils/seqlen_balancing.py b/verl/utils/seqlen_balancing.py deleted file mode 100644 index fee45da0d3..0000000000 --- a/verl/utils/seqlen_balancing.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Tuple, Callable -import heapq - -import torch -from torch import distributed as dist - -from tensordict import TensorDict -import copy - - -def karmarkar_karp(seqlen_list: List[int], k_partitions: int, equal_size: bool): - # see: https://en.wikipedia.org/wiki/Largest_differencing_method - class Set: - - def __init__(self) -> None: - self.sum = 0 - self.items = [] - - def add(self, idx: int, val: int): - self.items.append((idx, val)) - self.sum += val - - def merge(self, other): - for idx, val in other.items: - self.items.append((idx, val)) - self.sum += val - - def __lt__(self, other): - if self.sum != other.sum: - return self.sum < other.sum - if len(self.items) != len(other.items): - return len(self.items) < len(other.items) - return self.items < other.items - - class State: - - def __init__(self, items: List[Tuple[int, int]], k: int) -> None: - self.k = k - # sets should always be decreasing order - self.sets = [Set() for _ in range(k)] - assert len(items) in [1, k], f"{len(items)} not in [1, {k}]" - for i, (idx, seqlen) in enumerate(items): - self.sets[i].add(idx=idx, val=seqlen) - self.sets = sorted(self.sets, reverse=True) - - def spread(self): - return self.sets[0].sum - self.sets[-1].sum - - def get_partitions(self): - partitions = [] - for i in range(len(self.sets)): - cur_partition = [] - for idx, _ in self.sets[i].items: - cur_partition.append(idx) - partitions.append(cur_partition) - return partitions - - def merge(self, other): - for i in range(self.k): - self.sets[i].merge(other.sets[self.k - 1 - i]) - self.sets = sorted(self.sets, reverse=True) - - @property - def spread(self) -> int: - return self.sets[0].sum - self.sets[-1].sum - - def __lt__(self, other): - # least heap, let the state with largest spread to be popped first, - # if the spread is the same, let the state who has the largest set - # to be popped first. - if self.spread != other.spread: - return self.spread > other.spread - return self.sets[0] > other.sets[0] - - def __repr__(self) -> str: - repr_str = "[" - for i in range(self.k): - if i > 0: - repr_str += "," - repr_str += "{" - for j, (_, seqlen) in enumerate(self.sets[i].items): - if j > 0: - repr_str += "," - repr_str += str(seqlen) - repr_str += "}" - repr_str += "]" - return repr_str - - sorted_seqlen_list = sorted([(seqlen, i) for i, seqlen in enumerate(seqlen_list)]) - states_pq = [] - if equal_size: - assert len(seqlen_list) % k_partitions == 0, f"{len(seqlen_list)} % {k_partitions} != 0" - for offset in range(0, len(sorted_seqlen_list), k_partitions): - items = [] - for i in range(k_partitions): - seqlen, idx = sorted_seqlen_list[offset + i] - items.append((idx, seqlen)) - heapq.heappush(states_pq, State(items=items, k=k_partitions)) - else: - for seqlen, idx in sorted_seqlen_list: - heapq.heappush(states_pq, State(items=[(idx, seqlen)], k=k_partitions)) - - while len(states_pq) > 1: - state0 = heapq.heappop(states_pq) - state1 = heapq.heappop(states_pq) - # merge states - state0.merge(state1) - heapq.heappush(states_pq, state0) - - final_state = states_pq[0] - partitions = final_state.get_partitions() - if equal_size: - for i, partition in enumerate(partitions): - assert len(partition) * \ - k_partitions == len(seqlen_list), f"{len(partition)} * {k_partitions} != {len(seqlen_list)}" - return partitions - - -def greedy_partition(seqlen_list: List[int], k_partitions: int, equal_size: bool): - bias = sum(seqlen_list) + 1 if equal_size else 0 - sorted_seqlen = [(seqlen + bias, i) for i, seqlen in enumerate(seqlen_list)] - partitions = [[] for _ in range(k_partitions)] - partition_sums = [0 for _ in range(k_partitions)] - for seqlen, i in sorted_seqlen: - min_idx = None - for j in range(k_partitions): - if min_idx is None or partition_sums[j] < partition_sums[min_idx]: - min_idx = j - partitions[min_idx].append(i) - partition_sums[min_idx] += seqlen - if equal_size: - for i, partition in enumerate(partitions): - assert len(partition) * \ - k_partitions == len(seqlen_list), f"{len(partition)} * {k_partitions} != {len(seqlen_list)}" - return partitions - - -def get_seqlen_balanced_partitions(seqlen_list: List[int], k_partitions: int, equal_size: bool): - """ get order of seq lengths to make partitions balanced, this is - used in balacing sum of seqlength across dp ranks and microbatches - Parameters: - seqlen_list (List[int]): - seq lengths of each items - k_partitions (int): - resulting number of partitions - equal_size (bool): - if True, number of items in each partitions must be equal. - if False, only consider balancing the sum, each partition can have - variable number of items - Returns: - partitions (List[List[int]]): - return k_partitions list containing the index of items. - """ - assert len(seqlen_list) >= k_partitions, f"number of items:[{len(seqlen_list)}] < k_partitions:[{k_partitions}]" - - def _check_and_sort_partitions(partitions): - assert len(partitions) == k_partitions, f"{len(partitions)} != {k_partitions}" - seen_idx = set() - sorted_partitions = [None] * k_partitions - for i, partition in enumerate(partitions): - assert len(partition) > 0, f"the {i}-th partition is empty" - for idx in partition: - seen_idx.add(idx) - sorted_partitions[i] = sorted(partition) - assert seen_idx == set(range(len(seqlen_list))) - return sorted_partitions - - partitions = karmarkar_karp(seqlen_list=seqlen_list, k_partitions=k_partitions, equal_size=equal_size) - return _check_and_sort_partitions(partitions) - - -def log_seqlen_unbalance(seqlen_list: List[int], partitions: List[List[int]], prefix): - # add some metrics of seqlen sum on dp ranks - k_partition = len(partitions) - # assert len(seqlen_list) % k_partition == 0 - batch_size = len(seqlen_list) // k_partition - min_sum_seqlen = None - max_sum_seqlen = None - total_sum_seqlen = 0 - for offset in range(0, len(seqlen_list), batch_size): - cur_sum_seqlen = sum(seqlen_list[offset:offset + batch_size]) - if min_sum_seqlen is None or cur_sum_seqlen < min_sum_seqlen: - min_sum_seqlen = cur_sum_seqlen - if max_sum_seqlen is None or cur_sum_seqlen > max_sum_seqlen: - max_sum_seqlen = cur_sum_seqlen - total_sum_seqlen += cur_sum_seqlen - - balanced_sum_seqlen_list = [] - for partition in partitions: - cur_sum_seqlen_balanced = sum([seqlen_list[i] for i in partition]) - balanced_sum_seqlen_list.append(cur_sum_seqlen_balanced) - # print("balanced_sum_seqlen_list: ", balanced_sum_seqlen_list) - min_sum_seqlen_balanced = min(balanced_sum_seqlen_list) - max_sum_seqlen_balanced = max(balanced_sum_seqlen_list) - - return { - f'{prefix}/min': min_sum_seqlen, - f'{prefix}/max': max_sum_seqlen, - f'{prefix}/minmax_diff': max_sum_seqlen - min_sum_seqlen, - f'{prefix}/balanced_min': min_sum_seqlen_balanced, - f'{prefix}/balanced_max': max_sum_seqlen_balanced, - f'{prefix}/mean': total_sum_seqlen / len(partitions) - } - - -def ceildiv(a, b): - return -(a // -b) - - -def rearrange_micro_batches(batch: TensorDict, max_token_len, dp_group=None): - """Split the batch into a list of micro_batches, where the max_token_len is smaller than max_token_len - and the number of valid tokens in each micro batch is well balanced. - """ - # this is per local micro_bsz - max_seq_len = batch['attention_mask'].shape[-1] - assert max_token_len >= max_seq_len, \ - f'max_token_len must be greater than the sequence length. Got {max_token_len=} and {max_seq_len=}' - - seq_len_effective: torch.Tensor = batch['attention_mask'].sum(dim=1) - total_seqlen = seq_len_effective.sum().item() - num_micro_batches = ceildiv(total_seqlen, max_token_len) - if dist.is_initialized(): - num_micro_batches = torch.tensor([num_micro_batches], device='cuda') - dist.all_reduce(num_micro_batches, op=dist.ReduceOp.MAX, group=dp_group) - num_micro_batches = num_micro_batches.cpu().item() - - seq_len_effective = seq_len_effective.tolist() - assert num_micro_batches <= len(seq_len_effective) - - micro_bsz_idx = get_seqlen_balanced_partitions(seq_len_effective, num_micro_batches, equal_size=False) - - micro_batches = [] - - for partition in micro_bsz_idx: - curr_micro_batch = [] - for idx in partition: - curr_micro_batch.append(batch[idx:idx + 1]) - curr_micro_batch = torch.cat(curr_micro_batch) - - micro_batches.append(curr_micro_batch) - - return micro_batches, micro_bsz_idx - - -def get_reverse_idx(idx_map): - reverse_idx_map = copy.deepcopy(idx_map) - - for i, idx in enumerate(idx_map): - reverse_idx_map[idx] = i - - return reverse_idx_map diff --git a/verl/utils/swedev_utils.py b/verl/utils/swedev_utils.py deleted file mode 100644 index 813a305542..0000000000 --- a/verl/utils/swedev_utils.py +++ /dev/null @@ -1,422 +0,0 @@ -import asyncio -import requests -import hashlib -import random -import aiohttp -import torch -from tensordict import TensorDict - -OBS_ERROR_MSG = "obs too long" - - -def clear_suffix(text): - text = text.replace('<|im_end|>\n<|im_start|>user\n', '') - text = text.replace('<|im_end|>\n<|im_start|>assistant\n', '') - text = text.rstrip() - return text - - -def is_stop(response): - is_finish = "" in response - return is_finish - - -async def initialize_runtime(instance_id): - url = get_api(type="start") - payload = {"instance_hash": instance_id} - try: - async with aiohttp.ClientSession() as session: - async with session.post(url, json=payload, timeout=600) as response: - result = await response.json() - return result - except Exception as e: - print(f"Initializing - API call failed: {e}") - return None - - -async def call_observation_api(sid, text: str): - if isinstance(sid, torch.Tensor): - sid = sid.item() - url = get_api(type="action") - payload = { - "sid": sid, - "content": text, - } - try: - async with aiohttp.ClientSession() as session: - async with session.post(url, json=payload) as response: - return await response.json() - except Exception as e: - print(f"Observation - API call failed: {e}") - return None - - -async def call_postprocess_api(sid: str): - url = get_api(type="postprocess") - if isinstance(sid, torch.Tensor): - sid = sid.item() - payload = {"sid": sid} - try: - async with aiohttp.ClientSession() as session: - async with session.post(url, json=payload, timeout=600) as response: - return await response.json() - except Exception as e: - print(f"Postprocess - API call failed: {e}") - return None - - -def calc_reward(reward_json): - # patch_is_None - # patch_exists - # patch_succesfully_applied - # resolved - # f2p_count - # f2p_total - # p2p_count - # p2p_total - - if 'reward' in reward_json: - return reward_json['reward'] - else: - try: - return reward_json['f2p_count'] / reward_json['f2p_total'] - except: - return 0.0 - - -def get_generation_tokens(role, model="qwen"): - assert role in ["user", "assistant"], f"role {role} not supported" - if model == "qwen": - if role == "user": - return [151645, 198, 151644, 872, 198] - elif role == "assistant": - return [151645, 198, 151644, 77091, 198] - else: - raise NotImplementedError - - -def expand_tensor_dict(td, repeats: int): - # Access the TensorDict inside DataProto's 'batch' attribute - assert repeats >= 1 - duplicated_values = [] - for key in sorted(td.keys()): - for _ in range(repeats - 1): - duplicated_values.append(td[key]) - return duplicated_values - - -def get_api(type): - base_url = random.sample(["http://60.165.239.98:5000", "http://60.165.239.99:5000"], 1)[0] - # TODO: only support shouyun1 now - base_url = "http://60.165.239.98:5000" - assert type in ["reward", "action", "start", "postprocess"] - if type == "reward": - return f"{base_url}/compute_reward" - elif type == "action": - return f"{base_url}/process_action" - elif type == "start": - return f"{base_url}/start_instance" - elif type == "postprocess": - return f"{base_url}/postprocess" - - -def remove_trailing_pad_tokens(tensor, pad_token_id): - tensor = tensor.flip(dims=[0]) - non_pad_indices = (tensor != pad_token_id).nonzero(as_tuple=True)[0] - if len(non_pad_indices) > 0: - first_non_pad_index = non_pad_indices[0].item() - tensor = tensor[first_non_pad_index:] - tensor = tensor.flip(dims=[0]) - return tensor - - -def get_system_prompt(): - return "You are OpenHands agent, a helpful AI assistant that can interact with a computer to solve tasks.\n\n* If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.\n* When configuring git credentials, use \"openhands\" as the user.name and \"openhands@all-hands.dev\" as the user.email by default, unless explicitly instructed otherwise.\n* The assistant MUST NOT include comments in the code unless they are necessary to describe non-obvious behavior.\n\nYou have access to the following functions:\n\n---- BEGIN FUNCTION #1: execute_bash ----\nDescription: Execute a bash command in the terminal.\n* Long running commands: For commands that may run indefinitely, it should be run in the background and the output should be redirected to a file, e.g. command = `python3 app.py > server.log 2>&1 &`.\n* Interactive: If a bash command returns exit code `-1`, this means the process is not yet finished. The assistant must then send a second call to terminal with an empty `command` (which will retrieve any additional logs), or it can send additional text (set `command` to the text) to STDIN of the running process, or it can send command=`ctrl+c` to interrupt the process.\n* Timeout: If a command execution result says \"Command timed out. Sending SIGINT to the process\", the assistant should retry running the command in the background.\n\nParameters:\n (1) command (string, required): The bash command to execute. Can be empty to view additional logs when previous exit code is `-1`. Can be `ctrl+c` to interrupt the currently running process.\n---- END FUNCTION #1 ----\n\n---- BEGIN FUNCTION #2: finish ----\nDescription: Finish the interaction when the task is complete OR if the assistant cannot proceed further with the task.\nNo parameters are required for this function.\n---- END FUNCTION #2 ----\n\n---- BEGIN FUNCTION #3: str_replace_editor ----\nDescription: Custom editing tool for viewing, creating and editing files\n* State is persistent across command calls and discussions with the user\n* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep\n* The `create` command cannot be used if the specified `path` already exists as a file\n* If a `command` generates a long output, it will be truncated and marked with ``\n* The `undo_edit` command will revert the last edit made to the file at `path`\n\nNotes for using the `str_replace` command:\n* The `old_str` parameter should match EXACTLY one or more consecutive lines from the original file. Be mindful of whitespaces!\n* If the `old_str` parameter is not unique in the file, the replacement will not be performed. Make sure to include enough context in `old_str` to make it unique\n* The `new_str` parameter should contain the edited lines that should replace the `old_str`\n\nParameters:\n (1) command (string, required): The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`.\nAllowed values: [`view`, `create`, `str_replace`, `insert`, `undo_edit`]\n (2) path (string, required): Absolute path to file or directory, e.g. `/workspace/file.py` or `/workspace`.\n (3) file_text (string, optional): Required parameter of `create` command, with the content of the file to be created.\n (4) old_str (string, optional): Required parameter of `str_replace` command containing the string in `path` to replace.\n (5) new_str (string, optional): Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert.\n (6) insert_line (integer, optional): Required parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`.\n (7) view_range (array, optional): Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [11, 12] will show lines 11 and 12. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file.\n---- END FUNCTION #3 ----\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nvalue_1\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format, start with \n- Required parameters MUST be specified\n- Only call one function at a time\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after.\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" - - -def _get_swebench_workspace_dir_name(instance: dict) -> str: - if isinstance(instance["version"], int): - instance["version"] = f'{instance["version"]}.0' - return f'{instance["repo"]}__{instance["version"]}'.replace('/', '__') - - -IN_CONTEXT_LEARNING_EXAMPLE_PREFIX = """ -Here's a running example of how to perform a task with the provided tools. - ---------------------- START OF EXAMPLE --------------------- - -USER: Create a list of numbers from 1 to 10, and display them in a web page at port 5000. - -ASSISTANT: Sure! Let me first check the current directory: - - -pwd && ls - - - -USER: EXECUTION RESULT of [execute_bash]: -/workspace -openhands@runtime:~/workspace$ - -ASSISTANT: There is no `app.py` file in the current directory. Let me create a Python file `app.py`: - -create -/workspace/app.py - -from flask import Flask -app = Flask(__name__) - -@app.route('/') -def index(): - numbers = list(range(1, 11)) - return str(numbers) - -if __name__ == '__main__': - app.run(port=5000) - - - -USER: EXECUTION RESULT of [str_replace_editor]: -File created successfully at: /workspace/app.py - -ASSISTANT: I have created a Python file `app.py` that will display a list of numbers from 1 to 10 when you run it. Let me run the Python file for you: - - -python3 app.py > server.log 2>&1 & - - - -USER: EXECUTION RESULT of [execute_bash]: -[1] 121 -[1]+ Exit 1 python3 app.py > server.log 2>&1 - -ASSISTANT: Looks like the server is running with PID 121 then crashed. Let me check the server log: - - -cat server.log - - - -USER: EXECUTION RESULT of [execute_bash]: -Traceback (most recent call last): - File "/workspace/app.py", line 2, in - from flask import Flask -ModuleNotFoundError: No module named 'flask' - -ASSISTANT: Looks like the server crashed because the `flask` module is not installed. Let me install the `flask` module for you: - - -pip3 install flask - - - -USER: EXECUTION RESULT of [execute_bash]: -Defaulting to user installation because normal site-packages is not writeable -Collecting flask - Using cached flask-3.0.3-py3-none-any.whl (101 kB) -Collecting blinker>=1.6.2 - Using cached blinker-1.7.0-py3-none-any.whl (13 kB) -Collecting Werkzeug>=3.0.0 - Using cached werkzeug-3.0.2-py3-none-any.whl (226 kB) -Collecting click>=8.1.3 - Using cached click-8.1.7-py3-none-any.whl (97 kB) -Collecting itsdangerous>=2.1.2 - Using cached itsdangerous-2.2.0-py3-none-any.whl (16 kB) -Requirement already satisfied: Jinja2>=3.1.2 in /home/openhands/.local/lib/python3.10/site-packages (from flask) (3.1.3) -Requirement already satisfied: MarkupSafe>=2.0 in /home/openhands/.local/lib/python3.10/site-packages (from Jinja2>=3.1.2->flask) (2.1.5) -Installing collected packages: Werkzeug, itsdangerous, click, blinker, flask -Successfully installed Werkzeug-3.0.2 blinker-1.7.0 click-8.1.7 flask-3.0.3 itsdangerous-2.2.0 - -ASSISTANT: The `flask` module is installed successfully. Let me run the Python file again: - - -python3 app.py > server.log 2>&1 & - - - -USER: EXECUTION RESULT of [execute_bash]: -[1] 124 - -ASSISTANT: -Let me check the server log again: - - -cat server.log - - - -USER: EXECUTION RESULT of [execute_bash]: -* Serving Flask app 'app' - * Debug mode: off -WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. - * Running on http://127.0.0.1:5000 -Press CTRL+C to quit - -ASSISTANT: -The server is running on port 5000 with PID 124. You can access the list of numbers by visiting http://127.0.0.1:5000. If you have any further questions, feel free to ask! - -USER: Now kill the server, make it display the numbers in a table format. - -ASSISTANT: -Sure! Let me stop the server first: - - -kill 124 - - - -USER: EXECUTION RESULT of [execute_bash]: -[1]+ Terminated python3 app.py > server.log 2>&1 - -ASSISTANT: -Now let me display the numbers in a table format: - -str_replace -/workspace/app.py -return str(numbers) -return '' + ''.join([f'' for i in numbers]) + '
{i}
' - - -USER: EXECUTION RESULT of [str_replace_editor]: -The file /workspace/app.py has been edited. Here's the result of running `cat -n` on a snippet of /workspace/app.py: - 3 - 4 @app.route('/') - 5 def index(): - 6 numbers = list(range(1, 11)) - 7 return '' + ''.join([f'' for i in numbers]) + '
{i}
' - 8 - 9 if __name__ == '__main__': - 10 app.run(port=5000) -Review the changes and make sure they are as expected. Edit the file again if necessary. - -ASSISTANT: -Running the updated file: - - - -python3 app.py > server.log 2>&1 & - - - -USER: EXECUTION RESULT of [execute_bash]: -[1] 126 - -ASSISTANT: -The server is running on port 5000 with PID 126. You can access the list of numbers in a table format by visiting http://127.0.0.1:5000. Let me know if you have any further requests! - - - ---------------------- END OF EXAMPLE --------------------- - -Do NOT assume the environment is the same as in the example above. - ---------------------- NEW TASK DESCRIPTION --------------------- -""".lstrip() - -IN_CONTEXT_LEARNING_EXAMPLE_SUFFIX = """ ---------------------- END OF NEW TASK DESCRIPTION --------------------- - -PLEASE follow the format strictly! PLEASE EMIT ONE AND ONLY ONE FUNCTION CALL PER MESSAGE. -""".lstrip() - - -def get_instruction(instance: dict): - workspace_dir_name = _get_swebench_workspace_dir_name(instance) - instruction = ( - '\n' - f'/workspace/{workspace_dir_name}\n' - '\n' - f"I've uploaded a python code repository in the directory {workspace_dir_name}. Consider the following PR description:\n\n" - f'\n' - f'{instance["problem_statement"]}\n' - '\n\n' - 'Can you help me implement the necessary changes to the repository so that the requirements specified in the are met?\n' - "I've already taken care of all changes to any of the test files described in the . This means you DON'T have to modify the testing logic or any of the tests in any way!\n" - 'Your task is to make the minimal changes to non-tests files in the /workspace directory to ensure the is satisfied.\n' - 'Follow these steps to resolve the issue:\n' - '1. As a first step, it might be a good idea to explore the repo to familiarize yourself with its structure.\n' - '2. Create a script to reproduce the error and execute it with `python ` using the BashTool, to confirm the error\n' - '3. Edit the sourcecode of the repo to resolve the issue\n' - '4. Rerun your reproduce script and confirm that the error is fixed!\n' - '5. Think about edgecases and make sure your fix handles them as well\n' - "Your thinking should be thorough and so it's fine if it's very long.\n") - content = (IN_CONTEXT_LEARNING_EXAMPLE_PREFIX + instruction + IN_CONTEXT_LEARNING_EXAMPLE_SUFFIX) - return content - - -def split_array(array, split_size, dim=0): - """ - Split the array according to the specified split_size along the given dimension. - - Args: - array: List where each element may be of shape [response_count, 2] with varying response_count - split_size: int or list of ints, specifying the size of each chunk - dim: dimension along which to split, default is 0 (batch dimension) - - Returns: - List of split arrays - """ - # Ensure dim is 0, as we only handle splitting on batch dimension - if dim != 0: - raise ValueError("Only splitting on batch dimension (dim=0) is supported") - - batch_size = len(array) - split_sizes = [] - - # Calculate split points - if isinstance(split_size, int): - idx0 = 0 - while idx0 < batch_size: - idx1 = min(batch_size, idx0 + split_size) - split_sizes.append((idx0, idx1)) - idx0 = idx1 - elif isinstance(split_size, (list, tuple)): - if len(split_size) == 0: - raise RuntimeError("Insufficient number of elements in split_size.") - - idx0 = 0 - for size in split_size: - if not isinstance(size, int): - raise TypeError("split(): argument 'split_size' must be int or list of ints") - idx1 = idx0 + size - if idx1 > batch_size: - idx1 = batch_size - split_sizes.append((idx0, idx1)) - idx0 = idx1 - - if idx0 < batch_size: - raise RuntimeError( - f"Split method expects split_size to sum exactly to {batch_size} (batch size), but got split_size={split_size}" - ) - else: - raise TypeError("split(): argument 'split_size' must be int or list of ints") - - # Split the array based on calculated split points - result = [] - for start, end in split_sizes: - result.append(array[start:end]) - - return result - - -def string_to_hash_tensor(s: str) -> torch.Tensor: - hash_object = hashlib.md5(s.encode()) - hash_int = int(hash_object.hexdigest(), 16) - hash_int = hash_int % (2**63) - return torch.tensor(hash_int, dtype=torch.int64) - - -def clear_swe_non_batch(data, is_dict=False): - keys = [ - 'repo', 'base_commit', 'patch', 'test_patch', 'problem_statement', 'hints_text', 'created_at', 'version', - 'FAIL_TO_PASS', 'PASS_TO_PASS', 'environment_setup_commit', 'description' - ] - keys.append("__index_level_0__") # pandas specific - keys.append("index") # verl specific - for key in keys: - if not is_dict: - if key in data.non_tensor_batch.keys(): - del data.non_tensor_batch[key] - else: - if key in data.keys(): - del data[key] diff --git a/verl/utils/tokenizer.py b/verl/utils/tokenizer.py deleted file mode 100644 index b82d87a39d..0000000000 --- a/verl/utils/tokenizer.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utils for tokenization.""" -import warnings - -__all__ = ['hf_tokenizer', 'hf_processor'] - - -def set_pad_token_id(tokenizer): - """Set pad_token_id to eos_token_id if it is None. - - Args: - tokenizer (transformers.PreTrainedTokenizer): The tokenizer to be set. - - """ - if tokenizer.pad_token_id is None: - tokenizer.pad_token_id = tokenizer.eos_token_id - warnings.warn(f'tokenizer.pad_token_id is None. Now set to {tokenizer.eos_token_id}') - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - warnings.warn(f'tokenizer.pad_token is None. Now set to {tokenizer.eos_token}') - - -def hf_tokenizer(name_or_path, correct_pad_token=True, correct_gemma2=True, **kwargs): - """Create a huggingface pretrained tokenizer which correctness handles eos and pad tokens. - - Args: - - name (str): The name of the tokenizer. - correct_pad_token (bool): Whether to correct the pad token id. - correct_gemma2 (bool): Whether to correct the gemma2 tokenizer. - - Returns: - - transformers.PreTrainedTokenizer: The pretrained tokenizer. - - """ - from transformers import AutoTokenizer - if correct_gemma2 and isinstance(name_or_path, str) and 'gemma-2-2b-it' in name_or_path: - # the EOS token in gemma2 is ambiguious, which may worsen RL performance. - # https://huggingface.co/google/gemma-2-2b-it/commit/17a01657f5c87135bcdd0ec7abb4b2dece04408a - warnings.warn('Found gemma-2-2b-it tokenizer. Set eos_token and eos_token_id to and 107.') - kwargs['eos_token'] = '' - kwargs['eos_token_id'] = 107 - tokenizer = AutoTokenizer.from_pretrained(name_or_path, **kwargs) - if correct_pad_token: - set_pad_token_id(tokenizer) - return tokenizer - - -def hf_processor(name_or_path, **kwargs): - """Create a huggingface processor to process multimodal data. - - Args: - name_or_path (str): The name of the processor. - - Returns: - transformers.ProcessorMixin: The pretrained processor. - """ - from transformers import AutoProcessor - try: - processor = AutoProcessor.from_pretrained(name_or_path, **kwargs) - except Exception: - processor = None - # Avoid load tokenizer, see: - # https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/auto/processing_auto.py#L344 - if processor is not None and "Processor" not in processor.__class__.__name__: - processor = None - return processor diff --git a/verl/utils/torch_dtypes.py b/verl/utils/torch_dtypes.py deleted file mode 100644 index 242c8fb37d..0000000000 --- a/verl/utils/torch_dtypes.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Adapted from Cruise. -""" - -import torch - -from typing import Union - -HALF_LIST = [16, "16", "fp16", "float16", torch.float16] -FLOAT_LIST = [32, "32", "fp32", "float32", torch.float32] -BFLOAT_LIST = ["bf16", "bfloat16", torch.bfloat16] - - -class PrecisionType(object): - """Type of precision used. - - >>> PrecisionType.HALF == 16 - True - >>> PrecisionType.HALF in (16, "16") - True - """ - - HALF = "16" - FLOAT = "32" - FULL = "64" - BFLOAT = "bf16" - MIXED = "mixed" - - @staticmethod - def supported_type(precision: Union[str, int]) -> bool: - return any(x == precision for x in PrecisionType) - - @staticmethod - def supported_types() -> list[str]: - return [x.value for x in PrecisionType] - - @staticmethod - def is_fp16(precision): - return precision in HALF_LIST - - @staticmethod - def is_fp32(precision): - return precision in FLOAT_LIST - - @staticmethod - def is_bf16(precision): - return precision in BFLOAT_LIST - - @staticmethod - def to_dtype(precision): - if precision in HALF_LIST: - return torch.float16 - elif precision in FLOAT_LIST: - return torch.float32 - elif precision in BFLOAT_LIST: - return torch.bfloat16 - else: - raise RuntimeError(f"unexpected precision: {precision}") - - @staticmethod - def to_str(precision): - if precision == torch.float16: - return 'fp16' - elif precision == torch.float32: - return 'fp32' - elif precision == torch.bfloat16: - return 'bf16' - else: - raise RuntimeError(f"unexpected precision: {precision}") diff --git a/verl/utils/torch_functional.py b/verl/utils/torch_functional.py deleted file mode 100644 index 925edc3122..0000000000 --- a/verl/utils/torch_functional.py +++ /dev/null @@ -1,575 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Contain small torch utilities -""" - -from typing import Dict, Union, List, Optional - -import numpy as np -import torch -import torch.distributed -import torch.nn.functional as F -from tensordict import TensorDict -from torch import nn - -try: - from flash_attn.ops.triton.cross_entropy import cross_entropy_loss - FLAH_ATTN_CROSS_ENTROPY_LOSS_AVAILABLE = True -except ImportError: - FLAH_ATTN_CROSS_ENTROPY_LOSS_AVAILABLE = False - - -def gather_from_labels(data, label): - """Gather the label from data. The value in label should be [0, vocab_size) - - Args: - data: (..., vocab_size) - label (torch.IntTensor) : (...,) - - Returns: - - """ - - output = torch.gather(data, -1, label.unsqueeze(-1)).squeeze(-1) - return output - - -def logprobs_from_logits(logits, labels): - """ - See: https://github.com/pytorch/pytorch/issues/563#issuecomment-330103591 - """ - if FLAH_ATTN_CROSS_ENTROPY_LOSS_AVAILABLE: - batch_dim = logits.shape[:-1] - last_dim = logits.shape[-1] - logits = logits.reshape(-1, last_dim) - labels = labels.reshape(-1) - output = logprobs_from_logits_flash_attn(logits, labels) - output = output.view(*batch_dim) - else: - output = logprobs_from_logits_v2(logits, labels) - return output - - -def logprobs_from_logits_flash_attn(logits, labels): - output = cross_entropy_loss(logits, labels) - assert isinstance( - output, tuple), "please make sure flash-attn>=2.4.3 where cross_entropy_loss returns Tuple[losses, z_losses]." - return -output[0] - - -def logprobs_from_logits_naive(logits, labels): - logp = F.log_softmax(logits, dim=-1) - logpy = gather_from_labels(logp, labels) - return logpy - - -def logprobs_from_logits_v2(logits: torch.FloatTensor, labels): - """ - A memory efficient implementation of logprobs_from_logits - """ - if logits.dtype in [torch.float32, torch.float64]: - logits_labels = torch.gather(logits, dim=-1, index=labels.unsqueeze(-1)).squeeze(-1) - # loop to reduce peak mem consumption - logsumexp_values = torch.stack([torch.logsumexp(l, dim=-1) for l in logits]) - logprobs_labels = logits_labels - logsumexp_values # log_softmax(x_i) = x_i - logsumexp(x) - else: - # logsumexp approach is unstable with bfloat16, fall back to slightly less efficent approach - logprobs_labels = [] - for row_logits, row_labels in zip(logits, labels): # loop to reduce peak mem consumption - row_logprobs = F.log_softmax(row_logits, dim=-1) - row_logprobs_labels = row_logprobs.gather(dim=-1, index=row_labels.unsqueeze(-1)).squeeze(-1) - logprobs_labels.append(row_logprobs_labels) - logprobs_labels = torch.stack(logprobs_labels) - return logprobs_labels - - -def clip_by_value(x, tensor_min, tensor_max): - """ - Tensor extenstion to torch.clamp - https://github.com/pytorch/pytorch/issues/2793#issuecomment-428784713 - """ - clipped = torch.max(torch.min(x, tensor_max), tensor_min) - return clipped - - -def entropy_from_logits(logits: torch.Tensor): - """Calculate entropy from logits.""" - pd = torch.nn.functional.softmax(logits, dim=-1) - entropy = torch.logsumexp(logits, dim=-1) - torch.sum(pd * logits, dim=-1) - return entropy - - -def masked_sum(values, mask, axis=None): - """Compute mean of tensor with a masked values.""" - return (values * mask).sum(axis=axis) - - -def masked_mean(values, mask, axis=None, preserve_grad: bool = False): - """Compute mean of tensor with a masked values.""" - mask_sum = mask.sum(axis=axis) - if (mask_sum == 0).any(): - # If mask is all zeros, return zeros with the same shape as the result would have - if preserve_grad: - # preserves gradient from `values` and `mask` - return (values * mask).sum(axis=axis) - return torch.zeros_like((values * mask).sum(axis=axis)) - return (values * mask).sum(axis=axis) / mask_sum - - -def masked_var(values, mask, unbiased=True): - """Compute variance of tensor with masked values.""" - mean = masked_mean(values, mask) - centered_values = values - mean - variance = masked_mean(centered_values**2, mask) - if unbiased: - mask_sum = mask.sum() - if mask_sum == 0: - raise ValueError("At least one element in the mask has to be 1.") - # note that if mask_sum == 1, then there is a division by zero issue - # to avoid it you just need to use a larger minibatch_size - if mask_sum == 1: - raise ValueError("The sum of the mask is one, which can cause a division by zero.") - bessel_correction = mask_sum / (mask_sum - 1) - variance = variance * bessel_correction - return variance - - -def masked_whiten(values, mask, shift_mean=True): - """Whiten values with masked values.""" - mean, var = masked_mean(values, mask), masked_var(values, mask) - whitened = (values - mean) * torch.rsqrt(var + 1e-8) - if not shift_mean: - whitened += mean - return whitened - - -def get_response_mask(response_id: torch.Tensor, eos_token: Union[int, List[int]] = 2, dtype=torch.int64): - ''' - end of sentence token can be int or list: 1 or [1, 2] - e.g. - response_id = torch.tensor([[20, 10, 34, 1, 0, 0, 0], - [78, 0, 76, 2, 1, 0, 0], - [23, 98, 1, 0, 0, 0, 0], - [33, 3, 98, 45, 1, 0, 0]]) - #eos_token=1 - response_mask: tensor([[1, 1, 1, 1, 0, 0, 0], - [1, 1, 1, 1, 1, 0, 0], - [1, 1, 1, 0, 0, 0, 0], - [1, 1, 1, 1, 1, 0, 0]]) - #eos_token=[1,2] - response_mask: tensor([[1, 1, 1, 1, 0, 0, 0], - [1, 1, 1, 1, 0, 0, 0], - [1, 1, 1, 0, 0, 0, 0], - [1, 1, 1, 1, 1, 0, 0]]) - ''' - eos_mask = torch.isin(response_id, torch.tensor(eos_token, device=response_id.device)).int() - return (eos_mask.cumsum(dim=1) - eos_mask).eq(0).to(dtype) - - -def compute_grad_norm(model: nn.Module): - total_grad_square = 0 - total_params = 0 - for param in model.parameters(): - if param.grad is not None: - total_grad_square += torch.sum(torch.square(param.grad.detach())).item() - return total_grad_square - - -def broadcast_dict_tensor(tensors: Union[Dict[str, torch.Tensor], TensorDict], src, group): - """ - TODO: optimize this. Technically, we only need one broadcast - """ - - for key in tensors.sorted_keys: - torch.distributed.broadcast(tensors[key], src=src, group=group, async_op=False) - - -# def broadcast_dict_non_tensor(data: Dict[str, List], src, group): -# for key in data.keys(): -# torch.distributed.broadcast_object_list(data[key], src=src, group=group) - - -def broadcast_dict_non_tensor(data: Dict[str, List], src, group): - for key in data.keys(): - # Wrap the list in another list to broadcast it as a single object - object_list = [data[key]] - torch.distributed.broadcast_object_list(object_list, src=src, group=group) - # Update the original list with the received data - data[key] = object_list[0] - - -def allgather_dict_tensors(tensors: Union[Dict[str, torch.Tensor], TensorDict], size, group, dim=0): - """ - TODO: optimize this. - - We can use async ops - - We can use only one allgather - Args: - tensors: - size: - group: - - Returns: - - """ - if isinstance(tensors, TensorDict): - is_tensor_dict = True - tensors_as_dict = tensors.to_dict() - else: - tensors_as_dict = tensors - is_tensor_dict = False - - output = {} - sorted_keys = sorted(tensors_as_dict.keys()) - for key in sorted_keys: - val = tensors_as_dict[key] - output[key] = [torch.empty_like(val) for _ in range(size)] - torch.distributed.all_gather(output[key], val, group=group, async_op=False) - output[key] = torch.cat(output[key], dim=dim) - - if is_tensor_dict: - output = TensorDict(source=output, batch_size=tensors.batch_size[0] * size) - - return output - - -def all_gather_dict_non_tensors(data: Dict[str, List], size, group): - output = {} - sorted_keys = sorted(data.keys()) - for key in sorted_keys: - val = data[key] - output[key] = [None for _ in range(size)] - # print(f"nodedup all gathering {torch.distributed.get_rank()=} {key=} {val=}") - torch.distributed.all_gather_object(output[key], val, group=group) - # print(f"nodedup all gathering {torch.distributed.get_rank()=} {key=} {output[key]=}") - output[key] = np.concatenate(output[key], axis=0) - # print(f"nodedup all gathering {torch.distributed.get_rank()=} {output=}") - return output - - -# def all_gather_dict_non_tensors(data: Dict[str, List], size, group): -# output = {} -# sorted_keys = sorted(data.keys()) -# for key in sorted_keys: -# val = data[key] -# output[key] = [None for _ in range(size)] -# torch.distributed.all_gather_object(output[key], val, group=group) -# output[key] = np.concatenate(output[key], axis=0) -# return output - - -def split_dict_tensor_into_batches(tensors: TensorDict, batch_size) -> List[TensorDict]: - assert tensors.batch_size[0] % batch_size == 0, \ - f'input data batch size: {tensors.batch_size[0]}, split batch size: {batch_size}' - return tensors.split(batch_size) - - -def pad_2d_list_to_length(response, pad_token_id, max_length=None): - """ - pad a 2D list (e.g. responses, logprobs) to a 2D tensor. - """ - response_length = max(len(sub_list) for sub_list in response) - if max_length is not None and max_length > response_length: - target_length = max_length - else: - target_length = response_length - padded_response = [tuple(sub_list) + (pad_token_id,) * (target_length - len(sub_list)) for sub_list in response] - tensor = torch.tensor(padded_response) - return tensor - - -def pad_sequence_to_length(tensors, max_seq_len, pad_token_id, left_pad=False): - """ - pad a 2D tensors (e.g. responses, logprobs) in the last dim to max_seq_length. - input shape: [bs, seq_length] - output shape: [bs, max_seq_length] - (0, max_seq_len - tensors.shape[-1]) means right pad to max_seq_length and no left pad - """ - if tensors.shape[-1] >= max_seq_len: - return tensors - pad_tuple = (max_seq_len - tensors.shape[-1], 0) if left_pad else (0, max_seq_len - tensors.shape[-1]) - return F.pad(tensors, pad_tuple, 'constant', pad_token_id) - - -from transformers import PreTrainedTokenizer - - -def tokenize_and_postprocess_data(prompt: str, - tokenizer: PreTrainedTokenizer, - max_length: int, - pad_token_id: int, - left_pad=True, - truncation='error'): - """ - input_data is the output from tokenizer. - """ - assert truncation in ['left', 'right', 'error'] - - input_data = tokenizer(prompt, return_tensors='pt', add_special_tokens=False) - - input_ids = input_data['input_ids'] - attention_mask = input_data['attention_mask'] - - assert input_ids.ndim == 2 - - sequence_length = input_ids.shape[-1] - if sequence_length < max_length: - input_ids = pad_sequence_to_length(input_ids, - max_seq_len=max_length, - pad_token_id=pad_token_id, - left_pad=left_pad) - attention_mask = pad_sequence_to_length(attention_mask, - max_seq_len=max_length, - pad_token_id=0, - left_pad=left_pad) - elif sequence_length > max_length: - if truncation == 'left': - # actually, left truncation may not be reasonable - input_ids = input_ids[:, -max_length:] - attention_mask = attention_mask[:, -max_length:] - elif truncation == 'right': - input_ids = input_ids[:, :max_length] - attention_mask = attention_mask[:, :max_length] - elif truncation == 'error': - raise NotImplementedError(f'{sequence_length=} is larger than {max_length=}') - else: - raise NotImplementedError(f'Unknown truncation method {truncation}') - - return input_ids, attention_mask - - -def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor): - """ Remove the pad token. - - Args: - input_ids shape: [bs, seq_length] - attention_mask shape: [bs, seq_length] - Returns: - no_padding_batch(List[List[int]]): contains the rmpad token ids per query. - """ - no_padding_batch = [] - for ids, mask in zip(input_ids, attention_mask): - no_padding_batch.append((ids[len(ids) - mask.sum():]).cpu().numpy().tolist()) - return no_padding_batch - - -def log_probs_from_logits_response(input_ids, logits, response_length): - """Compute the response log_probs from full logits. Note that logits = model(input_ids) - - Args: - input_ids: [batch_size, seqlen] - logits: [batch_size, seqlen, vocab_size] - - Returns: - response_log_prob: - """ - response_logits = logits[:, -response_length - 1:-1] - response = input_ids[:, -response_length:] - response_log_prob = logprobs_from_logits(logits=response_logits, labels=response) - return response_log_prob - - -def log_probs_from_logits_response_rmpad(input_ids, attention_mask, logits_rmpad, response_length): - """Compute the log_probs from logits with rmpad logits and pad input. Note that - logits_rmpad = model(input_ids_rmpad). For each sentences, there is a shift between - logits and input_ids. - The reason for this function to is to compute logprobs_from_logits in rmpad mode because it is memory-intensive - for large vocab_size - - Args: - input_ids: [batch_size, seqlen] - attention_mask: [batch_size, seqlen] - logits_rmpad: [total_nnz, vocab_size] - response_length: int - """ - from flash_attn.bert_padding import pad_input, unpad_input - - batch_size, seqlen = input_ids.shape - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), attention_mask=attention_mask) - input_ids_rmpad = input_ids_rmpad.squeeze(-1) - input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=0) - full_log_probs_rmpad = logprobs_from_logits(logits=logits_rmpad, labels=input_ids_rmpad_rolled) # (total_nnz,) - full_output = pad_input(hidden_states=full_log_probs_rmpad.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen) - output = full_output.squeeze(-1)[:, -response_length - 1:-1] # [batch_size, response_length] - return output - - -def log_probs_from_logits_all_rmpad(input_ids_rmpad, logits_rmpad, indices, batch_size, seqlen, response_length): - """Compute the log_probs from logits with rmpad input_ids and logits. Note that - logits_rmpad = model(input_ids_rmpad). For each sentences, there is a shift between - logits and input_ids. - The reason for this function to is to compute logprobs_from_logits in rmpad mode because it is memory-intensive - for large vocab_size - - Args: - input_ids_rmpad: [1, total_nnz] - logits_rmpad: [total_nnz, vocab_size] - indices: [total_nnz] - batch_size: int - seqlen: int - response_length: int - """ - from flash_attn.bert_padding import pad_input - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # transpose back to [total_nnz, 1] - input_ids_rmpad = input_ids_rmpad.squeeze(-1) - input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=0) - full_log_probs_rmpad = logprobs_from_logits(logits=logits_rmpad, labels=input_ids_rmpad_rolled) # (total_nnz,) - full_output = pad_input(hidden_states=full_log_probs_rmpad.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen) - output = full_output.squeeze(-1)[:, -response_length - 1:-1] # [batch_size, response_length] - return output - - -from transformers.generation.logits_process import (TemperatureLogitsWarper, TopKLogitsWarper, TopPLogitsWarper) - - -def post_process_logits(input_ids, logits, temperature, top_k, top_p): - if temperature != 1.: - logits = logits.div_(temperature) # inplace operation to avoid OOM - # TODO: add them back - # if top_k is not None and top_k > 0: - # logits = TopKLogitsWarper(top_k=top_k)(input_ids, logits) - # if top_p is not None and top_p < 1.0 and top_p > 0.0: - # logits = TopPLogitsWarper(top_p=top_p)(input_ids, logits) - return logits - - -""" -Optimizer related -""" - -from torch.optim import Optimizer -from torch.optim.lr_scheduler import LambdaLR -import math - - -def get_cosine_schedule_with_warmup( - optimizer: Optimizer, - num_warmup_steps: int, - num_training_steps: int, - min_lr_ratio: float = 0.0, - num_cycles: float = 0.5, - last_epoch: int = -1, -): - """ - Create a schedule with a learning rate that decreases following the values of the cosine function between the - initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the - initial lr set in the optimizer. - Args: - optimizer (:class:`~torch.optim.Optimizer`): - The optimizer for which to schedule the learning rate. - num_warmup_steps (:obj:`int`): - The number of steps for the warmup phase. - num_training_steps (:obj:`int`): - The total number of training steps. - min_lr_ratio (:obj:`float`, `optional`, defaults to 0.0): - The minimum lr ratio w.r.t the maximum. - num_cycles (:obj:`float`, `optional`, defaults to 0.5): - The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0 - following a half-cosine). - last_epoch (:obj:`int`, `optional`, defaults to -1): - The index of the last epoch when resuming training. - Return: - :obj:`torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. - """ - assert min_lr_ratio >= 0 and min_lr_ratio <= 1. - coef = (1 - min_lr_ratio) * 0.5 - intercept = (1 + min_lr_ratio) * 0.5 - - def lr_lambda(current_step): - if current_step < num_warmup_steps: - return float(current_step) / float(max(1, num_warmup_steps)) - progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) - x = math.cos(math.pi * float(num_cycles) * 2.0 * progress) - return max(0.0, x * coef + intercept) - - return LambdaLR(optimizer, lr_lambda, last_epoch) - - -def get_constant_schedule_with_warmup( - optimizer: Optimizer, - num_warmup_steps: int, - last_epoch: int = -1, -): - - def lr_lambda(current_step): - return min(1, float(current_step) / float(max(1, num_warmup_steps))) - - return LambdaLR(optimizer, lr_lambda, last_epoch) - - -def prepare_decoder_attention_mask(attention_mask, input_shape, inputs_embeds): - # create causal mask - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - combined_attention_mask = None - if input_shape[-1] > 1: - combined_attention_mask = _make_causal_mask( - input_shape, - inputs_embeds.dtype, - device=inputs_embeds.device, - ) - - if attention_mask is not None: - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, - tgt_len=input_shape[-1]).to(inputs_embeds.device) - combined_attention_mask = (expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + - combined_attention_mask) - - return combined_attention_mask - - -# Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device): - """ - Make causal mask used for bi-directional self-attention. - """ - bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) - mask_cond = torch.arange(mask.size(-1), device=device) - mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) - mask = mask.to(dtype) - return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len) - - -# Copied from transformers.models.bart.modeling_bart._expand_mask -def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): - """ - Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. - """ - bsz, src_len = mask.size() - tgt_len = tgt_len if tgt_len is not None else src_len - - expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) - - inverted_mask = 1.0 - expanded_mask - - return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) - - -def get_unpad_data(attention_mask): - seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) - indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() - max_seqlen_in_batch = seqlens_in_batch.max().item() - cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) - return ( - indices, - cu_seqlens, - max_seqlen_in_batch, - ) diff --git a/verl/utils/tracking.py b/verl/utils/tracking.py deleted file mode 100644 index 3688173254..0000000000 --- a/verl/utils/tracking.py +++ /dev/null @@ -1,264 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -A unified tracking interface that supports logging data to different backend -""" -import dataclasses -from enum import Enum -from functools import partial -from pathlib import Path -from typing import List, Union, Dict, Any - - -class Tracking(object): - supported_backend = ["wandb", "mlflow", "swanlab", "vemlp_wandb", "tensorboard", "console"] - - def __init__(self, project_name, experiment_name, default_backend: Union[str, List[str]] = 'console', config=None): - if isinstance(default_backend, str): - default_backend = [default_backend] - for backend in default_backend: - if backend == 'tracking': - import warnings - warnings.warn("`tracking` logger is deprecated. use `wandb` instead.", DeprecationWarning) - else: - assert backend in self.supported_backend, f'{backend} is not supported' - - self.logger = {} - - if 'tracking' in default_backend or 'wandb' in default_backend: - import wandb - wandb.init( - project=project_name, - name=experiment_name, - config=config) - self.logger['wandb'] = wandb - - if 'mlflow' in default_backend: - import mlflow - import os - - MLFLOW_TRACKING_URI = os.environ.get("MLFLOW_TRACKING_URI", None) - if MLFLOW_TRACKING_URI: - mlflow.set_tracking_uri(MLFLOW_TRACKING_URI) - - # Project_name is actually experiment_name in MLFlow - # If experiment does not exist, will create a new experiment - experiment = mlflow.set_experiment(project_name) - mlflow.start_run(experiment_id=experiment.experiment_id, run_name=experiment_name) - mlflow.log_params(_compute_mlflow_params_from_objects(config)) - self.logger['mlflow'] = _MlflowLoggingAdapter() - - if "swanlab" in default_backend: - import swanlab - import os - - SWANLAB_API_KEY = os.environ.get("SWANLAB_API_KEY", None) - SWANLAB_LOG_DIR = os.environ.get("SWANLAB_LOG_DIR", "swanlog") - SWANLAB_MODE = os.environ.get("SWANLAB_MODE", "cloud") - if SWANLAB_API_KEY: - swanlab.login(SWANLAB_API_KEY) # NOTE: previous login information will be overwritten - swanlab.init(project=project_name, - experiment_name=experiment_name, - config={ - "FRAMEWORK": "veRL", - **config - }, - logdir=SWANLAB_LOG_DIR, - mode=SWANLAB_MODE) - self.logger["swanlab"] = swanlab - - if 'vemlp_wandb' in default_backend: - import os - import volcengine_ml_platform - from volcengine_ml_platform import wandb as vemlp_wandb - volcengine_ml_platform.init( - ak=os.environ["VOLC_ACCESS_KEY_ID"], - sk=os.environ["VOLC_SECRET_ACCESS_KEY"], - region=os.environ["MLP_TRACKING_REGION"], - ) - - vemlp_wandb.init( - project=project_name, - name=experiment_name, - config=config, - sync_tensorboard=True, - ) - self.logger['vemlp_wandb'] = vemlp_wandb - - if 'tensorboard' in default_backend: - self.logger['tensorboard'] = _TensorboardAdapter() - - if 'console' in default_backend: - from verl.utils.logger.aggregate_logger import LocalLogger - self.console_logger = LocalLogger(print_to_console=True) - self.logger['console'] = self.console_logger - - def log(self, data, step, backend=None): - for default_backend, logger_instance in self.logger.items(): - if backend is None or default_backend in backend: - if default_backend == "wandb": - # flush logs right away for wandb - logger_instance.log(data=data, step=step, commit=True) - else: - logger_instance.log(data=data, step=step) - - def __del__(self): - if 'wandb' in self.logger: - self.logger['wandb'].finish(exit_code=0) - if 'swanlab' in self.logger: - self.logger['swanlab'].finish() - if 'vemlp_wandb' in self.logger: - self.logger['vemlp_wandb'].finish(exit_code=0) - if 'tensorboard' in self.logger: - self.logger['tensorboard'].finish() - - -class _TensorboardAdapter: - - def __init__(self): - from torch.utils.tensorboard import SummaryWriter - import os - tensorboard_dir = os.environ.get("TENSORBOARD_DIR", "tensorboard_log") - os.makedirs(tensorboard_dir, exist_ok=True) - print(f"Saving tensorboard log to {tensorboard_dir}.") - self.writer = SummaryWriter(tensorboard_dir) - - def log(self, data, step): - for key in data: - self.writer.add_scalar(key, data[key], step) - - def finish(self): - self.writer.close() - - -class _MlflowLoggingAdapter: - - def log(self, data, step): - import mlflow - mlflow.log_metrics(metrics=data, step=step) - - -def _compute_mlflow_params_from_objects(params) -> Dict[str, Any]: - if params is None: - return {} - - return _flatten_dict(_transform_params_to_json_serializable(params, convert_list_to_dict=True), sep='/') - - -def _transform_params_to_json_serializable(x, convert_list_to_dict: bool): - _transform = partial(_transform_params_to_json_serializable, convert_list_to_dict=convert_list_to_dict) - - if dataclasses.is_dataclass(x): - return _transform(dataclasses.asdict(x)) - if isinstance(x, dict): - return {k: _transform(v) for k, v in x.items()} - if isinstance(x, list): - if convert_list_to_dict: - return {'list_len': len(x)} | {f'{i}': _transform(v) for i, v in enumerate(x)} - else: - return [_transform(v) for v in x] - if isinstance(x, Path): - return str(x) - if isinstance(x, Enum): - return x.value - - return x - - -def _flatten_dict(raw: Dict[str, Any], *, sep: str) -> Dict[str, Any]: - import pandas as pd - ans = pd.json_normalize(raw, sep=sep).to_dict(orient='records')[0] - assert isinstance(ans, dict) - return ans - - -@dataclasses.dataclass -class ValidationGenerationsLogger: - - def log(self, loggers, samples, step): - if 'wandb' in loggers: - self.log_generations_to_wandb(samples, step) - if 'swanlab' in loggers: - self.log_generations_to_swanlab(samples, step) - if 'mlflow' in loggers: - self.log_generations_to_mlflow(samples, step) - - def log_generations_to_wandb(self, samples, step): - """Log samples to wandb as a table""" - import wandb - - # Create column names for all samples - columns = ["step"] + sum([[f"input_{i+1}", f"output_{i+1}", f"score_{i+1}"] for i in range(len(samples))], []) - - if not hasattr(self, 'validation_table'): - # Initialize the table on first call - self.validation_table = wandb.Table(columns=columns) - - # Create a new table with same columns and existing data - # Workaround for https://github.com/wandb/wandb/issues/2981#issuecomment-1997445737 - new_table = wandb.Table(columns=columns, data=self.validation_table.data) - - # Add new row with all data - row_data = [] - row_data.append(step) - for sample in samples: - row_data.extend(sample) - - new_table.add_data(*row_data) - - # Update reference and log - wandb.log({"val/generations": new_table}, step=step) - self.validation_table = new_table - - def log_generations_to_swanlab(self, samples, step): - """Log samples to swanlab as text""" - import swanlab - - swanlab_text_list = [] - for i, sample in enumerate(samples): - row_text = f""" - input: {sample[0]} - - --- - - output: {sample[1]} - - --- - - score: {sample[2]} - """ - swanlab_text_list.append(swanlab.Text(row_text, caption=f"sample {i+1}")) - - # Log to swanlab - swanlab.log({"val/generations": swanlab_text_list}, step=step) - - def log_generations_to_mlflow(self, samples, step): - """Log validation generation to mlflow as artifacts""" - #https://mlflow.org/docs/latest/api_reference/python_api/mlflow.html?highlight=log_artifact#mlflow.log_artifact - - import mlflow - import tempfile - import json - try: - with tempfile.TemporaryDirectory() as tmp_dir: - validation_gen_step_file = Path(tmp_dir, f"val_step{step}.json") - row_data = [] - for sample in samples: - data = {"input": sample[0], "output": sample[1], "score": sample[2]} - row_data.append(data) - with open(validation_gen_step_file, "w") as file: - json.dump(row_data, file) - mlflow.log_artifact(validation_gen_step_file) - except Exception as e: - print(f"WARNING: save validation generation file to mlflow failed with error {e}") diff --git a/verl/utils/ulysses.py b/verl/utils/ulysses.py deleted file mode 100644 index 67aa150c8c..0000000000 --- a/verl/utils/ulysses.py +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Utilities for DeepSpeed Ulysses Sequence Parallelism. -DeepSpeed Ulysses Paper: https://arxiv.org/abs/2309.14509 -Inspired from: https://github.com/microsoft/DeepSpeed/blob/master/deepspeed/sequence/layer.py -""" -from typing import Any, Optional, List, Tuple - -import torch -from torch import Tensor -import torch.distributed as dist -from torch.distributed import ProcessGroup - -_ULYSSES_SEQUENCE_PARALLEL_GROUP = None - - -def set_ulysses_sequence_parallel_group(group: dist.ProcessGroup): - """ - Set ulysses sequence parallel process group. - """ - global _ULYSSES_SEQUENCE_PARALLEL_GROUP - _ULYSSES_SEQUENCE_PARALLEL_GROUP = group - - -def get_ulysses_sequence_parallel_group() -> Optional[dist.ProcessGroup]: - """ - Get ulysses sequence parallel process group. - """ - global _ULYSSES_SEQUENCE_PARALLEL_GROUP - return _ULYSSES_SEQUENCE_PARALLEL_GROUP - - -def get_ulysses_sequence_parallel_world_size(group: ProcessGroup = None) -> int: - """ - Get ulysses sequence parallel world size. - """ - group = get_ulysses_sequence_parallel_group() if group is None else group - return dist.get_world_size(group) if group else 1 - - -def get_ulysses_sequence_parallel_rank(group: ProcessGroup = None) -> int: - """ - Get ulysses sequence parallel rank. - """ - group = get_ulysses_sequence_parallel_group() if group is None else group - return dist.get_rank(group) if group else 0 - - -def gather_seq_scatter_heads( - x: Tensor, - seq_dim: int, - head_dim: int, - unpadded_dim_size: int = 0, - group: ProcessGroup = None, -) -> Tensor: - """ - A func to sync embedding input with alltoall in sequence parallel - gather sequence dimension and scatter head dim: - e.g. seq_dim: 1, head_dim: 2 - [bsz, seq/n, h, ...] -> [bsz, seq, h/n, ...] - """ - group = get_ulysses_sequence_parallel_group() if group is None else group - if not group: - return x - sp_world = get_ulysses_sequence_parallel_world_size(group) - x = SeqAllToAll.apply(group, x, head_dim, seq_dim) - if unpadded_dim_size and unpadded_dim_size % sp_world != 0: - padding_size = x.size(seq_dim) - unpadded_dim_size - x = _unpad_tensor(x, seq_dim, padding_size) - return x - - -def gather_heads_scatter_seq(x: Tensor, head_dim: int, seq_dim: int, group: ProcessGroup = None) -> Tensor: - """ - A func to sync attention result with alltoall in sequence parallel - gather head dimension and scatter seq dim: - e.g. seq_dim: 1, head_dim: 2 - [bsz, seq, h/n, ...] -> [bsz, seq/n, h, ...] - """ - group = get_ulysses_sequence_parallel_group() if group is None else group - if not group: - return x - dim_size = x.size(seq_dim) - sp_world = get_ulysses_sequence_parallel_world_size(group) - if dim_size % sp_world != 0: - padding_size = sp_world - (dim_size % sp_world) - x = _pad_tensor(x, seq_dim, padding_size) - return SeqAllToAll.apply(group, x, seq_dim, head_dim, False) - - -def _pad_tensor(x: Tensor, dim: int, padding_size: int) -> Tensor: - shape = list(x.shape) - shape[dim] = padding_size - pad = torch.zeros(shape, dtype=x.dtype, device=x.device) - return torch.cat([x, pad], dim=dim) - - -def _unpad_tensor(x: Tensor, dim: int, padding_size: int) -> Tensor: - slc = [slice(None)] * len(x.shape) - slc[dim] = slice(0, -padding_size) - return x[slc] - - -def slice_input_tensor(x: Tensor, dim: int, padding: bool = True, group: ProcessGroup = None) -> Tensor: - group = get_ulysses_sequence_parallel_group() if group is None else group - sp_world_size = dist.get_world_size(group) - sp_rank = get_ulysses_sequence_parallel_rank() - dim_size = x.size(dim) - # pad before slice - if padding and dim_size % sp_world_size: - padding_size = sp_world_size - (dim_size % sp_world_size) - x = _pad_tensor(x, dim, padding_size) - # slice the input tensor - parts = x.size(dim) // sp_world_size - slc = [slice(None)] * len(x.shape) - slc[dim] = slice(sp_rank * parts, (sp_rank + 1) * parts) - return x[slc].contiguous() - - -def all_to_all_tensor( - local_input: Tensor, - scatter_dim: int, - gather_dim: int, - group: Optional[dist.ProcessGroup] = None, - async_op: bool = False, -): - group = get_ulysses_sequence_parallel_group() if group is None else group - seq_world_size = dist.get_world_size(group) - input_list = [t.contiguous() for t in torch.tensor_split(local_input, seq_world_size, scatter_dim)] - output_list = [torch.empty_like(input_list[0]) for _ in range(seq_world_size)] - comm = dist.all_to_all(output_list, input_list, group=group, async_op=async_op) - if async_op: - - def wait(): - comm.wait() - return torch.cat(output_list, dim=gather_dim).contiguous() - - return wait - return torch.cat(output_list, dim=gather_dim).contiguous() - - -def all_gather_tensor(local_tensor: Tensor, group: Optional[dist.ProcessGroup] = None, async_op: bool = False): - group = get_ulysses_sequence_parallel_group() if group is None else group - sp_world_size = dist.get_world_size(group=group) - output_shape = list(local_tensor.shape) - output_shape[0] = output_shape[0] * sp_world_size - output = torch.empty(output_shape, dtype=local_tensor.dtype, device=local_tensor.device) - dist.all_gather_into_tensor(output, local_tensor, group=group, async_op=async_op) - return output - - -class SeqAllToAll(torch.autograd.Function): - - @staticmethod - def forward( - ctx: Any, - group: dist.ProcessGroup, - local_input: Tensor, - scatter_dim: int, - gather_dim: int, - async_op: bool = False, - ) -> Tensor: - ctx.group = group - ctx.scatter_dim = scatter_dim - ctx.gather_dim = gather_dim - ctx.async_op = async_op - return all_to_all_tensor(local_input, scatter_dim, gather_dim, group, async_op) - - @staticmethod - def backward(ctx: Any, *grad_output: Tensor) -> Tuple[None, Tensor, None, None]: - if ctx.async_op: - input_t = torch.cat(grad_output[1:], dim=ctx.gather_dim).contiguous() - else: - input_t = grad_output[0] - return ( - None, - all_to_all_tensor(input_t, ctx.gather_dim, ctx.scatter_dim, ctx.group, False), - None, - None, - None, - None, - ) - - -class Gather(torch.autograd.Function): - - @staticmethod - def forward(ctx: Any, - group: dist.ProcessGroup, - local_tensor: Tensor, - gather_dim: int, - grad_scaler: bool = True, - async_op=False) -> Tensor: - ctx.group = group - ctx.gather_dim = gather_dim - ctx.grad_scaler = grad_scaler - ctx.async_op = async_op - - sp_world_size = dist.get_world_size(group=group) - ctx.sp_world_size = sp_world_size - - sp_rank = dist.get_rank(group=group) - ctx.sp_rank = sp_rank - - local_shape = list(local_tensor.size()) - split_size = local_shape[0] - part_size = local_shape[gather_dim] # store original size - ctx.part_size = part_size - - output = all_gather_tensor(local_tensor, group, async_op) - return torch.cat(output.split(split_size, dim=0), dim=gather_dim) - - @staticmethod - def backward(ctx: Any, grad_output: Tensor) -> Any: - if ctx.grad_scaler: - grad_output = grad_output * ctx.sp_world_size - return (None, grad_output.split(ctx.part_size, - dim=ctx.gather_dim)[ctx.sp_rank].contiguous(), None, None, None, None) - - -def gather_outpus_and_unpad(x: Tensor, - gather_dim: int, - unpad_dim: int = None, - padding_size: int = 0, - grad_scaler: bool = True, - group: Optional[dist.ProcessGroup] = None): - group = get_ulysses_sequence_parallel_group() if group is None else group - sp_size = get_ulysses_sequence_parallel_world_size() - if group == None: - return x - x = Gather.apply(group, x, gather_dim, grad_scaler) - if unpad_dim is not None: - assert isinstance(padding_size, int), 'padding size is not given or is not an integer' - if padding_size == 0: - return x - x = _unpad_tensor(x, unpad_dim, padding_size) - return x - - -def ulysses_pad_and_slice_inputs(input_ids_rmpad: torch.Tensor, - position_ids_rmpad: Optional[torch.Tensor] = None, - sp_size: int = 1): - """ - Pad and slice input_ids to be divisible by sp_size - Pad position_ids to be divisible by sp_size. - - Note both input_ids_rmpad and position_ids_rmpad will be padded and sliced. - - The is the utility of pre-forward for ulysses sequence parallelism - - Args: - input_ids_rmpad: shape of [bsz, seqlen] - position_ids_rmpad: shape of [bsz, seqlen], where bsz must be 1 - sp_size (int): ulysses sequence parallelism size - - Returns: - torch.Tensor: padded and sliced input_ids - torch.Tensor: padded and sliced position_ids - int: pad size - """ - if position_ids_rmpad is not None: - assert position_ids_rmpad.size(0) == 1 - assert input_ids_rmpad.size(1) == position_ids_rmpad.size(1) - if sp_size <= 1: - return input_ids_rmpad, position_ids_rmpad, 0 - _, total_seq_len = input_ids_rmpad.shape - pad_size = (sp_size - total_seq_len % sp_size) % sp_size - if pad_size > 0: - input_ids_rmpad = torch.nn.functional.pad(input_ids_rmpad, (0, pad_size), value=0) - if position_ids_rmpad is not None: - pad_pos_ids = torch.arange(pad_size, device=position_ids_rmpad.device).unsqueeze(0) - position_ids_rmpad = torch.cat((position_ids_rmpad, pad_pos_ids), dim=-1) - input_ids_rmpad = slice_input_tensor(input_ids_rmpad, dim=1, padding=False) - if position_ids_rmpad is not None: - position_ids_rmpad = slice_input_tensor(position_ids_rmpad, dim=1, padding=False) - return input_ids_rmpad, position_ids_rmpad, pad_size - - -def validate_ulysses_config(num_heads, ulysses_sequence_size): - if ulysses_sequence_size > 1: - assert num_heads % ulysses_sequence_size == 0,\ - f"num_heads ({num_heads}) must be divisible by ulysses sequence size({ulysses_sequence_size})" diff --git a/verl/version/version b/verl/version/version deleted file mode 100644 index 8f91ec141a..0000000000 --- a/verl/version/version +++ /dev/null @@ -1 +0,0 @@ -0.2.0.dev diff --git a/verl/workers/__init__.py b/verl/workers/__init__.py deleted file mode 100644 index 1ce90c5eb3..0000000000 --- a/verl/workers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/verl/workers/actor/__init__.py b/verl/workers/actor/__init__.py deleted file mode 100644 index 7a1404e176..0000000000 --- a/verl/workers/actor/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base import BasePPOActor -from .dp_actor import DataParallelPPOActor - -__all__ = ["BasePPOActor", "DataParallelPPOActor"] diff --git a/verl/workers/actor/base.py b/verl/workers/actor/base.py deleted file mode 100644 index 144f0b90ef..0000000000 --- a/verl/workers/actor/base.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The base class for Actor -""" -from abc import ABC, abstractmethod -from typing import Iterable, Dict - -from verl import DataProto -import torch - -__all__ = ['BasePPOActor'] - - -class BasePPOActor(ABC): - - def __init__(self, config): - """The base class for PPO actor - - Args: - config (DictConfig): a config passed to the PPOActor. We expect the type to be - DictConfig (https://omegaconf.readthedocs.io/), but it can be any namedtuple in general. - """ - super().__init__() - self.config = config - - @abstractmethod - def compute_log_prob(self, data: DataProto) -> torch.Tensor: - """Compute logits given a batch of data. - - Args: - data (DataProto): a batch of data represented by DataProto. It must contain key ```input_ids```, - ```attention_mask``` and ```position_ids```. - - Returns: - DataProto: a DataProto containing the key ```log_probs``` - - - """ - pass - - @abstractmethod - def update_policy(self, data: DataProto) -> Dict: - """Update the policy with an iterator of DataProto - - Args: - data (DataProto): an iterator over the DataProto that returns by - ```make_minibatch_iterator``` - - Returns: - Dict: a dictionary contains anything. Typically, it contains the statistics during updating the model - such as ```loss```, ```grad_norm```, etc,. - - """ - pass diff --git a/verl/workers/actor/dp_actor.py b/verl/workers/actor/dp_actor.py deleted file mode 100644 index 33712a3de7..0000000000 --- a/verl/workers/actor/dp_actor.py +++ /dev/null @@ -1,392 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Single Process Actor -""" - -import itertools -from typing import Iterable, Tuple - -import torch -from torch import nn -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP - -from verl import DataProto -from verl.trainer.ppo import core_algos -from verl.trainer.ppo.core_algos import compute_policy_loss, kl_penalty, agg_loss -from verl.workers.actor import BasePPOActor -from verl.utils.py_functional import append_to_dict -from verl.utils.torch_functional import logprobs_from_logits, masked_mean -from verl.utils.ulysses import ulysses_pad_and_slice_inputs, gather_outpus_and_unpad -from verl.utils.seqlen_balancing import rearrange_micro_batches, get_reverse_idx -import verl.utils.torch_functional as verl_F - -from flash_attn.bert_padding import pad_input, unpad_input, rearrange, index_first_axis - -__all__ = ['DataParallelPPOActor'] - - -class DataParallelPPOActor(BasePPOActor): - - def __init__( - self, - config, - actor_module: nn.Module, - actor_optimizer: torch.optim.Optimizer = None, - ): - """When optimizer is None, it is Reference Policy""" - super().__init__(config) - self.actor_module = actor_module - self.actor_optimizer = actor_optimizer - self.use_remove_padding = self.config.get('use_remove_padding', False) - print(f'Actor use_remove_padding={self.use_remove_padding}') - self.ulysses_sequence_parallel_size = self.config.ulysses_sequence_parallel_size - self.use_ulysses_sp = self.ulysses_sequence_parallel_size > 1 - - self.compute_entropy_from_logits = ( - torch.compile(verl_F.entropy_from_logits, dynamic=True) - if self.config.get('use_torch_compile', True) # use torch compile by default - else verl_F.entropy_from_logits) - - def _forward_micro_batch(self, micro_batch, temperature) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Returns: - entropy: # (bs, response_len) - log_probs: # (bs, response_len) - """ - response_length = micro_batch['responses'].size(-1) - multi_modal_inputs = {} - if 'multi_modal_inputs' in micro_batch: - for key in micro_batch['multi_modal_inputs'][0].keys(): - multi_modal_inputs[key] = torch.cat([inputs[key] for inputs in micro_batch['multi_modal_inputs']], - dim=0) - - with torch.autocast(device_type='cuda', dtype=torch.bfloat16): - input_ids = micro_batch['input_ids'] - batch_size, seqlen = input_ids.shape - attention_mask = micro_batch['attention_mask'] - position_ids = micro_batch['position_ids'] - if position_ids.dim() == 3: # qwen2vl mrope - position_ids = position_ids.transpose(0, 1) # (bsz, 3, seqlen) -> (3, bsz, seqlen) - - if self.use_remove_padding: - print(f"using remove padding, before {input_ids.shape=} {attention_mask.sum(dim=-1)=}") - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - - # unpad the position_ids to align the rotary - if position_ids.dim() == 3: - position_ids_rmpad = index_first_axis(rearrange(position_ids, "c b s ... -> (b s) c ..."), - indices).transpose(0, 1).unsqueeze( - 1) # (3, bsz, seqlen) -> (3, 1, bsz * seqlen) - else: - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # for compute the log_prob - input_ids_rmpad_rolled = torch.roll(input_ids_rmpad, shifts=-1, dims=1) # (1, total_nnz) - - # pad and slice the inputs if sp > 1 - if self.use_ulysses_sp: - input_ids_rmpad, position_ids_rmpad, pad_size = ulysses_pad_and_slice_inputs(input_ids_rmpad, \ - position_ids_rmpad, \ - sp_size=self.ulysses_sequence_parallel_size) - input_ids_rmpad_rolled, _, _ = ulysses_pad_and_slice_inputs(input_ids_rmpad_rolled, None, - self.ulysses_sequence_parallel_size) - - input_ids_rmpad_rolled = input_ids_rmpad_rolled.squeeze(0) # ((total_nnz / sp) + pad) - - # only pass input_ids and position_ids to enable flash_attn_varlen - print(f"rmpad before forward {input_ids_rmpad.shape=} {position_ids_rmpad.shape=}") - output = self.actor_module(input_ids=input_ids_rmpad, - attention_mask=None, - position_ids=position_ids_rmpad, - **multi_modal_inputs, - use_cache=False) # prevent model thinks we are generating - logits_rmpad = output.logits.squeeze(0) # (total_nnz, vocab_size) - - logits_rmpad.div_(temperature) - - # compute entropy - entropy_rmpad = self.compute_entropy_from_logits(logits_rmpad) # ((total_nnz / sp) + pad) - - # if use_sp: ((total_nnz / sp) + pad) ; if not use_sp: (batch, seqlen) - log_probs = logprobs_from_logits(logits=logits_rmpad, labels=input_ids_rmpad_rolled) - - # gather log_prob if sp > 1 - if self.use_ulysses_sp: - # gather and unpad for the ulysses sp - log_probs = gather_outpus_and_unpad(log_probs, gather_dim=0, unpad_dim=0, padding_size=pad_size) - entropy_rmpad = gather_outpus_and_unpad(entropy_rmpad, - gather_dim=0, - unpad_dim=0, - padding_size=pad_size) - # pad back to (bsz, seqlen) - full_entropy = pad_input(hidden_states=entropy_rmpad.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen) - full_log_probs = pad_input(hidden_states=log_probs.unsqueeze(-1), - indices=indices, - batch=batch_size, - seqlen=seqlen) - - # only return response part: - entropy = full_entropy.squeeze(-1)[:, -response_length - 1:-1] # (bsz, response_length) - log_probs = full_log_probs.squeeze(-1)[:, -response_length - 1:-1] # (bsz, response_length) - - else: # not using rmpad and no ulysses sp - output = self.actor_module(input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - **multi_modal_inputs, - use_cache=False) # prevent model thinks we are generating - logits = output.logits - logits.div_(temperature) - logits = logits[:, -response_length - 1:-1, :] # (bsz, response_length, vocab_size) - log_probs = logprobs_from_logits(logits, micro_batch['responses']) - entropy = verl_F.entropy_from_logits(logits) # (bsz, response_length) - - return entropy, log_probs - - def _optimizer_step(self): - assert self.config.grad_clip is not None - - if isinstance(self.actor_module, FSDP): - grad_norm = self.actor_module.clip_grad_norm_(max_norm=self.config.grad_clip) - else: - grad_norm = torch.nn.utils.clip_grad_norm_(self.actor_module.parameters(), max_norm=self.config.grad_clip) - - # if grad_norm is not finite, skip the update - if not torch.isfinite(grad_norm): - print(f"WARN: grad_norm is not finite: {grad_norm}") - self.actor_optimizer.zero_grad() - else: - self.actor_optimizer.step() - return grad_norm - - def compute_log_prob(self, data: DataProto) -> torch.Tensor: - """Compute the log probability of the responses given input_ids, attention_mask and position_ids - - Args: - data (DataProto): a DataProto containing keys - - ``input_ids``: tensor of shape [batch_size, sequence_length]. torch.int64. Note that input_ids is the - concatenation of prompt and response. Note that ``sequence_length = prompt_length + response_length``. - - ``attention_mask``: tensor of shape [batch_size, sequence_length]. torch.int64. - - ``position_ids``: tensor of shape [batch_size, sequence_length]. torch.int64. - - ``responses``: tensor of shape [batch_size, response_length]. torch.int64. - - Returns: - torch.Tensor: the log_prob tensor - """ - # set to eval - self.actor_module.eval() - - micro_batch_size = data.meta_info['micro_batch_size'] - temperature = data.meta_info['temperature'] # temperature must be in the data.meta_info to avoid slient error - use_dynamic_bsz = data.meta_info['use_dynamic_bsz'] - - select_keys = ['responses', 'input_ids', 'attention_mask', 'position_ids'] - batch = data.select(batch_keys=select_keys).batch - has_multi_modal_inputs = 'multi_modal_inputs' in data.non_tensor_batch.keys() - - print(f"inside dp actor, {use_dynamic_bsz=}") - if has_multi_modal_inputs: - num_micro_batches = data.batch.batch_size[0] // micro_batch_size - non_tensor_select_keys = ['multi_modal_inputs'] - micro_batches = data.select(select_keys, non_tensor_select_keys).chunk(num_micro_batches) - elif use_dynamic_bsz: - # split using dynamic bsz - max_token_len = data.meta_info['max_token_len'] * self.ulysses_sequence_parallel_size - micro_batches, indices = rearrange_micro_batches(batch=batch, max_token_len=max_token_len) - else: - micro_batches = batch.split(micro_batch_size) - - log_probs_lst = [] - for micro_batch in micro_batches: - if isinstance(micro_batch, DataProto): - micro_batch = {**micro_batch.batch, **micro_batch.non_tensor_batch} - - with torch.no_grad(): - _, log_probs = self._forward_micro_batch(micro_batch, temperature=temperature) - log_probs_lst.append(log_probs) - log_probs = torch.concat(log_probs_lst, dim=0) - - if use_dynamic_bsz: - indices = list(itertools.chain.from_iterable(indices)) - assert len(indices) == log_probs.size(0), f"{len(indices)} vs. {log_probs.size()}" - revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) - log_probs = log_probs[revert_indices] - - return log_probs - - def update_policy(self, data: DataProto): - # make sure we are in training mode - self.actor_module.train() - - temperature = data.meta_info['temperature'] # temperature must be in the data.meta_info to avoid slient error - - select_keys = ['responses', 'input_ids', 'attention_mask', 'position_ids', 'old_log_probs', 'advantages'] - if self.config.masking: - select_keys.append('loss_mask') - if self.config.use_kl_loss: - select_keys.append('ref_log_prob') - batch = data.select(batch_keys=select_keys).batch - has_multi_modal_inputs = 'multi_modal_inputs' in data.non_tensor_batch.keys() - - # Split to make minibatch iterator for updating the actor - # See PPO paper for details. https://arxiv.org/abs/1707.06347 - if has_multi_modal_inputs: - num_mini_batches = data.batch.batch_size[0] // self.config.ppo_mini_batch_size - non_tensor_select_keys = ['multi_modal_inputs'] - dataloader = data.select(select_keys, non_tensor_select_keys).chunk(num_mini_batches) - else: - dataloader = batch.split(self.config.ppo_mini_batch_size) - - metrics = {} - for epoch in range(self.config.ppo_epochs): - for batch_idx, data in enumerate(dataloader): - # split batch into micro_batches - mini_batch = data - if has_multi_modal_inputs: - self.gradient_accumulation = self.config.ppo_mini_batch_size // self.config.ppo_micro_batch_size_per_gpu - num_micro_batches = mini_batch.batch.batch_size[0] // self.config.ppo_micro_batch_size_per_gpu - micro_batches = data.select(select_keys, non_tensor_select_keys).chunk(num_micro_batches) - elif self.config.use_dynamic_bsz: - max_token_len = self.config.ppo_max_token_len_per_gpu * self.ulysses_sequence_parallel_size - micro_batches, _ = rearrange_micro_batches(batch=mini_batch, max_token_len=max_token_len) - else: - self.gradient_accumulation = self.config.ppo_mini_batch_size // self.config.ppo_micro_batch_size_per_gpu - # split batch into micro_batches - micro_batches = mini_batch.split(self.config.ppo_micro_batch_size_per_gpu) - - self.actor_optimizer.zero_grad() - - for data in micro_batches: - # Support all hardwares - if isinstance(data, DataProto): - data = {**data.batch.to(torch.cuda.current_device()), **data.non_tensor_batch} - else: - data = data.to(torch.cuda.current_device()) # actor device is cpu when using offload - responses = data['responses'] - response_length = responses.size(1) - attention_mask = data['attention_mask'] - old_log_prob = data['old_log_probs'] - advantages = data['advantages'] - response_mask = attention_mask[:, -response_length:] - if self.config.masking: - response_mask = data['loss_mask'] - - clip_ratio = self.config.clip_ratio - clip_ratio_low = self.config.get('clip_ratio_low', clip_ratio) - clip_ratio_high = self.config.get('clip_ratio_high', clip_ratio) - clip_ratio_c = self.config.get('clip_ratio_c', 3.0) - entropy_coeff = self.config.entropy_coeff - loss_agg_mode = self.config.loss_agg_mode - - # all return: (bsz, response_length) - entropy, log_prob = self._forward_micro_batch(micro_batch=data, temperature=temperature) - - print( - f"inside dp actor {response_mask.shape=} {response_length=} {responses.shape=} {old_log_prob.shape=} {log_prob.shape=}" - ) - # NOTE (sumanthrh): THis is a quick fix when response_mask is zero - preserve_grad = True - - pg_loss, pg_clipfrac, ppo_kl, pg_clipfrac_lower = compute_policy_loss( - old_log_prob=old_log_prob, - log_prob=log_prob, - advantages=advantages, - response_mask=response_mask, - cliprange=clip_ratio, - cliprange_low=clip_ratio_low, - cliprange_high=clip_ratio_high, - clip_ratio_c=clip_ratio_c, - preserve_grad=preserve_grad) - - try: - print(f"PG Loss before backward pass: {pg_loss=}") - print(f"PG Loss requires grad?: {pg_loss.requires_grad}") - print(f"PG Loss gradient function: {pg_loss.grad_fn}") - print(f"PG Loss has NaN: {torch.isnan(pg_loss)}. Loss has Inf: {torch.isinf(pg_loss)}") - except Exception as e: - print(f"Cannot print PG loss details") - - # compute entropy loss from entropy - entropy_loss = agg_loss(loss_mat=entropy, loss_mask=response_mask, loss_agg_mode=loss_agg_mode, preserve_grad=preserve_grad) - - try: - print(f"Entropy Loss before backward pass: {entropy_loss=}") - print(f"Entropy Loss requires grad?: {entropy_loss.requires_grad}") - print(f"Entropy Loss gradient function: {entropy_loss.grad_fn}") - print(f"Entropy Loss has NaN: {torch.isnan(entropy_loss)}. Loss has Inf: {torch.isinf(entropy_loss)}") - except Exception as e: - print(f"Cannot print Entropy loss details") - - # compute policy loss - policy_loss = pg_loss - entropy_loss * entropy_coeff - - try: - print(f"Policy Loss before backward pass: {policy_loss=}") - print(f"Policy Loss requires grad?: {policy_loss.requires_grad}") - print(f"Policy Loss gradient function: {policy_loss.grad_fn}") - print(f"Policy Loss has NaN: {torch.isnan(policy_loss)}. Loss has Inf: {torch.isinf(policy_loss)}") - except Exception as e: - print(f"Cannot print Policy loss details") - - if self.config.use_kl_loss: - ref_log_prob = data['ref_log_prob'] - # compute kl loss - kld = core_algos.kl_penalty(logprob=log_prob, - ref_logprob=ref_log_prob, - kl_penalty=self.config.kl_loss_type) - print("kld.shape", kld.shape) - print("response_mask.shape", response_mask.shape) - kl_loss = masked_mean(kld, response_mask, preserve_grad=preserve_grad) - - policy_loss = policy_loss + kl_loss * self.config.kl_loss_coef - metrics['actor/kl_loss'] = kl_loss.detach().item() - metrics['actor/kl_coef'] = self.config.kl_loss_coef - - if self.config.use_dynamic_bsz: - # relative to the dynamic bsz - loss = policy_loss * (len(data) / self.config.ppo_mini_batch_size) - else: - loss = policy_loss / self.gradient_accumulation - - print(f"Loss before backward pass: {loss=}") - print(f"Loss requires grad?: {loss.requires_grad}") - print(f"Loss gradient function: {loss.grad_fn}") - print(f"Loss has NaN: {torch.isnan(loss)}. Loss has Inf: {torch.isinf(loss)}") - loss.backward() - - data = { - 'actor/entropy': entropy_loss.detach().item(), - 'actor/pg_loss': pg_loss.detach().item(), - 'actor/pg_clipfrac': pg_clipfrac.detach().item(), - 'actor/ppo_kl': ppo_kl.detach().item(), - 'actor/pg_clipfrac_lower': pg_clipfrac_lower.detach().item(), - } - append_to_dict(metrics, data) - - grad_norm = self._optimizer_step() - data = {'actor/grad_norm': grad_norm.detach().item()} - append_to_dict(metrics, data) - self.actor_optimizer.zero_grad() - return metrics diff --git a/verl/workers/actor/megatron_actor.py b/verl/workers/actor/megatron_actor.py deleted file mode 100644 index 2d0adc807f..0000000000 --- a/verl/workers/actor/megatron_actor.py +++ /dev/null @@ -1,413 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Megatron Actor. -In megatron actor, the differences are: -1. We only make minibatch - -Note that our model doesn't have to be `MegatronModule` because we don't share embedding in the last layer -""" - -import importlib -from functools import partial -from packaging.version import Version -from typing import Iterable, Dict - -import torch -from torch import nn -import torch.distributed -from megatron.core.optimizer import OptimizerConfig -from megatron.core import parallel_state as mpu -from megatron.core import ModelParallelConfig -from verl.utils.megatron_utils import get_model_config -from megatron.core.pipeline_parallel import get_forward_backward_func - -from megatron.core.distributed import finalize_model_grads -# from megatron.core.optimizer import DistributedOptimizer - -from megatron.core.optimizer import DistributedOptimizer - -from omegaconf import OmegaConf -from verl.utils.megatron.tensor_parallel import vocab_parallel_entropy, vocab_parallel_log_probs_from_logits -from verl.utils.megatron.pipeline_parallel import (compute_transformers_input_shapes, make_batch_generator) -from verl import DataProto -from verl.trainer.ppo.core_algos import compute_policy_loss, kl_penalty, agg_loss -from verl.workers.actor import BasePPOActor -from verl.utils.py_functional import append_to_dict -from verl.utils.torch_functional import logprobs_from_logits, masked_mean, broadcast_dict_tensor, split_dict_tensor_into_batches - -__all__ = ['MegatronPPOActor'] - - -class MegatronPPOActor(BasePPOActor): - - def __init__(self, config, model_config, megatron_config: ModelParallelConfig, actor_module: nn.ModuleList, - actor_optimizer: DistributedOptimizer, actor_optimizer_config: OptimizerConfig): - """MeagtronPPOActor class. This class implements the simple PPO logics when the model is built with Megatron. - - Args: - config (OmegaConf): the basic config that contains the hyper-parameters of PPO Actor. It must contain - - ``ppo_micro_batch_size_per_gpu``: micro batch size when updating ppo. - - ``ppo_mini_batch_size``: minibatch size when updating ppo using the batch data. - - ``ppo_epochs``: number of epochs to update the actor using the batch data. - - ``shuffle``: whether to shuffle the data after each ppo epoch. - - ``clip_ratio``: clip ratio of the ppo algorithm. See https://arxiv.org/abs/1707.06347. - - ``entropy_coeff``: entropy coefficient of the PPO loss. See https://arxiv.org/abs/1707.06347. - model_config (OmegaConf): model configuration. It must contains ``model_config.vocab_size`` and - ``model_config.hidden_size`` - megatron_config (OmegaConf): megatron configuration. It must contains - - ``sequence_parallel_enabled``: whether the sequence parallel is enabled. - - ``param_dtype``: the dtype of the parameters. - - ``virtual_pipeline_model_parallel_size``: virtual pipeline model parallel size. a.k.a number of chunks in each pp stage. - actor_module (nn.ModuleList): actor module is a ModuleList that contains a list of nn.Module in this pp stage. - each nn.Module in this rank holds a vpp module chunk. See https://arxiv.org/pdf/2104.04473.pdf for more details. - The actor module has some constraints to follow in order to use the updating logics implemented here - - 1. It must implement unpad_input before any computation and pad_input after all the computation. Remove padding is an - optimization that removes the padding tokens. See unpad_input and pad_input function in flash-attn - (https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/bert_padding.py). - - 2. Each pp stage must return the hidden state with the same shape [total_nnz, 1, hidden_size], - where total_nnz is the number of valid tokens in this batch. If sequence parallel is enabled, the size - of the hidden state is [total_nnz // tp, 1, hidden_size]. - actor_optimizer (DistributedOptimizer): currently, we only support DistributedOptimizer in Megatron. It implements - zero1 optimizer that shards the optimizer state across dp ranks. - - >>> def megatron_actor_model_provider(pre_process, post_process): - >>> vpp_rank = mpu.get_virtual_pipeline_model_parallel_rank() - >>> parallel_model = ParallelMistralForCausalLMRmPadPP(config=actor_model_config, - >>> megatron_config=megatron_config, - >>> pre_process=pre_process, - >>> post_process=post_process).cuda() - >>> return parallel_model - >>> from megatron.training import get_model - >>> from megatron.optimizer import get_megatron_optimizer - >>> actor_module = get_model(megatron_actor_model_provider, wrap_with_ddp=True) - >>> actor_module = nn.ModuleList(actor_module) - >>> actor_optimizer = get_megatron_optimizer(actor_module) - >>> actor = MegatronPPOActor(config=config, - >>> model_config=actor_model_config, - >>> megatron_config=megatron_config, - >>> actor_module=actor_module, - >>> actor_optimizer=actor_optimizer) - """ - super().__init__(config) - self._validate_config(config) - self.model_config = model_config - self.megatron_config = megatron_config - self.actor_module = actor_module - self.actor_optimizer: DistributedOptimizer = actor_optimizer - self.actor_optimizer_config = actor_optimizer_config - - self.optimizer_step_args = OmegaConf.create({ - 'skip_grad': None, - 'overlap_dp_param_comm': False, - 'overlap_dp_grad_comm': False, - 'gradient_accumulation_steps': 1, - 'sequence_parallel': self.megatron_config.sequence_parallel, - 'DDP_impl': 'local', - 'layernorm_allreduce_bucket_threshold': 0, - 'pipeline_model_parallel_split_rank': None, - 'reduce_grads_use_alltoall': False - }) - - config = get_model_config(self.actor_module[0]) - print(config) - config.finalize_model_grads_func = finalize_model_grads - - def _validate_config(self, config) -> None: - """Validate config options not implemented for Megatron backend""" - assert config.get('ulysses_sequence_parallel_size', 1) == 1 - - def compute_log_prob(self, data: DataProto) -> torch.Tensor: - """Compute the log probability of the responses given input_ids, attention_mask and position_ids - - Args: - data (DataProto): a DataProto containing keys - - ``input_ids``: tensor of shape [batch_size, sequence_length]. torch.int64. Note that input_ids is the - concatenation of prompt and response. Note that ``sequence_length = prompt_length + response_length``. - - ``attention_mask``: tensor of shape [batch_size, sequence_length]. torch.int64. - - ``position_ids``: tensor of shape [batch_size, sequence_length]. torch.int64. - - ``responses``: tensor of shape [batch_size, response_length]. torch.int64. - - Returns: - DataProto: torch.Tensor: the log_prob tensor - """ - data.batch = data.batch.contiguous() - - def compute_logprobs_fn(output, data): - response = data['responses'] - response_length = response.size(1) - logits = output - logits = logits[:, -response_length - 1:-1].contiguous() - log_probs = vocab_parallel_log_probs_from_logits(logits, response) - return {'log_probs': log_probs} - - # We make recompute_old_log_prob by default here. - # TODO (zhangchi.usc1992): actually, this function should only return log_prob and this logic should be handled by user outside - recompute_old_log_prob = self.config.get('recompute_old_log_prob', True) - - if recompute_old_log_prob or 'old_log_probs' not in data.batch.keys(): - select_keys = ['responses', 'input_ids', 'attention_mask', 'position_ids'] - batch = data.select(batch_keys=select_keys).batch - input_ids = batch['input_ids'] - batch_size = input_ids.size(0) - response = batch['responses'] - response_length = response.size(1) - with torch.no_grad(): - output = self.forward_backward_batch(data, forward_only=True, post_process_fn=compute_logprobs_fn) - if mpu.is_pipeline_last_stage(ignore_virtual=True): - # only on last rank. It should be on every tp rank - log_probs = torch.cat([o['log_probs'] for o in output], dim=0) # (bs, seq_size) - log_probs = log_probs.to(torch.float32) - else: - log_probs = torch.empty(size=(batch_size, response_length), - dtype=torch.float32, - device=input_ids.device) - - # broadcast across pp ranks - torch.distributed.broadcast(tensor=log_probs, - src=mpu.get_pipeline_model_parallel_last_rank(), - group=mpu.get_pipeline_model_parallel_group(), - async_op=False) - - # add empty cache after each compute - torch.cuda.empty_cache() - - return log_probs - - def make_minibatch_iterator(self, data: DataProto) -> Iterable[DataProto]: - """Make minibatch iterator for updating the actor - - Args: - data (DataProto): a DataProto containing keys - - ``input_ids``: tensor of shape [batch_size, sequence_length]. torch.int64, where ``sequence_length = prompt_length + response_length`` - - ``attention_mask``: tensor of shape [batch_size, sequence_length]. torch.int64 - - ``position_ids``: tensor of shape [batch_size, sequence_length]. torch.int64 - - ``responses``: tensor of shape [batch_size, response_length]. torch.int64. Note that responses = input_ids[:, -response_length:] - - ``old_log_probs``: tensor of shape [batch_size, response_length]. torch.float32. The log probability of responses. - - ``advantages``: tensor of shape [batch_size, response_length]. torch.float32. The advantages of responses. - See PPO paper for details. https://arxiv.org/abs/1707.06347 - - Returns: - - """ - select_keys = ['responses', 'input_ids', 'attention_mask', 'position_ids', 'old_log_probs', 'advantages'] - if self.config.use_kl_loss: - select_keys.append('ref_log_prob') - data = data.select(batch_keys=select_keys) - return data.make_iterator(mini_batch_size=self.config.ppo_mini_batch_size, - epochs=self.config.ppo_epochs, - dataloader_kwargs={'shuffle': self.config.shuffle}) - - def forward_backward_batch(self, data: DataProto, forward_only=False, post_process_fn=None): - """ - We assume: - - The model takes input: (input_ids, attention_mask, position_ids). No rmpad for the input - - The communication shape is (total_nnz_pad_to_sp // tp_size, 1, hidden_size) if sequence parallel is enabled - """ - # broadcast from last pp rank to all other pp ranks - # TODO: actually, we just need to control the sampling order. - broadcast_dict_tensor(data.batch, - src=mpu.get_pipeline_model_parallel_last_rank(), - group=mpu.get_pipeline_model_parallel_group()) - # split into micro-batches - data.batch['attention_mask'] = data.batch['attention_mask'].to(bool) - - if data.meta_info.get('micro_batch_size', None) is not None: - batch_size = data.meta_info['micro_batch_size'] - else: - batch_size = self.config.ppo_micro_batch_size_per_gpu - batches = split_dict_tensor_into_batches(data.batch, batch_size=batch_size) - # compute input shapes for pp stages - input_shapes = compute_transformers_input_shapes( - batches, - meta_info={ - 'sequence_parallel': self.megatron_config.sequence_parallel, - 'hidden_size': self.model_config.hidden_size - }) - n_micro_batch = len(batches) - seq_len = batches[0]['input_ids'].shape[1] - - forward_backward_func = get_forward_backward_func() - - def loss_func(output, data, meta_info): - if forward_only: - if post_process_fn is None: - return 1.0, {'logits': output} - else: - return 1.0, post_process_fn(output, data) - - responses = data['responses'] - response_length = responses.size(1) - attention_mask = data['attention_mask'] - response_mask = attention_mask[:, -response_length:] - old_log_prob = data['old_log_probs'] - advantages = data['advantages'] - - clip_ratio = meta_info['clip_ratio'] - clip_ratio_low = self.config.clip_ratio_low if self.config.clip_ratio_low is not None else clip_ratio - clip_ratio_high = self.config.clip_ratio_high if self.config.clip_ratio_high is not None else clip_ratio - clip_ratio_c = meta_info['clip_ratio_c'] - entropy_coeff = meta_info['entropy_coeff'] - loss_agg_mode = self.config.loss_agg_mode - - # compute policy loss - logits = output - logits = logits[:, -response_length - 1:-1].contiguous() - logits_back = logits.clone() - log_prob = vocab_parallel_log_probs_from_logits(logits, responses) - logits = logits_back - pg_loss, pg_clipfrac, ppo_kl, pg_clipfrac_lower = compute_policy_loss(old_log_prob=old_log_prob, - log_prob=log_prob, - advantages=advantages, - response_mask=response_mask, - cliprange=clip_ratio, - cliprange_low=clip_ratio_low, - cliprange_high=clip_ratio_high, - clip_ratio_c=clip_ratio_c, - loss_agg_mode=loss_agg_mode) - entropy = vocab_parallel_entropy(logits) - entropy_loss = agg_loss(loss_mat=entropy, loss_mask=response_mask, loss_agg_mode=loss_agg_mode) - policy_loss = pg_loss - entropy_loss * entropy_coeff - - metrics = {} - if self.config.use_kl_loss: - ref_log_prob = data['ref_log_prob'] - # compute kl loss - kld = kl_penalty(logprob=log_prob, ref_logprob=ref_log_prob, kl_penalty=self.config.kl_loss_type) - kl_loss = agg_loss(loss_mat=kld, loss_mask=response_mask, loss_agg_mode=self.config.loss_agg_mode) - - policy_loss = policy_loss + kl_loss * self.config.kl_loss_coef - metrics['actor/kl_loss'] = kl_loss.detach().item() - metrics['actor/kl_coef'] = self.config.kl_loss_coef - - # return loss and stats - stats = { - 'actor/entropy_loss': entropy_loss.detach().item(), - 'actor/pg_loss': pg_loss.detach().item(), - 'actor/pg_clipfrac': pg_clipfrac.detach().item(), - 'actor/ppo_kl': ppo_kl.detach().item(), - 'actor/pg_clipfrac_lower': pg_clipfrac_lower.detach().item() - } - append_to_dict(stats, metrics) - return policy_loss, stats - - def forward_step(batch_iter, model): - batch = next(batch_iter) - input_ids = batch['input_ids'] - attention_mask = batch['attention_mask'] - position_ids = batch['position_ids'] - from verl.models.mcore import gptmodel_forward - - output = gptmodel_forward(model, - input_ids, - attention_mask, - position_ids, - sequence_parallel=self.megatron_config.sequence_parallel) - if forward_only: - meta_info = None - else: - clip_ratio_c = self.config.get('clip_ratio_c', 3.0) - meta_info = { - 'clip_ratio': self.config.clip_ratio, - 'entropy_coeff': self.config.entropy_coeff, - 'clip_ratio_c': clip_ratio_c - } - return output, partial(loss_func, data=batch, meta_info=meta_info) - - # batch should be a list of batches inside micro-batches - batch_generator = make_batch_generator(batches, vpp_size=len(self.actor_module)) - - # TODO: we may use the new schedule instead - # for flash-attn: (seq_len, batch_size, hidden_size) = (mbs*seq_len, 1, hidden_size) - if mpu.get_pipeline_model_parallel_world_size() > 1: - losses_reduced = forward_backward_func( - forward_step_func=forward_step, - data_iterator=batch_generator, - model=self.actor_module, - num_microbatches=n_micro_batch, - seq_length=batch_size * seq_len, # no use when input_shapes was set - micro_batch_size=1, # no use when input_shapes was set - forward_only=forward_only, - ) - else: - losses_reduced = forward_backward_func( - forward_step_func=forward_step, - data_iterator=batch_generator, - model=self.actor_module, - num_microbatches=n_micro_batch, - seq_length=batch_size * seq_len, # in use for pp = 1 - micro_batch_size=1, # in use for pp = 1 - forward_only=forward_only, - ) - # loss_reduces contains the stats returned from loss_func - return losses_reduced - - def update_policy(self, dataloader: Iterable[DataProto]) -> Dict: - """Update the policy with an iterator of DataProto - - Args: - dataloader (Iterable[DataProto]): an iterator over the DataProto that returns by ``make_minibatch_iterator`` - The keys of each data batch is described in the make_minibatch_iterator. - - Returns: - Dict: a dictionary containing the statistics. Note that the statistics are only valid in the last pp stage - and users have to combine the output in each dp rank manually. - - """ - metrics = {} - for data in dataloader: - # data = data.batch.to(self.actor_module.device) - self.actor_optimizer.zero_grad() - # use use_contiguous_buffers_in_local_ddp and no overlap_dp_param_comm - for chunk in self.actor_module: - # if use distributed optimizer, zero grad buffer will be handled by optimizer - chunk.zero_grad_buffer() - - metric_micro_batch = self.forward_backward_batch(data) - for metric in metric_micro_batch: - append_to_dict(metrics, metric) # append the metric from this micro-batch to global metrics. - - update_successful, grad_norm, num_zeros_in_grad = self.actor_optimizer.step() - - if update_successful: - # allgather already execute in optimizer.step in new megatron - pass - else: - raise NotImplementedError - - # add empty cache after each compute - torch.cuda.empty_cache() - - return metrics diff --git a/verl/workers/agentic/async_rollout.py b/verl/workers/agentic/async_rollout.py deleted file mode 100644 index 50c537c73c..0000000000 --- a/verl/workers/agentic/async_rollout.py +++ /dev/null @@ -1,116 +0,0 @@ -# TODO(haoran): stuck in the loop -# TODO(haoran): time control; loss_mask -# TODO(haoran): check reason for loading weight -import logging -import os -from functools import partial -from json import JSONDecodeError - -import torch -import sglang as sgl -import torch.distributed -from omegaconf import DictConfig -from sglang.srt.function_call_parser import FunctionCallParser -from sglang.srt.openai_api.protocol import Tool -from torch.distributed import DeviceMesh -from tensordict import TensorDict - -from verl import DataProto -from verl.workers.rollout.base import BaseRollout -from verl.workers.agentic.codeact import CodeActAgentGroup - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_PPO_LOGGING_LEVEL', 'INFO')) - - -def _pre_process_inputs(pad_token_id, token_ids: torch.Tensor) -> list[int]: - # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = token_ids[non_pad_index:].tolist() - return token_ids - - -class AsyncRollout(BaseRollout): - - def __init__(self, model_path, config: DictConfig, device_mesh: DeviceMesh): - super().__init__() - torch.distributed.barrier() - # print(f"nodedup in AsyncRollout: {torch.distributed.is_initialized() = } {torch.distributed.get_rank() = }") - os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" - # os.environ["CUDA_LAUNCH_BLOCKING"] = "1" - self.tp_rank = device_mesh.get_local_rank(1) - self.device_mesh = device_mesh - cuda_visible_device = os.environ["CUDA_VISIBLE_DEVICES"] - visible_devices: list[str | None] = [None] * device_mesh.size(1) - torch.distributed.all_gather_object(visible_devices, cuda_visible_device, group=device_mesh.get_group(1)) - os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(visible_devices) - print( - f"nodedup in async rollout {os.environ['CUDA_VISIBLE_DEVICES']=} @ {torch.distributed.get_rank()=} {self.tp_rank=}" - ) - self.total_len = config.prompt_length + config.response_length - print(f"async rollout {config.gpu_memory_utilization=}") - torch.distributed.barrier() - # print(f"nodedup in async rollout {os.environ['CUDA_VISIBLE_DEVICES']=} @ {torch.distributed.get_rank()=} {self.tp_rank=}") - if self.tp_rank == 0: - self.engine = sgl.Engine( - model_path=model_path, - port=40000, - dtype=config.dtype, - max_total_tokens=60*self.total_len, - max_prefill_tokens=2*self.total_len, - enable_memory_saver=config.enable_memory_saver, - mem_fraction_static=config.gpu_memory_utilization, - tp_size=device_mesh.size(1), - log_level="INFO", - # enable_metrics=True, - ) - print(f"nodedup {torch.distributed.get_rank() = } releasing memory occupation") - self.engine.release_memory_occupation() - print(f"nodedup {torch.distributed.get_rank() = } engine initialized") - else: - self.engine = None - self.engine: sgl.srt.entrypoints.engine.Engine | None - os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_device - torch.distributed.barrier() - self.config = config - self.task_type = config.task_type - # print("Config sampling params:", config.sampling_params) - self.sampling_params = dict(config.sampling_params) - self.sampling_params.update({ - "skip_special_tokens": False, - }) - - def generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto | None: - # print(f"nodedup in generate seq {torch.distributed.get_rank()=} {self.tp_rank=} {prompts.non_tensor_batch=}") - logger.info(f"nodedup in generate seq {torch.distributed.get_rank()=} {self.tp_rank=}") - if self.tp_rank != 0: - return None - - sampling_params = self.sampling_params.copy() - # print("kwargs: ", kwargs) - sampling_params.update(kwargs) - print("final sampling params:", sampling_params) - device = torch.cuda.current_device() - - codeact_agent_group = CodeActAgentGroup( - batch=prompts, - num_trajectories=self.config.n_trajectories, - infer_engine=self.engine, - max_prompt_length=self.config.prompt_length, - max_response_length=self.config.response_length, - max_starting_message_length=self.config.max_starting_message_length, - max_parallel_agents=self.config.max_parallel_agents // self.device_mesh.size(0), - max_eval_parallel_agents=self.config.max_eval_parallel_agents // self.device_mesh.size(0), - max_iterations = self.config.max_iterations, - tokenizer=self.engine.tokenizer_manager.tokenizer, - sampling_params=sampling_params, - device=device, - log_messages_dir=self.config.log_messages_dir, - remove_think_tokens=self.config.remove_think_tokens, - qwen3_enable_thinking=self.config.qwen3_enable_thinking, - ) - - results = codeact_agent_group.run() - logger.info(f"nodedup finish generate seq {torch.distributed.get_rank()=} {self.tp_rank=}") - return results diff --git a/verl/workers/agentic/codeact.py b/verl/workers/agentic/codeact.py deleted file mode 100644 index 3d22f1ba02..0000000000 --- a/verl/workers/agentic/codeact.py +++ /dev/null @@ -1,1375 +0,0 @@ -import json -import asyncio -import uuid -from collections import deque -from pathlib import Path -from typing import Any, List, Dict, Optional, Set, Callable, Tuple -import os -import pandas as pd -import tempfile -import time - -import torch -from tensordict import TensorDict -from verl import DataProto -from verl.utils.model import compute_position_id_with_mask -import verl.utils.torch_functional as verl_F -import torch.nn.functional as F -from transformers import AutoTokenizer - -import openhands -import openhands.agenthub.codeact_agent.function_calling as codeact_function_calling -from openhands.controller.agent import Agent -from openhands.controller.state.state import State, AgentState -from openhands.core.config import LLMConfig, AgentConfig, SandboxConfig, AppConfig -from openhands.core.main import create_runtime, run_controller -from openhands.core.logger import openhands_logger as logger -from openhands.core.message import Message, TextContent -from openhands.core.message_utils import ( - events_to_messages, -) -from openhands.core.exceptions import ( - AgentStuckInLoopError, - FunctionCallNotExistsError, - FunctionCallValidationError, - LLMContextWindowExceedError, - LLMMalformedActionError, - LLMNoActionError, - LLMResponseError, -) -from openhands.events.action import ( - Action, - AgentFinishAction, - MessageAction, -) -from openhands.events.event import EventSource -from openhands.memory.condenser import Condenser -from openhands.core.config.condenser_config import ( - NoOpCondenserConfig, -) -from openhands.llm.fn_call_converter import ( - convert_fncall_messages_to_non_fncall_messages, - convert_non_fncall_messages_to_fncall_messages, -) -from openhands.llm.llm import LLM -from openhands.utils.prompt import PromptManager -from openhands.utils.async_utils import call_sync_from_async, call_async_from_sync - -from swegym.harness.test_spec import make_test_spec -from swegym.harness.run_evaluation import ( - APPLY_PATCH_FAIL, - APPLY_PATCH_PASS, -) -from swegym.harness.grading import get_eval_report -from openhands.events.action import CmdRunAction -from openhands.events.observation import CmdOutputObservation -from .utils import process_git_patch - -DOCKER_IMAGE_PREFIX = os.environ.get('EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/xingyaoww/') -logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}') - -# this is for the tokenizer.apply_chat_template to be able to generate assistant masks directly -# todo: this is a hack, we should find a better way to do this -chat_template = ( - "{% for message in messages %}" - "{% if (message['role'] != 'assistant') %}" - "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}" - "{% elif (message['role'] == 'assistant')%}" - "{{'<|im_start|>' + message['role'] + '\n'}}" - "{% generation %}" - "{{message['content'] + '<|im_end|>'}}" - "{% endgeneration %}" - "{{'\n'}}" - "{% endif %}" - "{% endfor %}" - ) - -# chat template for qwen3 thinking mode to remove think tokens similar to generation phase -chat_template_qwen3_thinking = ( - "{% for message in messages %}" - "{% if (message['role'] != 'assistant') %}" - "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}" - "{% elif (message['role'] == 'assistant')%}" - "{{'<|im_start|>' + message['role'] + '\n'}}" - "{% generation %}" - "{% set full_content = message['content'] %}" - "{% set mycontent = message['content'] %}" - "{% set is_last_message = loop.last and messages[-1]['role'] == 'assistant' %}" - "{% if '' in full_content and not is_last_message %}" - "{% set mycontent = full_content.split('')[-1].lstrip('\n') %}" - "{% endif %}" - "{{mycontent + '<|im_end|>'}}" - "{% endgeneration %}" - "{{'\n'}}" - "{% endif %}" - "{% endfor %}" -) - - -def convert_right_padding_to_left(tokenizer, input_ids, attention_mask, device, max_len=None): - """ - Converts right-padded tensors to left-padded tensors with optional custom length. - - Args: - tokenizer: The tokenizer object with pad_token_id attribute - input_ids (torch.Tensor): Right-padded input IDs tensor of shape [batch_size, seq_length] - attention_mask (torch.Tensor): Right-padded attention mask tensor of shape [batch_size, seq_length] - device: The device to place the new tensors on - max_len (int, optional): The desired maximum length of the returned tensors. - If None, uses the original sequence length. - - Returns: - tuple: (left_padded_input_ids, left_padded_attention_mask) - """ - batch_size, orig_seq_length = input_ids.size() - - # Use original length if max_len is not specified - seq_length = max_len if max_len is not None else orig_seq_length - - # Create new tensors with the desired size - left_padded_input_ids = torch.full((batch_size, seq_length), - tokenizer.pad_token_id, - dtype=input_ids.dtype, - device=device) - left_padded_attention_mask = torch.zeros((batch_size, seq_length), - dtype=attention_mask.dtype, - device=device) - - for i in range(batch_size): - # Get the non-padded length of this sequence - seq_len = attention_mask[i].sum().item() - - # Trim sequence if it's longer than max_len - if seq_len > seq_length: - logger.warning(f"Trimming sequence length from {seq_len} to {seq_length}") - seq_len = seq_length - - # Calculate the offset for left padding - offset = seq_length - seq_len - - # Copy the non-padded tokens to the end - left_padded_input_ids[i, offset:] = input_ids[i, :seq_len] - left_padded_attention_mask[i, offset:] = 1 # Set attention mask for non-padding tokens - - return left_padded_input_ids, left_padded_attention_mask - -def pad_to_max_length_right(tokenizer, encodings, max_length, device): - """ - Pads tokenizer outputs to a specific maximum length with configurable padding side. - - Args: - tokenizer: The tokenizer object with pad_token_id attribute - encodings (dict): Dictionary containing 'input_ids', 'attention_mask', and optionally 'assistant_masks' - max_length (int): The desired maximum length to pad to - device: The device to place the tensors on - - Returns: - dict: Dictionary with padded tensors for 'input_ids', 'attention_mask', and 'assistant_masks' if present - """ - batch_size = len(encodings['input_ids']) - - # Initialize output tensors - padded_input_ids = torch.full((batch_size, max_length), - tokenizer.pad_token_id, - dtype=torch.long, - device=device) - padded_attention_mask = torch.zeros((batch_size, max_length), - dtype=torch.long, - device=device) - padded_assistant_mask = torch.zeros((batch_size, max_length), - dtype=torch.long, - device=device) - - # Fill tensors with actual values - num_trimmed = 0 - for i in range(batch_size): - seq_len = encodings["attention_mask"][i].sum().item() if isinstance(encodings["attention_mask"][i], torch.Tensor) else sum(encodings["attention_mask"][i]) - # Trim if longer than max_length - actual_len = min(seq_len, max_length) - if seq_len > max_length: - logger.warning( - f"Trimming sequence length from {seq_len} to {actual_len} for batch item {i}" - ) - num_trimmed += 1 - - # Right padding - copy sequence data to the beginning - padded_input_ids[i, :actual_len] = torch.tensor(encodings['input_ids'][i][:actual_len], device=device) - padded_attention_mask[i, :actual_len] = torch.tensor(encodings['attention_mask'][i][:actual_len], device=device) - padded_assistant_mask[i, :actual_len] = torch.tensor(encodings['assistant_masks'][i][:actual_len], device=device) - - logger.info(f"Trimmed {num_trimmed*100 / max(batch_size, 1)}% of samples in the batch of size {batch_size}") - return padded_input_ids, padded_attention_mask, padded_assistant_mask - - -def codeact_user_response( - state: State, - encapsulate_solution: bool = False, - try_parse: Callable[[Action], str] | None = None, -) -> str: - encaps_str = ( - ( - 'Please encapsulate your final answer (answer ONLY) within and .\n' - 'For example: The answer to the question is 42 .\n' - ) - if encapsulate_solution - else '' - ) - msg = ( - 'Please continue working on the task on whatever approach you think is suitable.\n' - 'If you think you have solved the task, please first send your answer to user through message and then finish the interaction.\n' - f'{encaps_str}' - 'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP.\n' - ) - - if state.history: - # check if the last action has an answer, if so, early exit - if try_parse is not None: - last_action = next( - ( - event - for event in reversed(state.history) - if isinstance(event, Action) - ), - None, - ) - ans = try_parse(last_action) - if ans is not None: - return '/exit' - - # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up - user_msgs = [ - event - for event in state.history - if isinstance(event, MessageAction) and event.source == 'user' - ] - if len(user_msgs) >= 2: - # let the agent know that it can give up when it has tried 3 times - return ( - msg - + 'If you want to give up, use the "finish" tool to finish the interaction.\n' - ) - return msg - -def get_instance_docker_image(instance_id: str) -> str: - image_name = 'sweb.eval.x86_64.' + instance_id - image_name = image_name.replace( - '__', '_s_' - ) # to comply with docker image naming convention - return (DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + image_name).lower() - -# Helper function for sandbox config -def get_default_sandbox_config_for_eval(): - return SandboxConfig( - use_host_network=False, - timeout=300, - api_key=os.environ.get('ALLHANDS_API_KEY', None), - remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'), - keep_runtime_alive=False, - remote_runtime_init_timeout=3600, - remote_runtime_api_timeout=120, - remote_runtime_enable_retries=True, - remote_runtime_class='sysbox', - ) - -class OnlineCodeActAgent(Agent): - """ - An online implementation of CodeActAgent that leverages infer's asynchronous capabilities - for a single agent instance. - """ - - def __init__( - self, - instance_id: int, - trajectory_id: int, - max_prompt_length: int = 1024, - infer_engine=None, - tokenizer=None, - sampling_params=None, - qwen3_enable_thinking: bool = True, - ) -> None: - """ - Initialize a single OnlineCodeActAgent instance. - """ - # dummy value to let openhands tracks the name - llm = LLM(LLMConfig(model="dummy")) - - super().__init__(llm, AgentConfig()) - - self.tokenizer = tokenizer - self.max_prompt_length = max_prompt_length - self.reset() - self.step_count = 0 - self.infer_engine = infer_engine - self.sampling_params = sampling_params - - # Store instance and trajectory IDs separately - self.instance_id = instance_id - self.trajectory_id = trajectory_id - - # Initialize tools - self.tools = codeact_function_calling.get_tools( - codeact_enable_browsing=False, - codeact_enable_jupyter=False, - codeact_enable_llm_editor=False, - ) - - # Initialize prompt manager - self.prompt_manager = PromptManager( - microagent_dir=os.path.join( - os.path.dirname(os.path.dirname(openhands.__file__)), - 'microagents', - ), - prompt_dir=os.path.join(os.path.dirname(openhands.agenthub.codeact_agent.__file__), 'prompts'), - disabled_microagents=None, - ) - - # Initialize condenser - self.condenser = Condenser.from_config(NoOpCondenserConfig()) - - # Initialize state - self.pending_actions = deque() - - # will be set in _initialize_runtime_for_agent - self.runtime = None - self.instruction = None - self.config = None - - self.qwen3_enable_thinking = qwen3_enable_thinking - - def close(self): - """Close the agent runtime.""" - if self.runtime: - # remove all threads in event stream - self.runtime.event_stream.close() - - self.runtime.close() - - - - def _initial_messages(self) -> list[Message]: - """Creates the initial messages (including the system prompt) for the LLM conversation.""" - return [ - Message( - role='system', - content=[ - TextContent( - text=self.prompt_manager.get_system_message(), - cache_prompt=False, # Assuming caching is active - ) - ], - ) - ] - - def _enhance_messages(self, messages: list[Message]) -> list[Message]: - """Enhances the user message with additional context based on keywords matched.""" - results: list[Message] = [] - is_first_message_handled = False - - for msg in messages: - if msg.role == 'user' and not is_first_message_handled: - is_first_message_handled = True - # Compose the first user message with examples - self.prompt_manager.add_examples_to_initial_message(msg) - - # Add repo/runtime info if enabled - if self.config.get_agent_config().enable_prompt_extensions: - self.prompt_manager.add_info_to_initial_message(msg) - - # Enhance the user message with additional context based on keywords matched - if msg.role == 'user': - self.prompt_manager.enhance_message(msg) - - results.append(msg) - - return results - - def _get_messages(self, state: State) -> List[Message]: - """Get the message history for this agent.""" - # Start with initial messages (system prompt) - messages = self._initial_messages() - - # If using a condenser, condense the history - events = self.condenser.condensed_history(state) - - # Convert history events to messages - messages += events_to_messages( - events, - max_message_chars=32768, # Default value, adjust as needed - vision_is_active=False, # Assuming vision is not active - enable_som_visual_browsing=False, # Assuming SOM visual browsing is not enabled - ) - - messages = self._enhance_messages(messages) - - return messages - - - # Conversion utility function - def convert_str_to_completion_format(self, fn_call_messages): - # from types import SimpleNamespace - from litellm import ModelResponse - - role = fn_call_messages[0]['role'] - response_str = fn_call_messages[0]['content'] - tool_calls = fn_call_messages[0].get('tool_calls', None) - - return ModelResponse( - choices=[ - { - "index": 0, - "message": { - "content": response_str, - "role": role, - "tool_calls": tool_calls, - "function_calling": None - } - } - ] - ) - - async def generate(self, input_ids, sampling_params): - res = await self.infer_engine.async_generate(input_ids=input_ids, sampling_params=self.sampling_params) - response_str = res["text"] - return response_str - - - async def step(self, state: State) -> Action: - """Generate a response using batched infer.""" - self.step_count += 1 - print(f"instance id {self.instance_id}, trajectory {self.trajectory_id}, step {self.step_count}") - if self.pending_actions: - return self.pending_actions.popleft() - - # if we're done, go back - latest_user_message = state.get_last_user_message() - if latest_user_message and latest_user_message.content.strip() == '/exit': - return AgentFinishAction() - - # prepare what we want to send to the LLM - messages = self._get_messages(state) - messages = self.llm.format_messages_for_llm(messages) - messages = convert_fncall_messages_to_non_fncall_messages( - messages, self.tools - ) - - try: - input_ids = self.tokenizer.apply_chat_template( - messages, add_generation_prompt=True, tokenize=True, enable_thinking=self.qwen3_enable_thinking - ) - if len(input_ids) >= self.max_prompt_length: - return AgentFinishAction(thought="The context is too long. Exit now.") - - response_str = call_async_from_sync(self.generate, input_ids=input_ids, sampling_params=self.sampling_params) - - if not response_str: - # If we got an empty response (possible error), return a message action - self.pending_actions.append( - MessageAction( - content="I encountered an error processing your request. Let's try again.", - ) - ) - else: - # Convert to actions - message = [ - { - 'role': 'assistant', - 'content': response_str, - } - ] - fn_call_messages = convert_non_fncall_messages_to_fncall_messages( - message, self.tools - ) - actions = codeact_function_calling.response_to_actions( - self.convert_str_to_completion_format(fn_call_messages) - ) - print(f"Take action: {type(actions)}") - - for action in actions: - self.pending_actions.append(action) - - except ( - LLMMalformedActionError, - LLMNoActionError, - LLMResponseError, - FunctionCallValidationError, - FunctionCallNotExistsError, - ): - raise - - except Exception as e: - logger.error(f"Error in agent step: {str(e)}") - # Handle errors gracefully by creating a message action - self.pending_actions.append( - MessageAction( - content=f"An error: {str(e)} encountered. Please try a different approach.", - ) - ) - - # Return the first pending action - if not self.pending_actions: - # Fallback in case of empty actions - return AgentFinishAction() - - return self.pending_actions.popleft() - - def get_final_messages(self, state: State) -> List[Message]: - """Get the final messages for this agent.""" - messages = self._get_messages(state) - messages = self.llm.format_messages_for_llm(messages) - messages = convert_fncall_messages_to_non_fncall_messages( - messages, self.tools - ) - return messages - - def _is_last_action_finish(self, state: State) -> bool: - if state and state.history: - last_action = next( - ( - event - for event in reversed(state.history) - if isinstance(event, Action) - ), - None, - ) - if isinstance(last_action, AgentFinishAction): - return True - return False - - -Agent.register('OnlineCodeActAgent', OnlineCodeActAgent) - - - -class CodeActAgentGroup: - """ - A class that manages multiple CodeActAgent instances to generate trajectories in parallel. - """ - - def __init__( - self, - batch: DataProto, - num_trajectories: int, - infer_engine: Any, - max_prompt_length: int = 1024, - max_response_length: int = 1024, - max_starting_message_length: int = 12000, - max_parallel_agents: int = 1, - max_eval_parallel_agents: int = 1, - max_iterations: int = 10, - tokenizer: Any = None, - sampling_params: Any = None, - device: Any = None, - log_messages_dir: str = None, - remove_think_tokens: bool = False, - qwen3_enable_thinking: bool = True - ) -> None: - """ - Initialize the CodeActAgentGroup to manage multiple agent instances. - - Args: - batch: DataProto containing the batch of data - num_trajectories: Number of trajectories to generate per instance - infer_engine: The infer engine for generation - max_prompt_length: Maximum prompt length - max_parallel_agents: Maximum number of agents to run in parallel - max_iterations: Maximum number of iterations per agent - tokenizer: Tokenizer to use for text encoding/decoding - max_batch_size: Maximum batch size for LLM generation - """ - self.batch = batch - self.infer_engine = infer_engine - self.max_prompt_length = max_prompt_length - self.max_response_length = max_response_length - self.total_len = self.max_prompt_length + self.max_response_length - # todo: make it a config - self.max_starting_message_length = max_starting_message_length - self.max_parallel_agents = max_parallel_agents - self.max_eval_parallel_agents = max_eval_parallel_agents - print("max eval parallel agents: ", self.max_eval_parallel_agents) - if max_eval_parallel_agents <= 0: - print(f"`max_eval_parallel_agents` has not been set. Setting it to `max_parallel_agents` i.e {max_parallel_agents}") - self.max_eval_parallel_agents = max_parallel_agents - self.max_iterations = max_iterations - self.num_trajectories = num_trajectories - self.tokenizer = tokenizer - self.sampling_params = sampling_params - self.device = device - - # Map of instance ID to agent instance - self.agents = {} - - # Map of instance ID to agent results - self.results = {} - - self.qwen3_enable_thinking = qwen3_enable_thinking - self.log_messages_dir = None - if log_messages_dir: - self.log_messages_dir = Path(log_messages_dir) - logger.info(f"Logging all messages to {self.log_messages_dir}") - - # Initialize agents for each instance - self._initialize_agents() - - self.remove_think_tokens = remove_think_tokens - if self.remove_think_tokens: - logger.info("Removing think tokens....") - - - def _convert_results_to_dataproto(self) -> DataProto: - """ - Convert results to DataProto format for training. - - Args: - results: Dictionary of results, with structure {instance_id: {trajectory_id: result_dict}} - input_dataproto: The input DataProto that contains the original batch data - tokenizer: The tokenizer to use for encoding messages - - Returns: - DataProto: A DataProto object with the converted results - """ - - # Non-tensor data - git_patch_list = [] - success_list = [] - error_list = [] - resolved_list = [] - has_finish_action_list = [] - - # Create a mapping of instance_id -> list of trajectories - instance_trajectories = {} - for instance_id, trajectories in self.results.items(): - instance_trajectories[instance_id] = [] - for trajectory_id, result in trajectories.items(): - instance_trajectories[instance_id].append(result) - - # Create the final results in the same order as the batch - matched_results = [] - instance_list = [] - for batch_item in self.batch: - instance_id = batch_item.non_tensor_batch['instance']['instance_id'] - instance = batch_item.non_tensor_batch['instance'] - if instance_id in instance_trajectories: - # Add all trajectories for this instance - traj_results = instance_trajectories[instance_id] - matched_results.extend(traj_results) - instance_list.extend([instance] * len(traj_results)) - - assert len(matched_results) == self.num_trajectories * len(self.batch), f"Expected number of results {self.num_trajectories * len(self.batch)}, got {len(matched_results)}" - - # Group results by instance_id for message handling - results_by_instance = {} - for i, result in enumerate(matched_results): - instance_id = instance_list[i]['instance_id'] - if instance_id not in results_by_instance: - results_by_instance[instance_id] = [] - results_by_instance[instance_id].append((i, result)) - - # Handle empty messages by copying from another trajectory of the same instance - for instance_id, results in results_by_instance.items(): - # Find a valid messages list to use as fallback - valid_messages = None - valid_patch = None - for _, result in results: - messages = result.get('messages', []) - if messages and len(messages) > 0: - valid_messages = messages - valid_patch = result.get('git_patch', None) - valid_resolved = result.get('resolved', False) - valid_finish = result.get('finish', False) - valid_error = result.get('error', None) - break - - # If we found valid messages, use them for trajectories with empty messages - if valid_messages: - for idx, result in results: - if not result.get('messages') or len(result.get('messages', [])) == 0: - print(f"Got empty messages for instance_id {instance_id}, trajectory {idx}. Copying messages array from a valid trajectory. ") - # Copy messages from the valid trajectory - matched_results[idx]['messages'] = valid_messages.copy() - matched_results[idx]['git_patch'] = valid_patch - matched_results[idx]['resolved'] = valid_resolved - matched_results[idx]['error'] = valid_error - matched_results[idx]['finish'] = valid_finish - - # Get batch of messages - all_messages = [] - all_prompts = [] - all_responses = [] - for result in matched_results: - messages = result.get('messages', []) - all_messages.append(messages) - # get the response: starting from the first assistant message - starting_index = 0 - for i, msg in enumerate(messages): - if msg["role"] == 'assistant': - starting_index = i - break - if starting_index == 0: - # If we don't find an assistant, all messages are prompts and there are no responses - print(f'ERROR: Found no assistant message. len(messages) == {len(messages)} and roles are {[msg["role"] for msg in messages]}') - starting_index = len(messages) - prompt = messages[:starting_index] - all_prompts.append(prompt) - response = messages[starting_index:] - all_responses.append(response) - - # Also add non-tensor data - git_patch_list.append(result.get('git_patch', None)) - success_list.append(result.get('success', False)) - error_list.append(result.get('error', None)) - resolved_list.append(result.get('resolved', False)) - has_finish_action_list.append(result.get('finish', False)) - - - # Encode messages, get assitant mask and position ids - prompt_encodings = self.tokenizer.apply_chat_template( - all_prompts, - # return_tensors="pt", - add_generation_prompt=False, - return_dict=True, - padding=True - ) - prompt_input_ids = torch.tensor(prompt_encodings['input_ids'], device=self.device) - prompt_attention_mask = torch.tensor(prompt_encodings['attention_mask'], device=self.device) - prompt_input_ids, prompt_attention_mask = convert_right_padding_to_left(self.tokenizer, prompt_input_ids, prompt_attention_mask, self.device, self.max_starting_message_length) - - response_encodings = self.tokenizer.apply_chat_template( - all_responses, - chat_template=chat_template_qwen3_thinking if self.remove_think_tokens else chat_template, - # return_tensors="pt", - return_assistant_tokens_mask=True, - add_generation_prompt=False, - return_dict=True, - padding=True - ) - - response_ids, response_attention_mask, response_assistant_mask = pad_to_max_length_right( - self.tokenizer, response_encodings, self.total_len, self.device) - - - input_ids = torch.cat([prompt_input_ids, response_ids], dim=1) - attention_mask = torch.cat([prompt_attention_mask, response_attention_mask], dim=1) - position_ids = compute_position_id_with_mask(attention_mask) - - # Create tensor dictionary - logger.info(f"input_ids shape: {input_ids.shape}, response_ids shape: {response_ids.shape}, max_starting_message_length: {self.max_starting_message_length}, max_response_length: {self.total_len}") - assert input_ids.shape[1] == attention_mask.shape[1] == position_ids.shape[1], f"input_ids shape {input_ids.shape}, attention_mask shape {attention_mask.shape}, position_ids shape {position_ids.shape} do not match" - assert response_ids.shape[1] == response_assistant_mask.shape[1], f"response_ids shape {response_ids.shape}, response_assistant_mask shape {response_assistant_mask.shape} do not match" - tensor_dict = { - 'input_ids': input_ids, - 'responses': response_ids, - 'attention_mask': attention_mask, - 'position_ids': position_ids, - 'loss_mask': response_assistant_mask, - } - - # Create non-tensor dictionary - non_tensor_dict = { - 'git_patch': git_patch_list, - 'success': success_list, - 'error': error_list, - 'instance': instance_list, - 'resolved': resolved_list, - 'finish': has_finish_action_list, - "response_texts": full_response_texts, - } - - # Create and return DataProto - result_dataproto = DataProto.from_dict( - tensors=tensor_dict, - non_tensors=non_tensor_dict - ) - - return result_dataproto - - def close(self): - """Clean up resources""" - - # Close all agent instances - for instance_id in self.agents: - for trajectory_id in self.agents[instance_id]: - self._cleanup_agent(instance_id, trajectory_id) - - def _cleanup_agent(self, instance_id, trajectory_id): - try: - self.agents[instance_id][trajectory_id].close() - except Exception as e: - logger.warning(f"Error closing agent {instance_id}, trajectory {trajectory_id}: {str(e)}") - - def __del__(self): - """Destructor to ensure resources are cleaned up""" - self.close() - - def _initialize_agents(self) -> None: - """Initialize agent instances for each task.""" - for data_item in self.batch: - instance_id = data_item.non_tensor_batch['instance']['instance_id'] - self.agents[instance_id] = {} - for n in range(self.num_trajectories): - self.agents[instance_id][n] = OnlineCodeActAgent( - instance_id=instance_id, - trajectory_id=n, - max_prompt_length=self.max_prompt_length, - tokenizer=self.tokenizer, - infer_engine=self.infer_engine, - sampling_params=self.sampling_params, - qwen3_enable_thinking=self.qwen3_enable_thinking - ) - # Set the instance data for each agent - self.agents[instance_id][n].instance_data = data_item.non_tensor_batch['instance'] - self.agents[instance_id][n].max_iterations = self.max_iterations - - async def _initialize_runtime_for_agent(self, batch_id: int, trajectory_id: int) -> None: - """Initialize the runtime for a specific agent.""" - instance_id = self.batch[batch_id].non_tensor_batch['instance']['instance_id'] - instance = pd.Series(self.batch[batch_id].non_tensor_batch['instance']) - agent = self.agents[instance_id][trajectory_id] - - try: - # Configure sandbox - RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true' - SWE_BENCH_CONTAINER_IMAGE = 'ghcr.io/opendevin/eval-swe-bench:full-v1.2.1' - - if os.environ.get('USE_INSTANCE_IMAGE', 'true').lower() == 'true': - # Use a different instance image for each instance of swe-bench eval - base_container_image = get_instance_docker_image(instance_id) - logger.info( - f'Using instance container image: {base_container_image}. ' - f'Please make sure this image exists. ' - f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.' - ) - else: - base_container_image = SWE_BENCH_CONTAINER_IMAGE - logger.info(f'Using swe-bench container image: {base_container_image}') - - sandbox_config = get_default_sandbox_config_for_eval() - sandbox_config.base_container_image = base_container_image - sandbox_config.enable_auto_lint = True - sandbox_config.use_host_network = False - sandbox_config.platform = 'linux/amd64' - - app_config = AppConfig( - default_agent='OnlineCodeActAgent', - run_as_openhands=False, - max_iterations=self.max_iterations, - runtime='remote', - sandbox=sandbox_config, - workspace_base=None, - workspace_mount_path=None, - ) - agent_config = AgentConfig( - codeact_enable_jupyter=False, - codeact_enable_browsing=False, - codeact_enable_llm_editor=False, - condenser=NoOpCondenserConfig(), - enable_prompt_extensions=False, - ) - app_config.set_agent_config(agent_config) - agent.config = app_config - - # Create runtime - runtime = create_runtime(app_config) - - # Connect runtime - await runtime.connect() - - # Initialize runtime - from .utils import initialize_runtime, get_instruction - # initialize_runtime(runtime, instance) - await call_sync_from_async(initialize_runtime, runtime, instance) - - # Store the runtime and instruction - agent.runtime = runtime - agent.instruction = get_instruction(instance) - - logger.info(f"Successfully initialized runtime for instance {instance_id}") - except Exception as e: - logger.error(f"Failed to initialize runtime for instance {instance_id}: {str(e)}") - if 'runtime' in locals() and runtime: - runtime.event_stream.close() - runtime.close() - - # Update agent state to reflect error - agent.error = str(e) - agent.agent_state = AgentState.ERROR - raise - - async def _run_agent(self, batch_id: int, trajectory_id: int, pos_id: int) -> Dict[str, Any]: - instance_id = self.batch[batch_id].non_tensor_batch['instance']['instance_id'] - """Run a single agent to completion and return the results.""" - agent = self.agents[instance_id][trajectory_id] - assert agent is not None - instance = pd.Series(self.batch[batch_id].non_tensor_batch['instance']) - runtime = agent.runtime - - try: - # Run the agent controller - state = await run_controller( - config=agent.config, - initial_user_action=MessageAction(content=agent.instruction), - runtime=runtime, - agent=agent, - fake_user_response_fn=codeact_user_response, - ) - - if state: - print(state.last_error) - - from .utils import complete_runtime, is_fatal_evaluation_error - # Check for fatal errors - if state and is_fatal_evaluation_error(state.last_error): - logger.error(f"Fatal error in agent {instance_id}: {state.last_error}") - raise Exception('Fatal error detected: ' + state.last_error) - - final_messages = agent.get_final_messages(state) - # Complete the runtime and get the git patch - return_val = await call_sync_from_async(complete_runtime, runtime, instance) - # return_val = complete_runtime(runtime, instance) - # print patch - if return_val.get('git_patch', None): - print(f"Git patch for instance {instance_id}, traj {trajectory_id}:") - print('-' * 80) - print(return_val['git_patch']) - print('-' * 80) - - return_val = { - 'instance_id': instance_id, - 'trajectory_id': trajectory_id, - 'state': state, - 'git_patch': return_val.get('git_patch', None), - 'messages': final_messages, - 'success': not bool(state.last_error if state else True), - 'error': state.last_error if state and state.last_error else None, - 'finish': agent._is_last_action_finish(state) - } - except Exception as e: - logger.error(f"Error running agent {instance_id}: {str(e)}") - # Update agent state to reflect error - agent.error = str(e) - agent.agent_state = AgentState.ERROR - - if state: - final_messages = agent.get_final_messages(state) - else: - print(f'No final message state for instance {instance_id}, trajectory {trajectory_id}') - final_messages = [] - - if not final_messages or len(final_messages) == 0: - print(f'1095: Final messages are non-existent (or empty) for instance {instance_id}, trajectory {trajectory_id}') - - return_val = { - 'instance_id': instance_id, - 'trajectory_id': trajectory_id, - 'messages': final_messages, - 'state': state, - 'git_patch': None, - 'success': False, - 'error': str(e), - 'finish': agent._is_last_action_finish(state) - } - finally: - # cleanup agent resources - self._cleanup_agent(instance_id, trajectory_id) - - return return_val - - def _apply_patch_and_evaluate(self, runtime, model_patch, instance_id, trajectory_id, test_spec): - """Apply patch and evaluate the solution.""" - model_patch = process_git_patch(model_patch) - # Get patch and save it to /tmp/patch.diff - with tempfile.TemporaryDirectory() as temp_dir: - # Patch file - patch_file_path = os.path.join(temp_dir, 'patch.diff') - with open(patch_file_path, 'w') as f: - f.write(model_patch) - runtime.copy_to(patch_file_path, '/tmp') - # Eval script - eval_script_path = os.path.join(temp_dir, 'eval.sh') - with open(eval_script_path, 'w') as f: - f.write(test_spec.eval_script) - runtime.copy_to(eval_script_path, '/tmp') - - # Set +x - action = CmdRunAction(command='chmod +x /tmp/eval.sh') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert obs.exit_code == 0 - - # Apply patch - exec_command = ( - 'cd /testbed && ' - "(git apply -v /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " - "(echo 'Failed to apply patch with git apply, trying with patch command...' && " - "(patch --batch --fuzz=5 -p1 -i /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || " - "echo 'APPLY_PATCH_FAIL')))" - ) - action = CmdRunAction(command=exec_command) - action.set_hard_timeout(600) - obs = runtime.run_action(action) - assert isinstance(obs, CmdOutputObservation) - apply_patch_output = obs.content - assert isinstance(apply_patch_output, str) - # instance['test_result']['apply_patch_output'] = apply_patch_output - - if 'APPLY_PATCH_FAIL' in apply_patch_output: - raise Exception(f"Instance {instance_id}, trajectory {trajectory_id} {APPLY_PATCH_FAIL}:\n{apply_patch_output}") - elif 'APPLY_PATCH_PASS' in apply_patch_output: - logger.info(f'[{instance_id}, {trajectory_id}] {APPLY_PATCH_PASS}:\n{apply_patch_output}') - - # Run eval script in background and save output to log file - log_file = '/tmp/eval_output.log' - action = CmdRunAction(command=f'/tmp/eval.sh > {log_file} 2>&1 & echo $!') - action.set_hard_timeout(300) # Short timeout just to get the process ID - obs = runtime.run_action(action) - - if isinstance(obs, CmdOutputObservation) and obs.exit_code == 0: - pid = obs.content.split()[-1].strip() - logger.info( - f'[{instance_id}, {trajectory_id}] Evaluation process started with PID: {pid}' - ) - - # Poll for completion - start_time = time.time() - timeout = 1200 # 20 minutes - while True: - seconds_elapsed = time.time() - start_time - if seconds_elapsed > timeout: - raise Exception( - f'[{instance_id}, {trajectory_id}] Evaluation timed out after {timeout} seconds' - ) - check_action = CmdRunAction( - command=f'ps -p {pid} > /dev/null; echo $?' - ) - check_action.set_hard_timeout(300) - check_obs = runtime.run_action(check_action) - if ( - isinstance(check_obs, CmdOutputObservation) - and check_obs.content.split()[-1].strip() == '1' - ): - logger.info( - f'[{instance_id}, {trajectory_id}] Evaluation process completed after {seconds_elapsed} seconds' - ) - break - logger.info( - f'[{instance_id}, {trajectory_id}] [{seconds_elapsed:.0f}s] Evaluation still running, waiting...' - ) - time.sleep(30) # Wait for 30 seconds before checking again - - # Read the log file - cat_action = CmdRunAction(command=f'cat {log_file}') - cat_action.set_hard_timeout(300) - cat_obs = runtime.run_action(cat_action) - - # Grade answer - if isinstance(cat_obs, CmdOutputObservation) and cat_obs.exit_code == 0: - test_output = cat_obs.content - assert isinstance(test_output, str) - # instance['test_result']['test_output'] = test_output - - # Get report from test output - logger.info(f'[{instance_id}, {trajectory_id}] Grading answer...') - - with tempfile.TemporaryDirectory() as temp_dir: - # Create a directory structure that matches the expected format - # NOTE: this is a hack to make the eval report format consistent - # with the original SWE-Bench eval script - log_dir = os.path.join(temp_dir, 'logs', instance_id.lower()) - os.makedirs(log_dir, exist_ok=True) - test_output_path = os.path.join(log_dir, 'test_output.txt') - with open(test_output_path, 'w') as f: - f.write(test_output) - try: - _report = get_eval_report( - test_spec=test_spec, - prediction={ - 'model_patch': model_patch, - 'instance_id': instance_id, - }, - log_path=test_output_path, - include_tests_status=True, - ) - report = _report[instance_id] - logger.info( - f"[{instance_id}, {trajectory_id}] report: {report}\nResult for [{instance_id}, {trajectory_id}]: resolved: {report['resolved']}" - ) - self.results[instance_id][trajectory_id]['resolved'] = report[ - 'resolved' - ] - except Exception as e: - logger.error( - f'[{instance_id}, {trajectory_id}] Error when getting eval report: {e}' - ) - self.results[instance_id][trajectory_id]['resolved'] = False - else: - raise Exception(f'[{instance_id}, {trajectory_id}] Error when starting eval:\n{obs.content}') - else: - raise Exception( - f'[{instance_id}] Unexpected output when applying patch:\n{apply_patch_output}' - ) - - async def _evaluate_agent(self, batch_id: int, trajectory_id: int) -> None: - """Initialize the runtime for a specific agent.""" - instance_id = self.batch[batch_id].non_tensor_batch['instance']['instance_id'] - instance = pd.Series(self.batch[batch_id].non_tensor_batch['instance']) - test_spec = make_test_spec(instance=instance) - - try: - # Configure sandbox - # We use a different instance image for the each instance of swe-bench eval - base_container_image = get_instance_docker_image(instance_id) - logger.info( - f'Using instance container image: {base_container_image}. ' - f'Please make sure this image exists. ' - f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.' - ) - sandbox_config = get_default_sandbox_config_for_eval() - sandbox_config.base_container_image = base_container_image - sandbox_config.remote_runtime_resource_factor = 1 - config = AppConfig( - run_as_openhands=False, - runtime="remote", - sandbox=sandbox_config, - # do not mount workspace - workspace_base=None, - workspace_mount_path=None, - ) - - # Create runtime - runtime = create_runtime(config) - - # Connect runtime - await runtime.connect() - - assert instance_id in self.results and trajectory_id in self.results[instance_id], \ - f"Instance {instance_id} or trajectory {trajectory_id} not found in results" - - model_patch = self.results[instance_id][trajectory_id].get('git_patch', None) - if not model_patch: - raise Exception(f"No git patch found for instance {instance_id}, trajectory {trajectory_id}") - - - await call_sync_from_async(self._apply_patch_and_evaluate, runtime, model_patch, instance_id, trajectory_id, test_spec) - - except Exception as e: - logger.error(f"Failed to evaluate traj {trajectory_id} for instance {instance_id}: {str(e)}") - self.results[instance_id][trajectory_id]['resolved'] = False - self.results[instance_id][trajectory_id]['eval_error'] = str(e) - finally: - if 'runtime' in locals() and runtime: - runtime.event_stream.close() - runtime.close() - - if self.log_messages_dir: - result = self.results[instance_id][trajectory_id] - instance_dir = self.log_messages_dir / str(instance_id) - instance_dir.mkdir(exist_ok=True, parents=True) - with open(instance_dir / f"{trajectory_id}.json", "w") as f: - result_json = json.dumps(result, default=lambda x: str(x)) - f.write(result_json) - - async def generate_trajectories_pipeline(self) -> Dict[int, Dict[int, Dict[str, Any]]]: - """ - Generate trajectories with pipelined runtime initialization to improve efficiency. - """ - total_instances = len(self.batch) - print("Total instances:", total_instances) - - # Create two queues: one for initialization and one for running - init_queue = asyncio.Queue() - run_queue = asyncio.Queue(maxsize=self.max_parallel_agents) - eval_queue = asyncio.Queue(maxsize=self.max_parallel_agents) - - # Fill the initialization queue - for trajectory_id in range(self.num_trajectories): - for batch_idx in range(total_instances): - await init_queue.put((batch_idx, trajectory_id)) - - # Track active tasks - active_init_tasks = set() - active_run_tasks = set() - active_eval_tasks = set() - need_init_tasks = self.num_trajectories * total_instances - needed_run_tasks = self.num_trajectories * total_instances # Total tasks we'll eventually need - needed_eval_tasks = self.num_trajectories * total_instances - - # Helper function to initialize runtime - import time - async def initialize_one_runtime(): - start_time = time.time() - batch_idx, trajectory_id = await init_queue.get() - instance_id = self.batch[batch_idx].non_tensor_batch['instance']['instance_id'] - try: - logger.info(f"Initializing runtime for instance {instance_id}, trajectory {trajectory_id}") - await self._initialize_runtime_for_agent(batch_idx, trajectory_id) - # Add to run queue after successful initialization - await run_queue.put((batch_idx, trajectory_id)) - elpased_time = time.time() - start_time - print(f"Successfully initialized runtime for instance {instance_id}, trajectory {trajectory_id} in {elpased_time:.2f} seconds") - except Exception as e: - nonlocal needed_run_tasks - nonlocal needed_eval_tasks - needed_run_tasks -= 1 - needed_eval_tasks -= 1 - logger.error(f"Error initializing runtime for {instance_id}, trajectory {trajectory_id}: {str(e)}") - # Handle initialization error - if instance_id not in self.results: - self.results[instance_id] = {} - self.results[instance_id][trajectory_id] = { - 'instance_id': instance_id, - 'trajectory_id': trajectory_id, - 'messages': [], - 'state': None, - 'git_patch': None, - 'success': False, - 'error': str(e), - 'finish': False, - 'resolved': False - } - finally: - init_queue.task_done() - # Start another initialization task if available - nonlocal need_init_tasks - if not init_queue.empty() and need_init_tasks > 0: - need_init_tasks -= 1 - task = asyncio.create_task(initialize_one_runtime()) - active_init_tasks.add(task) - task.add_done_callback(lambda t: active_init_tasks.discard(t)) - - # Helper function to run one agent - async def run_one_agent(pos_id: int): - batch_idx, trajectory_id = await run_queue.get() - instance_id = self.batch[batch_idx].non_tensor_batch['instance']['instance_id'] - start_time = time.time() - try: - logger.info(f"Running agent for instance {instance_id}, trajectory {trajectory_id}") - result = await self._run_agent(batch_idx, trajectory_id, pos_id) - elapsed = time.time() - start_time - - # Store the result - if instance_id not in self.results: - self.results[instance_id] = {} - self.results[instance_id][trajectory_id] = result - - await eval_queue.put((batch_idx, trajectory_id)) - - print(f"Successfully completed instance {instance_id}, trajectory {trajectory_id} in {elapsed:.2f}s") - except Exception as e: - logger.error(f"[This line should not be reached!!] Error running agent for {instance_id}, trajectory {trajectory_id}: {str(e)}") - nonlocal needed_eval_tasks - needed_eval_tasks -= 1 - # Store error result - if instance_id not in self.results: - self.results[instance_id] = {} - self.results[instance_id][trajectory_id] = { - 'instance_id': instance_id, - 'trajectory_id': trajectory_id, - 'messages': [], - 'state': None, - 'git_patch': None, - 'success': False, - 'error': str(e), - 'finish': False, - 'resolved': False - } - finally: - run_queue.task_done() - nonlocal needed_run_tasks - # Start another run task if available - if needed_run_tasks > 0: - needed_run_tasks -= 1 - task = asyncio.create_task(run_one_agent(pos_id)) - active_run_tasks.add(task) - task.add_done_callback(lambda t: active_run_tasks.discard(t)) - - # Helper function to eval one trajectory - async def eval_one_agent(): - batch_idx, trajectory_id = await eval_queue.get() - instance_id = self.batch[batch_idx].non_tensor_batch['instance']['instance_id'] - start_time = time.time() - try: - logger.info(f"Evaluating agent for instance {instance_id}, trajectory {trajectory_id}") - await self._evaluate_agent(batch_idx, trajectory_id) - elapsed = time.time() - start_time - - print(f"Successfully completed evaluating instance {instance_id}, trajectory {trajectory_id} in {elapsed:.2f}s") - except Exception as e: - logger.error(f"Error evaluating agent for {instance_id}, trajectory {trajectory_id}: {str(e)}") - # Store error result - self.results[instance_id][trajectory_id]['resolved'] = False - finally: - eval_queue.task_done() - nonlocal needed_eval_tasks - # Start another run task if available - if needed_eval_tasks > 0: - needed_eval_tasks -= 1 - task = asyncio.create_task(eval_one_agent()) - active_eval_tasks.add(task) - task.add_done_callback(lambda t: active_eval_tasks.discard(t)) - - # Start initial batch of initialization tasks - max_parallel_init = self.max_parallel_agents # Use some parallel initialization tasks - for _ in range(min(max_parallel_init, init_queue.qsize())): - need_init_tasks -= 1 - task = asyncio.create_task(initialize_one_runtime()) - active_init_tasks.add(task) - task.add_done_callback(lambda t: active_init_tasks.discard(t)) - - # Start a few agent run tasks (they'll wait on the run_queue) - for pos_id in range(self.max_parallel_agents): - needed_run_tasks -= 1 - task = asyncio.create_task(run_one_agent(pos_id)) - active_run_tasks.add(task) - task.add_done_callback(lambda t: active_run_tasks.discard(t)) - - for _ in range(self.max_eval_parallel_agents): - needed_eval_tasks -= 1 - task = asyncio.create_task(eval_one_agent()) - active_eval_tasks.add(task) - task.add_done_callback(lambda t: active_eval_tasks.discard(t)) - - # Wait for all initialization tasks to complete - if init_queue.qsize() > 0: - await init_queue.join() - - # Wait for all run tasks to complete - if run_queue.qsize() > 0: - await run_queue.join() - - # Wait for all eval tasks to complete - if eval_queue.qsize() > 0: - await eval_queue.join() - - # Wait for any remaining active tasks - all_tasks = active_init_tasks.union(active_run_tasks) - all_tasks = all_tasks.union(active_eval_tasks) - if all_tasks: - logger.info(f"Waiting for {len(all_tasks)} (init: {len(active_init_tasks)}, run: {len(active_run_tasks)}, eval: {len(active_eval_tasks)}) remaining tasks to complete") - await asyncio.wait(all_tasks) - - results_dataproto = self._convert_results_to_dataproto() - return results_dataproto - - def run(self) -> Dict[int, Dict[int, Dict[str, Any]]]: - """ - Run the agent group synchronously by creating a new event loop if necessary. - - Returns: - Dict mapping instance ID to a dict of trajectory ID to results - """ - try: - # Try to get the current event loop - loop = asyncio.get_event_loop() - except RuntimeError: - # No event loop exists in this thread, create a new one - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - # Run the generate_trajectories coroutine in the event loop - try: - return loop.run_until_complete(self.generate_trajectories_pipeline()) - finally: - # Close the batch manager to ensure cleanup - self.close() - # loop.close() \ No newline at end of file diff --git a/verl/workers/agentic/fsdp_sgl.py b/verl/workers/agentic/fsdp_sgl.py deleted file mode 100644 index f1c7f7401d..0000000000 --- a/verl/workers/agentic/fsdp_sgl.py +++ /dev/null @@ -1,331 +0,0 @@ -import logging -import os -import time -from typing import List, Any, Optional -import torch -from tqdm import tqdm -from sglang.srt.entrypoints.engine import Engine -from sglang.srt.utils import init_custom_process_group -from tensordict import TensorDict -from torch.distributed.device_mesh import DeviceMesh -from torch.distributed.fsdp.api import ShardedStateDictConfig, StateDictType, FullStateDictConfig -from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP -from torch.distributed.tensor import DTensor -from torch.distributed import ProcessGroup - -from verl import DataProto -from verl.utils.debug import log_gpu_memory_usage -from verl.utils.torch_functional import (broadcast_dict_tensor, allgather_dict_tensors, all_gather_dict_non_tensors, - broadcast_dict_non_tensor) -from ..sharding_manager.base import BaseShardingManager - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_PPO_LOGGING_LEVEL', 'INFO')) - - -class FSDPSGLShardingManager(BaseShardingManager): - - def __init__( - self, - module: FSDP, - inference_engine: Engine, - model_config, - full_params: bool = False, - device_mesh: DeviceMesh = None, - role: str = "actor_rollout", - rollout_count: int = 0, - exchange_size: int | float | None = None, - ): - self.module = module - self.inference_engine = inference_engine - self.model_config = model_config - self.device_mesh = device_mesh - self.exchange_size = exchange_size - # If negative, then set to None - if self.exchange_size is not None and self.exchange_size <= 0: - self.exchange_size = None - - # Full params - self.full_params = full_params - self.role = role - - if "actor" in role: - if full_params: - FSDP.set_state_dict_type(self.module, - state_dict_type=StateDictType.FULL_STATE_DICT, - state_dict_config=FullStateDictConfig()) - else: - FSDP.set_state_dict_type(self.module, - state_dict_type=StateDictType.SHARDED_STATE_DICT, - state_dict_config=ShardedStateDictConfig()) - - dp_rank = device_mesh.get_local_rank(0) - addr = os.environ["MASTER_ADDR"] - port = 40000 - print(f"in sharding manager {role=} {device_mesh=} {dp_rank=}") - if role == "actor": - print(f"nodedup sharding manager starting weight pg {dp_rank=} {addr=} {port=} {role=} {rollout_count=}") - if dp_rank == 0: - self.update_weight_pg: ProcessGroup = init_custom_process_group( - backend="nccl", - init_method=f"tcp://{addr}:{port}", - world_size=rollout_count + 1, - rank=0, - group_name="update_weight_group", - ) - if role == "rollout": - tp_rank = device_mesh.get_local_rank(1) - assert rollout_count == device_mesh.size(0), f"{rollout_count=}, {device_mesh.size(0)=}" - if tp_rank == 0: - print( - f"nodedup sharding manager starting weight pg {dp_rank=} {addr=} {port=} {role=} {rollout_count=}") - # self.inference_engine.init_weights_update_group( - # master_address=addr, - # master_port=port, - # rank_offset=dp_rank * device_mesh.size(1) + 1, - # world_size=device_mesh.size(0) + 1, - # group_name=f"weight_update_group_{dp_rank}", - # backend="nccl", - # ) - self.update_weight_pg = init_custom_process_group( - backend="nccl", - init_method=f"tcp://{addr}:{port}", - world_size=rollout_count + 1, - rank=1 + device_mesh.get_local_rank(0), - group_name="update_weight_group", - ) - print(f"nodedup sharding manager started weight pg dp_rank: {dp_rank}, addr: {addr}, port: {port}") - - # Note that torch_random_states may be different on each dp rank - self.torch_random_states = torch.cuda.get_rng_state() - # get a random rng states - if self.device_mesh is not None and "rollout" in role: - gen_dp_rank = self.device_mesh['dp'].get_local_rank() - torch.cuda.manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states - self.gen_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.torch_random_states) - else: - self.gen_random_states = None - - def __enter__(self): - local_rank = self.device_mesh.get_local_rank(1) - if "actor" in self.role: - start = time.time() - log_gpu_memory_usage('Before state_dict() in sharding manager memory', logger=logger) - st = self.module.state_dict() - k, v = next(iter(st.items())) - device = v.device - print(f"state_dict dtype, device of {k}: {v.dtype=} {device=}") - log_gpu_memory_usage('After state_dict() in sharding manager memory', logger=logger) - # print(f'Weight keys: {st.keys()}') - target_device = torch.device("cpu") if self.exchange_size else device - tensor_list = [] - for k, v in tqdm(st.items()): - if isinstance(v, DTensor): - v = v.full_tensor() - if local_rank == 0: - v_bf16 = v.to(dtype=torch.bfloat16) - v_target = v_bf16.to(device=target_device) - tensor_list.append((k, v_target)) - del v_bf16 - else: - del v - del st - torch.cuda.empty_cache() - log_gpu_memory_usage('After del state_dict and empty_cache in sharding manager', logger=logger) - param_count = sum([v.numel() for k, v in tensor_list]) - print(f"param count: {param_count}; used {time.time() - start} seconds to prepare tensor list") - if "rollout" in self.role and self.device_mesh.get_local_rank(1) == 0: - print("resuming memory occupation") - self.inference_engine.resume_memory_occupation() - print("resumed memory occupation") - torch.cuda.synchronize() - - def tensor_loader(): - for k, v in tensor_list: - yield (k, v.to(device)), v.numel() * v.element_size() - - loader = tensor_loader() - done = False - loop_count = 0 - - log_gpu_memory_usage('Before sync model weights in sharding manager', logger=logger) - - while not done: - if "actor" in self.role and local_rank != 0: - del tensor_list - torch.cuda.empty_cache() - torch.cuda.synchronize() - break - each_loop_start_time = time.time() - if "actor" in self.role and local_rank == 0: - count = 0 - if self.exchange_size is None: - gpu_tensor_list = tensor_list - done = True - else: - gpu_tensor_list = [] - for item, size in loader: - gpu_tensor_list.append(item) - count += size - if count > self.exchange_size: - break - else: - done = True - print(f"got gpu_tensor_list {self.exchange_size=} {count=} {done=}") - - if self.role == "actor_rollout": - if local_rank == 0: - print("Using `update_weights_from_tensor`") - self.inference_engine.update_weights_from_tensor(gpu_tensor_list) - del gpu_tensor_list - else: - if self.role == "actor": - if self.device_mesh.get_rank() == 0: - assert local_rank == 0 - descriptions = {k: (v.shape, v.dtype) for k, v in gpu_tensor_list} - lst = [descriptions] - torch.distributed.barrier(group=self.update_weight_pg) - print(f"sending descriptions: {torch.distributed.get_rank()=} {self.update_weight_pg.rank()=}") - torch.distributed.broadcast_object_list(lst, group_src=0, group=self.update_weight_pg) - print(f"sent descriptions completed {len(lst[0])=}") - for _, v in gpu_tensor_list: - torch.distributed.broadcast(v, group_src=0, group=self.update_weight_pg) - lst = [done] - torch.distributed.broadcast_object_list(lst, group_src=0, group=self.update_weight_pg) - if local_rank == 0: - del gpu_tensor_list - else: - if self.device_mesh.get_local_rank(1) == 0: - lst = [None] - tensor_list = [] - torch.distributed.barrier(group=self.update_weight_pg) - print( - f"receiving descriptions: {torch.distributed.get_rank()=} {self.update_weight_pg.rank()=}") - torch.distributed.broadcast_object_list(lst, group_src=0, group=self.update_weight_pg) - print(f"receiving descriptions completed {lst=}") - for k, (shape, dtype) in lst[0].items(): - v = torch.empty(shape, dtype=dtype, device='cuda') - torch.distributed.broadcast(v, group_src=0, group=self.update_weight_pg) - tensor_list.append((k, v)) - self.inference_engine.update_weights_from_tensor(tensor_list) - lst = [None] - torch.distributed.object_list(lst, group_src=0, group=self.update_weight_pg) - assert lst[0] is not None - done = lst[0] - del tensor_list, v - torch.cuda.empty_cache() - torch.cuda.synchronize() - loop_consumed_time = time.time() - each_loop_start_time - log_gpu_memory_usage(f'After loop {loop_count} {done=} {loop_consumed_time=} in sharding manager', - logger=logger) - loop_count += 1 - - log_gpu_memory_usage('After sync model weights in sharding manager', logger=logger) - - torch.distributed.barrier() - - # important: need to manually set the random states of each tp to be identical. - if self.device_mesh is not None and "rollout" in self.role: - self.torch_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.gen_random_states) - - def __exit__(self, exc_type, exc_value, traceback): - log_gpu_memory_usage('Before sglang offload in sharding manager', logger=logger) - if self.device_mesh.get_local_rank(1) == 0 and "rollout" in self.role: - self.inference_engine.release_memory_occupation() - log_gpu_memory_usage('After sglang offload in sharding manager', logger=logger) - - # self.module.to('cuda') - # if torch.distributed.get_rank() == 0: - # print(f'after actor module to cuda in sharding manager memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB') - - if "actor" in self.role: - self.module.train() - - # add empty cache after each compute - torch.cuda.empty_cache() - - # restore random states - if self.device_mesh is not None and "rollout" in self.role: - self.gen_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.torch_random_states) - - def preprocess_data(self, data: DataProto) -> DataProto: - data.batch = allgather_dict_tensors( - data.batch.contiguous(), - size=self.device_mesh.size(1), - group=self.device_mesh.get_group(1), - dim=0, - ) - data.non_tensor_batch = all_gather_dict_non_tensors( - data.non_tensor_batch, - size=self.device_mesh.size(1), - group=self.device_mesh.get_group(1), - ) - - return data - - def postprocess_data(self, data: DataProto) -> DataProto: - # prevent nccl timeout - torch.distributed.barrier() - tp_size = self.device_mesh.size(1) - tp_rank = self.device_mesh.get_local_rank(1) - src_rank = self.device_mesh.get_local_rank(0) * tp_size - - # First, broadcast information about the batch and whether non_tensor_batch exists - if tp_rank == 0: - description: dict = {k: (v.shape, v.dtype) for k, v in data.batch.items()} - description['batch_size'] = data.batch.batch_size - # Add information about whether non_tensor_batch exists and its keys - has_non_tensor = hasattr(data, 'non_tensor_batch') and data.non_tensor_batch is not None - description['has_non_tensor'] = has_non_tensor - if has_non_tensor: - description['non_tensor_keys'] = list(data.non_tensor_batch.keys()) - lst = [description] - else: - lst = [None] - - torch.distributed.broadcast_object_list(lst, src=src_rank, group=self.device_mesh.get_group(1)) - description = lst[0] - print(f"{self.device_mesh.get_rank()=} {tp_size=} {src_rank=} {tp_rank=}, description: {description=}") - - # Create or reconstruct data on non-source ranks - if tp_rank != 0: - batch_size = description.pop('batch_size') - has_non_tensor = description.pop('has_non_tensor') - non_tensor_keys = description.pop('non_tensor_keys', []) - - batch = TensorDict( - { - k: torch.empty(shape, dtype=dtype, device='cuda') for k, (shape, dtype) in description.items() - }, - batch_size=batch_size) - - data = DataProto(batch=batch) - - # Initialize non_tensor_batch if it should exist - if has_non_tensor: - data.non_tensor_batch = {key: [] for key in non_tensor_keys} - - # Broadcast tensor batch - broadcast_dict_tensor( - data.batch, - src=src_rank, - group=self.device_mesh.get_group(1), - ) - - # Only broadcast non_tensor_batch if it exists - if hasattr(data, 'non_tensor_batch') and data.non_tensor_batch is not None: - broadcast_dict_non_tensor( - data.non_tensor_batch, - src=src_rank, - group=self.device_mesh.get_group(1), - ) - - if tp_size > 1: - # TODO: shall we build a micro_dp group for vllm when integrating with vLLM? - local_prompts = data.chunk(chunks=tp_size) - data = local_prompts[tp_rank] - - return data \ No newline at end of file diff --git a/verl/workers/agentic/utils.py b/verl/workers/agentic/utils.py deleted file mode 100644 index bb558e38d3..0000000000 --- a/verl/workers/agentic/utils.py +++ /dev/null @@ -1,408 +0,0 @@ -import pandas as pd -import numpy as np -from typing import Any -import os -import json -import tempfile -from time import sleep - -from openhands.utils.shutdown_listener import sleep_if_should_continue -from openhands.core.logger import openhands_logger as logger -from openhands.events.action import CmdRunAction, MessageAction -from openhands.events.observation import CmdOutputObservation, ErrorObservation -from openhands.runtime.base import Runtime -from openhands.core.exceptions import ( - AgentRuntimeBuildError, - AgentRuntimeDisconnectedError, - AgentRuntimeError, - AgentRuntimeNotFoundError, - AgentRuntimeNotReadyError, - AgentRuntimeTimeoutError, - AgentRuntimeUnavailableError, -) - -import json -import numpy as np - -def ensure_serializable(obj): - """Recursively convert non-serializable objects to JSON serializable formats.""" - if isinstance(obj, np.ndarray): # Convert numpy arrays to lists - return obj.tolist() - elif isinstance(obj, (np.integer, int)): # Convert numpy int to Python int - return int(obj) - elif isinstance(obj, (np.floating, float)): # Convert numpy float to Python float - return float(obj) - elif isinstance(obj, dict): # Recursively process dictionaries - return {key: ensure_serializable(value) for key, value in obj.items()} - elif isinstance(obj, list): # Recursively process lists - return [ensure_serializable(item) for item in obj] - elif isinstance(obj, tuple): # Convert tuples to lists - return tuple(ensure_serializable(item) for item in obj) - return obj # Return as is if already serializable - -class EvalException(Exception): - pass - -def assert_and_raise(condition: bool, msg: str): - """Raise an EvalException if the condition is not met. - - This will be used in conjunction with _process_instance_wrapper to handle retries. An EvalException should trigger a retry. - """ - if not condition: - raise EvalException(msg) - -RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true' - -def _get_swebench_workspace_dir_name(instance: pd.Series) -> str: - return f'{instance.repo}__{instance.version}'.replace('/', '__') - -def get_instruction(instance: pd.Series): - workspace_dir_name = _get_swebench_workspace_dir_name(instance) - instruction = f""" - -/workspace/{workspace_dir_name} - - -I've uploaded a python code repository in the directory {workspace_dir_name}. Consider the following issue description: - - -{instance.problem_statement} - - -Can you help me implement the necessary changes to the repository so that the requirements specified in the are met? -I've already taken care of all changes to any of the test files described in the . This means you DON'T have to modify the testing logic or any of the tests in any way! -Also the development Python environment is already set up for you (i.e., all dependencies already installed), so you don't need to install other packages. -Your task is to make the minimal changes to non-test files in the /workspace/{workspace_dir_name} directory to ensure the is satisfied. - -Follow these steps to resolve the issue: - -1. EXPLORATION: First, thoroughly explore the repository structure using tools like `find` and `grep`. - - Identify all files mentioned in the problem statement - - Locate where the issue occurs in the codebase - - Understand the surrounding context and dependencies - - Use `grep` to search for relevant functions, classes, or error messages - -2. ANALYSIS: Based on your exploration, think carefully about the problem and propose 2-5 possible approaches to fix the issue. - - Analyze the root cause of the problem - - Consider trade-offs between different solutions - - Select the most promising approach and explain your reasoning - -3. TEST CREATION: Before implementing any fix, create a script to reproduce and verify the issue. - - Look at existing test files in the repository to understand the test format/structure - - Create a minimal reproduction script that demonstrates the issue - - Run your script to confirm the error exists - -4. IMPLEMENTATION: Edit the source code to implement your chosen solution. - - Make minimal, focused changes to fix the issue - -5. VERIFICATION: Test your implementation thoroughly. - - Run your reproduction script to verify the fix works - - Add edge cases to your test script to ensure comprehensive coverage - - Run existing tests related to the modified code to ensure you haven't broken anything - -6. FINAL REVIEW: Carefully re-read the problem description and compare your changes with the base commit {instance["base_commit"]}. - - Ensure you've fully addressed all requirements - - Run any tests in the repository related to: - * The issue you are fixing - * The files you modified - * The functions you changed - - If any tests fail, revise your implementation until all tests pass - -Be thorough in your exploration, testing, and reasoning. It's fine if your thinking process is lengthy - quality and completeness are more important than brevity. -""" - - if RUN_WITH_BROWSING: - instruction += """ - -You SHOULD NEVER attempt to browse the web. - -""" - return instruction - -def initialize_runtime( - runtime: Runtime, - instance: pd.Series, # this argument is not required -): - """Initialize the runtime for the agent. - - This function is called before the runtime is used to run the agent. - """ - logger.info('-' * 30) - logger.info('BEGIN Runtime Initialization Fn') - logger.info('-' * 30) - workspace_dir_name = _get_swebench_workspace_dir_name(instance) - obs: CmdOutputObservation - - # Set instance id - action = CmdRunAction( - command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc""" - ) - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - obs.exit_code == 0, f'Failed to export SWE_INSTANCE_ID: {str(obs)}' - ) - - action = CmdRunAction(command="""export USER=$(whoami); echo USER=${USER} """) - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise(obs.exit_code == 0, f'Failed to export USER: {str(obs)}') - - if os.environ.get('USE_INSTANCE_IMAGE', 'true').lower() == 'true': - # inject the init script - script_dir = os.path.dirname(__file__) - - # inject the instance info - action = CmdRunAction(command='mkdir -p /swe_util/eval_data/instances') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - obs.exit_code == 0, - f'Failed to create /swe_util/eval_data/instances: {str(obs)}', - ) - - swe_instance_json_name = 'swe-bench-instance.json' - with tempfile.TemporaryDirectory() as temp_dir: - # Construct the full path for the desired file name within the temporary directory - temp_file_path = os.path.join(temp_dir, swe_instance_json_name) - # Write to the file with the desired name within the temporary directory - with open(temp_file_path, 'w') as f: - if not isinstance(instance, dict): - serializable_instance = ensure_serializable(instance.to_dict()) - json.dump([serializable_instance], f) - else: - json.dump([instance], f) - - # Copy the file to the desired location - runtime.copy_to(temp_file_path, '/swe_util/eval_data/instances/') - - # inject the instance swe entry - runtime.copy_to( - str(os.path.join(script_dir, 'scripts/setup/instance_swe_entry.sh')), - '/swe_util/', - ) - action = CmdRunAction(command='cat ~/.bashrc') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise(obs.exit_code == 0, f'Failed to cat ~/.bashrc: {str(obs)}') - - action = CmdRunAction(command='source ~/.bashrc') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - if isinstance(obs, ErrorObservation): - logger.error(f'Failed to source ~/.bashrc: {str(obs)}') - assert_and_raise(obs.exit_code == 0, f'Failed to source ~/.bashrc: {str(obs)}') - - action = CmdRunAction(command='source /swe_util/instance_swe_entry.sh') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - obs.exit_code == 0, - f'Failed to source /swe_util/instance_swe_entry.sh: {str(obs)}', - ) - else: - action = CmdRunAction(command='source /swe_util/swe_entry.sh') - action.set_hard_timeout(1800) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - obs.exit_code == 0, - f'Failed to source /swe_util/swe_entry.sh: {str(obs)}', - ) - - action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - obs.exit_code == 0, - f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}', - ) - - action = CmdRunAction(command='git reset --hard') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise(obs.exit_code == 0, f'Failed to git reset --hard: {str(obs)}') - - action = CmdRunAction( - command='for remote_name in $(git remote); do git remote remove "${remote_name}"; done' - ) - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise(obs.exit_code == 0, f'Failed to remove git remotes: {str(obs)}') - - action = CmdRunAction(command='which python') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - obs.exit_code == 0 and 'testbed' in obs.content, - f'Expected to find python interpreter from testbed, but got: {str(obs)}', - ) - - logger.info('-' * 30) - logger.info('END Runtime Initialization Fn') - logger.info('-' * 30) - -def complete_runtime( - runtime: Runtime, - instance: pd.Series, # this argument is not required, but it is used to get the workspace_dir_name -) -> dict[str, Any]: - """Complete the runtime for the agent. - - This function is called before the runtime is used to run the agent. - If you need to do something in the sandbox to get the correctness metric after - the agent has run, modify this function. - """ - logger.info('-' * 30) - logger.info('BEGIN Runtime Completion Fn') - logger.info('-' * 30) - obs: CmdOutputObservation - workspace_dir_name = _get_swebench_workspace_dir_name(instance) - - action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - - if obs.exit_code == -1: - # The previous command is still running - # We need to kill previous command - logger.info('The previous command is still running, trying to kill it...') - action = CmdRunAction(command='C-c') - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - - # Then run the command again - action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - - assert_and_raise( - isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, - f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}', - ) - - action = CmdRunAction(command='git config --global core.pager ""') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, - f'Failed to git config --global core.pager "": {str(obs)}', - ) - - action = CmdRunAction(command='git add -A') - action.set_hard_timeout(600) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - assert_and_raise( - isinstance(obs, CmdOutputObservation) and obs.exit_code == 0, - f'Failed to git add -A: {str(obs)}', - ) - - n_retries = 0 - git_patch = None - while n_retries < 5: - action = CmdRunAction( - command=f'git diff --no-color --cached {instance["base_commit"]}' - ) - action.set_hard_timeout(max(300 + 100 * n_retries, 600)) - logger.info(action, extra={'msg_type': 'ACTION'}) - obs = runtime.run_action(action) - logger.info(obs, extra={'msg_type': 'OBSERVATION'}) - n_retries += 1 - if isinstance(obs, CmdOutputObservation): - if obs.exit_code == 0: - git_patch = obs.content.strip() - break - else: - logger.info('Failed to get git diff, retrying...') - sleep_if_should_continue(10) - elif isinstance(obs, ErrorObservation): - logger.error(f'Error occurred: {obs.content}. Retrying...') - sleep_if_should_continue(10) - else: - assert_and_raise(False, f'Unexpected observation type: {str(obs)}') - - assert_and_raise(git_patch is not None, 'Failed to get git diff (None)') - - logger.info('-' * 30) - logger.info('END Runtime Completion Fn') - logger.info('-' * 30) - return {'git_patch': git_patch} - -def is_fatal_evaluation_error(error: str | None) -> bool: - if not error: - return False - - FATAL_EXCEPTIONS = [ - AgentRuntimeError, - AgentRuntimeBuildError, - AgentRuntimeTimeoutError, - AgentRuntimeUnavailableError, - AgentRuntimeNotReadyError, - AgentRuntimeDisconnectedError, - AgentRuntimeNotFoundError, - ConnectionError, - ] - - if any(exception.__name__ in error for exception in FATAL_EXCEPTIONS): - logger.error(f'Fatal evaluation error detected: {error}') - return True - - return False - -def process_git_patch(patch): - if not isinstance(patch, str): - return '' - - if not patch.strip(): - print(f'Skipping empty patch....') - # skip empty patches - return '' - - patch = patch.replace('\r\n', '\n') - # There might be some weird characters at the beginning of the patch - # due to some OpenHands inference command outputs - - # FOR EXAMPLE: - # git diff --no-color --cached 895f28f9cbed817c00ab68770433170d83132d90 - # 0 - # diff --git a/django/db/models/sql/.backup.query.py b/django/db/models/sql/.backup.query.py - # new file mode 100644 - # index 0000000000..fc13db5948 - - # We "find" the first line that starts with "diff" and then we remove lines before it - lines = patch.split('\n') - for i, line in enumerate(lines): - if line.startswith('diff --git'): - patch = '\n'.join(lines[i:]) - break - - patch = patch.rstrip() + '\n' # Make sure the last line ends with a newline - return patch \ No newline at end of file diff --git a/verl/workers/critic/__init__.py b/verl/workers/critic/__init__.py deleted file mode 100644 index 80808f1063..0000000000 --- a/verl/workers/critic/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base import BasePPOCritic -from .dp_critic import DataParallelPPOCritic - -__all__ = ["BasePPOCritic", "DataParallelPPOCritic"] diff --git a/verl/workers/critic/base.py b/verl/workers/critic/base.py deleted file mode 100644 index 9d1055df4e..0000000000 --- a/verl/workers/critic/base.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Base class for a critic -""" -from abc import ABC, abstractmethod - -import torch - -from verl import DataProto - -__all__ = ['BasePPOCritic'] - - -class BasePPOCritic(ABC): - - def __init__(self, config): - super().__init__() - self.config = config - - @abstractmethod - def compute_values(self, data: DataProto) -> torch.Tensor: - """Compute values""" - pass - - @abstractmethod - def update_critic(self, data: DataProto): - """Update the critic""" - pass diff --git a/verl/workers/critic/dp_critic.py b/verl/workers/critic/dp_critic.py deleted file mode 100644 index d100425b3c..0000000000 --- a/verl/workers/critic/dp_critic.py +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Implement a multiprocess PPOCritic -""" -import itertools -from typing import Iterable - -import torch -import torch.distributed -from torch import nn, optim - -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP - -from verl import DataProto -from verl.trainer.ppo import core_algos -from verl.workers.critic import BasePPOCritic -from verl.utils.py_functional import append_to_dict -from verl.utils.torch_functional import masked_mean -from verl.utils.ulysses import ulysses_pad_and_slice_inputs, gather_outpus_and_unpad -from verl.utils.seqlen_balancing import rearrange_micro_batches, get_reverse_idx - -from flash_attn.bert_padding import pad_input, unpad_input, rearrange, index_first_axis - -__all__ = ['DataParallelPPOCritic'] - - -class DataParallelPPOCritic(BasePPOCritic): - - def __init__(self, config, critic_module: nn.Module, critic_optimizer: optim.Optimizer): - super().__init__(config=config) - self.critic_module = critic_module - self.critic_optimizer = critic_optimizer - self.use_remove_padding = self.config.model.get('use_remove_padding', False) - print(f'Critic use_remove_padding={self.use_remove_padding}') - - self.ulysses_sequence_parallel_size = self.config.get('ulysses_sequence_parallel_size', 1) - - def _forward_micro_batch(self, micro_batch): - response_length = micro_batch['responses'].size(-1) - multi_modal_inputs = {} - if 'multi_modal_inputs' in micro_batch: - for key in micro_batch['multi_modal_inputs'][0].keys(): - multi_modal_inputs[key] = torch.cat([inputs[key] for inputs in micro_batch['multi_modal_inputs']], - dim=0) - - with torch.autocast(device_type='cuda', dtype=torch.bfloat16): - input_ids = micro_batch['input_ids'] - batch, seqlen = input_ids.shape - attention_mask = micro_batch['attention_mask'] - position_ids = micro_batch['position_ids'] - if position_ids.dim() == 3: # qwen2vl mrope - position_ids = position_ids.transpose(0, 1) - - if self.use_remove_padding: - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - - # unpad the position_ids to align the rotary - if position_ids.dim() == 3: - position_ids_rmpad = index_first_axis(rearrange(position_ids, "c b s ... -> (b s) c ..."), - indices).transpose(0, 1).unsqueeze( - 1) # (3, bsz, seqlen) -> (3, 1, bsz * seqlen) - else: - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # pad and slice the inputs if sp > 1 - if self.ulysses_sequence_parallel_size > 1: - input_ids_rmpad, position_ids_rmpad, pad_size = ulysses_pad_and_slice_inputs(input_ids_rmpad, \ - position_ids_rmpad, \ - sp_size=self.ulysses_sequence_parallel_size) - - # only pass input_ids and position_ids to enable flash_attn_varlen - output = self.critic_module(input_ids=input_ids_rmpad, - attention_mask=None, - position_ids=position_ids_rmpad, - **multi_modal_inputs, - use_cache=False) # prevent model thinks we are generating - values_rmpad = output.logits - values_rmpad = values_rmpad.squeeze(0) # (total_nnz) - - # gather output if sp > 1 - if self.ulysses_sequence_parallel_size > 1: - values_rmpad = gather_outpus_and_unpad(values_rmpad, - gather_dim=0, - unpad_dim=0, - padding_size=pad_size) - - # pad it back - values = pad_input(values_rmpad, indices=indices, batch=batch, seqlen=seqlen).squeeze(-1) - values = values[:, -response_length - 1:-1] - else: - output = self.critic_module(input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - **multi_modal_inputs, - use_cache=False) # prevent model thinks we are generating - values = output.logits - values = values[:, -response_length - 1:-1].squeeze(-1) - return values - - def _optimizer_step(self): - assert self.config.grad_clip is not None - - if isinstance(self.critic_module, FSDP): - grad_norm = self.critic_module.clip_grad_norm_(self.config.grad_clip) - else: - grad_norm = torch.nn.utils.clip_grad_norm_(self.critic_module.parameters(), max_norm=self.config.grad_clip) - - # if grad_norm is not finite, skip the update - if not torch.isfinite(grad_norm): - print(f"WARN: grad_norm is not finite: {grad_norm}") - self.critic_optimizer.zero_grad() - else: - self.critic_optimizer.step() - return grad_norm - - def compute_values(self, data: DataProto) -> torch.Tensor: - self.critic_module.eval() - micro_batch_size = data.meta_info['micro_batch_size'] - select_keys = ['responses', 'input_ids', 'attention_mask', 'position_ids'] - batch = data.select(batch_keys=select_keys).batch - use_dynamic_bsz = data.meta_info['use_dynamic_bsz'] - has_multi_modal_inputs = 'multi_modal_inputs' in data.non_tensor_batch.keys() - - if has_multi_modal_inputs: - num_micro_batches = data.batch.batch_size[0] // micro_batch_size - non_tensor_select_keys = ['multi_modal_inputs'] - micro_batches = data.select(select_keys, non_tensor_select_keys).chunk(num_micro_batches) - elif use_dynamic_bsz: - # split using dynamic bsz - max_token_len = data.meta_info['max_token_len'] * self.ulysses_sequence_parallel_size - micro_batches, indices = rearrange_micro_batches(batch=batch, max_token_len=max_token_len) - else: - micro_batches = batch.split(micro_batch_size) - - values_lst = [] - for micro_batch in micro_batches: - if isinstance(micro_batch, DataProto): - micro_batch = {**micro_batch.batch, **micro_batch.non_tensor_batch} - - with torch.no_grad(): - values = self._forward_micro_batch(micro_batch) - values_lst.append(values) - values = torch.concat(values_lst, dim=0) - responses = data.batch['responses'] - attention_mask = data.batch['attention_mask'] - response_length = responses.size(1) - values = values * attention_mask[:, -response_length - 1:-1] - - if use_dynamic_bsz: - indices = list(itertools.chain.from_iterable(indices)) - assert len(indices) == values.size(0), f"{len(indices)} vs. {values.size()}" - revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) - values = values[revert_indices] - - return values - - def update_critic(self, data: DataProto): - # make sure we are in training mode - self.critic_module.train() - metrics = {} - - select_keys = ['input_ids', 'responses', 'attention_mask', 'position_ids', 'values', 'returns'] - batch = data.select(batch_keys=select_keys).batch - has_multi_modal_inputs = 'multi_modal_inputs' in data.non_tensor_batch.keys() - - # Split to make minibatch iterator for updating the actor - # See PPO paper for details. https://arxiv.org/abs/1707.06347 - if has_multi_modal_inputs: - num_mini_batches = data.batch.batch_size[0] // self.config.ppo_mini_batch_size - non_tensor_select_keys = ['multi_modal_inputs'] - dataloader = data.select(select_keys, non_tensor_select_keys).chunk(num_mini_batches) - else: - dataloader = batch.split(self.config.ppo_mini_batch_size) - - for epoch in range(self.config.ppo_epochs): - for batch_idx, data in enumerate(dataloader): - # split batch into micro_batches - mini_batch = data - if has_multi_modal_inputs: - num_micro_batches = mini_batch.batch.batch_size[0] // self.config.ppo_micro_batch_size_per_gpu - micro_batches = data.select(select_keys, non_tensor_select_keys).chunk(num_micro_batches) - elif self.config.use_dynamic_bsz: - max_token_len = self.config.ppo_max_token_len_per_gpu * self.ulysses_sequence_parallel_size - micro_batches, _ = rearrange_micro_batches(batch=mini_batch, max_token_len=max_token_len) - else: - micro_batches = mini_batch.split(self.config.ppo_micro_batch_size_per_gpu) - self.gradient_accumulation = self.config.ppo_mini_batch_size // self.config.ppo_micro_batch_size_per_gpu - - self.critic_optimizer.zero_grad() - - for data in micro_batches: - #Support all devices - if isinstance(data, DataProto): - data = {**data.batch.to(torch.cuda.current_device()), **data.non_tensor_batch} - else: - data = data.to(torch.cuda.current_device()) # critic device is cpu when using offload - input_ids = data['input_ids'] - responses = data['responses'] - attention_mask = data['attention_mask'] - position_ids = data['position_ids'] - values = data['values'] - returns = data['returns'] - response_length = responses.size(1) - - response_mask = attention_mask[:, -response_length - 1:-1] - - vpreds = self._forward_micro_batch(data) - - # assert not torch.any(torch.isnan(vpreds)).item() - - vf_loss, vf_clipfrac = core_algos.compute_value_loss(vpreds=vpreds, - values=values, - returns=returns, - response_mask=response_mask, - cliprange_value=self.config.cliprange_value) - if self.config.use_dynamic_bsz: - # relative to the dynamic bsz - loss = vf_loss * (len(data) / self.config.ppo_mini_batch_size) - else: - loss = vf_loss / self.gradient_accumulation - - loss.backward() - - data = { - 'critic/vf_loss': vf_loss.detach().item(), - 'critic/vf_clipfrac': vf_clipfrac.detach().item(), - 'critic/vpred_mean': masked_mean(vpreds, response_mask).detach().item(), - } - - append_to_dict(metrics, data) - - grad_norm = self._optimizer_step() - data = {'critic/grad_norm': grad_norm.detach().item()} - append_to_dict(metrics, data) - self.critic_optimizer.zero_grad() - return metrics diff --git a/verl/workers/critic/megatron_critic.py b/verl/workers/critic/megatron_critic.py deleted file mode 100644 index 55076075c5..0000000000 --- a/verl/workers/critic/megatron_critic.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Implement a multiprocess PPOCritic -""" - -import importlib -from functools import partial -from packaging.version import Version -from typing import Iterable - -import torch -import torch.distributed -from omegaconf import OmegaConf -from torch import nn - -from verl import DataProto -from verl.trainer.ppo import core_algos -from verl.workers.critic import BasePPOCritic -from verl.utils.megatron.pipeline_parallel import (compute_transformers_input_shapes, make_batch_generator) -from verl.utils.py_functional import append_to_dict -from verl.utils.torch_dtypes import PrecisionType -from verl.utils.torch_functional import masked_mean, broadcast_dict_tensor, split_dict_tensor_into_batches -from verl.utils.megatron import sequence_parallel as sp_utils -from megatron.core.optimizer import OptimizerConfig - -from megatron.core import parallel_state as mpu -from megatron.core.pipeline_parallel import get_forward_backward_func -from megatron.core.optimizer import DistributedOptimizer - - -class MegatronPPOCritic(BasePPOCritic): - - def __init__(self, config, model_config, megatron_config, critic_module: nn.ModuleList, - critic_optimizer: DistributedOptimizer, critic_optimizer_config: OptimizerConfig): - super().__init__(config=config) - self._validate_config(config) - self.model_config = model_config - self.megatron_config = megatron_config - - self.critic_module = critic_module - self.critic_optimizer = critic_optimizer - self.critic_optimizer_config = critic_optimizer_config - - # we create a separate nametuple for optimizer step so that global args won't affect it. - self.optimizer_step_args = OmegaConf.create({ - 'skip_grad': None, - 'overlap_dp_param_comm': False, - 'overlap_dp_grad_comm': False, - 'gradient_accumulation_steps': 1, - 'sequence_parallel': self.megatron_config.sequence_parallel, - 'DDP_impl': 'local', - 'layernorm_allreduce_bucket_threshold': 0, - 'pipeline_model_parallel_split_rank': None, - 'reduce_grads_use_alltoall': False - }) - - def _validate_config(self, config) -> None: - """Validate config options not implemented for Megatron backend""" - assert config.get('ulysses_sequence_parallel_size', 1) == 1 - - def compute_values(self, data: DataProto) -> DataProto: - # data.batch = data.batch.to(self.critic_module.module.device) - responses = data.batch['responses'] - attention_mask = data.batch['attention_mask'] - response_length = responses.size(1) - with torch.no_grad(): - output = self.forward_backward_batch(data=data, forward_only=True) - if mpu.is_pipeline_last_stage(ignore_virtual=True): - # only on last rank. It should be on every tp rank - values = torch.cat([o['vpreds'] for o in output], dim=0) # (bs, seq_size, vocal_size) - values = values.to(torch.float32) - else: - values = torch.empty_like(attention_mask, dtype=torch.float32) - - # each tp ranks should contain the same value - values = values * attention_mask - values = values[:, -response_length - 1:-1] - values = values.contiguous() - - # sync among pp ranks - torch.distributed.broadcast(tensor=values, - src=mpu.get_pipeline_model_parallel_last_rank(), - group=mpu.get_pipeline_model_parallel_group()) - - # add empty cache after each compute - torch.cuda.empty_cache() - - return values - - def make_minibatch_iterator(self, data: DataProto) -> Iterable[DataProto]: - select_keys = ['input_ids', 'responses', 'attention_mask', 'position_ids', 'values', 'returns'] - data = data.select(batch_keys=select_keys) - return data.make_iterator(mini_batch_size=self.config.ppo_mini_batch_size, - epochs=self.config.ppo_epochs, - dataloader_kwargs={'shuffle': self.config.shuffle}) - - def forward_backward_batch(self, data: DataProto, forward_only=False): - # broadcast from last pp rank to all other pp ranks - data.batch = data.batch.contiguous() - broadcast_dict_tensor(data.batch, - src=mpu.get_pipeline_model_parallel_last_rank(), - group=mpu.get_pipeline_model_parallel_group()) - # split into micro-batches - data.batch['attention_mask'] = data.batch['attention_mask'].to(bool) - batches = split_dict_tensor_into_batches(data.batch, batch_size=self.config.ppo_micro_batch_size_per_gpu) - n_micro_batch = len(batches) - seq_len = batches[0]['input_ids'].shape[1] - - # compute input shapes for pp stages - input_shapes = compute_transformers_input_shapes( - batches, - meta_info={ - 'sequence_parallel': self.megatron_config.sequence_parallel, - 'hidden_size': self.model_config.hidden_size - }) - - forward_backward_func = get_forward_backward_func() - - def loss_func(output, data, meta_info): - if forward_only: - return 1.0, {'vpreds': output} - - responses = data['responses'] - attention_mask = data['attention_mask'] - values = data['values'] - returns = data['returns'] - response_length = responses.size(1) - - response_mask = attention_mask[:, -response_length:] - - cliprange_value = self.config.cliprange_value - - vpreds = output # (bs, sequence_length) - vpreds = vpreds[:, -response_length - 1:-1] - - vf_loss, vf_clipfrac = core_algos.compute_value_loss(vpreds=vpreds, - values=values, - returns=returns, - response_mask=response_mask, - cliprange_value=cliprange_value) - stats = { - 'critic/vf_loss': vf_loss.detach().item(), - 'critic/vf_clipfrac': vf_clipfrac.detach().item(), - 'critic/vpred_mean': masked_mean(vpreds, response_mask).detach().item(), - } - - return vf_loss, stats - - def forward_step(batch_iter, model): - batch = next(batch_iter) - input_ids = batch['input_ids'] - attention_mask = batch['attention_mask'] - position_ids = batch['position_ids'] - from verl.models.mcore import gptmodel_forward - - output = gptmodel_forward(model, - input_ids, - attention_mask, - position_ids, - sequence_parallel=self.megatron_config.sequence_parallel, - value_model=True) - - return output, partial(loss_func, data=batch, meta_info={}) - - # batch should be a list of batches inside micro-batches - batch_generator = make_batch_generator(batches, vpp_size=len(self.critic_module)) - - # TODO: we may use the new schedule instead - # for flash-attn: (seq_len, batch_size, hidden_size) = (mbs*seq_len, 1, hidden_size) - if mpu.get_pipeline_model_parallel_world_size() > 1: - losses_reduced = forward_backward_func( - forward_step_func=forward_step, - data_iterator=batch_generator, - model=self.critic_module, - num_microbatches=n_micro_batch, - seq_length=self.config.ppo_micro_batch_size_per_gpu * seq_len, # no use when input_shapes was set - micro_batch_size=1, # no use when input_shapes was set - forward_only=forward_only, - ) - else: - losses_reduced = forward_backward_func( - forward_step_func=forward_step, - data_iterator=batch_generator, - model=self.critic_module, - num_microbatches=n_micro_batch, - seq_length=self.config.ppo_micro_batch_size_per_gpu * seq_len, # in use for pp = 1 - micro_batch_size=1, # in use for pp = 1 - forward_only=forward_only, - ) - # loss_reduces contains the stats returned from loss_func - return losses_reduced - - def update_critic(self, dataloader: Iterable[DataProto]): - metrics = {} - - for data in dataloader: - # data = data.batch.to(self.critic_module.device) - self.critic_optimizer.zero_grad() - # use use_contiguous_buffers_in_local_ddp and no overlap_dp_param_comm - for chunk in self.critic_module: - chunk.zero_grad_buffer() - - metric_micro_batch = self.forward_backward_batch(data) - - update_successful, grad_norm, num_zeros_in_grad = self.critic_optimizer.step() - - if update_successful: - # allgather already execute in optimizer.step in new megatron - pass - else: - raise NotImplementedError - - for metric in metric_micro_batch: - append_to_dict(metrics, metric) # append the metric from this micro-batch to global metrics. - - # add empty cache after each compute - torch.cuda.empty_cache() - return metrics diff --git a/verl/workers/fsdp_workers.py b/verl/workers/fsdp_workers.py deleted file mode 100644 index 76db53f3f3..0000000000 --- a/verl/workers/fsdp_workers.py +++ /dev/null @@ -1,1221 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The main entry point to run the PPO algorithm -""" - -import logging -import os -import warnings -import psutil - -import torch -import torch.distributed -from torch.distributed.device_mesh import init_device_mesh -import verl.utils.torch_functional as verl_F -from omegaconf import DictConfig, open_dict -from verl import DataProto -from verl.single_controller.base import Worker -from verl.single_controller.base.decorator import register, Dispatch -from verl.utils import hf_tokenizer, hf_processor -from verl.utils.debug import log_gpu_memory_usage -from verl.utils.fs import copy_to_local -from verl.utils.fsdp_utils import get_fsdp_wrap_policy, init_fn, get_init_weight_context_manager -from verl.utils.fsdp_utils import offload_fsdp_optimizer, offload_fsdp_model_to_cpu, load_fsdp_optimizer, \ - load_fsdp_model_to_gpu -from verl.utils.import_utils import import_external_libs -from verl.utils.model import compute_position_id_with_mask -from verl.utils.flops_counter import FlopsCounter -from verl.utils.checkpoint.fsdp_checkpoint_manager import FSDPCheckpointManager -from verl.workers.agentic.fsdp_sgl import FSDPSGLShardingManager -from verl.workers.sharding_manager.fsdp_ulysses import FSDPUlyssesShardingManager - -from codetiming import Timer - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_PPO_LOGGING_LEVEL', 'WARN')) - - -def create_device_mesh(world_size, fsdp_size): - if fsdp_size < 0 or fsdp_size >= world_size: - device_mesh = init_device_mesh('cuda', mesh_shape=(world_size,), mesh_dim_names=['fsdp']) - else: - device_mesh = init_device_mesh('cuda', - mesh_shape=(world_size // fsdp_size, fsdp_size), - mesh_dim_names=['ddp', 'fsdp']) - return device_mesh - - -def get_sharding_strategy(device_mesh): - from torch.distributed.fsdp import ShardingStrategy - if device_mesh.ndim == 1: - sharding_strategy = ShardingStrategy.FULL_SHARD - elif device_mesh.ndim == 2: - sharding_strategy = ShardingStrategy.HYBRID_SHARD - else: - raise NotImplementedError(f"Get device mesh ndim={device_mesh.ndim}, but only support 1 or 2") - return sharding_strategy - - -class ActorRolloutRefWorker(Worker): - """ - This worker can be instantiated as a standalone actor or a standalone rollout or a standalone reference policy - or a hybrid engine based on the config.rollout - """ - - def __init__(self, config: DictConfig, role: str): - super().__init__() - self.config = config - import torch.distributed - if not torch.distributed.is_initialized(): - from datetime import timedelta - torch.distributed.init_process_group(timeout=timedelta(minutes=120)) - - # build device mesh for FSDP - world_size = torch.distributed.get_world_size() - # TODO(sgm): support FSDP hybrid shard for larger model - self.device_mesh = create_device_mesh(world_size=world_size, fsdp_size=self.config.actor.fsdp_config.fsdp_size) - - print( - f"nodedup INIT ActorRolloutRefWorker: {role=} {torch.distributed.get_rank()=} {world_size=} {self.device_mesh=} {os.environ['CUDA_VISIBLE_DEVICES']=}" - ) - - # build device mesh for Ulysses Sequence Parallel - self.ulysses_device_mesh = None - self.ulysses_sequence_parallel_size = self.config.actor.get('ulysses_sequence_parallel_size', 1) - dp = world_size // self.ulysses_sequence_parallel_size - if self.ulysses_sequence_parallel_size > 1: - self.ulysses_device_mesh = init_device_mesh('cuda', - mesh_shape=(dp, self.ulysses_sequence_parallel_size), - mesh_dim_names=['dp', 'sp']) - - self.ulysses_sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) - - self.role = role - assert self.role in ['actor', 'rollout', 'ref', 'actor_rollout', 'actor_rollout_ref'] - - self._is_actor = self.role in ['actor', 'actor_rollout', 'actor_rollout_ref'] - self._is_rollout = self.role in ['rollout', 'actor_rollout', 'actor_rollout_ref'] - self._is_ref = self.role in ['ref', 'actor_rollout_ref'] - - self._is_offload_param = False - self._is_offload_optimizer = False - if self._is_actor: - self._is_offload_param = self.config.actor.fsdp_config.get('param_offload', False) - self._is_offload_optimizer = self.config.actor.fsdp_config.get('optimizer_offload', False) - elif self._is_ref: - # TODO: it seems that manual offload is slowly than FSDP offload - self._is_offload_param = self.config.ref.fsdp_config.get('param_offload', False) - - # normalize config - if self._is_actor: - self.config.actor.ppo_mini_batch_size *= self.config.rollout.n - self.config.actor.ppo_mini_batch_size //= (self.device_mesh.size() // self.ulysses_sequence_parallel_size) - assert self.config.actor.ppo_mini_batch_size > 0, f'ppo_mini_batch_size {self.config.actor.ppo_mini_batch_size} should be larger than 0 after normalization' - # micro bsz - if self.config.actor.ppo_micro_batch_size is not None: - self.config.actor.ppo_micro_batch_size //= (self.device_mesh.size() // - self.ulysses_sequence_parallel_size) - self.config.actor.ppo_micro_batch_size_per_gpu = self.config.actor.ppo_micro_batch_size - assert self.config.actor.ppo_mini_batch_size % self.config.actor.ppo_micro_batch_size_per_gpu == 0, \ - f'normalized ppo_mini_batch_size {self.config.actor.ppo_mini_batch_size} should be divisible by ppo_micro_batch_size_per_gpu {self.config.actor.ppo_micro_batch_size_per_gpu}' - assert self.config.actor.ppo_mini_batch_size // self.config.actor.ppo_micro_batch_size_per_gpu > 0, \ - f'normalized ppo_mini_batch_size {self.config.actor.ppo_mini_batch_size} should be larger than ppo_micro_batch_size_per_gpu {self.config.actor.ppo_micro_batch_size_per_gpu}' - - # normalize rollout config - if self._is_rollout and self.config.rollout.log_prob_micro_batch_size is not None: - self.config.rollout.log_prob_micro_batch_size //= (self.device_mesh.size() // - self.ulysses_sequence_parallel_size) - self.config.rollout.log_prob_micro_batch_size_per_gpu = self.config.rollout.log_prob_micro_batch_size - # normalize ref config - if self._is_ref and self.config.ref.log_prob_micro_batch_size is not None: - self.config.ref.log_prob_micro_batch_size //= (self.device_mesh.size() // - self.ulysses_sequence_parallel_size) - self.config.ref.log_prob_micro_batch_size_per_gpu = self.config.ref.log_prob_micro_batch_size - - def _build_model_optimizer(self, - model_path, - fsdp_config, - optim_config, - override_model_config, - use_remove_padding=False, - enable_gradient_checkpointing=False, - trust_remote_code=False, - use_liger=False, - role='actor'): - from verl.utils.model import print_model_size, update_model_config, get_generation_config - from verl.utils.torch_dtypes import PrecisionType - from transformers import AutoModelForCausalLM, AutoConfig, AutoModelForVision2Seq - from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy, MixedPrecision, CPUOffload - from torch import optim - - assert role in ['actor', 'ref'] - - log_gpu_memory_usage('Before init from HF AutoModel', logger=logger) - local_path = copy_to_local(model_path) - - # note that we have to create model in fp32. Otherwise, the optimizer is in bf16, which is incorrect - # TODO(zhangchi.usc1992): 1. support create from random initialized model. 2. Support init with FSDP directly - self.tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) - self.processor = hf_processor(local_path, trust_remote_code=trust_remote_code) - - torch_dtype = fsdp_config.get('model_dtype', None) - if torch_dtype is None: - torch_dtype = torch.float32 if self._is_actor else torch.bfloat16 - else: - torch_dtype = PrecisionType.to_dtype(torch_dtype) - - # override model kwargs - actor_model_config = AutoConfig.from_pretrained(local_path, trust_remote_code=trust_remote_code) - - self.generation_config = get_generation_config(local_path, trust_remote_code=trust_remote_code) - - override_config_kwargs = { - 'bos_token_id': self.tokenizer.bos_token_id, - 'eos_token_id': self.tokenizer.eos_token_id, - 'pad_token_id': self.tokenizer.pad_token_id, - } - override_config_kwargs.update(override_model_config) - update_model_config(actor_model_config, override_config_kwargs=override_config_kwargs) - if self.rank == 0: - print(f'Model config after override: {actor_model_config}') - - # NOTE(fix me): tie_word_embedding causes meta_tensor init to hang - init_context = get_init_weight_context_manager(use_meta_tensor=not actor_model_config.tie_word_embeddings, - mesh=self.device_mesh) - - with init_context(), warnings.catch_warnings(): - warnings.simplefilter("ignore") - if type(actor_model_config) in AutoModelForVision2Seq._model_mapping.keys(): - actor_module_class = AutoModelForVision2Seq - else: - actor_module_class = AutoModelForCausalLM - - actor_module = actor_module_class.from_pretrained(pretrained_model_name_or_path=local_path, - torch_dtype=torch_dtype, - config=actor_model_config, - attn_implementation='flash_attention_2', - trust_remote_code=trust_remote_code) - - if use_remove_padding or self.ulysses_sequence_parallel_size > 1: - from verl.models.transformers.monkey_patch import apply_monkey_patch - apply_monkey_patch(model=actor_module, ulysses_sp_size=self.ulysses_sequence_parallel_size) - - # Apply Liger kernel to the model if use_liger is set to True - if use_liger: - from liger_kernel.transformers.monkey_patch import _apply_liger_kernel_to_instance - _apply_liger_kernel_to_instance(model=actor_module) - - # some parameters may not in torch_dtype. TODO(zhangchi.usc1992) remove this after we switch to fsdp2 - actor_module.to(torch_dtype) - - if enable_gradient_checkpointing: - actor_module.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant': False}) - torch.distributed.barrier() - - if self.rank == 0: - print_model_size(actor_module) - - log_gpu_memory_usage('After init from HF AutoModel', logger=logger) - - # We wrap FSDP for rollout as well - mixed_precision_config = fsdp_config.get('mixed_precision', None) - if mixed_precision_config is not None: - param_dtype = PrecisionType.to_dtype(mixed_precision_config.get('param_dtype', 'bf16')) - reduce_dtype = PrecisionType.to_dtype(mixed_precision_config.get('reduce_dtype', 'fp32')) - buffer_dtype = PrecisionType.to_dtype(mixed_precision_config.get('buffer_dtype', 'fp32')) - else: - param_dtype = torch.bfloat16 - reduce_dtype = torch.float32 - buffer_dtype = torch.bfloat16 - - mixed_precision = MixedPrecision(param_dtype=param_dtype, reduce_dtype=reduce_dtype, buffer_dtype=buffer_dtype) - print(f'actor module Mixed precision: {mixed_precision}') - - auto_wrap_policy = get_fsdp_wrap_policy(module=actor_module, config=fsdp_config.get('wrap_policy', None)) - - if self._is_rollout and self.config.rollout.name == 'hf': - # TODO(zhangchi.usc1992, shengguangming) fix me. Current, auto_wrap_policy causes HFRollout to hang in Gemma - auto_wrap_policy = None - - # print(f'wrap_policy: {auto_wrap_policy}') - - fsdp_mesh = self.device_mesh - sharding_strategy = get_sharding_strategy(fsdp_mesh) - - # TODO: add transformer policy - # We force reference policy to use CPUOffload to save memory. - # We force turn off CPUOffload for actor because it causes incorrect results when using grad accumulation - cpu_offload = None if role == 'actor' else CPUOffload(offload_params=True) - actor_module_fsdp = FSDP( - actor_module, - cpu_offload=cpu_offload, - param_init_fn=init_fn, - use_orig_params=False, - auto_wrap_policy=auto_wrap_policy, - device_id=torch.cuda.current_device(), - sharding_strategy=sharding_strategy, # zero3 - mixed_precision=mixed_precision, - sync_module_states=True, - device_mesh=self.device_mesh, - forward_prefetch=False) - - log_gpu_memory_usage('After Actor FSDP init', logger=logger) - - # TODO: add more optimizer args into config - if role == 'actor' and optim_config is not None: - from verl.utils.torch_functional import get_constant_schedule_with_warmup - actor_optimizer = optim.AdamW(actor_module_fsdp.parameters(), - lr=optim_config.lr, - betas=optim_config.get('betas', (0.9, 0.999)), - weight_decay=optim_config.get('weight_decay', 1e-2)) - - total_steps = optim_config.get('total_training_steps', 0) - num_warmup_steps = int(optim_config.get('lr_warmup_steps', -1)) - if num_warmup_steps < 0: - num_warmup_steps_ratio = optim_config.get('lr_warmup_steps_ratio', 0.) - num_warmup_steps = int(num_warmup_steps_ratio * total_steps) - - print(f'Total steps: {total_steps}, num_warmup_steps: {num_warmup_steps}') - - actor_lr_scheduler = get_constant_schedule_with_warmup(optimizer=actor_optimizer, - num_warmup_steps=num_warmup_steps) - else: - actor_optimizer = None - actor_lr_scheduler = None - - log_gpu_memory_usage('After actor optimizer init', logger=logger) - - return actor_module_fsdp, actor_optimizer, actor_lr_scheduler, actor_model_config - - def _build_rollout(self, trust_remote_code=False): - from torch.distributed.device_mesh import init_device_mesh - # TODO(sgm): support FSDP hybrid shard for larger model - infer_tp = self.config.rollout.tensor_model_parallel_size - dp = self.world_size // infer_tp - assert self.world_size % infer_tp == 0, f'rollout world_size: {self.world_size} is not divisible by infer_tp: {infer_tp}' - # print(f"{self.world_size=} {dp=} {infer_tp=}") - rollout_device_mesh = init_device_mesh('cuda', mesh_shape=(dp, infer_tp), mesh_dim_names=['dp', 'infer_tp']) - rollout_name = self.config.rollout.name - if rollout_name == 'hf': - from verl.workers.rollout import HFRollout - from verl.workers.sharding_manager import BaseShardingManager - rollout = HFRollout(module=self.actor_module_fsdp, config=self.config.rollout) - rollout_sharding_manager = BaseShardingManager() - # TODO: a sharding manager that do nothing? - - elif rollout_name == 'vllm': - from verl.workers.rollout.vllm_rollout import vLLMRollout, vllm_mode - from verl.workers.sharding_manager import FSDPVLLMShardingManager - log_gpu_memory_usage(f'Before building {rollout_name} rollout', logger=None) - local_path = copy_to_local(self.config.model.path) - if vllm_mode == 'customized': - rollout = vLLMRollout(actor_module=self.actor_module_fsdp, - config=self.config.rollout, - tokenizer=self.tokenizer, - model_hf_config=self.actor_model_config) - elif vllm_mode == 'spmd': - rollout = vLLMRollout(model_path=local_path, - config=self.config.rollout, - tokenizer=self.tokenizer, - model_hf_config=self.actor_model_config, - device_mesh=rollout_device_mesh, - trust_remote_code=trust_remote_code) - else: - raise NotImplementedError("vllm_mode must be 'customized' or 'spmd'") - log_gpu_memory_usage(f'After building {rollout_name} rollout', logger=None) - if torch.distributed.get_world_size() == 1: - self.config.rollout.load_format = 'dummy_hf' - rollout_sharding_manager = FSDPVLLMShardingManager(module=self.actor_module_fsdp, - inference_engine=rollout.inference_engine, - model_config=self.actor_model_config, - full_params='hf' in self.config.rollout.load_format, - device_mesh=rollout_device_mesh) - log_gpu_memory_usage('After building sharding manager', logger=None) - - elif rollout_name == 'sglang': - from verl.workers.rollout.sglang_rollout import SGLangRollout - # NOTE(linjunrong): Due to recent fp8 support in SGLang. Now importing any symbol relate to SGLang's model_runner would check CUDA device capability. - # However, due to veRL's setting, the main process of ray can not find any CUDA device, which would potentially lead to: - # "RuntimeError: No CUDA GPUs are available". - # For this reason, sharding_manager.__init__ should not import FSDPSGLangShardingManager and we import it here use the abs path. - # check: https://github.com/sgl-project/sglang/blob/00f42707eaddfc2c0528e5b1e0094025c640b7a0/python/sglang/srt/layers/quantization/fp8_utils.py#L76 - from verl.workers.sharding_manager.fsdp_sglang import FSDPSGLangShardingManager - log_gpu_memory_usage(f'Before building {rollout_name} rollout', logger=None) - rollout = SGLangRollout(actor_module=self.config.model.path, - config=self.config.rollout, - tokenizer=self.tokenizer, - model_hf_config=self.actor_model_config) - log_gpu_memory_usage(f'After building {rollout_name} rollout', logger=None) - - if torch.distributed.get_world_size() == 1: - self.config.rollout.load_format = 'dummy_hf' - rollout_sharding_manager = FSDPSGLangShardingManager(module=self.actor_module_fsdp, - inference_engine=rollout.inference_engine, - model_config=self.actor_model_config, - full_params='hf' in self.config.rollout.load_format, - device_mesh=rollout_device_mesh) - log_gpu_memory_usage('After building sharding manager', logger=None) - - elif self.config.rollout.name == "async": - from verl.workers.agentic.async_rollout import AsyncRollout - from verl.workers.agentic.fsdp_sgl import FSDPSGLShardingManager - local_path = copy_to_local(self.config.model.path) - # print(f"nodedup creating async rollout instance, {torch.distributed.get_rank()=} {rollout_device_mesh.get_rank()=} {rollout_device_mesh.shape=}") - rollout = AsyncRollout(model_path=local_path, config=self.config.rollout, device_mesh=rollout_device_mesh) - rollout_sharding_manager = FSDPSGLShardingManager( - module=self.actor_module_fsdp, - inference_engine=rollout.engine, - model_config=self.actor_model_config, - full_params='hf' in self.config.rollout.load_format, - device_mesh=rollout_device_mesh, - role=self.role, - rollout_count=rollout_device_mesh.size(0), - exchange_size=self.config.get("exchange_size"), - ) - else: - raise NotImplementedError(f"Rollout name: {self.config.rollout.name} is not supported") - return rollout, rollout_sharding_manager - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - from verl.workers.actor import DataParallelPPOActor - # This is used to import external_lib into the huggingface systems - import_external_libs(self.config.model.get('external_lib', None)) - - from omegaconf import OmegaConf - override_model_config = OmegaConf.to_container(self.config.model.get('override_config', OmegaConf.create())) - - use_remove_padding = self.config.model.get('use_remove_padding', False) - - if self._is_actor or self._is_rollout: - # we need the model for actor and rollout - if self._is_actor: - optim_config = self.config.actor.optim - fsdp_config = self.config.actor.fsdp_config - else: - optim_config = None - fsdp_config = OmegaConf.create() - self.actor_module_fsdp, self.actor_optimizer, self.actor_lr_scheduler, self.actor_model_config = self._build_model_optimizer( - model_path=self.config.model.path, - fsdp_config=fsdp_config, - optim_config=optim_config, - override_model_config=override_model_config, - use_remove_padding=use_remove_padding, - enable_gradient_checkpointing=self.config.model.get('enable_gradient_checkpointing', False), - trust_remote_code=self.config.model.get('trust_remote_code', False), - use_liger=self.config.model.get('use_liger', False), - role='actor') - - # get the original unwrapped module - self.actor_module = self.actor_module_fsdp._fsdp_wrapped_module - - if self._is_offload_optimizer: - offload_fsdp_optimizer(optimizer=self.actor_optimizer) - log_gpu_memory_usage('After offload actor optimizer during init', logger=logger) - # load from checkpoint - if self._is_actor: - OmegaConf.set_struct(self.config.actor, True) - with open_dict(self.config.actor): - self.config.actor.use_remove_padding = use_remove_padding - self.actor = DataParallelPPOActor(config=self.config.actor, - actor_module=self.actor_module_fsdp, - actor_optimizer=self.actor_optimizer) - - if self._is_rollout: - # if not self._is_actor: - # self.actor_module_fsdp = None - # self.actor_optimizer = None - # self.actor_lr_scheduler = None - # self.actor_model_config = None - self.rollout, self.rollout_sharding_manager = self._build_rollout( - trust_remote_code=self.config.model.get('trust_remote_code', False)) - elif self._is_actor: - self.rollout_sharding_manager = FSDPSGLShardingManager( - module=self.actor_module_fsdp, - inference_engine=None, - model_config=self.actor_model_config, - full_params='hf' in self.config.rollout.load_format, - device_mesh=self.device_mesh, - role=self.role, - rollout_count=self.device_mesh.size(0), - exchange_size=self.config.get("exchange_size"), - ) - - if self._is_ref: - self.ref_module_fsdp = self._build_model_optimizer(model_path=self.config.model.path, - fsdp_config=self.config.ref.fsdp_config, - optim_config=None, - override_model_config=override_model_config, - use_remove_padding=use_remove_padding, - trust_remote_code=self.config.model.get( - 'trust_remote_code', False), - use_liger=self.config.model.get('use_liger', False), - role='ref')[0] - OmegaConf.set_struct(self.config.ref, True) - with open_dict(self.config.ref): - self.config.ref.use_remove_padding = use_remove_padding - self.ref_policy = DataParallelPPOActor(config=self.config.ref, actor_module=self.ref_module_fsdp) - - if self._is_actor: - self.flops_counter = FlopsCounter(self.actor_model_config) - self.checkpoint_manager = FSDPCheckpointManager( - model=self.actor_module_fsdp, - optimizer=self.actor.actor_optimizer, - lr_scheduler=self.actor_lr_scheduler, - processing_class=self.processor if self.processor is not None else self.tokenizer, - checkpoint_contents=self.config.actor.checkpoint.contents) - torch.cuda.empty_cache() - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def update_actor(self, data: DataProto): - # Support all hardwares - data = data.to(torch.cuda.current_device()) - - assert self._is_actor - if self._is_offload_param: - load_fsdp_model_to_gpu(self.actor_module_fsdp) - if self._is_offload_optimizer: - load_fsdp_optimizer(optimizer=self.actor_optimizer, device_id=torch.cuda.current_device()) - - log_gpu_memory_usage('Before update policy', logger=logger) - - with self.ulysses_sharding_manager: - data = self.ulysses_sharding_manager.preprocess_data(data=data) - # perform training - with Timer(name='update_policy', logger=None) as timer: - metrics = self.actor.update_policy(data=data) - delta_time = timer.last - global_num_tokens = data.meta_info['global_token_num'] - estimated_flops, promised_flops = self.flops_counter.estimate_flops(global_num_tokens, delta_time) - metrics[ - 'perf/mfu/actor'] = estimated_flops * self.config.actor.ppo_epochs / promised_flops / self.world_size - metrics['perf/max_memory_allocated_gb'] = torch.cuda.max_memory_allocated() / (1024**3) - metrics['perf/max_memory_reserved_gb'] = torch.cuda.max_memory_reserved() / (1024**3) - metrics['perf/cpu_memory_used_gb'] = psutil.virtual_memory().used / (1024**3) - - self.actor_lr_scheduler.step() - lr = self.actor_lr_scheduler.get_last_lr()[0] - metrics['actor/lr'] = lr - - log_gpu_memory_usage('After update policy', logger=logger) - - # TODO: here, we should return all metrics - output = DataProto(meta_info={'metrics': metrics}) - - output = self.ulysses_sharding_manager.postprocess_data(data=output) - output = output.to('cpu') - - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.actor_module_fsdp) - if self._is_offload_optimizer: - offload_fsdp_optimizer(optimizer=self.actor_optimizer) - - return output - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def generate_sequences(self, prompts: DataProto): - prompts = prompts.to(torch.cuda.current_device()) - - if self._is_actor and not self._is_rollout: - with self.rollout_sharding_manager: - pass - return - - assert self._is_rollout - if self._is_offload_param: - load_fsdp_model_to_gpu(self.actor_module_fsdp) - - meta_info = { - 'eos_token_id': - self.generation_config.eos_token_id - if self.generation_config is not None else self.tokenizer.eos_token_id, - 'pad_token_id': - self.generation_config.pad_token_id - if self.generation_config is not None else self.tokenizer.pad_token_id, - } - prompts.meta_info.update(meta_info) - with self.rollout_sharding_manager: - - # after parameters sync with rollout, offload actor model to CPU - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.actor_module_fsdp) - if self._is_offload_optimizer: - offload_fsdp_optimizer(optimizer=self.actor_optimizer) - - log_gpu_memory_usage('After entering rollout sharding manager', logger=logger) - - prompts = self.rollout_sharding_manager.preprocess_data(prompts) - output = self.rollout.generate_sequences(prompts=prompts) - log_gpu_memory_usage('After rollout generation', logger=logger) - - output = self.rollout_sharding_manager.postprocess_data(output) - - output = output.to('cpu') - - # clear kv cache - log_gpu_memory_usage('After generate_sequences', logger=logger) - return output - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_log_prob(self, data: DataProto): - assert self._is_actor - if self._is_offload_param: - load_fsdp_model_to_gpu(self.actor_module_fsdp) - - # Support all hardwares - data = data.to(torch.cuda.current_device()) - # we should always recompute old_log_probs when it is HybridEngine - data.meta_info['micro_batch_size'] = self.config.rollout.log_prob_micro_batch_size_per_gpu - data.meta_info['max_token_len'] = self.config.rollout.log_prob_max_token_len_per_gpu - data.meta_info['use_dynamic_bsz'] = self.config.rollout.log_prob_use_dynamic_bsz - data.meta_info['temperature'] = self.config.rollout.temperature - # perform recompute log_prob - with self.ulysses_sharding_manager: - data = self.ulysses_sharding_manager.preprocess_data(data) - output = self.actor.compute_log_prob(data=data) - output = DataProto.from_dict(tensors={'old_log_probs': output}, - meta_info={'temperature': self.config.rollout.temperature}) - output = self.ulysses_sharding_manager.postprocess_data(output) - - output = output.to('cpu') - - # https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes - # unshard the root FSDP module - if self.world_size > 1: - self.actor.actor_module._handle.reshard(True) - - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.actor_module_fsdp) - - log_gpu_memory_usage('After compute_log_prob', logger=logger) - return output - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_ref_log_prob(self, data: DataProto): - assert self._is_ref - - # Support all hardwares - data = data.to(torch.cuda.current_device()) - - micro_batch_size = self.config.ref.log_prob_micro_batch_size_per_gpu - data.meta_info['micro_batch_size'] = micro_batch_size - data.meta_info['temperature'] = self.config.rollout.temperature - data.meta_info['max_token_len'] = self.config.ref.log_prob_max_token_len_per_gpu - data.meta_info['use_dynamic_bsz'] = self.config.ref.log_prob_use_dynamic_bsz - with self.ulysses_sharding_manager: - data = self.ulysses_sharding_manager.preprocess_data(data) - output = self.ref_policy.compute_log_prob(data=data) - output = DataProto.from_dict(tensors={'ref_log_prob': output}) - output = self.ulysses_sharding_manager.postprocess_data(output) - - output = output.to('cpu') - - # https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes - # unshard the root FSDP module - if self.world_size > 1: - self.ref_policy.actor_module._handle.reshard(True) - - return output - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): - # only support save and load ckpt for actor - assert self._is_actor - import torch - if self._is_offload_param: - load_fsdp_model_to_gpu(self.actor_module_fsdp) - - self.checkpoint_manager.save_checkpoint(local_path=local_path, - hdfs_path=hdfs_path, - global_step=global_step, - max_ckpt_to_keep=max_ckpt_to_keep) - - torch.distributed.barrier() - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.actor_module_fsdp) - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=False): - if self._is_offload_param: - load_fsdp_model_to_gpu(self.actor_module_fsdp) - - self.checkpoint_manager.load_checkpoint(local_path=local_path, - hdfs_path=hdfs_path, - del_local_after_load=del_local_after_load) - - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.actor_module_fsdp) - - if self._is_offload_optimizer: - offload_fsdp_optimizer(self.actor_optimizer) - - -class CriticWorker(Worker): - - def __init__(self, config): - super().__init__() - import torch.distributed - if not torch.distributed.is_initialized(): - torch.distributed.init_process_group(backend="nccl") - self.config = config - - # build device mesh for Ulysses Sequence Parallel - world_size = torch.distributed.get_world_size() - from torch.distributed.device_mesh import init_device_mesh - - fsdp_size = self.config.model.fsdp_config.fsdp_size - self.device_mesh = create_device_mesh(world_size=world_size, fsdp_size=fsdp_size) - - self.ulysses_device_mesh = None - self.ulysses_sequence_parallel_size = self.config.get('ulysses_sequence_parallel_size', 1) - dp = world_size // self.ulysses_sequence_parallel_size - if self.ulysses_sequence_parallel_size > 1: - self.ulysses_device_mesh = init_device_mesh('cuda', - mesh_shape=(dp, self.ulysses_sequence_parallel_size), - mesh_dim_names=['dp', 'sp']) - - self.ulysses_sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) - - # set FSDP offload params - self._is_offload_param = self.config.model.fsdp_config.param_offload - self._is_offload_optimizer = self.config.model.fsdp_config.optimizer_offload - - # normalize config - self.config.ppo_mini_batch_size *= self.config.rollout_n - self.config.ppo_mini_batch_size //= (torch.distributed.get_world_size() // self.ulysses_sequence_parallel_size) - if self.config.ppo_micro_batch_size is not None: - self.config.ppo_micro_batch_size //= (torch.distributed.get_world_size() // - self.ulysses_sequence_parallel_size) - self.config.forward_micro_batch_size //= (torch.distributed.get_world_size() // - self.ulysses_sequence_parallel_size) - self.config.ppo_micro_batch_size_per_gpu = self.config.ppo_micro_batch_size - self.config.forward_micro_batch_size_per_gpu = self.config.forward_micro_batch_size - assert self.config.ppo_mini_batch_size % self.config.ppo_micro_batch_size_per_gpu == 0, \ - f'normalized ppo_mini_batch_size {self.config.ppo_mini_batch_size} should be divisible by ppo_micro_batch_size_per_gpu {self.config.ppo_micro_batch_size_per_gpu}' - assert self.config.ppo_mini_batch_size // self.config.ppo_micro_batch_size_per_gpu > 0, \ - f'normalized ppo_mini_batch_size {self.config.ppo_mini_batch_size} should be larger than ppo_micro_batch_size_per_gpu {self.config.ppo_micro_batch_size_per_gpu}' - - def _build_critic_model_optimizer(self, config): - # the following line is necessary - from verl.utils.model import LambdaLayer, print_model_size, squeeze - from verl.utils.torch_dtypes import PrecisionType - from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy, MixedPrecision - from torch import optim - - local_path = copy_to_local(config.model.path) - # note that the tokenizer between actor and critic may be different. So override tokenizer info with actor info - # using random initialized model from any architecture. May not be the same as Actor. - - tokenizer_path = copy_to_local(config.model.tokenizer_path) - self.tokenizer = hf_tokenizer(tokenizer_path, trust_remote_code=config.model.get('trust_remote_code', False)) - self.processor = hf_processor(tokenizer_path, trust_remote_code=config.model.get('trust_remote_code', False)) - - from omegaconf import OmegaConf - override_config = OmegaConf.to_container(self.config.model.get('override_config', OmegaConf.create())) - override_config_kwargs = { - 'bos_token_id': self.tokenizer.bos_token_id, - 'eos_token_id': self.tokenizer.eos_token_id, - 'pad_token_id': self.tokenizer.pad_token_id, - } - override_config_kwargs.update(override_config) - if self.rank == 0: - print(f'Critic overriding config {override_config_kwargs}') - - torch_dtype = self.config.model.fsdp_config.get('model_dtype', 'fp32') - torch_dtype = PrecisionType.to_dtype(torch_dtype) - - from transformers import AutoConfig, AutoModelForTokenClassification - from torch import nn - - trust_remote_code = False - critic_model_config = AutoConfig.from_pretrained(local_path, trust_remote_code=trust_remote_code) - critic_model_config.num_labels = 1 - - init_context = get_init_weight_context_manager(use_meta_tensor=not critic_model_config.tie_word_embeddings, - mesh=self.device_mesh) - - with init_context(), warnings.catch_warnings(): - warnings.simplefilter("ignore") - setattr(critic_model_config, 'classifier_dropout', 0.) - setattr(critic_model_config, 'hidden_dropout', '0') - critic_module = AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path=local_path, - torch_dtype=torch_dtype, - config=critic_model_config, - attn_implementation='flash_attention_2', - trust_remote_code=trust_remote_code) - - use_remove_padding = config.model.get('use_remove_padding', False) - if use_remove_padding or self.ulysses_sequence_parallel_size > 1: - from verl.models.transformers.monkey_patch import apply_monkey_patch - apply_monkey_patch(model=critic_module, ulysses_sp_size=self.ulysses_sequence_parallel_size) - - # some parameters may not in torch_dtype - critic_module.to(torch_dtype) - - if config.model.get('enable_gradient_checkpointing', False): - critic_module.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant': False}) - if self.rank == 0: - print_model_size(critic_module) - - self.critic_model_config = critic_model_config - - fsdp_config = self.config.model.fsdp_config - mixed_precision_config = fsdp_config.get('mixed_precision', None) - if mixed_precision_config is not None: - param_dtype = PrecisionType.to_dtype(mixed_precision_config.get('param_dtype', 'bf16')) - reduce_dtype = PrecisionType.to_dtype(mixed_precision_config.get('reduce_dtype', 'fp32')) - buffer_dtype = PrecisionType.to_dtype(mixed_precision_config.get('buffer_dtype', 'fp32')) - else: - param_dtype = torch.bfloat16 - reduce_dtype = torch.float32 - buffer_dtype = torch.float32 - - mixed_precision = MixedPrecision(param_dtype=param_dtype, reduce_dtype=reduce_dtype, buffer_dtype=buffer_dtype) - - auto_wrap_policy = get_fsdp_wrap_policy(module=critic_module, config=self.config.model.fsdp_config.wrap_policy) - - log_gpu_memory_usage('Before critic FSDP', logger=None) - - fsdp_mesh = self.device_mesh - sharding_strategy = get_sharding_strategy(fsdp_mesh) - - # Note: We force turn off CPUOffload for critic because it causes incorrect results when using grad accumulation - critic_module = FSDP(critic_module, - param_init_fn=init_fn, - use_orig_params=False, - auto_wrap_policy=auto_wrap_policy, - device_id=torch.cuda.current_device(), - sharding_strategy=sharding_strategy, - mixed_precision=mixed_precision, - sync_module_states=True, - forward_prefetch=False, - device_mesh=self.device_mesh, - cpu_offload=None) - - log_gpu_memory_usage('After critic FSDP', logger=None) - - critic_optimizer = optim.AdamW(critic_module.parameters(), - lr=config.optim.lr, - betas=config.optim.get('betas', (0.9, 0.999)), - weight_decay=config.optim.get('weight_decay', 1e-2)) - - total_steps = config.optim.get('total_training_steps', 0) - num_warmup_steps = int(config.optim.get('lr_warmup_steps', -1)) - if num_warmup_steps < 0: - num_warmup_steps_ratio = config.optim.get('lr_warmup_steps_ratio', 0.) - num_warmup_steps = int(num_warmup_steps_ratio * total_steps) - - print(f'Total steps: {total_steps}, num_warmup_steps: {num_warmup_steps}') - - from verl.utils.torch_functional import get_constant_schedule_with_warmup - critic_lr_scheduler = get_constant_schedule_with_warmup(optimizer=critic_optimizer, - num_warmup_steps=num_warmup_steps) - - return critic_module, critic_optimizer, critic_lr_scheduler - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - # This is used to import external_lib into the huggingface systems - import_external_libs(self.config.model.get('external_lib', None)) - - from verl.workers.critic import DataParallelPPOCritic - self.critic_module, self.critic_optimizer, self.critic_lr_scheduler = self._build_critic_model_optimizer( - self.config) - - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.critic_module) - if self._is_offload_optimizer: - offload_fsdp_optimizer(optimizer=self.critic_optimizer) - - self.critic = DataParallelPPOCritic(config=self.config, - critic_module=self.critic_module, - critic_optimizer=self.critic_optimizer) - - self.flops_counter = FlopsCounter(self.critic_model_config) - self.checkpoint_manager = FSDPCheckpointManager( - model=self.critic_module, - optimizer=self.critic_optimizer, - lr_scheduler=self.critic_lr_scheduler, - processing_class=self.processor if self.processor is not None else self.tokenizer, - checkpoint_contents=self.config.checkpoint.contents) - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_values(self, data: DataProto): - - # Support all hardwares - data = data.to(torch.cuda.current_device()) - - if self._is_offload_param: - load_fsdp_model_to_gpu(self.critic_module) - micro_batch_size = self.config.forward_micro_batch_size_per_gpu - data.meta_info['micro_batch_size'] = micro_batch_size - data.meta_info['max_token_len'] = self.config.forward_max_token_len_per_gpu - data.meta_info['use_dynamic_bsz'] = self.config.use_dynamic_bsz - # perform forward computation - with self.ulysses_sharding_manager: - data = self.ulysses_sharding_manager.preprocess_data(data=data) - values = self.critic.compute_values(data=data) - output = DataProto.from_dict(tensors={'values': values}) - output = self.ulysses_sharding_manager.postprocess_data(data=output) - - output = output.to('cpu') - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.critic_module) - return output - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def update_critic(self, data: DataProto): - # Support all hardwares - data = data.to(torch.cuda.current_device()) - if self._is_offload_param: - load_fsdp_model_to_gpu(self.critic_module) - if self._is_offload_optimizer: - load_fsdp_optimizer(optimizer=self.critic_optimizer, device_id=torch.cuda.current_device()) - - # perform forward computation - with self.ulysses_sharding_manager: - data = self.ulysses_sharding_manager.preprocess_data(data=data) - - with Timer(name='update_critic', logger=None) as timer: - metrics = self.critic.update_critic(data=data) - delta_time = timer.last - - global_num_tokens = data.meta_info['global_token_num'] - estimated_flops, promised_flops = self.flops_counter.estimate_flops(global_num_tokens, delta_time) - metrics['perf/mfu/critic'] = estimated_flops * self.config.ppo_epochs / promised_flops / self.world_size - - self.critic_lr_scheduler.step() - lr = self.critic_lr_scheduler.get_last_lr()[0] - metrics['critic/lr'] = lr - - output = DataProto(batch=None, meta_info={'metrics': metrics}) - output = self.ulysses_sharding_manager.postprocess_data(data=output) - - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.critic_module) - if self._is_offload_optimizer: - offload_fsdp_optimizer(optimizer=self.critic_optimizer) - - output = output.to('cpu') - return output - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def save_checkpoint(self, local_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): - import torch - if self._is_offload_param: - load_fsdp_model_to_gpu(self.critic_module) - - self.checkpoint_manager.save_checkpoint(local_path=local_path, - hdfs_path=hdfs_path, - global_step=global_step, - max_ckpt_to_keep=max_ckpt_to_keep) - - torch.distributed.barrier() - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.critic_module) - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def load_checkpoint(self, local_path, hdfs_path=None, del_local_after_load=True): - import torch - if self._is_offload_param: - load_fsdp_model_to_gpu(self.critic_module) - - self.checkpoint_manager.load_checkpoint(local_path=local_path, - hdfs_path=hdfs_path, - del_local_after_load=del_local_after_load) - - torch.distributed.barrier() - if self._is_offload_param: - offload_fsdp_model_to_cpu(self.critic_module) - - if self._is_offload_optimizer: - offload_fsdp_optimizer(self.critic_optimizer) - - -# TODO(sgm): we may need to extract it to dp_reward_model.py -class RewardModelWorker(Worker): - """ - Note that we only implement the reward model that is subclass of AutoModelForTokenClassification. - """ - - def __init__(self, config): - super().__init__() - import torch.distributed - if not torch.distributed.is_initialized(): - torch.distributed.init_process_group(backend="nccl") - self.config = config - - # build device mesh for Ulysses Sequence Parallel - world_size = torch.distributed.get_world_size() - from torch.distributed.device_mesh import init_device_mesh - - fsdp_size = self.config.model.fsdp_config.fsdp_size - self.device_mesh = create_device_mesh(world_size=world_size, fsdp_size=fsdp_size) - - self.ulysses_device_mesh = None - self.ulysses_sequence_parallel_size = self.config.get('ulysses_sequence_parallel_size', 1) - dp = world_size // self.ulysses_sequence_parallel_size - if self.ulysses_sequence_parallel_size > 1: - self.ulysses_device_mesh = init_device_mesh('cuda', - mesh_shape=(dp, self.ulysses_sequence_parallel_size), - mesh_dim_names=['dp', 'sp']) - - self.ulysses_sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) - - self.use_remove_padding = self.config.model.get('use_remove_padding', False) - - # normalize config - if self.config.micro_batch_size is not None: - self.config.micro_batch_size //= torch.distributed.get_world_size() - self.config.micro_batch_size_per_gpu = self.config.micro_batch_size - - def _build_model(self, config): - # the following line is necessary - from transformers import AutoModelForTokenClassification, AutoConfig - from torch.distributed.fsdp import FullyShardedDataParallel as FSDP, ShardingStrategy, CPUOffload - - # download the checkpoint from hdfs - local_path = copy_to_local(config.model.path) - - if self.config.model.input_tokenizer is None: - self._do_switch_chat_template = False - else: - self._do_switch_chat_template = True - input_tokenizer_local_path = copy_to_local(config.model.input_tokenizer) - self.input_tokenizer = hf_tokenizer(input_tokenizer_local_path, - trust_remote_code=config.model.get('trust_remote_code', False)) - self.tokenizer = hf_tokenizer(local_path, trust_remote_code=config.model.get('trust_remote_code', False)) - - trust_remote_code = config.model.get('trust_remote_code', False) - model_config = AutoConfig.from_pretrained(local_path, trust_remote_code=trust_remote_code) - model_config.num_labels = 1 - - # note that we have to create model in fp32. Otherwise, the optimizer is in bf16, which is incorrect - init_context = get_init_weight_context_manager(use_meta_tensor=not model_config.tie_word_embeddings, - mesh=self.device_mesh) - - with init_context(), warnings.catch_warnings(): - warnings.simplefilter("ignore") - setattr(model_config, 'classifier_dropout', 0.) - reward_module = AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path=local_path, - config=model_config, - torch_dtype=torch.bfloat16, - attn_implementation='flash_attention_2', - trust_remote_code=trust_remote_code) - - if config.model.get('use_remove_padding', False) or self.ulysses_sequence_parallel_size > 1: - from verl.models.transformers.monkey_patch import apply_monkey_patch - apply_monkey_patch(model=reward_module, ulysses_sp_size=self.ulysses_sequence_parallel_size) - - reward_module.to(torch.bfloat16) - - auto_wrap_policy = get_fsdp_wrap_policy(module=reward_module, config=self.config.model.fsdp_config) - - fsdp_mesh = self.device_mesh - sharding_strategy = get_sharding_strategy(fsdp_mesh) - - reward_module = FSDP( - reward_module, - param_init_fn=init_fn, - use_orig_params=False, - auto_wrap_policy=auto_wrap_policy, - device_id=torch.cuda.current_device(), - sharding_strategy=sharding_strategy, # zero3 - sync_module_states=True, - cpu_offload=CPUOffload(offload_params=True), - forward_prefetch=False, - device_mesh=self.device_mesh) - - return reward_module - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - # This is used to import external_lib into the huggingface systems - import_external_libs(self.config.model.get('external_lib', None)) - self.reward_module = self._build_model(config=self.config) - - def _forward_micro_batch(self, micro_batch): - from flash_attn.bert_padding import pad_input, unpad_input, index_first_axis, rearrange - from verl.utils.ulysses import ulysses_pad_and_slice_inputs, gather_outpus_and_unpad - - with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.bfloat16): - input_ids = micro_batch['input_ids'] - batch_size, seqlen = input_ids.shape - attention_mask = micro_batch['attention_mask'] - position_ids = micro_batch['position_ids'] - - if self.use_remove_padding: - input_ids_rmpad, indices, *_ = unpad_input(input_ids.unsqueeze(-1), - attention_mask) # input_ids_rmpad (total_nnz, ...) - input_ids_rmpad = input_ids_rmpad.transpose(0, 1) # (1, total_nnz) - - # unpad the position_ids to align the rotary - position_ids_rmpad = index_first_axis(rearrange(position_ids.unsqueeze(-1), "b s ... -> (b s) ..."), - indices).transpose(0, 1) - - # pad and slice the inputs if sp > 1 - if self.ulysses_sequence_parallel_size > 1: - input_ids_rmpad, position_ids_rmpad, pad_size = ulysses_pad_and_slice_inputs(input_ids_rmpad, \ - position_ids_rmpad, \ - sp_size=self.ulysses_sequence_parallel_size) - - # only pass input_ids and position_ids to enable flash_attn_varlen - output = self.reward_module(input_ids=input_ids_rmpad, - attention_mask=None, - position_ids=position_ids_rmpad, - use_cache=False) # prevent model thinks we are generating - reward_rmpad = output.logits - reward_rmpad = reward_rmpad.squeeze(0) # (total_nnz) - - # gather output if sp > 1 - if self.ulysses_sequence_parallel_size > 1: - reward_rmpad = gather_outpus_and_unpad(reward_rmpad, - gather_dim=0, - unpad_dim=0, - padding_size=pad_size) - - # pad it back - rm_score = pad_input(reward_rmpad, indices=indices, batch=batch_size, seqlen=seqlen).squeeze(-1) - else: - output = self.reward_module(input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - use_cache=False) - rm_score = output.logits # (batch_size, seq_len, 1) - rm_score = rm_score.squeeze(-1) - - # extract the result of the last valid token - eos_mask_idx = torch.argmax(position_ids * attention_mask, dim=-1) # (bsz,) - rm_score = rm_score[torch.arange(batch_size), eos_mask_idx] - return rm_score - - def _expand_to_token_level(self, data: DataProto, scores: torch.Tensor): - batch_size = data.batch.batch_size[0] - # expand as token_level_reward - attention_mask = data.batch['attention_mask'] - position_ids = data.batch['position_ids'] - response_length = data.batch['responses'].shape[-1] - eos_mask_idx = torch.argmax(position_ids * attention_mask, dim=-1) # (bsz,) - token_level_scores = torch.zeros_like(attention_mask, dtype=scores.dtype) # (bsz, seqlen) - token_level_scores[torch.arange(batch_size), eos_mask_idx] = scores - - # select the response part - token_level_scores = token_level_scores[:, -response_length:] - - return token_level_scores - - def _switch_chat_template(self, data: DataProto): - src_max_length = data.batch['attention_mask'].shape[-1] - - src_tokenizer = self.input_tokenizer - target_tokenizer = self.tokenizer - - rm_input_ids = [] - rm_attention_mask = [] - - for i in range(data.batch.batch_size[0]): - # extract raw prompt - chat: list = data.non_tensor_batch['raw_prompt'][i].tolist() - - # extract response - response_ids = data.batch['responses'][i] - response_length = response_ids.shape[-1] - valid_response_length = data.batch['attention_mask'][i][-response_length:].sum() - valid_response_ids = response_ids[:valid_response_length] - - # decode - response = src_tokenizer.decode(valid_response_ids) - # remove bos and eos - response = response.replace(src_tokenizer.eos_token, '') - - chat.append({'role': 'assistant', 'content': response}) - - prompt_with_chat_template = target_tokenizer.apply_chat_template(chat, - add_generation_prompt=False, - tokenize=False) - if self.rank == 0 and i == 0: - # for debugging purpose - print(f'Switch template. chat: {prompt_with_chat_template}') - - # the maximum length is actually determined by the reward model itself - max_length = self.config.get('max_length', src_max_length) - if max_length is None: - max_length = src_max_length - input_ids, attention_mask = verl_F.tokenize_and_postprocess_data( - prompt=prompt_with_chat_template, - tokenizer=target_tokenizer, - max_length=max_length, - pad_token_id=target_tokenizer.pad_token_id, - left_pad=False, # right padding - truncation=self.config.get('truncation', 'right')) # truncate from the right - - rm_input_ids.append(input_ids) - rm_attention_mask.append(attention_mask) - - rm_input_ids = torch.cat(rm_input_ids, dim=0) - rm_attention_mask = torch.cat(rm_attention_mask, dim=0) - - rm_position_ids = compute_position_id_with_mask(rm_attention_mask) - - rm_inputs = {'input_ids': rm_input_ids, 'attention_mask': rm_attention_mask, 'position_ids': rm_position_ids} - - return DataProto.from_dict(rm_inputs) - - @register(dispatch_mode=Dispatch.DP_COMPUTE_PROTO) - def compute_rm_score(self, data: DataProto): - import itertools - from verl.utils.seqlen_balancing import rearrange_micro_batches, get_reverse_idx - # Support all hardwares - data = data.to(torch.cuda.current_device()) - if self._do_switch_chat_template: - rm_data = self._switch_chat_template(data) - - # Support all hardwares - rm_data.batch = rm_data.batch.to(torch.cuda.current_device()) - - # perform forward computation - with self.ulysses_sharding_manager: - rm_data = self.ulysses_sharding_manager.preprocess_data(data=rm_data) - data = self.ulysses_sharding_manager.preprocess_data(data=data) - - use_dynamic_bsz = self.config.use_dynamic_bsz - if use_dynamic_bsz: - max_token_len = self.config.forward_max_token_len_per_gpu * self.ulysses_sequence_parallel_size - micro_batches, indices = rearrange_micro_batches(batch=rm_data.batch, max_token_len=max_token_len) - else: - micro_batches = rm_data.batch.split(self.config.micro_batch_size_per_gpu) - output = [] - for micro_batch in micro_batches: - rm_score = self._forward_micro_batch(micro_batch) - output.append(rm_score) - scores = torch.cat(output, dim=0) # (batch_size) - - if use_dynamic_bsz: - indices = list(itertools.chain.from_iterable(indices)) - assert len(indices) == scores.size(0), f"{len(indices)} vs. {scores.size()}" - revert_indices = torch.tensor(get_reverse_idx(indices), dtype=torch.long) - scores = scores[revert_indices] - - token_level_scores = self._expand_to_token_level(data, scores) - # Note that this is only the scores, may not be the final rewards used to train RL - output = DataProto.from_dict(tensors={'rm_scores': token_level_scores}) - output = self.ulysses_sharding_manager.postprocess_data(data=output) - - # https://pytorch.org/docs/stable/notes/fsdp.html#fsdp-notes - # unshard the root FSDP module - self.reward_module._handle.reshard(True) - - output = output.to('cpu') - return output diff --git a/verl/workers/megatron_workers.py b/verl/workers/megatron_workers.py deleted file mode 100644 index 04f562ecf8..0000000000 --- a/verl/workers/megatron_workers.py +++ /dev/null @@ -1,832 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The main entry point to run the PPO algorithm -""" - -import os -import logging -import ray -import torch -import torch.distributed -import torch.nn as nn -from omegaconf import DictConfig - -from verl.single_controller.base.megatron.worker import MegatronWorker -from verl.workers.actor.megatron_actor import MegatronPPOActor -from verl.workers.critic.megatron_critic import MegatronPPOCritic -from verl.workers.sharding_manager import AllGatherPPModel -from verl.workers.reward_model.megatron.reward_model import MegatronRewardModel - -from verl.single_controller.base.decorator import register, Dispatch -from verl import DataProto -from verl.utils.fs import copy_to_local -from verl.utils.debug import log_gpu_memory_usage -from verl.utils.model import load_megatron_model_weights, load_megatron_gptmodel_weights -from verl.utils.flops_counter import FlopsCounter -from verl.utils.checkpoint.megatron_checkpoint_manager import MegatronCheckpointManager -from verl.utils.megatron_utils import init_model_parallel_config -from verl.utils.megatron_utils import offload_megatron_param_and_grad, load_megatron_param_and_grad -from verl.utils import hf_tokenizer - -from codetiming import Timer - -from megatron.core import parallel_state as mpu -from megatron.core import ModelParallelConfig - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_PPO_LOGGING_LEVEL', 'WARN')) - - -def set_random_seed(seed): - import torch - import numpy as np - import random - torch.manual_seed(seed) - np.random.seed(seed) - random.seed(seed) - if torch.cuda.device_count() > 0: - from megatron.core import tensor_parallel - tensor_parallel.model_parallel_cuda_manual_seed(seed) - # FIXME: torch cumsum not support deterministic (used in vllm sampler), - # https://github.com/pytorch/pytorch/issues/89492 - # torch.use_deterministic_algorithms(True, warn_only=True) - # os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8' - - -class ActorRolloutRefWorker(MegatronWorker): - """ - This worker can be instantiated as a standalone actor or a standalone rollout or a standalone reference policy - or a hybrid engine based on the config.rollout - """ - - def __init__(self, config: DictConfig, role: str): - super().__init__() - self.config = config - - # NOTE(sgm): We utilize colocate WorkerGroup by default. - # As a result, Workers for different model share the same process. - # Therefore, we only require one distribute initialization. - # To utilize different parallel startegy in different models: - # 1, users should disable WorkerDict; 2.assign different ResourcePool to different models, - # 3. and apply the following patch in ray==2.10, https://github.com/ray-project/ray/pull/44385 - if not torch.distributed.is_initialized(): - rank = int(os.environ['LOCAL_RANK']) - torch.distributed.init_process_group(backend="nccl") - torch.cuda.set_device(rank) - - if self.config.actor.megatron.sequence_parallel: - os.environ['CUDA_DEVICE_MAX_CONNECTIONS'] = '1' - mpu.initialize_model_parallel( - tensor_model_parallel_size=self.config.actor.megatron.tensor_model_parallel_size, - pipeline_model_parallel_size=self.config.actor.megatron.pipeline_model_parallel_size, - virtual_pipeline_model_parallel_size=self.config.actor.megatron.virtual_pipeline_model_parallel_size, - pipeline_model_parallel_split_rank=None, - use_sharp=False, - context_parallel_size=1, - expert_model_parallel_size=1, - nccl_communicator_config_path=None, - ) - - set_random_seed(seed=self.config.actor.megatron.seed) - - self.role = role - assert self.role in ['actor', 'rollout', 'ref', 'actor_rollout', 'actor_rollout_ref'] - - self._is_actor = self.role in ['actor', 'actor_rollout', 'actor_rollout_ref'] - self._is_rollout = self.role in ['rollout', 'actor_rollout', 'actor_rollout_ref'] - self._is_ref = self.role in ['ref', 'actor_rollout_ref'] - - # TODO(sgm): Currently, we only support reference model param offload - # will support other offload later - self._is_offload_param = False - self._is_offload_grad = False - self._is_offload_optimizer = False - - # normalize config - if self._is_actor and self._is_rollout: - self.config.actor.ppo_mini_batch_size *= self.config.rollout.n - self.config.actor.ppo_mini_batch_size //= mpu.get_data_parallel_world_size() - if self.config.actor.get('ppo_micro_batch_size', None): - self.config.actor.ppo_micro_batch_size //= mpu.get_data_parallel_world_size() - self.config.rollout.log_prob_micro_batch_size //= mpu.get_data_parallel_world_size() - self.config.actor.ppo_micro_batch_size_per_gpu = self.config.actor.ppo_micro_batch_size - self.config.rollout.log_prob_micro_batch_size_per_gpu = self.config.rollout.log_prob_micro_batch_size - - self._is_offload_param = self.config.actor.get('param_offload', False) - self._is_offload_grad = self.config.actor.get('grad_offload', False) - self._is_offload_optimizer = self.config.actor.get('optimizer_offload', False) - elif self._is_ref: - if self.config.ref.get('ppo_micro_batch_size', None): - self.config.ref.log_prob_micro_batch_size //= mpu.get_data_parallel_world_size() - self.config.ref.ppo_micro_batch_size_per_gpu = self.config.ref.ppo_micro_batch_size - self._is_offload_param = self.config.ref.get('param_offload', False) - - def _build_model_optimizer(self, - model_path, - megatron_config: ModelParallelConfig, - optim_config, - override_model_config, - enable_gradient_checkpointing=False): - from verl.utils.megatron.optimizer import get_megatron_optimizer - from megatron.core.models.gpt.gpt_model import ModelType - from verl.utils.model import print_model_size, update_model_config, get_generation_config - from verl.utils.megatron_utils import get_model, init_megatron_optim_config, convert_config - from transformers import AutoConfig - - # Step 1: initialize the tokenizer - local_path = copy_to_local(model_path) - self.tokenizer = hf_tokenizer(local_path) - - # Step 2: get the actor_model_config - actor_model_config = AutoConfig.from_pretrained(local_path, - trust_remote_code=self.config.model.get( - 'trust_remote_code', False)) - - self.generation_config = get_generation_config(local_path, - trust_remote_code=self.config.model.get( - 'trust_remote_code', False)) - - override_config_kwargs = { - 'bos_token_id': self.tokenizer.bos_token_id, - 'eos_token_id': self.tokenizer.eos_token_id, - 'pad_token_id': self.tokenizer.pad_token_id, - } - override_config_kwargs.update(override_model_config) - update_model_config(actor_model_config, override_config_kwargs=override_config_kwargs) - - if self.rank == 0: - print(f'Model config after override: {actor_model_config}') - - self.share_embeddings_and_output_weights = getattr(actor_model_config, "tie_word_embeddings", False) - self.architectures = getattr(actor_model_config, "architectures", None) - - tfconfig = convert_config(actor_model_config, megatron_config) - self.hf_config = actor_model_config - print(f'TF config: {tfconfig}') - self.hf_config = actor_model_config - - def megatron_actor_model_provider(pre_process, post_process): - from verl.utils.model import get_parallel_gptmodel_from_config - parallel_model = get_parallel_gptmodel_from_config( - tfconfig, - actor_model_config, - pre_process, - post_process, - share_embeddings_and_output_weights=self.share_embeddings_and_output_weights, - value=False) - parallel_model.cuda() - return parallel_model - - # Step 3: initialize the megatron model - if self._is_actor and self._is_rollout: - # Initialize the 3D HybridEngine - hybrid_engine = AllGatherPPModel( - model_provider=megatron_actor_model_provider, - use_distributed_optimizer=self.config.actor.megatron.use_distributed_optimizer) - # Fetch the model at current rank - actor_module = hybrid_engine.this_rank_models - actor_modules_list = [] - if isinstance(actor_module, nn.ModuleList): - for module in actor_module: - actor_modules_list.append(module) - actor_module = actor_modules_list - print(f'actor_module: {len(actor_module)}') - if self.config.actor.load_weight: - load_megatron_gptmodel_weights(self.config, - actor_model_config, - actor_module, - params_dtype=megatron_config.params_dtype, - is_value_model=False) - - if self.rank == 0: - print_model_size(actor_module[0]) - log_gpu_memory_usage('After AllGatherPPModel init', logger=logger) - elif self._is_ref: - print(f'self.config.ref.load_weight: {self.config.ref.load_weight}') - ref_module = get_model(model_provider_func=megatron_actor_model_provider, - model_type=ModelType.encoder_or_decoder, - wrap_with_ddp=False, - use_distributed_optimizer=self.config.ref.megatron.use_distributed_optimizer) - # ref_module = nn.ModuleList(ref_module) - - if self.config.ref.load_weight: # should align with the actor: - assert self.config.actor.load_weight == self.config.ref.load_weight - print(f'load ref weight start') - load_megatron_gptmodel_weights(self.config, - actor_model_config, - ref_module, - params_dtype=megatron_config.params_dtype, - is_value_model=False) - log_gpu_memory_usage('After ref module init', logger=logger) - return ref_module, actor_model_config - - # TODO: add more optimizer args into config - if self._is_actor: - optim_config = init_megatron_optim_config(optim_config) - actor_optimizer = get_megatron_optimizer(model=actor_module, config=optim_config) - else: - optim_config = None - actor_optimizer = None - - log_gpu_memory_usage('After actor optimizer init', logger=logger) - - return actor_module, hybrid_engine, actor_optimizer, actor_model_config, optim_config - - def _build_rollout(self): - if self.config.rollout.name == 'vllm': - from verl.workers.rollout.vllm_rollout import vLLMRollout, vllm_mode - from verl.workers.sharding_manager import MegatronVLLMShardingManager - from verl.utils.model import normalize_pp_vpp_params - - # NOTE(sgm): If the QKV and gate_up projection layer are concate together in actor, - # we will reorganize their weight format when resharding from actor to rollout. - layer_name_mapping = { - "qkv_layer_name": "self_attention.linear_qkv.", - "gate_proj_layer_name": "linear_fc1.weight", - } - - # reshard the weight partition from actor to rollout to initialize the rollout class - # create a new cuda space for parameters not in this pp rank - self.hybrid_engine.load_params_to_cuda() - # broadcast the parameters from pp rank to other ranks - self.hybrid_engine.allgather_params() - # obtain name to parameters in pp/vpp - params = self.hybrid_engine.get_all_params() - # update the param name for the - params = normalize_pp_vpp_params(params=params, - num_hidden_layers=self.actor_model_config.num_hidden_layers, - layer_name='layers') - assert vllm_mode == 'customized', "Support for vllm>=0.7 for Megatron-LM backend has not been implemented yet." - rollout = vLLMRollout(actor_module=params, - config=self.config.rollout, - tokenizer=self.tokenizer, - model_hf_config=self.actor_model_config, - train_tp=mpu.get_tensor_model_parallel_world_size()) - log_gpu_memory_usage('After building vllm rollout', logger=logger) - - # perform weight resharding between actor and rollout - sharding_manager = MegatronVLLMShardingManager(module=self.hybrid_engine, - inference_engine=rollout.inference_engine, - model_config=self.actor_model_config, - layer_name_mapping=layer_name_mapping) - log_gpu_memory_usage('After building sharding manager', logger=logger) - else: - raise NotImplementedError('Only vllmRollout is supported with Megatron now') - - return rollout, sharding_manager - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - if self.config.model.get('external_lib', None) is not None: - # This is used to import external_lib into the huggingface systems - import importlib - importlib.import_module(self.config.model.external_lib) - - from omegaconf import OmegaConf - from verl.utils.torch_dtypes import PrecisionType - override_model_config = OmegaConf.to_container(self.config.model.get('override_config', OmegaConf.create())) - self.param_dtype = torch.bfloat16 - - megatron_config = OmegaConf.create({ - 'sequence_parallel': self.config.actor.megatron.get('sequence_parallel', True), - 'param_dtype': PrecisionType.to_str(self.param_dtype), - 'tensor_model_parallel_size': mpu.get_tensor_model_parallel_world_size(), - 'pipeline_model_parallel_rank': mpu.get_pipeline_model_parallel_rank(), - 'pipeline_model_parallel_size': mpu.get_pipeline_model_parallel_world_size(), - 'virtual_pipeline_model_parallel_rank': mpu.get_virtual_pipeline_model_parallel_rank(), - 'virtual_pipeline_model_parallel_size': mpu.get_virtual_pipeline_model_parallel_world_size() - }) - - megatron_config = init_model_parallel_config(megatron_config) - - if self._is_actor or self._is_rollout: - # we need the model for actor and rollout - if self._is_actor: - optim_config = self.config.actor.optim - else: - optim_config = None - self.actor_module, self.hybrid_engine, self.actor_optimizer, \ - self.actor_model_config, self.actor_optim_config = self._build_model_optimizer( - model_path=self.config.model.path, - megatron_config=megatron_config, - optim_config=optim_config, - override_model_config=override_model_config, - ) - - if self._is_actor: - self.actor = MegatronPPOActor(config=self.config.actor, - model_config=self.actor_model_config, - megatron_config=megatron_config, - actor_module=self.actor_module, - actor_optimizer=self.actor_optimizer, - actor_optimizer_config=self.actor_optim_config) - - if self._is_rollout: - self.rollout, self.sharding_manager = self._build_rollout() - - if self._is_ref: - self.ref_module, self.ref_model_config = self._build_model_optimizer( - model_path=self.config.model.path, - megatron_config=megatron_config, - optim_config=None, - override_model_config=override_model_config, - ) - self.ref_policy = MegatronPPOActor(config=self.config.ref, - model_config=self.ref_model_config, - megatron_config=megatron_config, - actor_module=self.ref_module, - actor_optimizer=None, - actor_optimizer_config=None) - - if self._is_actor: - self.flops_counter = FlopsCounter(self.actor_model_config) - self.checkpoint_mananager = MegatronCheckpointManager( - config=self.config, - model_config=self.actor_model_config, - role='actor', - model=self.actor_module, - arch=self.architectures[0], - hf_config=self.hf_config, - param_dtype=self.param_dtype, - share_embeddings_and_output_weights=self.share_embeddings_and_output_weights, - tokenizer=self.tokenizer, - optimizer=self.actor_optimizer, - use_distributed_optimizer=self.config.actor.megatron.use_distributed_optimizer, - checkpoint_contents=self.config.actor.checkpoint.contents) - - torch.cuda.empty_cache() - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def update_actor(self, data: DataProto): - assert self._is_actor - - data.batch = data.batch.cuda() - - log_gpu_memory_usage('Before update policy', logger=logger) - - dataloader = self.actor.make_minibatch_iterator(data=data) - with Timer(name='update_policy', logger=None) as timer: - metrics = self.actor.update_policy(dataloader=dataloader) - delta_time = timer.last - global_num_tokens = data.meta_info['global_token_num'] - estimated_flops, promised_flops = self.flops_counter.estimate_flops(global_num_tokens, delta_time) - metrics['perf/mfu/actor'] = estimated_flops * self.config.actor.ppo_epochs / promised_flops / self.world_size - - log_gpu_memory_usage('After update policy', logger=logger) - - # TODO: here, we should return all metrics - output = DataProto(meta_info={'metrics': metrics}) - output = output.to('cpu') - torch.cuda.empty_cache() - return output - - @register(dispatch_mode=Dispatch.MEGATRON_PP_AS_DP_PROTO) - def generate_sequences(self, prompts: DataProto): - assert self._is_rollout - - prompts.batch = prompts.batch.cuda() - meta_info = { - 'eos_token_id': - self.generation_config.eos_token_id - if self.generation_config is not None else self.tokenizer.eos_token_id, - 'pad_token_id': - self.generation_config.pad_token_id - if self.generation_config is not None else self.tokenizer.pad_token_id, - } - prompts.meta_info.update(meta_info) - with self.sharding_manager: - log_gpu_memory_usage('After entering sharding manager', logger=logger) - - prompts = self.sharding_manager.preprocess_data(prompts) - output = self.rollout.generate_sequences(prompts=prompts) - - log_gpu_memory_usage('After rollout generation', logger=logger) - - output = self.sharding_manager.postprocess_data(output) - - output = output.to('cpu') - # clear kv cache - torch.cuda.empty_cache() - log_gpu_memory_usage('After generate_sequences', logger=logger) - return output - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def compute_ref_log_prob(self, data: DataProto): - data = data.to('cuda') - - assert self._is_ref - if self._is_offload_param: - load_megatron_param_and_grad(self.ref_module, torch.cuda.current_device(), self._is_offload_grad) - - micro_batch_size = self.config.rollout.log_prob_micro_batch_size_per_gpu - data.meta_info['micro_batch_size'] = micro_batch_size - data.meta_info['temperature'] = self.config.rollout.temperature - output = self.ref_policy.compute_log_prob(data=data) - output = DataProto.from_dict(tensors={'ref_log_prob': output}) - output = output.to('cpu') - if self._is_offload_param: - offload_megatron_param_and_grad(self.ref_module, self._is_offload_grad) - torch.cuda.empty_cache() - return output - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def compute_log_prob(self, data: DataProto): - assert self._is_actor - data = data.to('cuda') - output = data - # we should always recompute old_log_probs when it is HybridEngine - output.meta_info['micro_batch_size'] = self.config.rollout.log_prob_micro_batch_size_per_gpu - output.meta_info['temperature'] = self.config.rollout.temperature - old_log_probs = self.actor.compute_log_prob(data=output) - output.batch['old_log_probs'] = old_log_probs - output = output.to('cpu') - # clear kv cache - torch.cuda.empty_cache() - log_gpu_memory_usage('After generate_sequences', logger=logger) - return output - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def load_checkpoint(self, checkpoint_path, hdfs_path=None, del_local_after_load=True): - self.checkpoint_mananager.load_checkpoint(local_path=checkpoint_path, - hdfs_path=hdfs_path, - del_local_after_load=del_local_after_load) - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def load_pretrained_model(self, checkpoint_path, del_local_after_load=True): - pass - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def save_checkpoint(self, checkpoint_path, hdfs_path=None, global_step=0, max_ckpt_to_keep=None): - self.checkpoint_mananager.save_checkpoint(local_path=checkpoint_path, - hdfs_path=hdfs_path, - global_step=global_step, - max_ckpt_to_keep=max_ckpt_to_keep) - - -class CriticWorker(MegatronWorker): - - def __init__(self, config): - super().__init__() - self.config = config - - # NOTE(sgm): We utilize colocate WorkerGroup by default. - # As a result, Workers for different model share the same process. - # Therefore, we only require one distribute initialization. - # To utilize different parallel startegy in different models: - # 1, users should disable WorkerDict; 2.assign different ResourcePool to different models, - # 3. and apply the following patch in ray==2.10, https://github.com/ray-project/ray/pull/44385 - if not torch.distributed.is_initialized(): - rank = int(os.environ['LOCAL_RANK']) - torch.distributed.init_process_group(backend="nccl") - torch.cuda.set_device(rank) - - if self.config.megatron.sequence_parallel: - os.environ['CUDA_DEVICE_MAX_CONNECTIONS'] = '1' - mpu.initialize_model_parallel( - tensor_model_parallel_size=self.config.megatron.tensor_model_parallel_size, - pipeline_model_parallel_size=self.config.megatron.pipeline_model_parallel_size, - virtual_pipeline_model_parallel_size=self.config.megatron.virtual_pipeline_model_parallel_size, - pipeline_model_parallel_split_rank=None, - use_sharp=False, - context_parallel_size=1, - expert_model_parallel_size=1, - nccl_communicator_config_path=None, - ) - - set_random_seed(seed=self.config.megatron.seed) - - # normalize config - self.config.ppo_mini_batch_size *= self.config.rollout_n - self.config.ppo_mini_batch_size //= mpu.get_data_parallel_world_size() - if self.config.get('ppo_micro_batch_size', None): - self.config.ppo_micro_batch_size //= mpu.get_data_parallel_world_size() - self.config.ppo_micro_batch_size_per_gpu = self.config.ppo_micro_batch_size - - # TODO(sgm): support critic model offload - - def _build_critic_model_optimizer(self, - model_path, - megatron_config: ModelParallelConfig, - optim_config, - override_model_config, - enable_gradient_checkpointing=False): - from megatron.core.models.gpt.gpt_model import ModelType - from verl.utils.model import print_model_size, update_model_config - from verl.utils.megatron.optimizer import get_megatron_optimizer - from verl.utils.megatron_utils import get_model, init_megatron_optim_config, init_model_parallel_config, convert_config - from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig - - # Step 1: initialize the tokenizer - local_path = copy_to_local(model_path) - self.tokenizer = hf_tokenizer(local_path) - - # Step 2: get the actor_model_config - critic_model_config = AutoConfig.from_pretrained(local_path, - trust_remote_code=self.config.model.get( - 'trust_remote_code', False)) - - override_config_kwargs = { - 'bos_token_id': self.tokenizer.bos_token_id, - 'eos_token_id': self.tokenizer.eos_token_id, - 'pad_token_id': self.tokenizer.pad_token_id, - } - override_config_kwargs.update(override_model_config) - self.share_embeddings_and_output_weights = getattr(critic_model_config, "tie_word_embeddings", False) - update_model_config(critic_model_config, override_config_kwargs=override_config_kwargs) - self.architectures = getattr(critic_model_config, "architectures", None) - if self.rank == 0: - print(f'Model config after override: {critic_model_config}') - tfconfig = convert_config(critic_model_config, megatron_config) - self.hf_config = critic_model_config - print(f'TF config: {tfconfig}') - - def megatron_critic_model_provider(pre_process, post_process): - from verl.utils.model import get_parallel_gptmodel_from_config - parallel_model = get_parallel_gptmodel_from_config(tfconfig, - critic_model_config, - pre_process, - post_process, - share_embeddings_and_output_weights=False, - value=True) - parallel_model.cuda() - return parallel_model - - # Step 3: initialize the megatron model - critic_module = get_model(model_provider_func=megatron_critic_model_provider, - model_type=ModelType.encoder_or_decoder, - wrap_with_ddp=True, - use_distributed_optimizer=self.config.megatron.use_distributed_optimizer) - # note that here critic_module will be a list to be compatible with the construction of interleaved pp (vpp). - # but here, we do not use pp (vpp) yet. For simplicity, we remove the list - # critic_module = nn.ModuleList(critic_module) - - if self.config.load_weight: - load_megatron_gptmodel_weights(self.config, - critic_model_config, - critic_module, - params_dtype=megatron_config.params_dtype, - is_value_model=True) - if self.rank == 0: - print_model_size(critic_module[0]) - - # TODO: add more optimizer args into config - optim_config = init_megatron_optim_config(optim_config) - critic_optimizer = get_megatron_optimizer(model=critic_module, config=optim_config) - torch.cuda.empty_cache() - return critic_module, critic_optimizer, critic_model_config, optim_config - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - # create critic - from omegaconf import OmegaConf - from verl.utils.torch_dtypes import PrecisionType - - if self.config.model.get('external_lib', None) is not None: - # This is used to import external_lib into the huggingface systems - import importlib - importlib.import_module(self.config.model.external_lib) - override_model_config = OmegaConf.to_container(self.config.model.get('override_config', OmegaConf.create())) - self.param_dtype = torch.bfloat16 - - megatron_config = OmegaConf.create({ - 'sequence_parallel': self.config.megatron.get('sequence_parallel', True), - 'param_dtype': PrecisionType.to_str(self.param_dtype), - 'tensor_model_parallel_size': mpu.get_tensor_model_parallel_world_size(), - 'pipeline_model_parallel_rank': mpu.get_pipeline_model_parallel_rank(), - 'pipeline_model_parallel_size': mpu.get_pipeline_model_parallel_world_size(), - 'virtual_pipeline_model_parallel_rank': mpu.get_virtual_pipeline_model_parallel_rank(), - 'virtual_pipeline_model_parallel_size': mpu.get_virtual_pipeline_model_parallel_world_size() - }) - - megatron_config = init_model_parallel_config(megatron_config) - self.critic_module, self.critic_optimizer, self.critic_model_config, critic_optimizer_config = self._build_critic_model_optimizer( - model_path=self.config.model.path, - megatron_config=megatron_config, - optim_config=self.config.optim, - override_model_config=override_model_config) - self.critic = MegatronPPOCritic(config=self.config, - model_config=self.critic_model_config, - megatron_config=megatron_config, - critic_module=self.critic_module, - critic_optimizer=self.critic_optimizer, - critic_optimizer_config=critic_optimizer_config) - self.flops_counter = FlopsCounter(self.critic_model_config) - self.checkpoint_mananager = MegatronCheckpointManager( - config=self.config, - model_config=self.critic_model_config, - role='critic', - model=self.critic_module, - arch=self.architectures[0], - hf_config=self.hf_config, - param_dtype=self.param_dtype, - share_embeddings_and_output_weights=False, - tokenizer=self.tokenizer, - optimizer=self.critic_optimizer, - use_distributed_optimizer=self.config.megatron.use_distributed_optimizer, - checkpoint_contents=self.config.checkpoint.contents) - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def compute_values(self, data: DataProto): - data = data.to('cuda') - values = self.critic.compute_values(data=data) - output = DataProto.from_dict(tensors={'values': values}) - output = output.to('cpu') - return output - - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def update_critic(self, data: DataProto): - data = data.to('cuda') - dataloader = self.critic.make_minibatch_iterator(data) - with Timer(name='update_critic', logger=None) as timer: - metrics = self.critic.update_critic(dataloader=dataloader) - delta_time = timer.last - global_num_tokens = data.meta_info['global_token_num'] - estimated_flops, promised_flops = self.flops_counter.estimate_flops(global_num_tokens, delta_time) - metrics['perf/mfu/critic'] = estimated_flops * self.config.ppo_epochs / promised_flops / self.world_size - output = DataProto(batch=None, meta_info={'metrics': metrics}) - output = output.to('cpu') - return output - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def load_checkpoint(self, checkpoint_path, hdfs_path=None, del_local_after_load=True): - self.checkpoint_mananager.load_checkpoint(local_path=checkpoint_path, - hdfs_path=hdfs_path, - del_local_after_load=del_local_after_load) - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def save_checkpoint(self, checkpoint_path, hdfs_path=None, global_steps=0, max_ckpt_to_keep=None): - self.checkpoint_mananager.save_checkpoint(local_path=checkpoint_path, - hdfs_path=hdfs_path, - global_step=global_steps, - max_ckpt_to_keep=max_ckpt_to_keep) - - -class RewardModelWorker(MegatronWorker): - """ - Note that we only implement the reward model that is subclass of AutoModelForSequenceClassification. - """ - - def __init__(self, config): - super().__init__() - self.config = config - - # NOTE(sgm): We utilize colocate WorkerGroup by default. - # As a result, Workers for different model share the same process. - # Therefore, we only require one distribute initialization. - # To utilize different parallel startegy in different models: - # 1, users should disable WorkerDict; 2.assign different ResourcePool to different models, - # 3. and apply the following patch in ray==2.10, https://github.com/ray-project/ray/pull/44385 - if not torch.distributed.is_initialized(): - rank = int(os.environ['LOCAL_RANK']) - torch.distributed.init_process_group(backend="nccl") - torch.cuda.set_device(rank) - - if self.config.megatron.sequence_parallel: - os.environ['CUDA_DEVICE_MAX_CONNECTIONS'] = '1' - mpu.initialize_model_parallel( - tensor_model_parallel_size=self.config.megatron.tensor_model_parallel_size, - pipeline_model_parallel_size=self.config.megatron.pipeline_model_parallel_size, - virtual_pipeline_model_parallel_size=self.config.megatron.virtual_pipeline_model_parallel_size, - pipeline_model_parallel_split_rank=None, - use_sharp=False, - context_parallel_size=1, - expert_model_parallel_size=1, - nccl_communicator_config_path=None, - ) - - set_random_seed(seed=self.config.megatron.seed) - - # normalize config - if self.config.micro_batch_size is not None: - self.config.micro_batch_size //= mpu.get_data_parallel_world_size() - self.config.micro_batch_size_per_gpu = self.config.micro_batch_size - - def _build_rm_model(self, model_path, megatron_config: ModelParallelConfig, override_model_config): - from megatron.core.models.gpt.gpt_model import ModelType - from verl.utils.model import update_model_config - from verl.utils.megatron_utils import get_model - from transformers import AutoConfig - - # Step 1: initialize the tokenizer - local_path = copy_to_local(model_path) - self.tokenizer = hf_tokenizer(local_path) - - # Step 2: get the actor_model_config - rm_model_config = AutoConfig.from_pretrained(local_path, - trust_remote_code=self.config.model.get( - 'trust_remote_code', False)) - - override_config_kwargs = { - 'bos_token_id': self.tokenizer.bos_token_id, - 'eos_token_id': self.tokenizer.eos_token_id, - 'pad_token_id': self.tokenizer.pad_token_id, - } - override_config_kwargs.update(override_model_config) - update_model_config(rm_model_config, override_config_kwargs=override_config_kwargs) - - if self.rank == 0: - print(f'Model config after override: rm_model_config {rm_model_config}') - - def megatron_rm_model_provider(pre_process, post_process): - from verl.utils.model import get_parallel_model_from_config - # vpp is not supported yet because it will hang for some reason. Need debugging - vpp_rank = mpu.get_virtual_pipeline_model_parallel_rank() # this will be set inside get_model - # this_megatron_config = copy.deepcopy(megatron_config) - # this_megatron_config.virtual_pipeline_model_parallel_rank = vpp_rank - parallel_model = get_parallel_model_from_config(config=rm_model_config, - megatron_config=megatron_config, - pre_process=pre_process, - post_process=post_process, - share_embeddings_and_output_weights=False, - value=True) - parallel_model.cuda() - return parallel_model - - # Step 3: initialize the megatron model - reward_model = get_model(model_provider_func=megatron_rm_model_provider, - model_type=ModelType.encoder_or_decoder, - wrap_with_ddp=False, - use_distributed_optimizer=self.config.reward_model.use_distributed_optimizer) - # note that here critic_module will be a list to be compatible with the construction of interleaved pp (vpp). - # but here, we do not use pp (vpp) yet. For simplicity, we remove the list - # reward_model = nn.ModuleList(reward_model) - - if self.config.load_weight: - load_megatron_model_weights(self.config, - rm_model_config, - reward_model, - params_dtype=megatron_config.params_dtype, - is_value_model=True) - - # TODO: add more optimizer args into config - torch.cuda.empty_cache() - return reward_model, rm_model_config - - @register(dispatch_mode=Dispatch.ONE_TO_ALL) - def init_model(self): - # create critic - from omegaconf import OmegaConf - from verl.utils.torch_dtypes import PrecisionType - - if self.config.model.get('external_lib', None) is not None: - # This is used to import external_lib into the huggingface systems - import importlib - importlib.import_module(self.config.model.external_lib) - override_model_config = OmegaConf.to_container(self.config.model.get('override_config', OmegaConf.create())) - - sft_tokenizer_local_path = copy_to_local(self.config.model.input_tokenizer) - sft_tokenizer = hf_tokenizer(sft_tokenizer_local_path) - rm_tokenizer_path = self.config.model.get('rm_tokenizer', None) - rm_tokenizer = None - if rm_tokenizer_path is not None: - rm_tokenizer_local_path = copy_to_local(rm_tokenizer_path) - rm_tokenizer = hf_tokenizer(rm_tokenizer_local_path) - - self.param_dtype = torch.bfloat16 - - megatron_config = OmegaConf.create({ - 'sequence_parallel': self.config.megatron.get('sequence_parallel', True), - 'param_dtype': PrecisionType.to_str(self.param_dtype), - 'tensor_model_parallel_size': mpu.get_tensor_model_parallel_world_size(), - 'pipeline_model_parallel_rank': mpu.get_pipeline_model_parallel_rank(), - 'pipeline_model_parallel_size': mpu.get_pipeline_model_parallel_world_size(), - 'virtual_pipeline_model_parallel_rank': mpu.get_virtual_pipeline_model_parallel_rank(), - 'virtual_pipeline_model_parallel_size': mpu.get_virtual_pipeline_model_parallel_world_size() - }) - - megatron_config = init_model_parallel_config(megatron_config) - - reward_model_module, reward_model_config = self._build_rm_model( - model_path=self.config.model.path, - megatron_config=megatron_config, - override_model_config=override_model_config, - ) - # FIXME(sgm): reward model param offload is implemented in MegatronRewardModel - # should be implemented in workers - self.rm = MegatronRewardModel(config=self.config, - reward_model_module=reward_model_module, - model_config=reward_model_config, - megatron_config=megatron_config, - sft_tokenizer=sft_tokenizer, - rm_tokenizer=rm_tokenizer) - - # TODO: reward model use itself tokenizer instead of sft tokenizer - # the input_ids, responses, attention_mask and position_ids may be different! - @register(dispatch_mode=Dispatch.MEGATRON_COMPUTE_PROTO) - def compute_rm_score(self, data: DataProto): - data.batch = data.batch.cuda() - output = self.rm.compute_reward(data) - output = output.to('cpu') - return output diff --git a/verl/workers/reward_manager/__init__.py b/verl/workers/reward_manager/__init__.py deleted file mode 100644 index 4ae0fa4552..0000000000 --- a/verl/workers/reward_manager/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .naive import NaiveRewardManager -from .prime import PrimeRewardManager -from .dapo import DAPORewardManager -from .swebench import SWEBenchRewardManager diff --git a/verl/workers/reward_manager/dapo.py b/verl/workers/reward_manager/dapo.py deleted file mode 100644 index f98558d998..0000000000 --- a/verl/workers/reward_manager/dapo.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl import DataProto -from verl.utils.reward_score import _default_compute_score -import torch -from collections import defaultdict - - -class DAPORewardManager: - """The reward manager. - """ - - def __init__(self, - tokenizer, - num_examine, - compute_score=None, - reward_fn_key='data_source', - max_resp_len=None, - overlong_buffer_cfg=None) -> None: - self.tokenizer = tokenizer - self.num_examine = num_examine # the number of batches of decoded responses to print to the console - self.compute_score = compute_score or _default_compute_score - self.reward_fn_key = reward_fn_key - self.overlong_buffer_cfg = overlong_buffer_cfg - self.max_resp_len = max_resp_len - - if self.overlong_buffer_cfg is not None: - assert self.max_resp_len is not None, f"max_resp_len must be provided if {overlong_buffer_cfg=}, but got None" - - def __call__(self, data: DataProto, return_dict: bool = False): - """We will expand this function gradually based on the available datasets""" - - # If there is rm score, we directly return rm score. Otherwise, we compute via rm_score_fn - if 'rm_scores' in data.batch.keys(): - if return_dict: - return {"reward": data.batch['rm_scores']} - else: - return data.batch['rm_scores'] - - reward_tensor = torch.zeros_like(data.batch['responses'], dtype=torch.float32) - reward_extra_info = defaultdict(list) - - already_print_data_sources = {} - - for i in range(len(data)): - data_item = data[i] # DataProtoItem - - prompt_ids = data_item.batch['prompts'] - - prompt_length = prompt_ids.shape[-1] - - valid_prompt_length = data_item.batch['attention_mask'][:prompt_length].sum() - valid_prompt_ids = prompt_ids[-valid_prompt_length:] - - response_ids = data_item.batch['responses'] - valid_response_length = data_item.batch['attention_mask'][prompt_length:].sum() - valid_response_ids = response_ids[:valid_response_length] - - # decode - prompt_str = self.tokenizer.decode(valid_prompt_ids, skip_special_tokens=True) - response_str = self.tokenizer.decode(valid_response_ids, skip_special_tokens=True) - eos_token = self.tokenizer.eos_token - if response_str.endswith(eos_token): - response_str = response_str[:-len(eos_token)] - - ground_truth = data_item.non_tensor_batch['reward_model']['ground_truth'] - - data_source = data_item.non_tensor_batch[self.reward_fn_key] - - extra_info = data_item.non_tensor_batch.get('extra_info', None) - - result = self.compute_score( - data_source=data_source, - solution_str=response_str, - ground_truth=ground_truth, - extra_info=extra_info, - ) - - score: float - if isinstance(result, dict): - score = result["score"] - # Store the information including original reward - for key, value in result.items(): - reward_extra_info[key].append(value) - else: - score = result - - reward = score - - if self.overlong_buffer_cfg.enable: - overlong_buffer_len = self.overlong_buffer_cfg.len - expected_len = self.max_resp_len - overlong_buffer_len - exceed_len = valid_response_length - expected_len - overlong_penalty_factor = self.overlong_buffer_cfg.penalty_factor - overlong_reward = min(-exceed_len / overlong_buffer_len * overlong_penalty_factor, 0) - reward += overlong_reward - if self.overlong_buffer_cfg.log: - reward_extra_info["overlong_reward"].append(overlong_reward) - reward_extra_info["overlong"].append(overlong_reward < 0) - - reward_tensor[i, valid_response_length - 1] = reward - - if data_source not in already_print_data_sources: - already_print_data_sources[data_source] = 0 - - if already_print_data_sources[data_source] < self.num_examine: - already_print_data_sources[data_source] += 1 - print("[prompt]", prompt_str) - print("[response]", response_str) - print("[ground_truth]", ground_truth) - if isinstance(result, dict): - for key, value in result.items(): - print(f"[{key}]", value) - else: - print(f"[score]", score) - - if return_dict: - return { - "reward_tensor": reward_tensor, - "reward_extra_info": reward_extra_info, - } - else: - return reward_tensor diff --git a/verl/workers/reward_manager/naive.py b/verl/workers/reward_manager/naive.py deleted file mode 100644 index f9ebbb9d97..0000000000 --- a/verl/workers/reward_manager/naive.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl import DataProto -from verl.utils.reward_score import _default_compute_score -import torch -from collections import defaultdict - - -class NaiveRewardManager: - """The reward manager. - """ - - def __init__(self, - tokenizer, - num_examine, - compute_score=None, - reward_fn_key='data_source', - use_parallel=False) -> None: - self.tokenizer = tokenizer - self.num_examine = num_examine # the number of batches of decoded responses to print to the console - self.compute_score = compute_score or _default_compute_score - self.reward_fn_key = reward_fn_key - self.use_parallel = use_parallel - - def __call__(self, data: DataProto, return_dict=False): - """We will expand this function gradually based on the available datasets""" - # If there is rm score, we directly return rm score. Otherwise, we compute via rm_score_fn - - if 'rm_scores' in data.batch.keys(): - if return_dict: - return {"reward": data.batch['rm_scores']} - else: - return data.batch['rm_scores'] - - reward_tensor = torch.zeros_like(data.batch['responses'], dtype=torch.float32) - reward_extra_info = defaultdict(list) - - already_print_data_sources = {} - - if "rm_final_scores" in data.batch.keys(): - # valid_response_lengths = [] - # unfaith_penaltys = [] - for i in range(len(data)): - item = data.batch[i] - prompt_ids = item['prompts'] - prompt_length = prompt_ids.shape[-1] - valid_response_length = item['attention_mask'][prompt_length:].sum() - # valid_response_lengths.append(valid_response_length) - # unfaith_penaltys.append(item['unfaith_penalty']) - reward_tensor[i, valid_response_length - 1] = item['rm_final_scores'] - - return reward_tensor - - import concurrent.futures - - def compute_score_for_item(i, data_item): - prompt_ids = data_item.batch['prompts'] - prompt_length = prompt_ids.shape[-1] - valid_prompt_length = data_item.batch['attention_mask'][:prompt_length].sum() - valid_prompt_ids = prompt_ids[-valid_prompt_length:] - - response_ids = data_item.batch['responses'] - valid_response_length = data_item.batch['attention_mask'][prompt_length:].sum() - valid_response_ids = response_ids[:valid_response_length] - - # decode - prompt_str = self.tokenizer.decode(valid_prompt_ids, skip_special_tokens=True) - response_str = self.tokenizer.decode(valid_response_ids, skip_special_tokens=True) - - ground_truth = data_item.non_tensor_batch['reward_model']['ground_truth'] - - data_source = data_item.non_tensor_batch[self.reward_fn_key] - - extra_info = data_item.non_tensor_batch.get('extra_info', None) - - # print(f"naive reward manager {self.tokenizer=}") - score = self.compute_score(data_source=data_source, - solution_str=response_str, - ground_truth=ground_truth, - extra_info=extra_info, - question=prompt_str, - tokenizer=self.tokenizer) - - return i, valid_response_length, score, data_source, prompt_str, response_str, ground_truth - - if self.use_parallel: - with concurrent.futures.ThreadPoolExecutor(max_workers=len(data)) as executor: - futures = [executor.submit(compute_score_for_item, i, data[i]) for i in range(len(data))] - results = [future.result() for future in concurrent.futures.as_completed(futures)] - else: - results = [compute_score_for_item(i, data[i]) for i in range(len(data))] - - for i, valid_response_length, score, data_source, prompt_str, response_str, ground_truth in results: - if isinstance(score, dict): - reward = score["score"] - for key, value in score.items(): - reward_extra_info[key].append(value) - else: - reward = score - - reward_tensor[i, valid_response_length - 1] = reward - - if data_source not in already_print_data_sources: - already_print_data_sources[data_source] = 0 - - if already_print_data_sources[data_source] < self.num_examine: - already_print_data_sources[data_source] += 1 - print("[prompt]", prompt_str) - print("[response]", response_str) - print("[ground_truth]", ground_truth) - if isinstance(score, dict): - for key, value in score.items(): - print(f"[{key}]", value) - else: - print(f"[score]", score) - - if return_dict: - return { - "reward_tensor": reward_tensor, - "reward_extra_info": reward_extra_info, - } - else: - return reward_tensor diff --git a/verl/workers/reward_manager/prime.py b/verl/workers/reward_manager/prime.py deleted file mode 100644 index 00b105dda2..0000000000 --- a/verl/workers/reward_manager/prime.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2024 PRIME team and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -from concurrent.futures import ProcessPoolExecutor -from functools import partial - -import torch - -from verl import DataProto -from verl.utils.reward_score import _default_compute_score - -async def single_compute_score(evaluation_func, completion, reference, task, task_extra_info, executor, timeout=300.): - loop = asyncio.get_running_loop() - try: - # Ensure process_completion is called properly - tasks = [ - asyncio.wait_for( - loop.run_in_executor( - executor, - partial(evaluation_func, task, completion, reference, task_extra_info) # Ensure synchronous - ), - timeout=timeout) - ] - return await asyncio.gather(*tasks) - except asyncio.TimeoutError: - print(f"Timeout occurred for completion: {completion}") - return None # Default value for timed-out rows - except Exception as e: - print(f"Error processing completion: {completion[:10]}, Error: {e}") - return None # Default value for failed rows - - -async def parallel_compute_score_async(evaluation_func, - completions, - references, - tasks, - extra_info=None, - num_processes=64): - scores = [] - with ProcessPoolExecutor(max_workers=num_processes) as executor: - if extra_info is None: - extra_info = [None] * len(tasks) - # Create tasks for all rows - tasks_async = [ - single_compute_score(evaluation_func, completion, reference, task, task_extra_info, executor, timeout=300.) - for completion, reference, task, task_extra_info in zip(completions, references, tasks, extra_info) - ] - # to prevent very occasional starvation caused by some anomalous programs ( like infinite loop ), the exceptions in async programs will instantly halt the evaluation, and all summoned processes will be killed. - try: - results = await asyncio.gather(*tasks_async, return_exceptions=False) - except: - for pid, proc in executor._processes.items(): - try: - proc.kill() - except Exception as kill_err: - print('shut down failed: ' + str(kill_err)) - raise - - # Process results - for result, completion, reference, task in zip(results, completions, references, tasks): - if isinstance(result, Exception) or result is None: - # Handle failed or timed-out tasks - scores.append(0.0) - elif isinstance(result[0], (int, float, bool)): - scores.append(float(result[0])) - else: - scores.append(float(result[0][0])) - return scores - - -class PrimeRewardManager: - """ - The Reward Manager used in https://github.com/PRIME-RL/PRIME - """ - - def __init__(self, tokenizer, num_examine, compute_score=None) -> None: - self.tokenizer = tokenizer - self.num_examine = num_examine # the number of batches of decoded responses to print to the console - self.compute_score = compute_score or _default_compute_score - - def verify(self, data): - """ - verify the batch and save as ``acc`` tensor - """ - # batched scoring - prompt_ids = data.batch['prompts'] - - response_ids = data.batch['responses'] - sequences_str = self.tokenizer.batch_decode(response_ids, skip_special_tokens=True) - ground_truth = [data_item.non_tensor_batch['reward_model']['ground_truth'] for data_item in data] - data_sources = data.non_tensor_batch['data_source'] - extra_info = data.non_tensor_batch.get('extra_info', None) - - assert len(sequences_str) == len(ground_truth) == len(data_sources) - try: - scores = asyncio.run( - parallel_compute_score_async(self.compute_score, - sequences_str, - ground_truth, - data_sources, - extra_info=extra_info, - num_processes=64)) - except asyncio.TimeoutError as e: - print('Global timeout in reward computing! Setting all as 0.') - scores = [0. for _ in range(len(sequences_str))] - except Exception as e: - print(f"Unexpected error in batched reward computing. Setting all as 0.: {e}") - scores = [0. for _ in range(len(sequences_str))] - data.batch['acc'] = torch.tensor(scores, dtype=torch.float32, device=prompt_ids.device) - return scores - - def __call__(self, data: DataProto, return_dict: bool = False): - """We will expand this function gradually based on the available datasets""" - - # If there is rm score, we directly return rm score. Otherwise, we compute via rm_score_fn - if 'rm_scores' in data.batch.keys(): - return data.batch['rm_scores'] - - reward_tensor = torch.zeros_like(data.batch['responses'], dtype=torch.float32) - - already_print_data_sources = {} - - # batched scoring - prompt_ids = data.batch['prompts'] - prompt_length = prompt_ids.shape[-1] - - response_ids = data.batch['responses'] - valid_response_length = data.batch['attention_mask'][:, prompt_length:].sum(dim=-1) - sequences_str = self.tokenizer.batch_decode(response_ids, skip_special_tokens=True) - data_sources = data.non_tensor_batch['data_source'] - - scores = self.verify(data) - # scores = self.verify_ray(data) - - for i in range(len(data)): - data_source = data_sources[i] - reward_tensor[i, valid_response_length[i].item() - 1] = scores[i] - - if data_source not in already_print_data_sources: - already_print_data_sources[data_source] = 0 - - if already_print_data_sources[data_source] < self.num_examine: - already_print_data_sources[data_source] += 1 - print(sequences_str) - - if return_dict: - return {"reward_tensor": reward_tensor} - else: - return reward_tensor diff --git a/verl/workers/reward_manager/swebench.py b/verl/workers/reward_manager/swebench.py deleted file mode 100644 index 88324b82d7..0000000000 --- a/verl/workers/reward_manager/swebench.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import statistics -from typing import List -import re - -from ray.util import scheduling_strategies -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy -from verl import DataProto -from verl.utils.reward_score import _default_compute_score -import torch -import ray -from ray.util import ActorPool - -import asyncio -import multiprocessing -from concurrent.futures import ProcessPoolExecutor -from functools import partial - - -# From PRIME team -async def single_compute_score(evaluation_func, completion, reference, task, task_extra_info, executor, timeout=300.): - loop = asyncio.get_running_loop() - try: - # Ensure process_completion is called properly - tasks = [ - asyncio.wait_for( - loop.run_in_executor( - executor, - partial(evaluation_func, task, completion, reference, task_extra_info) # Ensure synchronous - ), - timeout=timeout) - ] - return await asyncio.gather(*tasks) - except asyncio.TimeoutError: - print(f"Timeout occurred for completion: {completion}") - return None # Default value for timed-out rows - except Exception as e: - print(f"Error processing completion: {completion[:100]}, Error: {e}") - return None # Default value for failed rows - - -async def parallel_compute_score_async(evaluation_func, - completions, - references, - tasks, - extra_info=None, - num_processes=64): - # TODO: use spawn in ray application code: https://docs.ray.io/en/latest/ray-core/patterns/fork-new-processes.html - ctx = multiprocessing.get_context("spawn") - scores = [] - with ProcessPoolExecutor(max_workers=num_processes, mp_context=ctx) as executor: - if extra_info is None: - extra_info = [None] * len(tasks) - # Create tasks for all rows - """ - results = [] - for completion, reference, task, task_extra_info in zip(completions, references, tasks, extra_info): - print(f"Using eval func: {evaluation_func}") - cur_score = evaluation_func(task, completion, reference, task_extra_info) - results.append([cur_score]) - print(f"Get score: {cur_score}") - """ - # results.append(single_compute_score(evaluation_func, completion, reference, task, task_extra_info, executor, timeout=300.)) - # """ - tasks_async = [ - single_compute_score(evaluation_func, completion, reference, task, task_extra_info, executor, timeout=300.) - for completion, reference, task, task_extra_info in zip(completions, references, tasks, extra_info) - ] - # to prevent very occasional starvation caused by some anomalous programs ( like infinite loop ), the exceptions in async programs will instantly halt the evaluation, and all summoned processes will be killed. - try: - results = await asyncio.gather(*tasks_async, return_exceptions=False) - except: - for pid, proc in executor._processes.items(): - try: - proc.kill() - except Exception as kill_err: - print('shut down failed: ' + str(kill_err)) - raise - # """ - - # Process results - for result, completion, reference, task in zip(results, completions, references, tasks): - # print(f"Looping result: {result}") - if isinstance(result, Exception) or result is None: - # Handle failed or timed-out tasks - scores.append(0.0) - elif isinstance(result[0], (int, float, bool)): - scores.append(float(result[0])) - else: - scores.append(float(result[0][0])) - return scores - - -class SWEBenchRewardManager: - """The reward manager. - """ - - def __init__(self, tokenizer, num_examine, config, compute_score=None) -> None: - self.data_source = "SWE-Gym/SWE-Gym" - - self.tokenizer = tokenizer - self.num_examine = num_examine # the number of batches of decoded responses to print to the console - self.verifier_func = compute_score or _default_compute_score - self.config = config - - - def verify(self, data): - resolved = data.non_tensor_batch['resolved'] - error = data.non_tensor_batch['error'] - print(error) - has_finish_action = data.non_tensor_batch['finish'] - print("has_finish_action:", has_finish_action) - score = [0. for _ in range(len(resolved))] - for i, r in enumerate(resolved): - if r: - score[i] = 1.0 - - print("scores:", score) - reward_metrics = {} - reward_metrics['max_turn_ratio'] = sum("RuntimeError: Agent reached maximum iteration in headless mode" in e for e in error if e) / len(error) - reward_metrics['finish_action_ratio'] = sum(has_finish_action) / len(has_finish_action) - reward_metrics['stuck_ratio'] = sum("stuck in a loop" in e for e in error if e) / len(error) - - data.batch['acc'] = torch.tensor(score, dtype=torch.float32, device=data.batch['responses'].device) - for ability in list(set(data.non_tensor_batch['ability'])): - score_ = [data.batch['acc'][i].item() for i in range(len(data.batch['acc'])) if - data.non_tensor_batch['ability'][i] == ability] - reward_metrics[f'{ability}'] = statistics.mean(score_) - reward_metrics['all'] = data.batch['acc'].mean().item() - - return score, reward_metrics - - def __call__(self, data: DataProto): - """We will expand this function gradually based on the available datasets""" - - reward_tensor_dict={} - reward_metrics={} - reward_tensor = torch.zeros_like(data.batch['responses'], dtype=torch.float32) - - verifier_reward=torch.zeros_like(data.batch['responses'], dtype=torch.float32) - - response_ids = data.batch['responses'] - response_length = response_ids.shape[-1] - valid_response_length = data.batch['attention_mask'][:, -response_length:].sum(-1) - - # if the batch already contains evaluation results, the verification is skipped here. - if 'acc' in data.batch: - verifier_score = data.batch['acc'].cpu().numpy().tolist() - else: - # verifier_score, verifier_metrics = self.verify(data) - # Use ray based concurrency - verifier_score, verifier_metrics = self.verify(data) - reward_metrics.update(verifier_metrics) - for i in range(verifier_reward.shape[0]): - verifier_reward[i, valid_response_length[i] - 1] = verifier_score[i] - - reward_tensor_dict['gt_scores'] = verifier_reward - - if 'rm_scores' in data.batch.keys(): - reward_tensor_dict['rm_scores'] = data.batch['rm_scores'] - reward_metrics['reward_model']=data.batch['rm_scores'].sum(dim=1).mean().item() - if self.config.reward_model.rm_coef!=0: - reward_tensor += self.config.reward_model.rm_coef * reward_tensor_dict['rm_scores'] - - if self.config.verifier.reward_coef!=0: - reward_metrics['verifier'] = reward_tensor_dict['gt_scores'].sum(dim=1).mean().item() - reward_tensor += self.config.verifier.reward_coef * reward_tensor_dict['gt_scores'] - - reward_tensor_dict['all'] = reward_tensor - reward_metrics['reward_all'] = reward_tensor.sum(dim=-1).mean(dim=0).item() - - return reward_tensor_dict, reward_metrics - diff --git a/verl/workers/reward_model/__init__.py b/verl/workers/reward_model/__init__.py deleted file mode 100644 index a0b48a7508..0000000000 --- a/verl/workers/reward_model/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base import BasePPORewardModel diff --git a/verl/workers/reward_model/base.py b/verl/workers/reward_model/base.py deleted file mode 100644 index c02487db38..0000000000 --- a/verl/workers/reward_model/base.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The base class for reward model -""" - -from abc import ABC, abstractmethod - -from verl import DataProto - - -class BasePPORewardModel(ABC): - - def __init__(self, config): - self.config = config - - @abstractmethod - def compute_reward(self, data: DataProto) -> DataProto: - """Computing reward given input_ids. The transformers should output a tensor with shape - [batch_size, sequence_length], and the value at [EOS] mask should be gathered. - - Args: - data: must contain keys "input_ids", "attention_mask" and "position_ids". - - input_ids: [batch_size, sequence_length] - - attention_mask: [batch_size, sequence_length] - - position_ids: [batch_size, sequence_length] - - Returns: a data pass protocol containing "reward". Only the [EOS] position contains the reward. - Other position should have zero reward. Note that this may change in the future if we use - dense reward. So, we leave the interface for general case. - - reward: [batch_size, sequence_length]. - - """ - pass diff --git a/verl/workers/reward_model/megatron/__init__.py b/verl/workers/reward_model/megatron/__init__.py deleted file mode 100644 index b0956b4cc5..0000000000 --- a/verl/workers/reward_model/megatron/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .reward_model import MegatronRewardModel diff --git a/verl/workers/reward_model/megatron/reward_model.py b/verl/workers/reward_model/megatron/reward_model.py deleted file mode 100644 index a890f28843..0000000000 --- a/verl/workers/reward_model/megatron/reward_model.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Megatron Reward Model. -""" - -from tensordict import TensorDict -from verl import DataProto -import torch -import torch.distributed - -from verl.utils.torch_functional import pad_sequence_to_length -from verl.utils.megatron.pipeline_parallel import (compute_transformers_input_shapes, make_batch_generator) -from verl import DataProto -from verl.utils.torch_functional import broadcast_dict_tensor, split_dict_tensor_into_batches -from verl.workers.reward_model.base import BasePPORewardModel -from megatron.core import parallel_state as mpu -from megatron.core.pipeline_parallel import get_forward_backward_func - - -class MegatronRewardModel(BasePPORewardModel): - - def __init__(self, - config, - model_config, - reward_model_module: torch.nn.ModuleList, - megatron_config, - sft_tokenizer=None, - rm_tokenizer=None): - self.config = config - self.reward_model_module = reward_model_module - self.megatron_config = megatron_config - self.model_config = model_config - self.device = 'cuda' - self.sft_tokenizer = sft_tokenizer - self.rm_tokenizer = rm_tokenizer - self.use_different_tokenizer = rm_tokenizer is not None - - if self.config.param_offload: - self.offload_params_to_cpu() - - def re_encode_by_rm_tokenizer(self, data: DataProto) -> DataProto: - assert self.use_different_tokenizer, 're-encode need rm tokenizer not be None!' - # need to use rm tokenizer to re-generate input_ids, attention_mask and position_ids - # 1. remove pad for each sequence - # 2. decode by sft_tokenizer, remove sft system prompts - # 3. encode by rm_tokenizer with rm system prompts, get rm_input_ids - # 4. generate attention_mask and position_ids - input_ids = data.batch['input_ids'] # (bs, seq_len) - attention_mask = data.batch['attention_mask'] - position_ids = data.batch['position_ids'] - ori_values = {'input_ids': input_ids, 'attention_mask': attention_mask, 'position_ids': position_ids} - ori_bs, ori_seqlen = input_ids.size(0), input_ids.size(1) - input_ids_for_rm = [] - attention_mask_for_rm = [] - position_ids_for_rm = [] - print_decode = True - ori_seqlen = ori_seqlen + 128 - for id, mask in zip(input_ids, attention_mask): - # 1. remove pad for each sequence - non_zero_indices = torch.nonzero(mask).view(-1) - begin_pos, end_pos = non_zero_indices[0].item(), non_zero_indices[-1].item() - valid_id = id[begin_pos:end_pos + 1] - # 2. decode by sft_tokenizer, remove sft system prompts - decode_result = self.sft_tokenizer.decode(valid_id) - # workaround - decode_with_rm_chat = decode_result.replace("<|user|>\n", "[INST] ").replace( - "\n<|assistant|>\n", " [/INST]").replace(" \n<|assistant|>\n", " [/INST]") + "" - if print_decode and torch.distributed.get_rank() == 0: - # only print first decode result - print(f'device {torch.cuda.current_device()}: sft decode result:\n{decode_result}\n \ - \ndevice {torch.cuda.current_device()}: sft decode result with rm chat template:\n{decode_with_rm_chat}\n\n' - ) - print_decode = False - # 3. encode by rm_tokenizer - rm_input_ids = self.rm_tokenizer(decode_with_rm_chat, - return_tensors='pt')['input_ids'][0].to(input_ids.device) - # 4. generate attention_mask and position_ids - rm_attention_mask = torch.ones_like(rm_input_ids, device=input_ids.device) - cur_seqlen = rm_input_ids.shape[-1] - # NOTE(gh): the later reward compute will process the shape (bs, seqlen_pad_128) - if cur_seqlen > ori_seqlen: - print(f'warninig: rm encode seqlen {cur_seqlen} > sft encode seqlen {ori_seqlen}') - rm_input_ids = rm_input_ids[:ori_seqlen] - rm_attention_mask = rm_attention_mask[:ori_seqlen] - else: - # right padding - rm_input_ids = pad_sequence_to_length(rm_input_ids, ori_seqlen, self.rm_tokenizer.pad_token_id) - rm_attention_mask = pad_sequence_to_length(rm_attention_mask, ori_seqlen, 0) - rm_position_ids = torch.arange(0, ori_seqlen, device=input_ids.device) - input_ids_for_rm.append(torch.unsqueeze(rm_input_ids, dim=0)) - attention_mask_for_rm.append(torch.unsqueeze(rm_attention_mask, dim=0)) - position_ids_for_rm.append(torch.unsqueeze(rm_position_ids, dim=0)) - input_ids_for_rm = torch.cat(input_ids_for_rm, dim=0) - attention_mask_for_rm = torch.cat(attention_mask_for_rm, dim=0) - position_ids_for_rm = torch.cat(position_ids_for_rm, dim=0) - - # (bs, seqlen) will not change, but input_ids, attention_mask and position_ids will change - # NOTE(gh): need to replace into origin values after compute reward! - data.batch['input_ids'] = input_ids_for_rm - data.batch['attention_mask'] = attention_mask_for_rm - data.batch['position_ids'] = position_ids_for_rm - - return data, ori_values - - @torch.no_grad() - def compute_reward(self, data: DataProto) -> DataProto: - if self.config.param_offload: - self.load_params_to_cuda() - - if self.use_different_tokenizer: - data, ori_values = self.re_encode_by_rm_tokenizer(data) - - input_ids = data.batch['input_ids'] # (bs, seq_len') - attention_mask = data.batch['attention_mask'] - position_ids = data.batch['position_ids'] - - responses = data.batch['responses'] - batch_size = responses.size(0) - response_length = responses.size(1) - - with torch.no_grad(): - output = self.forward_batch(data) - if mpu.is_pipeline_last_stage(ignore_virtual=True): - logits = torch.cat([o['logits'] for o in output], dim=0) - else: - logits = torch.empty( - (input_ids.shape[0], input_ids.shape[1]), - dtype=torch.bfloat16, # TODO(sgm): check why is bfloat16 - device=input_ids.device) - # broadcast across pp ranks - torch.distributed.broadcast(tensor=logits, - src=mpu.get_pipeline_model_parallel_last_rank(), - group=mpu.get_pipeline_model_parallel_group(), - async_op=False) - - # (bs, seqlen', hidden_size) -> (bs, seqlen', 1) -> (bs, seqlen') - token_level_rewards = logits - # find the last token reward - ends = attention_mask.cumsum(dim=-1).argmax(dim=-1).view(-1, 1) # (bs, 1) - rewards = torch.gather(token_level_rewards, dim=1, index=ends) # (bs, 1) - - if self.use_different_tokenizer: - data.batch.update(ori_values) - input_ids = ori_values['input_ids'] - attention_mask = ori_values['attention_mask'] - position_ids = ori_values['position_ids'] - - token_level_rewards = rewards.expand(attention_mask.shape[0], attention_mask.shape[1]) # (bs, ori_seqlen) - - # assign last valid token reward to ori position - eos_mask_idx = torch.argmax(position_ids * attention_mask, dim=-1) # (bs,) - eos_mask = torch.zeros_like(attention_mask) - eos_mask[torch.arange(batch_size), eos_mask_idx] = 1. - - token_level_rewards = token_level_rewards * eos_mask - token_level_rewards = token_level_rewards[:, -response_length:] - - if self.config.param_offload: - self.offload_params_to_cpu() - else: - # add empty cache after each compute - torch.cuda.empty_cache() - - batch = TensorDict({'rm_scores': token_level_rewards}, batch_size=input_ids.shape[0]) - - return DataProto(batch=batch) - - def forward_batch(self, data: DataProto): - """ - We assume: - - The model takes input: (input_ids, attention_mask, position_ids). No rmpad for the input - - The communication shape is (total_nnz_pad_to_sp // tp_size, 1, hidden_size) if sequence parallel is enabled - """ - # broadcast from last pp rank to all other pp ranks - # TODO: actually, we just need to control the sampling order. - data.batch = data.batch.contiguous() - broadcast_dict_tensor(data.batch, - src=mpu.get_pipeline_model_parallel_last_rank(), - group=mpu.get_pipeline_model_parallel_group()) - - # split into micro-batches - if self.config is not None and 'ppo_micro_batch_size_per_gpu' in self.config: - infer_batch_size = self.config.ppo_micro_batch_size_per_gpu - else: - infer_batch_size = data.batch.batch_size[0] - - data.batch['attention_mask'] = data.batch['attention_mask'].to(bool) - batches = split_dict_tensor_into_batches(data.batch, batch_size=infer_batch_size) - n_micro_batch = len(batches) - seq_len = batches[0]['input_ids'].shape[1] - - # compute input shapes for pp stages - input_shapes = compute_transformers_input_shapes( - batches, - meta_info={ - 'sequence_parallel': self.megatron_config.sequence_parallel, - 'hidden_size': self.model_config.hidden_size - }) - # compute input shapes for pp stages - forward_backward_func = get_forward_backward_func() - - def loss_func(output): - return 1., {'logits': output.logits} - - def forward_step(batch_iter, model): - batch = next(batch_iter) - input_ids = batch['input_ids'] - attention_mask = batch['attention_mask'] - position_ids = batch['position_ids'] - output = model(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids) - return output, loss_func - - # batch should be a list of batches inside micro-batches - batch_generator = make_batch_generator(batches, vpp_size=len(self.reward_model_module)) - - # TODO: we may use the new schedule instead - # for flash-attn: (seq_len, batch_size, hidden_size) = (mbs*seq_len, 1, hidden_size) - if mpu.get_pipeline_model_parallel_world_size() > 1: - losses_reduced = forward_backward_func( - forward_step_func=forward_step, - data_iterator=batch_generator, - model=self.reward_model_module, - num_microbatches=n_micro_batch, - seq_length=infer_batch_size * seq_len, # no use when input_shapes was set - micro_batch_size=1, # no use when input_shapes was set - forward_only=True, - ) - else: - losses_reduced = forward_backward_func( - forward_step_func=forward_step, - data_iterator=batch_generator, - model=self.reward_model_module, - num_microbatches=n_micro_batch, - seq_length=infer_batch_size * seq_len, # in use for pp = 1 - micro_batch_size=1, # in use for pp = 1 - forward_only=True, - ) - # loss_reduces contains the stats returned from loss_func - - return losses_reduced - - def offload_params_to_cpu(self): - if self.device == 'cuda': - for reward_model_module in self.reward_model_module: - for name, param in reward_model_module.named_parameters(): - param.data = param.data.to('cpu', non_blocking=True) - self.device = 'cpu' - torch.cuda.empty_cache() - - def load_params_to_cuda(self): - if self.device == 'cpu': - for reward_model_module in self.reward_model_module: - for name, param in reward_model_module.named_parameters(): - param.data = param.data.to(torch.cuda.current_device(), non_blocking=True) - self.device = 'cuda' diff --git a/verl/workers/rollout/__init__.py b/verl/workers/rollout/__init__.py deleted file mode 100644 index 083848c77f..0000000000 --- a/verl/workers/rollout/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base import BaseRollout -from .naive import NaiveRollout -from .hf_rollout import HFRollout - -__all__ = ["BaseRollout", "NaiveRollout", "HFRollout"] diff --git a/verl/workers/rollout/base.py b/verl/workers/rollout/base.py deleted file mode 100644 index 8c2733325b..0000000000 --- a/verl/workers/rollout/base.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod -from typing import Iterable, Union - -from verl import DataProto - -__all__ = ['BaseRollout'] - - -class BaseRollout(ABC): - - def __init__(self): - """ - - Args: - dataloader: an Iterable of TensorDict that consistently generates prompts. Note that the dataloader - should handle when the training stops. - """ - super().__init__() - - @abstractmethod - def generate_sequences(self, prompts: DataProto) -> DataProto: - """Generate sequences""" - pass diff --git a/verl/workers/rollout/hf_rollout.py b/verl/workers/rollout/hf_rollout.py deleted file mode 100644 index 061a76180c..0000000000 --- a/verl/workers/rollout/hf_rollout.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Rollout with huggingface models. -TODO: refactor this class. Currently, it will hang when using FSDP HybridShard. We should actually create a single GPU model. -Then, get full state_dict and bind the state_dict to the single GPU model. Then, use the single GPU model to perform generation. -""" -import contextlib -import torch -import torch.distributed -from tensordict import TensorDict -from torch import nn -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP - -from verl import DataProto -from verl.utils.torch_functional import get_response_mask -from .base import BaseRollout - -from transformers import GenerationConfig - -__all__ = ['HFRollout'] - - -class HFRollout(BaseRollout): - - def __init__(self, module: nn.Module, config): - super().__init__() - self.config = config - self.module = module - - def generate_sequences(self, prompts: DataProto) -> DataProto: - batch_size = prompts.batch.batch_size[0] - num_chunks = max(batch_size // self.config.get('micro_batch_size', batch_size), 1) - batch_prompts = prompts.chunk(chunks=num_chunks) - output = [self._generate_minibatch(p) for p in batch_prompts] - output = DataProto.concat(output) - return output - - @torch.no_grad() - def _generate_minibatch(self, prompts: DataProto) -> DataProto: - idx = prompts.batch['input_ids'] # (bs, prompt_length) - attention_mask = prompts.batch['attention_mask'] # left-padded attention_mask - position_ids = prompts.batch['position_ids'] - - # used to construct attention_mask - eos_token_id = prompts.meta_info['eos_token_id'] - pad_token_id = prompts.meta_info['pad_token_id'] - - batch_size = idx.size(0) - prompt_length = idx.size(1) - - self.module.eval() - param_ctx = contextlib.nullcontext() - - # make sampling args can be overriden by inputs - do_sample = prompts.meta_info.get('do_sample', self.config.do_sample) - response_length = prompts.meta_info.get('response_length', self.config.response_length) - top_p = prompts.meta_info.get('top_p', self.config.get('top_p', 1.0)) - top_k = prompts.meta_info.get('top_k', self.config.get('top_k', 0)) - - if top_k is None: - top_k = 0 - top_k = max(0, top_k) # to be compatible with vllm - - temperature = prompts.meta_info.get('temperature', self.config.temperature) - - generation_config = GenerationConfig(temperature=temperature, top_p=top_p, top_k=top_k) - - if isinstance(self.module, FSDP): - # recurse need to set to False according to https://github.com/pytorch/pytorch/issues/100069 - param_ctx = FSDP.summon_full_params(self.module, writeback=False, recurse=False) - with param_ctx: - with torch.autocast(device_type='cuda', dtype=torch.bfloat16): - output = self.module.generate( - input_ids=idx, - attention_mask=attention_mask, - do_sample=do_sample, - max_new_tokens=response_length, - # max_length=max_length, - eos_token_id=eos_token_id, - pad_token_id=pad_token_id, - generation_config=generation_config, - # renormalize_logits=True, - output_scores=False, # this is potentially very large - return_dict_in_generate=True, - use_cache=True) - # TODO: filter out the seq with no answers like ds-chat - seq = output.sequences - - # huggingface generate will stop generating when all the batch reaches [EOS]. - # We have to pad to response_length - sequence_length = prompt_length + self.config.response_length - delta_length = sequence_length - seq.shape[1] - - if delta_length > 0: - delta_tokens = torch.ones(size=(batch_size, delta_length), device=seq.device, dtype=seq.dtype) - delta_tokens = pad_token_id * delta_tokens - seq = torch.cat((seq, delta_tokens), dim=1) - - assert seq.shape[1] == sequence_length - - prompt = seq[:, :prompt_length] # (bs, prompt_length) - response = seq[:, prompt_length:] # (bs, response_length) - - response_length = response.size(1) - delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) - delta_position_id = delta_position_id.unsqueeze(0).repeat(batch_size, 1) - - response_position_ids = position_ids[:, -1:] + delta_position_id - position_ids = torch.cat([position_ids, response_position_ids], dim=-1) - - response_attention_mask = get_response_mask(response_id=response, - eos_token=eos_token_id, - dtype=attention_mask.dtype) - attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) - - batch = TensorDict( - { - 'prompts': prompt, - 'responses': response, - 'input_ids': seq, - 'attention_mask': attention_mask, - 'position_ids': position_ids - }, - batch_size=batch_size) - - # empty cache before compute old_log_prob - torch.cuda.empty_cache() - - self.module.train() - return DataProto(batch=batch) diff --git a/verl/workers/rollout/naive/__init__.py b/verl/workers/rollout/naive/__init__.py deleted file mode 100644 index df81c8603f..0000000000 --- a/verl/workers/rollout/naive/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .naive_rollout import NaiveRollout diff --git a/verl/workers/rollout/naive/naive_rollout.py b/verl/workers/rollout/naive/naive_rollout.py deleted file mode 100644 index 2c57bce107..0000000000 --- a/verl/workers/rollout/naive/naive_rollout.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -In single GPU rollout, the sequences are generated directly by sampling from the model. -The output will contain -1. output_ids -2. attention_masks (left padding) -3. eos_masks -4. log_probs -""" -from typing import Iterable, Union - -import torch -import torch.nn.functional as F -from tensordict import TensorDict -from torch import nn - -from verl import DataProto -from verl.utils.torch_functional import logprobs_from_logits -from ..base import BaseRollout - -__all__ = ['NaiveRollout'] - - -class NaiveRollout(BaseRollout): - - def __init__(self, module: nn.Module, config): - """A naive rollout. It requires the module to be compatible with huggingface APIs. That is: - The module should define __call__ to receive input_ids, attention_mask and position_ids. - It outputs a structure that contains logits field. - - Args: - module: module here follows huggingface APIs - config: DictConfig - """ - super().__init__() - self.config = config - self.module = module - - @torch.no_grad() - def generate_sequences(self, prompts: DataProto) -> DataProto: - """Generate sequences""" - idx = prompts.batch['input_ids'] # (bs, prompt_length) - attention_mask = prompts.batch['attention_mask'] # left-padded attention_mask - position_ids = prompts.batch['position_ids'] - - # used to construct attention_mask - eos_token_id = prompts.meta_info['eos_token_id'] - if isinstance(eos_token, int): - eos_token = [eos_token] - - batch_size = idx.size(0) - prompt_length = idx.size(1) - - self.module.eval() - - prev_attention_mask = torch.ones(size=(batch_size, 1), dtype=attention_mask.dtype, device=attention_mask.device) - - logits_lst = [] - for _ in range(self.config.response_length): - # if the sequence context is growing too long we must crop it at block_size - # idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:] - idx_cond = idx - # forward the model to get the logits for the index in the sequence - # we use huggingface APIs here - output = self.module(input_ids=idx_cond, attention_mask=attention_mask, position_ids=position_ids) - logits = output.logits - # pluck the logits at the final step and scale by desired temperature - logits = logits[:, -1, :] / self.config.temperature # (bs, vocab_size) - # optionally crop the logits to only the top k options - if self.config.top_k is not None: - v, _ = torch.topk(logits, min(self.config.top_k, logits.size(-1))) - logits[logits < v[:, [-1]]] = -float('Inf') - # apply softmax to convert logits to (normalized) probabilities - probs = F.softmax(logits, dim=-1) - # sample from the distribution - if self.config.do_sample: - idx_next = torch.multinomial(probs, num_samples=1) - else: - idx_next = torch.argmax(probs, dim=-1, keepdim=True) - - attention_mask = torch.cat((attention_mask, prev_attention_mask), dim=-1) - - for token_id in eos_token_id: - prev_attention_mask = torch.logical_and(idx_next != token_id, prev_attention_mask.bool()) - prev_attention_mask.to(attention_mask.dtype) - - position_ids = torch.cat((position_ids, position_ids[:, -1:] + 1), dim=-1) - - # append sampled index to the running sequence and continue - idx = torch.cat((idx, idx_next), dim=1) - logits_lst.append(logits) - - logits = torch.stack(logits_lst, dim=1) # (bs, response_length, vocab_size) - prompts = idx[:, :prompt_length] # (bs, prompt_length) - response = idx[:, prompt_length:] # (bs, response_length) - log_probs = logprobs_from_logits(logits=logits, labels=response) - batch = TensorDict( - { - 'input_ids': prompts, - 'responses': response, - 'sequences': idx, - 'old_log_probs': log_probs, - 'attention_mask': attention_mask, - 'position_ids': position_ids, - }, - batch_size=batch_size) - - self.module.train() - - return DataProto(batch=batch) diff --git a/verl/workers/rollout/sglang_rollout/__init__.py b/verl/workers/rollout/sglang_rollout/__init__.py deleted file mode 100644 index 81320e8aec..0000000000 --- a/verl/workers/rollout/sglang_rollout/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and - -from .sglang_rollout import SGLangRollout diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py deleted file mode 100644 index af4dfbbc44..0000000000 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations -import os -from contextlib import contextmanager -from typing import TYPE_CHECKING, List -from omegaconf import DictConfig -from tensordict import TensorDict -from verl import DataProto -from verl.workers.rollout.base import BaseRollout -from verl.utils.torch_functional import get_response_mask, pad_sequence_to_length, pad_2d_list_to_length -from sglang.srt.entrypoints.verl_engine import VerlEngine -from torch.distributed.device_mesh import init_device_mesh -from sglang.srt.sampling.sampling_params import SamplingParams -from verl.third_party.sglang import parallel_state as sglang_ps -import torch.distributed -from torch.nn.utils.rnn import pad_sequence - -if TYPE_CHECKING: - from torch import nn - - -# NOTE(sgm): add for verl. We can optimize it by making the dataloader yield List[int] without padding. -def _pre_process_inputs(pad_token_id, prompt_token_ids: torch.Tensor) -> List[int]: - # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = prompt_token_ids[non_pad_index:].tolist() - return token_ids - - -# NOTE(linjunrong): adhoc -def _post_process_outputs(tokenizer, output): - - def _map_each_response(l): - # output_token_ids = torch.tensor(l['token_ids']) - log_probs = [] - output_token_ids = [] - for log_prob, token_ids, _ in l["meta_info"]["output_token_logprobs"]: - log_probs.append(log_prob) - output_token_ids.append(token_ids) - log_probs = torch.tensor(log_probs) - output_token_ids = torch.tensor(output_token_ids) - return output_token_ids, log_probs - - out_map = map(lambda x: _map_each_response(x), output) - batched_output_token_ids = [] - batched_logprobs = [] - for output_token_ids, log_probs in out_map: - batched_output_token_ids.append(output_token_ids) - batched_logprobs.append(log_probs) - pad_token_id = (tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id) - batched_output_token_ids = pad_sequence(batched_output_token_ids, batch_first=True, padding_value=pad_token_id) - if len(batched_logprobs) > 0: - batched_logprobs = pad_sequence(batched_logprobs, batch_first=True, padding_value=pad_token_id) - return batched_output_token_ids, batched_logprobs - - -class SGLangRollout(BaseRollout): - - def __init__( - self, - actor_module: nn.Module | str, - config: DictConfig, - tokenizer, - model_hf_config, - **kwargs, - ): - """A SGLang rollout. It requires the module is supported by the SGLang. - - Args: - actor_module: module here follows huggingface APIs - config: DictConfig - tokenizer: the task/model tokenizer - model_hf_config: the huggingface config to initiallize the generating model in SGLang - **kwargs: train_tp, for Megatron Backend to initialize hybrid engine (zero redundancy) process group - """ - super().__init__() - self.config = config - - assert not (not config.enforce_eager and - config.free_cache_engine), "disable CUDA graph (enforce_eager = False) if free cache engine" - - tensor_parallel_size = self.config.get("tensor_model_parallel_size", 1) - assert (tensor_parallel_size <= torch.distributed.get_world_size() - ), "tensor parallel size should be less than or equal to the world size" - - if kwargs.get("train_tp", None) is not None: - # deployed with megatron - os.environ["CUDA_TIMER_STREAM_KAFKA_ENABLE"] = "0" - os.environ["MEGATRON_IMPORT_TIMERS"] = "0" - train_tp = kwargs.get("train_tp", None) - num_tp_per_train_tp = train_tp // tensor_parallel_size - sglang_ps.initialize_parallel_state( - tensor_model_parallel_size=tensor_parallel_size, - num_tp_per_train_tp=num_tp_per_train_tp, - ) - - assert (model_hf_config.max_position_embeddings >= config.prompt_length + - config.response_length), "model context length should be greater than total sequence length" - - tp_size = tensor_parallel_size - world_size = int(os.getenv("WORLD_SIZE", "-1")) - - # init device mesh - device_mesh_kwargs = dict( - mesh_shape=(world_size // tp_size, tp_size, 1), - mesh_dim_names=["dp", "tp", "pp"], - ) - device_mesh_cpu = init_device_mesh("cpu", **device_mesh_kwargs) - # device_mesh_device = init_device_mesh("cuda", **device_mesh_kwargs) - - # get tp_rank of this process in this tp group - visible_devices = [None] * device_mesh_cpu.size(1) - torch.distributed.all_gather_object(visible_devices, os.environ["CUDA_VISIBLE_DEVICES"], - device_mesh_cpu.get_group("tp")) - os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(visible_devices) - - self.inference_engine = VerlEngine( - model_path=actor_module, - dtype=config.dtype, - mem_fraction_static=config.gpu_memory_utilization, - device_mesh_cpu=device_mesh_cpu["tp"], - enable_memory_saver=True, - base_gpu_id=0, - gpu_id_step=1, - # NOTE(Chenyang): if you want to debug the sglang engine - # please set the following parameters - # Otherwise, it will make the engine run too slow - # log_level="INFO", - # log_requests=True, - # log_requests_level=2, - # max_running_requests=1, - ) - - # offload - self.inference_engine.release_memory_occupation() - - kwargs = dict(n=1, - max_new_tokens=config.response_length, - presence_penalty=0.0, - frequency_penalty=0.0, - repetition_penalty=1.0) - # supporting adding any sampling params from the config file - for k in config.keys(): - if hasattr(SamplingParams(), str(k)): - kwargs[k] = config.get(k) - print(f"kwargs: {kwargs}") - self.sampling_params = kwargs - - self.tokenizer = tokenizer - self.pad_token_id = tokenizer.pad_token_id - - @contextmanager - def update_sampling_params(self, **kwargs): - # update sampling params - old_sampling_params_args = {} - if kwargs: - for key, value in kwargs.items(): - if key in self.sampling_params: - old_value = self.sampling_params[key] - old_sampling_params_args[key] = old_value - self.sampling_params[key] = value - yield - # roll back to previous sampling params - # if len(old_sampling_params_args): - for key, value in old_sampling_params_args.items(): - self.sampling_params[key] = value - - @torch.no_grad() - def generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: - # if self.config.free_cache_engine: - - idx = prompts.batch["input_ids"] # (bs, prompt_length) - # left-padded attention_mask - attention_mask = prompts.batch["attention_mask"] - position_ids = prompts.batch["position_ids"] - - # used to construct attention_mask - eos_token_id = prompts.meta_info["eos_token_id"] - - batch_size = idx.size(0) - idx_list = [] - # parse idx from torch.Tensor to List[List[str]] - for i in range(batch_size): - idx_list.append(_pre_process_inputs(self.pad_token_id, idx[i])) - - do_sample = prompts.meta_info.get("do_sample", True) - if not do_sample: - # kwargs = { - # 'top_p': 1.0, - # 'top_k': -1, - # 'min_p': 0.0, - # 'temperature': 0, - # 'n': 1 # if greedy, only 1 response - # } - kwargs = dict( - n=1, - presence_penalty=0.0, - frequency_penalty=0.0, - repetition_penalty=1.0, - temperature=0, - top_p=1, - top_k=-1, - ignore_eos=False, - min_new_tokens=0, - max_new_tokens=self.config.response_length, - skip_special_tokens=True, - spaces_between_special_tokens=True, - ) - # users can customize different sampling_params at different run - with self.update_sampling_params(**kwargs): - print(f"{self.sampling_params=}") - output = self.inference_engine.generate( - prompt=None, # because we have already convert it to prompt token id - sampling_params=self.sampling_params, - return_logprob=True, - input_ids=idx_list, - ) - - out = _post_process_outputs(self.tokenizer, output) - - response = out[0].to(idx.device) - log_probs = out[1].to(idx.device) - - if response.shape[1] < self.config.response_length: - response = pad_sequence_to_length(response, self.config.response_length, self.pad_token_id) - log_probs = pad_sequence_to_length(log_probs, self.config.response_length, self.pad_token_id) - if self.config.n > 1 and do_sample: - idx = idx.repeat_interleave(self.config.n, dim=0) - attention_mask = attention_mask.repeat_interleave(self.config.n, dim=0) - position_ids = position_ids.repeat_interleave(self.config.n, dim=0) - batch_size = batch_size * self.config.n - seq = torch.cat([idx, response], dim=-1) - - response_length = response.size(1) - delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) - delta_position_id = delta_position_id.unsqueeze(0).repeat(batch_size, 1) - - # TODO(sgm): fix position_ids on right_pad - # prompt: left pad + response: right pad - # attention_mask: [0,0,0,0,1,1,1,1, | 1,1,1,0,0,0,0,0] - # position_ids: [0,0,0,0,0,1,2,3, | 4,5,6,7,8,9,10,11] - response_position_ids = position_ids[:, -1:] + delta_position_id - position_ids = torch.cat([position_ids, response_position_ids], dim=-1) - response_attention_mask = get_response_mask(response_id=response, - eos_token=eos_token_id, - dtype=attention_mask.dtype) - attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) - - # all the tp ranks should contain the same data here. data in all ranks are valid - batch = TensorDict( - { - "prompts": idx, - "responses": response, - "input_ids": seq, # here input_ids become the whole sentences - # 'old_log_probs': log_probs, # we will recompute old log prob with actor - "attention_mask": attention_mask, - "position_ids": position_ids, - }, - batch_size=batch_size, - ) - - # free cache engine - if self.config.free_cache_engine and self.inference_engine._engine is not None: - self.inference_engine._engine.tokenizer_manager.flush_cache() - - return DataProto(batch=batch) diff --git a/verl/workers/rollout/tokenizer.py b/verl/workers/rollout/tokenizer.py deleted file mode 100644 index c0dfa3a530..0000000000 --- a/verl/workers/rollout/tokenizer.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The base tokenizer class, required for any hybrid engine based rollout or inference with vLLM. -""" -from abc import ABC, abstractmethod -from typing import Dict, List, Union - -__all__ = ['HybridEngineBaseTokenizer'] - - -class HybridEngineBaseTokenizer(ABC): - """the tokenizer property and function name should align with HF's to meet vllm requirement""" - - @property - @abstractmethod - def vocab_size(self): - """ - `int`: Size of the base vocabulary (without the added tokens). - """ - pass - - @property - @abstractmethod - def pad_token_id(self): - """ - `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been set. - """ - pass - - @property - @abstractmethod - def eos_token_id(self): - """ - `Optional[int]`: Id of the end of sentence token in the vocabulary. Returns `None` if the token has not been - set. - """ - pass - - @property - @abstractmethod - def all_special_ids(self) -> List[int]: - """ - `List[int]`: List the ids of the special tokens(`''`, `''`, etc.) mapped to class attributes. - """ - pass - - @property - @abstractmethod - def all_special_tokens(self) -> List[str]: - """ - `List[str]`: A list of the unique special tokens (`''`, `''`, ..., etc.). - - Convert tokens of `tokenizers.AddedToken` type to string. - """ - pass - - @abstractmethod - def encode(self, text): - """ - Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary. - - Args: - text (`str`, `List[str]` or `List[int]`): - The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the - `tokenize` method) or a list of integers. - - text_pair (`str`, `List[str]` or `List[int]`, *optional*): - Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using - the `tokenize` method) or a list of integers. - """ - pass - - @abstractmethod - def decode( - self, - token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"], - skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = None, - **kwargs, - ) -> str: - """ - Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special - tokens and clean up tokenization spaces. - - Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`. - - Args: - token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): - List of tokenized input ids. Can be obtained using the `__call__` method. - skip_special_tokens (`bool`, *optional*, defaults to `False`): - Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*): - Whether or not to clean up the tokenization spaces. If `None`, will default to - `self.clean_up_tokenization_spaces`. - kwargs (additional keyword arguments, *optional*): - Will be passed to the underlying model specific decode method. - - Returns: - `str`: The decoded sentence. - """ - pass - - @abstractmethod - def convert_ids_to_tokens(self, - ids: Union[int, List[int]], - skip_special_tokens: bool = False) -> Union[str, List[str]]: - """ - Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and - added tokens. - - Args: - ids (`int` or `List[int]`): - The token id (or token ids) to convert to tokens. - skip_special_tokens (`bool`, *optional*, defaults to `False`): - Whether or not to remove special tokens in the decoding. - - Returns: - `str` or `List[str]`: The decoded token(s). - """ - pass - - @abstractmethod - def get_added_vocab(self) -> Dict[str, int]: - """ - Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from - the fast call because for now we always add the tokens even if they are already in the vocabulary. This is - something we should change. - - Returns: - `Dict[str, int]`: The added tokens. - """ - pass - - @abstractmethod - def convert_tokens_to_string(self, tokens: List[str]) -> str: - """ - Converts a sequence of tokens in a single string. The most simple way to do it is `" ".join(tokens)` but we - often want to remove sub-word tokenization artifacts at the same time. - - Args: - tokens (`List[str]`): The token to join in a string. - - Returns: - `str`: The joined tokens. - """ - pass - - @property - def is_fast(self): - return False diff --git a/verl/workers/rollout/vllm_rollout/__init__.py b/verl/workers/rollout/vllm_rollout/__init__.py deleted file mode 100644 index 0d6d4c3d81..0000000000 --- a/verl/workers/rollout/vllm_rollout/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from importlib.metadata import version, PackageNotFoundError - -### -# [SUPPORT AMD:] -import torch -### - - -def get_version(pkg): - try: - return version(pkg) - except PackageNotFoundError: - return None - - -package_name = 'vllm' -package_version = get_version(package_name) - -### -# package_version = get_version(package_name) -# [SUPPORT AMD:] -if "AMD" in torch.cuda.get_device_name(): - import re - package_version = version(package_name) - package_version = re.match(r'(\d+\.\d+\.?\d*)', package_version).group(1) -else: - package_version = get_version(package_name) -### - -if package_version <= '0.6.3': - vllm_mode = 'customized' - from .vllm_rollout import vLLMRollout - from .fire_vllm_rollout import FIREvLLMRollout -else: - vllm_mode = 'spmd' - from .vllm_rollout_spmd import vLLMRollout diff --git a/verl/workers/rollout/vllm_rollout/fire_vllm_rollout.py b/verl/workers/rollout/vllm_rollout/fire_vllm_rollout.py deleted file mode 100644 index c65af5a5b3..0000000000 --- a/verl/workers/rollout/vllm_rollout/fire_vllm_rollout.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The vllm_rollout that can be applied in different backend -When working with FSDP: -- Use DTensor weight loader (recommended) or HF weight loader -- Utilize state_dict from the FSDP to synchronize the weights among tp ranks in vLLM -When working with Megatron: -- Use Megatron weight loader -- During training, only the current pp stage holds the parameters -- Before inference, broadcast the parameters of the current pp rank to all other pp ranks (all pp ranks holds all the parameters) -- Bind the parameters to the inference engine -- Do inference in tp. pp is treated as additional dp -- After inference, all the parameters that doesn't belong to this pp rank is freed. -""" -from typing import List -from contextlib import contextmanager -from omegaconf import DictConfig -import torch -import torch.distributed -from tensordict import TensorDict -from torch import nn - -from verl import DataProto -from verl.utils.torch_functional import get_response_mask, pad_sequence_to_length -from verl.workers.rollout.base import BaseRollout -from verl.workers.rollout.vllm_rollout.vllm_rollout import vLLMRollout -from verl.third_party.vllm import LLM, vllm_version -from verl.third_party.vllm import parallel_state as vllm_ps -from vllm import SamplingParams - -# TODO -# 1. support pp in vllm -# 2. passing tokenizer is not necessary? no encoding/decoding is happending here -# 3. simplify init logics - - -# NOTE(sgm): add for verl. We can optimize it by making the dataloader yield List[int] without padding. -def _pre_process_inputs(pad_token_id, prompt_token_ids: torch.Tensor) -> List[int]: - # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = prompt_token_ids[non_pad_index:].tolist() - return token_ids - - -class FIREvLLMRollout(vLLMRollout): - - def __init__(self, actor_module: nn.Module, config: DictConfig, tokenizer, model_hf_config, **kwargs): - """A vLLM rollout. It requires the module is supported by the vllm. - - Args: - module: module here follows huggingface APIs - config: DictConfig - tokenizer: the task/model tokenizer - model_hf_config: the huggingface config to initiallize the generating model in vllm - **kwargs: train_tp, for Megatron Backend to initialize hybrid engine (zero redundancy) process group - """ - super().__init__(actor_module, config, tokenizer, model_hf_config, **kwargs) - - self.use_fire_sampling = config.get('use_fire_sampling', False) - if self.use_fire_sampling: - kwargs_0 = kwargs.copy() - kwargs_0['temperature'] = 30 - kwargs_0['max_tokens'] = 1 - if 'top_k' not in kwargs_0 or kwargs_0['top_k'] <= 0: - kwargs_0['top_k'] = 16 - self.sampling_params.max_tokens = config.response_length - 1 - for k in config.keys(): - if hasattr(SamplingParams(), str(k)): - kwargs_0[k] = config.get(k) - self.sampling_params_0 = SamplingParams(**kwargs_0) - - @contextmanager - def update_sampling_params(self, **kwargs): - # update sampling params - old_sampling_params_args = {} - if kwargs: - for key, value in kwargs.items(): - if hasattr(self.sampling_params, key): - old_value = getattr(self.sampling_params, key) - old_sampling_params_args[key] = old_value - setattr(self.sampling_params, key, value) - if self.use_fire_sampling: - old_sampling_params_args_0 = {} - if kwargs: - for key, value in kwargs.items(): - if hasattr(self.sampling_params_0, key): - old_value = getattr(self.sampling_params_0, key) - old_sampling_params_args_0[key] = old_value - setattr(self.sampling_params_0, key, value) - yield - # roll back to previous sampling params - # if len(old_sampling_params_args): - for key, value in old_sampling_params_args.items(): - setattr(self.sampling_params, key, value) - if self.use_fire_sampling: - for key, value in old_sampling_params_args_0.items(): - setattr(self.sampling_params_0, key, value) - - @torch.no_grad() - def generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: - # rebuild vllm cache engine - if self.config.free_cache_engine: - self.inference_engine.init_cache_engine() - - idx = prompts.batch['input_ids'] # (bs, prompt_length) - # left-padded attention_mask - attention_mask = prompts.batch['attention_mask'] - position_ids = prompts.batch['position_ids'] - - # used to construct attention_mask - eos_token_id = prompts.meta_info['eos_token_id'] - - batch_size = idx.size(0) - - idx_list = [] - # parse idx from torch.Tensor to List[List[str]] - for i in range(batch_size): - idx_list.append(_pre_process_inputs(self.pad_token_id, idx[i])) - - do_sample = prompts.meta_info.get('do_sample', True) - if not do_sample: - kwargs = { - 'best_of': 1, - 'top_p': 1.0, - 'top_k': -1, - 'min_p': 0.0, - 'temperature': 0, - 'n': 1 # if greedy, only 1 response - } - - if not self.use_fire_sampling: - # users can customize different sampling_params at different run - with self.update_sampling_params(**kwargs): - output = self.inference_engine.generate( - prompts=None, # because we have already convert it to prompt token id - sampling_params=self.sampling_params, - prompt_token_ids=idx_list, - use_tqdm=False) - - response = output[0].to(idx.device) # (bs, response_length) - log_probs = output[1].to(idx.device) # (bs, response_length) - else: - with self.update_sampling_params(**kwargs): - output_0 = self.inference_engine.generate( - prompts=None, # because we have already convert it to prompt token id - sampling_params=self.sampling_params_0, - prompt_token_ids=idx_list, - use_tqdm=False) - new_idx_list = [] - for i in range(batch_size): - new_idx_list.append(idx_list[i] + output_0[0][i].tolist()) - output = self.inference_engine.generate( - prompts=None, # because we have already convert it to prompt token id - sampling_params=self.sampling_params, - prompt_token_ids=new_idx_list, - use_tqdm=False) - - response = torch.cat([output_0[0], output[0]], dim=1).to(idx.device) # (bs, response_length) - # log_probs = torch.cat([output_0[1], output[1]], dim=1).to(idx.device) # (bs, response_length) - - if response.shape[1] < self.config.response_length: - response = pad_sequence_to_length(response, self.config.response_length, self.pad_token_id) - # log_probs = pad_sequence_to_length(log_probs, self.config.response_length, self.pad_token_id) - - if self.config.n > 1 and do_sample: - idx = idx.repeat_interleave(self.config.n, dim=0) - attention_mask = attention_mask.repeat_interleave(self.config.n, dim=0) - position_ids = position_ids.repeat_interleave(self.config.n, dim=0) - batch_size = batch_size * self.config.n - seq = torch.cat([idx, response], dim=-1) - - response_length = response.size(1) - delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) - delta_position_id = delta_position_id.unsqueeze(0).repeat(batch_size, 1) - - # TODO(sgm): fix position_ids on right_pad - # prompt: left pad + response: right pad - # attention_mask: [0,0,0,0,1,1,1,1, | 1,1,1,0,0,0,0,0] - # position_ids: [0,0,0,0,0,1,2,3, | 4,5,6,7,8,9,10,11] - response_position_ids = position_ids[:, -1:] + delta_position_id - position_ids = torch.cat([position_ids, response_position_ids], dim=-1) - response_attention_mask = get_response_mask(response_id=response, - eos_token=eos_token_id, - dtype=attention_mask.dtype) - attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) - - # all the tp ranks should contain the same data here. data in all ranks are valid - batch = TensorDict( - { - 'prompts': idx, - 'responses': response, - 'input_ids': seq, # here input_ids become the whole sentences - # 'old_log_probs': log_probs, # we will recompute old log prob with actor - 'attention_mask': attention_mask, - 'position_ids': position_ids - }, - batch_size=batch_size) - - # free vllm cache engine - if self.config.free_cache_engine: - self.inference_engine.free_cache_engine() - - return DataProto(batch=batch) diff --git a/verl/workers/rollout/vllm_rollout/vllm_rollout.py b/verl/workers/rollout/vllm_rollout/vllm_rollout.py deleted file mode 100644 index ea05f93436..0000000000 --- a/verl/workers/rollout/vllm_rollout/vllm_rollout.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The vllm_rollout that can be applied in different backend -When working with FSDP: -- Use DTensor weight loader (recommended) or HF weight loader -- Utilize state_dict from the FSDP to synchronize the weights among tp ranks in vLLM -When working with Megatron: -- Use Megatron weight loader -- During training, only the current pp stage holds the parameters -- Before inference, broadcast the parameters of the current pp rank to all other pp ranks (all pp ranks holds all the parameters) -- Bind the parameters to the inference engine -- Do inference in tp. pp is treated as additional dp -- After inference, all the parameters that doesn't belong to this pp rank is freed. -""" -from typing import List -from contextlib import contextmanager -from omegaconf import DictConfig -import torch -import torch.distributed -from tensordict import TensorDict -from torch import nn - -from verl import DataProto -from verl.utils.torch_functional import get_response_mask, pad_sequence_to_length -from verl.workers.rollout.base import BaseRollout -from verl.third_party.vllm import LLM, vllm_version -from verl.third_party.vllm import parallel_state as vllm_ps -from vllm import SamplingParams - -# TODO -# 1. support pp in vllm -# 2. passing tokenizer is not necessary? no encoding/decoding is happending here -# 3. simplify init logics - - -# NOTE(sgm): add for verl. We can optimize it by making the dataloader yield List[int] without padding. -def _pre_process_inputs(pad_token_id, prompt_token_ids: torch.Tensor) -> List[int]: - # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = prompt_token_ids[non_pad_index:].tolist() - return token_ids - - -class vLLMRollout(BaseRollout): - - def __init__(self, actor_module: nn.Module, config: DictConfig, tokenizer, model_hf_config, **kwargs): - """A vLLM rollout. It requires the module is supported by the vllm. - - Args: - module: module here follows huggingface APIs - config: DictConfig - tokenizer: the task/model tokenizer - model_hf_config: the huggingface config to initiallize the generating model in vllm - **kwargs: train_tp, for Megatron Backend to initialize hybrid engine (zero redundancy) process group - """ - super().__init__() - self.config = config - assert not (not config.enforce_eager and config.free_cache_engine), \ - "disable CUDA graph (enforce_eager = False) if free cache engine" - - tensor_parallel_size = self.config.get('tensor_model_parallel_size', 1) - assert tensor_parallel_size <= torch.distributed.get_world_size(), \ - "tensor parallel size should be less than or equal to the world size" - max_num_batched_tokens = int(self.config.get('max_num_batched_tokens', 8192)) - - if kwargs.get('train_tp', None) is not None: - # deployed with megatron - import os - os.environ['CUDA_TIMER_STREAM_KAFKA_ENABLE'] = '0' - os.environ['MEGATRON_IMPORT_TIMERS'] = '0' - train_tp = kwargs.get('train_tp', None) - num_tp_per_train_tp = train_tp // tensor_parallel_size - if vllm_version in ('0.4.2', '0.5.4', '0.6.3'): - vllm_ps.initialize_parallel_state(tensor_model_parallel_size=tensor_parallel_size, - num_tp_per_train_tp=num_tp_per_train_tp) - - assert model_hf_config.max_position_embeddings >= config.prompt_length + config.response_length, \ - "model context length should be greater than total sequence length" - - max_model_len = self.config.max_model_len if self.config.max_model_len \ - else config.prompt_length + config.response_length - max_model_len = int(max_model_len) - - if max_num_batched_tokens < max_model_len and self.config.enable_chunked_prefill: - raise ValueError('Enable chunked prefill, max_num_batched_tokens is smaller than max_model_len, \ - please increase max_num_batched_tokens or disable chunked prefill') - - self.inference_engine = LLM( - actor_module, - tokenizer=tokenizer, - model_hf_config=model_hf_config, - tensor_parallel_size=tensor_parallel_size, - dtype=config.dtype, - enforce_eager=config.enforce_eager, - gpu_memory_utilization=config.gpu_memory_utilization, - skip_tokenizer_init=False, - max_model_len=max_model_len, - load_format=config.load_format, - disable_log_stats=config.disable_log_stats, - max_num_batched_tokens=max_num_batched_tokens, - enable_chunked_prefill=config.enable_chunked_prefill, - ) - - # Offload vllm model to reduce peak memory usage - self.inference_engine.offload_model_weights() - - kwargs = dict( - n=1, - logprobs=0, # can be set to 0 and let actor to recompute - max_tokens=config.response_length, - ) - - # we may detokenize the result all together later - if vllm_version in ('0.4.2', '0.5.4', '0.6.3'): - kwargs['detokenize'] = False - - # supporting adding any sampling params from the config file - for k in config.keys(): - if hasattr(SamplingParams(), str(k)): - kwargs[k] = config.get(k) - - print(f"kwargs: {kwargs}") - self.sampling_params = SamplingParams(**kwargs) - - self.pad_token_id = tokenizer.pad_token_id - - @contextmanager - def update_sampling_params(self, **kwargs): - # update sampling params - old_sampling_params_args = {} - if kwargs: - for key, value in kwargs.items(): - if hasattr(self.sampling_params, key): - old_value = getattr(self.sampling_params, key) - old_sampling_params_args[key] = old_value - setattr(self.sampling_params, key, value) - yield - # roll back to previous sampling params - # if len(old_sampling_params_args): - for key, value in old_sampling_params_args.items(): - setattr(self.sampling_params, key, value) - - @torch.no_grad() - def generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: - # rebuild vllm cache engine - if self.config.free_cache_engine: - self.inference_engine.init_cache_engine() - - idx = prompts.batch['input_ids'] # (bs, prompt_length) - # left-padded attention_mask - attention_mask = prompts.batch['attention_mask'] - position_ids = prompts.batch['position_ids'] - - # used to construct attention_mask - eos_token_id = prompts.meta_info['eos_token_id'] - - batch_size = idx.size(0) - - idx_list = [] - # parse idx from torch.Tensor to List[List[str]] - for i in range(batch_size): - idx_list.append(_pre_process_inputs(self.pad_token_id, idx[i])) - - do_sample = prompts.meta_info.get('do_sample', True) - is_validate = prompts.meta_info.get('validate', False) - if not do_sample: - kwargs = { - 'best_of': 1, - 'top_p': 1.0, - 'top_k': -1, - 'min_p': 0.0, - 'temperature': 0, - 'n': 1 # if greedy, only 1 response - } - elif is_validate: - # TODO: try ** - kwargs = { - 'top_k': self.config.val_kwargs.top_k, - 'top_p': self.config.val_kwargs.top_p, - 'temperature': self.config.val_kwargs.temperature, - 'n': 1, # if validate, already repeat in ray_trainer - } - - # users can customize different sampling_params at different run - with self.update_sampling_params(**kwargs): - output = self.inference_engine.generate( - prompts=None, # because we have already convert it to prompt token id - sampling_params=self.sampling_params, - prompt_token_ids=idx_list, - use_tqdm=False) - - # TODO(sgm): disable logprob when recompute_log_prob is enable - # if n = 1: (bs, response_length) ; if n > 1: (bs * n, response_length) - response = output[0].to(idx.device) - # log_probs = output[1].to(idx.device) - - if response.shape[1] < self.config.response_length: - response = pad_sequence_to_length(response, self.config.response_length, self.pad_token_id) - # log_probs = pad_sequence_to_length(log_probs, self.config.response_length, self.pad_token_id) - - # utilize current sampling params - if self.sampling_params.n > 1 and do_sample: - idx = idx.repeat_interleave(self.sampling_params.n, dim=0) - attention_mask = attention_mask.repeat_interleave(self.sampling_params.n, dim=0) - position_ids = position_ids.repeat_interleave(self.sampling_params.n, dim=0) - batch_size = batch_size * self.sampling_params.n - seq = torch.cat([idx, response], dim=-1) - - response_length = response.size(1) - delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) - delta_position_id = delta_position_id.unsqueeze(0).repeat(batch_size, 1) - - # TODO(sgm): fix position_ids on right_pad - # prompt: left pad + response: right pad - # attention_mask: [0,0,0,0,1,1,1,1, | 1,1,1,0,0,0,0,0] - # position_ids: [0,0,0,0,0,1,2,3, | 4,5,6,7,8,9,10,11] - response_position_ids = position_ids[:, -1:] + delta_position_id - position_ids = torch.cat([position_ids, response_position_ids], dim=-1) - response_attention_mask = get_response_mask(response_id=response, - eos_token=eos_token_id, - dtype=attention_mask.dtype) - attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) - - # all the tp ranks should contain the same data here. data in all ranks are valid - batch = TensorDict( - { - 'prompts': idx, - 'responses': response, - 'input_ids': seq, # here input_ids become the whole sentences - # 'old_log_probs': log_probs, # we will recompute old log prob with actor - 'attention_mask': attention_mask, - 'position_ids': position_ids - }, - batch_size=batch_size) - - # free vllm cache engine - if self.config.free_cache_engine: - self.inference_engine.free_cache_engine() - - return DataProto(batch=batch) \ No newline at end of file diff --git a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py b/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py deleted file mode 100644 index 72ef78bb95..0000000000 --- a/verl/workers/rollout/vllm_rollout/vllm_rollout_spmd.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The vllm_rollout that can be applied in different backend -When working with FSDP: -- Use DTensor weight loader (recommended) or HF weight loader -- Utilize state_dict from the FSDP to synchronize the weights among tp ranks in vLLM -When working with Megatron: -- Use Megatron weight loader -- During training, only the current pp stage holds the parameters -- Before inference, broadcast the parameters of the current pp rank to all other pp ranks (all pp ranks holds all the parameters) -- Bind the parameters to the inference engine -- Do inference in tp. pp is treated as additional dp -- After inference, all the parameters that doesn't belong to this pp rank is freed. -""" -import numpy as np -from typing import List -from contextlib import contextmanager -from omegaconf import DictConfig -import torch -import torch.distributed -from tensordict import TensorDict -from torch import nn -from typing import Any, Union -from verl import DataProto -from verl.utils.torch_functional import get_response_mask, pad_2d_list_to_length -from verl.workers.rollout.base import BaseRollout -from vllm.distributed import parallel_state as vllm_ps -from vllm import LLM, SamplingParams -from verl.third_party.vllm import vllm_version - -# TODO -# 1. support pp in vllm -# 2. passing tokenizer is not necessary? no encoding/decoding is happending here -# 3. simplify init logics - - -# NOTE(sgm): add for verl. We can optimize it by making the dataloader yield List[int] without padding. -def _pre_process_inputs(pad_token_id, prompt_token_ids: torch.Tensor) -> List[int]: - # remove the left padding in the prompt token_id - # pad_token_id = self.llm_engine.tokenizer.pad_token_id if self.llm_engine.tokenizer.pad_token_id is not None else self.llm_engine.tokenizer.eos_token_id - non_pad_index = torch.nonzero(prompt_token_ids != pad_token_id, as_tuple=False)[0][0] - token_ids = prompt_token_ids[non_pad_index:].tolist() - return token_ids - - -def _repeat_interleave(value: Union[torch.Tensor, np.ndarray], repeats: int) -> Union[torch.Tensor, List[Any]]: - if isinstance(value, torch.Tensor): - return value.repeat_interleave(repeats, dim=0) - else: - return np.repeat(value, repeats, axis=0) - - -class vLLMRollout(BaseRollout): - - def __init__(self, model_path: str, config: DictConfig, tokenizer, model_hf_config, **kwargs): - """A vLLM rollout. It requires the module is supported by the vllm. - - Args: - module: module here follows huggingface APIs - config: DictConfig - tokenizer: the task/model tokenizer - model_hf_config: the huggingface config to initiallize the generating model in vllm - **kwargs: train_tp, for Megatron Backend to initialize hybrid engine (zero redundancy) process group - """ - super().__init__() - self.config = config - assert not (not config.enforce_eager and config.free_cache_engine), \ - "disable CUDA graph (enforce_eager = False) if free cache engine" - - tensor_parallel_size = self.config.get('tensor_model_parallel_size', 1) - assert tensor_parallel_size <= torch.distributed.get_world_size(), \ - "tensor parallel size should be less than or equal to the world size" - max_num_batched_tokens = self.config.get('max_num_batched_tokens', 8192) - - if kwargs.get('train_tp', None) is not None: - # deployed with megatron - import os - os.environ['CUDA_TIMER_STREAM_KAFKA_ENABLE'] = '0' - os.environ['MEGATRON_IMPORT_TIMERS'] = '0' - train_tp = kwargs.get('train_tp', None) - num_tp_per_train_tp = train_tp // tensor_parallel_size - vllm_ps.initialize_parallel_state(tensor_model_parallel_size=tensor_parallel_size, - num_tp_per_train_tp=num_tp_per_train_tp) - - assert model_hf_config.max_position_embeddings >= config.prompt_length + config.response_length, \ - "model context length should be greater than total sequence length" - - max_model_len = self.config.max_model_len if self.config.max_model_len \ - else config.prompt_length + config.response_length - max_model_len = int(max_model_len) - - if max_num_batched_tokens < max_model_len and self.config.enable_chunked_prefill: - raise ValueError('Enable chunked prefill, max_num_batched_tokens is smaller than max_model_len, \ - please increase max_num_batched_tokens or disable chunked prefill') - - trust_remote_code = kwargs.get('trust_remote_code', False) - - self.inference_engine = LLM( - model=model_path, - enable_sleep_mode=True, - tensor_parallel_size=tensor_parallel_size, - distributed_executor_backend="external_launcher", - dtype=config.dtype, - enforce_eager=config.enforce_eager, - gpu_memory_utilization=config.gpu_memory_utilization, - disable_custom_all_reduce=True, - disable_mm_preprocessor_cache=True, - skip_tokenizer_init=False, - max_model_len=max_model_len, - disable_log_stats=config.disable_log_stats, - max_num_batched_tokens=max_num_batched_tokens, - enable_chunked_prefill=config.enable_chunked_prefill, - enable_prefix_caching=True, - trust_remote_code=trust_remote_code, - ) - - # Offload vllm model to reduce peak memory usage - self.inference_engine.sleep(level=1) - - kwargs = dict( - n=1, - logprobs=0, # can be set to 0 and let actor to recompute - max_tokens=config.response_length, - ) - - # # we may detokenize the result all together later - if vllm_version != '0.3.1': - kwargs['detokenize'] = False - - # supporting adding any sampling params from the config file - for k in config.keys(): - if hasattr(SamplingParams(), str(k)): - kwargs[k] = config.get(k) - - print(f"kwargs: {kwargs}") - self.sampling_params = SamplingParams(**kwargs) - - self.pad_token_id = tokenizer.pad_token_id - - @contextmanager - def update_sampling_params(self, **kwargs): - # update sampling params - old_sampling_params_args = {} - if kwargs: - for key, value in kwargs.items(): - if hasattr(self.sampling_params, key): - old_value = getattr(self.sampling_params, key) - old_sampling_params_args[key] = old_value - setattr(self.sampling_params, key, value) - yield - # roll back to previous sampling params - # if len(old_sampling_params_args): - for key, value in old_sampling_params_args.items(): - setattr(self.sampling_params, key, value) - - @torch.no_grad() - def generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: - # rebuild vllm cache engine - if vllm_version in ('0.3.1', '0.4.2', '0.5.4', '0.6.3') and self.config.free_cache_engine: - self.inference_engine.init_cache_engine() - - idx = prompts.batch['input_ids'] # (bs, prompt_length) - # left-padded attention_mask - attention_mask = prompts.batch['attention_mask'] - position_ids = prompts.batch['position_ids'] - - # used to construct attention_mask - eos_token_id = prompts.meta_info['eos_token_id'] - - batch_size = idx.size(0) - - non_tensor_batch = prompts.non_tensor_batch - if 'raw_prompt_ids' not in non_tensor_batch: - non_tensor_batch['raw_prompt_ids'] = np.array( - [_pre_process_inputs(self.pad_token_id, idx[i]) for i in range(batch_size)], dtype=object) - - if batch_size != len(non_tensor_batch['raw_prompt_ids']): - raise RuntimeError('vllm sharding manager is not work properly.') - - if 'multi_modal_data' in non_tensor_batch: - vllm_inputs = [] - for raw_prompt_ids, multi_modal_data in zip(non_tensor_batch.pop('raw_prompt_ids'), - non_tensor_batch.pop('multi_modal_data')): - vllm_inputs.append({'prompt_token_ids': raw_prompt_ids, 'multi_modal_data': multi_modal_data}) - else: - vllm_inputs = [{ - 'prompt_token_ids': raw_prompt_ids - } for raw_prompt_ids in non_tensor_batch.pop('raw_prompt_ids')] - - # ensure the type of `prompt_token_ids` passed to vllm is list[int] - # https://github.com/volcengine/verl/pull/772 - for input_data in vllm_inputs: - if isinstance(input_data['prompt_token_ids'], np.ndarray): - input_data['prompt_token_ids'] = input_data['prompt_token_ids'].tolist() - elif not isinstance(input_data['prompt_token_ids'], list): - raise TypeError( - f"prompt_token_ids must be a list or numpy array, got {type(input_data['prompt_token_ids'])}") - - do_sample = prompts.meta_info.get('do_sample', True) - is_validate = prompts.meta_info.get('validate', False) - if not do_sample: - kwargs = { - 'best_of': 1, - 'top_p': 1.0, - 'top_k': -1, - 'min_p': 0.0, - 'temperature': 0, - 'n': 1 # if greedy, only 1 response - } - elif is_validate: - # TODO: try ** - kwargs = { - 'top_k': self.config.val_kwargs.top_k, - 'top_p': self.config.val_kwargs.top_p, - 'temperature': self.config.val_kwargs.temperature, - 'n': 1, # if validate, already repeat in ray_trainer - } - - # users can customize different sampling_params at different run - with self.update_sampling_params(**kwargs): - outputs = self.inference_engine.generate( - prompts=vllm_inputs, # because we have already convert it to prompt token id - sampling_params=self.sampling_params, - use_tqdm=False) - - # TODO(sgm): disable logprob when recompute_log_prob is enable - # if n = 1: (bs, response_length) ; if n > 1: (bs * n, response_length) - - response = [] - for output in outputs: - for sample_id in range(len(output.outputs)): - response.append(output.outputs[sample_id].token_ids) - - response = pad_2d_list_to_length(response, self.pad_token_id, - max_length=self.config.response_length).to(idx.device) - - if self.sampling_params.n > 1 and do_sample: - idx = _repeat_interleave(idx, self.sampling_params.n) - attention_mask = _repeat_interleave(attention_mask, self.sampling_params.n) - position_ids = _repeat_interleave(position_ids, self.sampling_params.n) - batch_size = batch_size * self.sampling_params.n - if 'multi_modal_inputs' in non_tensor_batch.keys(): - non_tensor_batch['multi_modal_inputs'] = _repeat_interleave(non_tensor_batch['multi_modal_inputs'], - self.sampling_params.n) - - seq = torch.cat([idx, response], dim=-1) - - response_length = response.size(1) - delta_position_id = torch.arange(1, response_length + 1, device=position_ids.device) - delta_position_id = delta_position_id.unsqueeze(0).expand(batch_size, -1) - if position_ids.dim() == 3: # qwen2vl mrope - delta_position_id = delta_position_id.view(batch_size, 1, -1).expand(batch_size, 3, -1) - - # TODO(sgm): fix position_ids on right_pad - # prompt: left pad + response: right pad - # attention_mask: [0,0,0,0,1,1,1,1, | 1,1,1,0,0,0,0,0] - # position_ids: [0,0,0,0,0,1,2,3, | 4,5,6,7,8,9,10,11] - response_position_ids = position_ids[:, -1:] + delta_position_id - position_ids = torch.cat([position_ids, response_position_ids], dim=-1) - response_attention_mask = get_response_mask(response_id=response, - eos_token=eos_token_id, - dtype=attention_mask.dtype) - attention_mask = torch.cat((attention_mask, response_attention_mask), dim=-1) - - # all the tp ranks should contain the same data here. data in all ranks are valid - batch = TensorDict( - { - 'prompts': idx, - 'responses': response, - 'input_ids': seq, # here input_ids become the whole sentences - # 'old_log_probs': log_probs, # we will recompute old log prob with actor - 'attention_mask': attention_mask, - 'position_ids': position_ids - }, - batch_size=batch_size) - - # free vllm cache engine - if vllm_version in ('0.3.1', '0.4.2', '0.5.4', '0.6.3') and self.config.free_cache_engine: - self.inference_engine.free_cache_engine() - - return DataProto(batch=batch, non_tensor_batch=non_tensor_batch) diff --git a/verl/workers/sharding_manager/__init__.py b/verl/workers/sharding_manager/__init__.py deleted file mode 100644 index a9c441f594..0000000000 --- a/verl/workers/sharding_manager/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from verl.utils.import_utils import ( - is_vllm_available, - is_sglang_available, - is_megatron_core_available, -) - -from .base import BaseShardingManager -from .fsdp_ulysses import FSDPUlyssesShardingManager - -AllGatherPPModel = None - -if is_megatron_core_available() and is_vllm_available(): - # from .megatron_vllm import AllGatherPPModel, MegatronVLLMShardingManager - pass -elif AllGatherPPModel is not None: - pass -else: - AllGatherPPModel = None - MegatronVLLMShardingManager = None - -if is_vllm_available(): - from .fsdp_vllm import FSDPVLLMShardingManager -else: - FSDPVLLMShardingManager = None - -# NOTE(linjunrong): Due to recent fp8 support in SGLang. Now importing any symbol relate to SGLang's model_runner would check CUDA device capability. -# However, due to veRL's setting, the main process of ray can not find any CUDA device, which would potentially lead to: -# "RuntimeError: No CUDA GPUs are available". -# For this reason, sharding_manager.__init__ should not import SGLangShardingManager and user need to import use the abs path. -# check: https://github.com/sgl-project/sglang/blob/00f42707eaddfc2c0528e5b1e0094025c640b7a0/python/sglang/srt/layers/quantization/fp8_utils.py#L76 -# if is_sglang_available(): -# from .fsdp.fsdp_sglang import FSDPSGLangShardingManager -# else: -# FSDPSGLangShardingManager = None diff --git a/verl/workers/sharding_manager/base.py b/verl/workers/sharding_manager/base.py deleted file mode 100644 index d8717890f2..0000000000 --- a/verl/workers/sharding_manager/base.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Sharding manager to implement HybridEngine -""" - -from verl import DataProto - - -class BaseShardingManager: - - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_value, traceback): - pass - - def preprocess_data(self, data: DataProto) -> DataProto: - return data - - def postprocess_data(self, data: DataProto) -> DataProto: - return data diff --git a/verl/workers/sharding_manager/fsdp_sglang.py b/verl/workers/sharding_manager/fsdp_sglang.py deleted file mode 100644 index 630bd454e7..0000000000 --- a/verl/workers/sharding_manager/fsdp_sglang.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2023-2024 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import logging -import torch -from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP -from torch.distributed.fsdp.api import ShardingStrategy, ShardedStateDictConfig, StateDictType, FullStateDictConfig -from torch.distributed.device_mesh import DeviceMesh - -from verl import DataProto -from verl.utils.torch_functional import (broadcast_dict_tensor, allgather_dict_tensors) -from verl.utils.debug import log_gpu_memory_usage -from sglang.srt.entrypoints.verl_engine import VerlEngine -from .base import BaseShardingManager -from verl.third_party.sglang import parallel_state as sglang_ps -# from vllm.distributed import parallel_state as sglang_ps - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_PPO_LOGGING_LEVEL', 'WARN')) - - -class FSDPSGLangShardingManager(BaseShardingManager): - - def __init__(self, - module: FSDP, - inference_engine: VerlEngine, - model_config, - full_params: bool = False, - device_mesh: DeviceMesh = None): - self.module = module - self.inference_engine = inference_engine - self.model_config = model_config - self.device_mesh = device_mesh - - # Full params - self.full_params = full_params - if full_params: - FSDP.set_state_dict_type(self.module, - state_dict_type=StateDictType.FULL_STATE_DICT, - state_dict_config=FullStateDictConfig()) - else: - FSDP.set_state_dict_type(self.module, - state_dict_type=StateDictType.SHARDED_STATE_DICT, - state_dict_config=ShardedStateDictConfig()) - - # Note that torch_random_states may be different on each dp rank - self.torch_random_states = torch.cuda.get_rng_state() - # get a random rng states - if self.device_mesh is not None: - gen_dp_rank = self.device_mesh['dp'].get_local_rank() - torch.cuda.manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states - self.gen_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.torch_random_states) - else: - self.gen_random_states = None - - def __enter__(self): - torch.cuda.empty_cache() - log_gpu_memory_usage('Before state_dict() in sharding manager memory', logger=logger) - params = self.module.state_dict() - log_gpu_memory_usage('After state_dict() in sharding manager memory', logger=logger) - # Copy, not share memory - load_format = None if self.full_params else 'dtensor' - self.inference_engine.resume_memory_occupation() - - self.inference_engine.update_weights_from_tensor([(k, v) for k, v in params.items()], load_format=None) - log_gpu_memory_usage('After sync model weights in sharding manager', logger=logger) - - del params - torch.cuda.empty_cache() - log_gpu_memory_usage('After del state_dict and empty_cache in sharding manager', logger=logger) - - # TODO: offload FSDP model weights - # self.module.cpu() - # torch.cuda.empty_cache() - # if torch.distributed.get_rank() == 0: - # print(f'after model to cpu in sharding manager memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB') - - # important: need to manually set the random states of each tp to be identical. - if self.device_mesh is not None: - self.torch_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.gen_random_states) - - def __exit__(self, exc_type, exc_value, traceback): - log_gpu_memory_usage('Before SGLang offload in sharding manager', logger=logger) - self.inference_engine.release_memory_occupation() - log_gpu_memory_usage('After SGLang offload in sharding manager', logger=logger) - - # self.module.to('cuda') - # if torch.distributed.get_rank() == 0: - # print(f'after actor module to cuda in sharding manager memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB') - - self.module.train() - - # add empty cache after each compute - torch.cuda.empty_cache() - - # restore random states - if self.device_mesh is not None: - self.gen_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.torch_random_states) - - def preprocess_data(self, data: DataProto) -> DataProto: - # TODO: Current impl doesn't consider FSDP with torch micro-dp - data.batch = allgather_dict_tensors(data.batch.contiguous(), - size=self.device_mesh["infer_tp"].mesh.size()[0], - group=self.device_mesh["infer_tp"].get_group(), - dim=0) - return data - - def postprocess_data(self, data: DataProto) -> DataProto: - # TODO: Current impl doesn't consider FSDP with torch micro-dp - global_rank = self.device_mesh.get_rank() - tp_rank = self.device_mesh["infer_tp"].get_local_rank() - tp_size = self.device_mesh["infer_tp"].mesh.size()[0] - src_rank = global_rank // tp_size * tp_size - broadcast_dict_tensor(data.batch, src=src_rank, group=self.device_mesh["infer_tp"].get_group()) - if tp_size > 1: - local_prompts = data.chunk(chunks=tp_size) - data = local_prompts[tp_rank] - return data diff --git a/verl/workers/sharding_manager/fsdp_ulysses.py b/verl/workers/sharding_manager/fsdp_ulysses.py deleted file mode 100644 index 34398d7edb..0000000000 --- a/verl/workers/sharding_manager/fsdp_ulysses.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Contains a resharding manager that binds weights from FSDP zero3 to XPerfGPT -""" -from .base import BaseShardingManager - -from torch.distributed.device_mesh import DeviceMesh - -from verl.utils.torch_functional import allgather_dict_tensors -from verl.protocol import all_gather_data_proto -from verl.utils.ulysses import set_ulysses_sequence_parallel_group, get_ulysses_sequence_parallel_group -import numpy as np - -import torch -import torch.distributed - -from verl import DataProto - - -class FSDPUlyssesShardingManager(BaseShardingManager): - """ - Sharding manager to support data resharding when using FSDP + Ulysses - """ - - def __init__(self, device_mesh: DeviceMesh): - super().__init__() - self.device_mesh = device_mesh - self.seed_offset = 12345 - - def __enter__(self): - if self.device_mesh is not None: - # We have a global SP group - # so we have to change to use model-specific sp group - self.prev_sp_group = get_ulysses_sequence_parallel_group() - set_ulysses_sequence_parallel_group(self.device_mesh['sp'].get_group()) - # TODO: check how to set seed for each model - - def __exit__(self, exc_type, exc_value, traceback): - # restore random states - if self.device_mesh is not None: - # revert to previous sp group - set_ulysses_sequence_parallel_group(self.prev_sp_group) - # TODO: check how to set seed for each model - - def preprocess_data(self, data: DataProto) -> DataProto: - """ - AllGather data from sp region - This is because the data is first sharded along the FSDP dimension as we utilize the DP_COMPUTE - In Ulysses, we need to make sure the same data is used across a SP group - """ - if self.device_mesh is not None: - sp_size = self.device_mesh['sp'].size() - group = self.device_mesh['sp'].get_group() - - all_gather_data_proto(data=data, process_group=group) - return data - - def postprocess_data(self, data: DataProto) -> DataProto: - """ - Split the data to follow FSDP partition - """ - if self.device_mesh is not None: - sp_size = self.device_mesh['sp'].size() - sp_rank = self.device_mesh['sp'].get_local_rank() - data = data.chunk(chunks=sp_size)[sp_rank] - return data \ No newline at end of file diff --git a/verl/workers/sharding_manager/fsdp_vllm.py b/verl/workers/sharding_manager/fsdp_vllm.py deleted file mode 100644 index fac07fb507..0000000000 --- a/verl/workers/sharding_manager/fsdp_vllm.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import logging -import torch -import numpy as np -from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP -from torch.distributed.fsdp.api import ShardingStrategy, ShardedStateDictConfig, StateDictType, FullStateDictConfig -from torch.distributed.device_mesh import DeviceMesh - -from verl.third_party.vllm import LLM -from verl.third_party.vllm import parallel_state as vllm_ps -from verl import DataProto -from verl.utils.torch_functional import (broadcast_dict_tensor, allgather_dict_tensors) -from verl.protocol import all_gather_data_proto -from verl.utils.debug import log_gpu_memory_usage -from verl.third_party.vllm import vllm_version - -from .base import BaseShardingManager - -logger = logging.getLogger(__file__) -logger.setLevel(os.getenv('VERL_PPO_LOGGING_LEVEL', 'WARN')) - - -class FSDPVLLMShardingManager(BaseShardingManager): - - def __init__(self, - module: FSDP, - inference_engine: LLM, - model_config, - full_params: bool = False, - device_mesh: DeviceMesh = None): - self.module = module - self.inference_engine = inference_engine - self.model_config = model_config - self.device_mesh = device_mesh - - # Full params - self.full_params = full_params - if full_params: - FSDP.set_state_dict_type(self.module, - state_dict_type=StateDictType.FULL_STATE_DICT, - state_dict_config=FullStateDictConfig()) - else: - FSDP.set_state_dict_type(self.module, - state_dict_type=StateDictType.SHARDED_STATE_DICT, - state_dict_config=ShardedStateDictConfig()) - - self.tp_size = vllm_ps.get_tensor_model_parallel_world_size() - self.tp_rank = vllm_ps.get_tensor_model_parallel_rank() - - # Note that torch_random_states may be different on each dp rank - self.torch_random_states = torch.cuda.get_rng_state() - # get a random rng states - if self.device_mesh is not None: - gen_dp_rank = self.device_mesh['dp'].get_local_rank() - torch.cuda.manual_seed(gen_dp_rank + 1000) # make sure all tp ranks have the same random states - self.gen_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.torch_random_states) - else: - self.gen_random_states = None - - def __enter__(self): - # NOTE: Basically, we only need `torch.cuda.empty_cache()` before vllm wake_up and - # after vllm sleep, since vllm has its own caching memory allocator CuMemAllocator. - # Out of vllm scope, we should avoid empty cache to let pytorch using caching memory - # to speed up memory allocations. - # - # pytorch: https://pytorch.org/docs/stable/notes/cuda.html#memory-management - # vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/device_allocator/cumem.py#L103 - torch.cuda.empty_cache() - - log_gpu_memory_usage('Before state_dict() in sharding manager memory', logger=logger) - params = self.module.state_dict() - log_gpu_memory_usage('After state_dict() in sharding manager memory', logger=logger) - # Copy, not share memory - load_format = 'hf' if self.full_params else 'dtensor' - - if vllm_version in ('0.4.2', '0.5.4', '0.6.3'): - self.inference_engine.sync_model_weights(params, load_format=load_format) - else: - self.inference_engine.wake_up() - world_size = torch.distributed.get_world_size() - model = self.inference_engine.llm_engine.model_executor.driver_worker.worker.model_runner.model - loaded_params = model.load_weights( - ((name, param.full_tensor() if world_size != 1 else param) for name, param in params.items())) - logger.info(f"vLLM load weights, loaded_params: {len(loaded_params)}") - - log_gpu_memory_usage('After sync model weights in sharding manager', logger=logger) - - del params - log_gpu_memory_usage('After del state_dict and empty_cache in sharding manager', logger=logger) - - # TODO: offload FSDP model weights - # self.module.cpu() - # torch.cuda.empty_cache() - # if torch.distributed.get_rank() == 0: - # print(f'after model to cpu in sharding manager memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB') - - # important: need to manually set the random states of each tp to be identical. - if self.device_mesh is not None: - self.torch_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.gen_random_states) - - def __exit__(self, exc_type, exc_value, traceback): - log_gpu_memory_usage('Before vllm offload in sharding manager', logger=logger) - # TODO(ZSL): check this - if vllm_version in ('0.4.2', '0.5.4', '0.6.3'): - self.inference_engine.offload_model_weights() - else: - self.inference_engine.sleep(level=1) - log_gpu_memory_usage('After vllm offload in sharding manager', logger=logger) - - # self.module.to('cuda') - # if torch.distributed.get_rank() == 0: - # print(f'after actor module to cuda in sharding manager memory allocated: {torch.cuda.memory_allocated() / 1e9}GB, reserved: {torch.cuda.memory_reserved() / 1e9}GB') - - self.module.train() - - # add empty cache after each compute - torch.cuda.empty_cache() - - # restore random states - if self.device_mesh is not None: - self.gen_random_states = torch.cuda.get_rng_state() - torch.cuda.set_rng_state(self.torch_random_states) - - def preprocess_data(self, data: DataProto) -> DataProto: - """All gather across tp group to make each rank has identical input.""" - if self.tp_size == 1: - return data - - # TODO: Current impl doesn't consider FSDP with torch micro-dp - if vllm_version in ('0.3.1', '0.4.2', '0.5.4', '0.6.3'): - group = vllm_ps.get_tensor_model_parallel_group() - else: - group = vllm_ps.get_tensor_model_parallel_group().device_group - - all_gather_data_proto(data=data, process_group=group) - return data - - def postprocess_data(self, data: DataProto) -> DataProto: - """Get chunk data of this tp rank since we do all gather in preprocess.""" - if self.tp_size == 1: - return data - - return data.chunk(chunks=self.tp_size)[self.tp_rank] diff --git a/verl/workers/sharding_manager/megatron_vllm.py b/verl/workers/sharding_manager/megatron_vllm.py deleted file mode 100644 index d7c684d5e1..0000000000 --- a/verl/workers/sharding_manager/megatron_vllm.py +++ /dev/null @@ -1,454 +0,0 @@ -# Copyright 2024 Bytedance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This file contains a Megatron style Hybrid Engine that shares the weights of the actor with the inference engine. -""" - -import importlib -from packaging.version import Version -import torch -import torch.distributed as dist - -from torch import nn - -from megatron.core import parallel_state as mpu -from megatron.core import DistributedDataParallel as LocalDDP -from megatron.core.transformer.module import Float16Module -from torch.nn.parallel.distributed import DistributedDataParallel as torchDDP -from verl.utils.megatron_utils import get_model, unwrap_model -from verl.utils.memory_buffer import ( - build_memory_buffer, - build_memory_reference_from_module, - get_weight_buffer_meta_from_module, -) - - -class AllGatherPPModel: - - def __init__(self, model_provider, use_distributed_optimizer=True) -> None: - - self._pp_group = mpu.get_pipeline_model_parallel_group() - self._pp_rank = mpu.get_pipeline_model_parallel_rank() - self._pp_size = mpu.get_pipeline_model_parallel_world_size() - self._vpp_size = mpu.get_virtual_pipeline_model_parallel_world_size() - self._model_chunk_size = self._vpp_size or 1 - - # each one holds a list of model_chunks in this pp stage - self._pp_models = [None] * self.pp_size - - rank_list = list(range(self.pp_size)) - # make current rank the last one to initialize - rank_list[self.pp_rank], rank_list[-1] = rank_list[-1], rank_list[self.pp_rank] - self._this_rank_models = None - - # store the parameter of each pp stage - self.memory_buffers = [None] * self.pp_size - for cur_pp_rank in rank_list: - print( - f'create pp model', f'torch allocated {torch.cuda.memory_allocated() / 1e9:.4f} GB, ' - f'reserved {torch.cuda.memory_reserved() / 1e9:.4f} GB') - # since the last initialized rank is the current pp rank, after init, the pp rank is still correct - mpu.set_pipeline_model_parallel_rank(cur_pp_rank) - if cur_pp_rank != self.pp_rank: - models = get_model(model_provider, wrap_with_ddp=False, use_distributed_optimizer=False) - models = nn.ModuleList(models) - assert len(models) == self._model_chunk_size, f"{len(models)} != {self._model_chunk_size}" - self.pp_models[cur_pp_rank] = models - else: - # for regular model, we wrapped it with DDP - models = get_model(model_provider, - wrap_with_ddp=True, - use_distributed_optimizer=use_distributed_optimizer) - assert len(models) == self._model_chunk_size, f"{len(models)} != {self._model_chunk_size}" - self._this_rank_models = nn.ModuleList(models) - self.pp_models[cur_pp_rank] = nn.ModuleList(unwrap_model(models, (torchDDP, LocalDDP))) - - self._build_param_buffer(cur_pp_rank) - self._build_param_references(cur_pp_rank, maintain_weight=cur_pp_rank == self.pp_rank) - - # TODO: after binding to the memory buffer, we can load the checkpoint here - if cur_pp_rank != self.pp_rank: - for model in self.pp_models[cur_pp_rank]: - model.eval() - self._offload_params_to_cpu(cur_pp_rank) - - def _build_param_buffer(self, pp_rank): - """Build the parameter buffer in each pp rank""" - if pp_rank == self._pp_rank: - from verl.utils.memory_buffer import MemoryBuffer - # The code here is very hard-coded, based on the following assumptions: - # 1. `len(_this_rank_models) == 1` - # 2. `_this_rank_models[0]` is a instance of `DistributedDataParallel` and `use_distributed_optimizer=True` - # 3. Only bfloat16 data type is used in parameters - source = self._this_rank_models[0].buffers[0].param_data - self.memory_buffers[pp_rank] = { - torch.bfloat16: MemoryBuffer(source.numel(), source.numel(), torch.bfloat16, source) - } - else: - model = self.pp_models[pp_rank] - weight_buffer_meta = get_weight_buffer_meta_from_module(model) - self.memory_buffers[pp_rank] = build_memory_buffer(weight_buffer_meta) - - def _build_param_references(self, pp_rank, maintain_weight=False): - if pp_rank == self._pp_rank: - return - model = self.pp_models[pp_rank] - build_memory_reference_from_module(model, self.memory_buffers[pp_rank], maintain_weight=maintain_weight) - - def _load_params_to_cuda(self, pp_rank, to_empty=False): - assert pp_rank != self.pp_rank, f"unexpected to load current pp rank [{pp_rank}] back to cuda" - for buffer in self.memory_buffers[pp_rank].values(): - if not to_empty: - buffer.data = buffer.data.to(torch.cuda.current_device(), non_blocking=True) - else: - buffer.data = torch.empty_like(buffer.data, device='cuda') - # rebuild reference after loading to CUDA - self._build_param_references(pp_rank) - - def _offload_params_to_cpu(self, pp_rank, to_empty=False): - assert pp_rank != self.pp_rank, f"unexpected to offload current pp rank [{pp_rank}] to cpu" - for buffer in self.memory_buffers[pp_rank].values(): - if not to_empty: - # offload the whole memory buffer to CPU - buffer.data = buffer.data.to('cpu', non_blocking=True) - else: - buffer.data = torch.empty_like(buffer.data, device='cpu') - self._build_param_references(pp_rank) - - def load_params_to_cuda(self, to_empty=False): - """load all model params to cuda""" - for cur_pp_rank in range(self.pp_size): - if cur_pp_rank != self.pp_rank: - self._load_params_to_cuda(cur_pp_rank, to_empty=to_empty) - - def allgather_params(self): - """allgather params of all pp ranks. Return a list of handles""" - for cur_pp_rank in range(self.pp_size): - global_src = dist.get_global_rank(group=self.pp_group, group_rank=cur_pp_rank) - - # NOTE(sgm): the async op may cause memory leakage of the memory_buffer/pp_models - - for _, param in sorted(self.pp_models[cur_pp_rank].named_parameters()): - dist.broadcast(tensor=param.data, src=global_src, group=self.pp_group, async_op=False) - - def forward(self, *inputs, **kwargs): - try: - prev_output = None - for cur_chunk_rank in range(self._model_chunk_size): - if self._vpp_size: - mpu.set_virtual_pipeline_model_parallel_rank(cur_chunk_rank) - - for cur_pp_rank in range(self.pp_size): - mpu.set_pipeline_model_parallel_rank(cur_pp_rank) - self.pp_models[cur_pp_rank][cur_chunk_rank].set_input_tensor(prev_output) - ret = self.pp_models[cur_pp_rank][cur_chunk_rank](*inputs, **kwargs) - self.pp_models[cur_pp_rank][cur_chunk_rank].set_input_tensor(None) - prev_output = ret - finally: - if self._vpp_size: - mpu.set_virtual_pipeline_model_parallel_rank(0) - mpu.set_pipeline_model_parallel_rank(self.pp_rank) - return ret - - def __call__(self, *inputs, **kwargs): - return self.forward(*inputs, **kwargs) - - def eval(self): - for model in self.pp_models[self.pp_rank]: - model.eval() - - def train(self): - for model in self.pp_models[self.pp_rank]: - model.train() - - def offload_params_to_cpu(self, to_empty=False): - """offload params of models that are not of current pp rank to cpu""" - for cur_pp_rank in range(self.pp_size): - if cur_pp_rank != self.pp_rank: - self._offload_params_to_cpu(cur_pp_rank, to_empty=to_empty) - - def get_all_params(self): - """Get all the parameters of the models in all pp ranks - - Returns: - params: List[List[Dict[str, Tensor]]]: a list of parameters in all pp, where each is a list of dict - tensors of each model chunk - - """ - params = [] - for pp_rank in range(self.pp_size): - params.append([]) - for model_chunk_idx in range(len(self.pp_models[pp_rank])): - params[pp_rank].append({}) - pp_model = self.pp_models[pp_rank][model_chunk_idx] - pp_model = unwrap_model(pp_model, ((torchDDP, LocalDDP, Float16Module))) # not use Float16Module - for name, param in pp_model.named_parameters(): - # NOTE(gh) workaround: should not get lora params for inference - if 'lora' in name: - continue - params[pp_rank][model_chunk_idx][name] = param - - return params - - def update_this_rank_models(self, new_models): - self._this_rank_models = new_models - self._pp_models[self.pp_rank] = unwrap_model(new_models, (torchDDP, LocalDDP)) - - @property - def this_rank_models(self): - return self._this_rank_models - - @property - def pp_size(self): - return self._pp_size - - @property - def pp_rank(self): - return self._pp_rank - - @property - def pp_group(self): - return self._pp_group - - @property - def pp_models(self): - return self._pp_models - - -""" -Megatron Hybrid Engine: -- During training, only the current pp stage holds the parameters -- Before inference, broadcast the parameters of the current pp rank to all other pp ranks (all pp ranks holds all the parameters) -- Bind the parameters to the inference engine -- Do inference in tp. pp is treated as additional dp -- After inference, all the parameters that doesn't belong to this pp rank is freed. -""" - -from .base import BaseShardingManager - -import torch -from torch import nn -import torch.distributed -from torch.distributed import new_group - -from verl import DataProto -from verl.utils.torch_functional import (broadcast_dict_tensor, allgather_dict_tensors) -import verl.utils.megatron.tensor_parallel as tp_utils -from verl.third_party.vllm import parallel_state as vllm_ps -from verl.third_party.vllm import LLM -from verl.utils.model import normalize_pp_vpp_params -# Micro Data parallel group. Micro data parallel group is additional dp group that origins from splitting training tp -# into infer_tp and micro_tp. By default, we use order micro_dp - tp -_MICRO_DATA_PARALLEL_GROUP = None - - -class MegatronVLLMShardingManager(BaseShardingManager): - - def __init__(self, module: AllGatherPPModel, inference_engine: LLM, model_config, layer_name_mapping): - self.module = module - self.inference_engine = inference_engine - self.model_config = model_config - self.layer_name_mapping = layer_name_mapping - - # initialize micro_dp group for vllm inference - global _MICRO_DATA_PARALLEL_GROUP - world_size = torch.distributed.get_world_size() - rank = torch.distributed.get_rank() - train_tensor_parallel_size = mpu.get_tensor_model_parallel_world_size() - infer_tensor_parallel_size = vllm_ps.get_tensor_model_parallel_world_size() - - # TODO(sgm): this may not be true for FSDP -> vLLM - assert infer_tensor_parallel_size <= train_tensor_parallel_size, \ - 'Not implemented for infer_tp > train_tp' - assert train_tensor_parallel_size % infer_tensor_parallel_size == 0 - - micro_dp_size = train_tensor_parallel_size // infer_tensor_parallel_size - num_micro_dp_groups = world_size // micro_dp_size - assert _MICRO_DATA_PARALLEL_GROUP is None, ("micro data parallel group is already initialized") - for i in range(num_micro_dp_groups): - ranks = range(i * micro_dp_size, (i + 1) * micro_dp_size) - group = new_group(ranks=ranks) - if rank in ranks: - _MICRO_DATA_PARALLEL_GROUP = group - - def default_tp_concat_fn(self, name, param, infer_params, model_config): - """ - name: name of the parameter - param: training parameters - infer_params (List[torch.Tensor]): a list of parameters all-gathered from micro_dp_group - model_config: huggingface model_config - TODO(zhangchi.usc1992): currently, the implementation is adhoc. We can move this function to the model - definition so that it is model-agnostic. If the model doesn't implement this function, - we can throw an error to force user disable TP HybridEngine. - """ - - if self.layer_name_mapping.get("qkv_layer_name") in name and "layer_norm" not in name: - # if the tensor is qkv, for each param on tp, split into q, k, v - # concat q, k, v separately. - q_lst = [] - k_lst = [] - v_lst = [] - assert model_config.num_attention_heads % model_config.num_key_value_heads == 0 - num_q_per_kv = model_config.num_attention_heads // model_config.num_key_value_heads - assert infer_params[0].shape[0] % (num_q_per_kv + 2) == 0 - kv_size_per_tp = infer_params[0].shape[0] // (num_q_per_kv + 2) - split_size = [kv_size_per_tp * num_q_per_kv, kv_size_per_tp, kv_size_per_tp] - for infer_param in infer_params: - num_query_groups_per_partition = model_config.num_key_value_heads // mpu.get_tensor_model_parallel_world_size( - ) - for chunk in infer_param.chunk(num_query_groups_per_partition): - split_size = [ - kv_size_per_tp * num_q_per_kv // num_query_groups_per_partition, - kv_size_per_tp // num_query_groups_per_partition, - kv_size_per_tp // num_query_groups_per_partition - ] - q, k, v = chunk.split(split_size) - q_lst.append(q) - k_lst.append(k) - v_lst.append(v) - q = torch.cat(q_lst, dim=0) - k = torch.cat(k_lst, dim=0) - v = torch.cat(v_lst, dim=0) - - infer_params = torch.cat((q, k, v), dim=0) - - elif self.layer_name_mapping.get("gate_proj_layer_name") in name: - # if the tensor is gate and proj - gate_lst = [] - up_lst = [] - for infer_param in infer_params: - gate, up = infer_param.chunk(2) - gate_lst.append(gate) - up_lst.append(up) - gate = torch.cat(gate_lst, dim=0) - up = torch.cat(up_lst, dim=0) - infer_params = torch.cat((gate, up), dim=0) - - else: - # concat tensor - infer_params = torch.cat(infer_params, dim=tp_utils.get_tensor_parallel_partition_dim(param)) - - return infer_params - - def _post_process_params(self, params): - """ - For each param, if it is a tp-splited param, we all-gather from micro_dp group. - """ - # here the params are in train tp format. we iterate params and all-gather - # TODO(zhangchi.usc1992) We can consider copy non-tp weight to another infer buffer. - # In this way, all the params in the original memory_buffers and can be offload. - micro_dp_size = get_micro_data_parallel_world_size() - micro_dp_group = get_micro_data_parallel_group() - - origin_params = {} - for name in params.keys(): - param = params[name] - if tp_utils.is_tensor_parallel_param(param): - # allocate a new tensor with proper size - if micro_dp_size <= 1: - infer_params = [param] - else: - infer_params = [torch.empty_like(param) for _ in range(micro_dp_size)] - torch.distributed.all_gather(infer_params, param, group=micro_dp_group) - infer_params = self.default_tp_concat_fn(name, param, infer_params, self.model_config) - # replace with original param - params[name] = infer_params - origin_params[name] = param - - return origin_params - - def __enter__(self): - # create a new cuda space for parameters not in this pp rank - self.module.load_params_to_cuda() - # broadcast the parameters from pp rank to other ranks - self.module.allgather_params() - # obtain name to parameters in pp/vpp - params = self.module.get_all_params() - - # bind the params to inference engine - self.params = normalize_pp_vpp_params(params=params, - num_hidden_layers=self.model_config.num_hidden_layers, - layer_name='layers') - self.origin_params = self._post_process_params(self.params) - self.inference_engine.sync_model_weights(self.params, load_format='megatron') - - def __exit__(self, exc_type, exc_value, traceback): - # offload parameters doesn't belong to this pp rank - self.module.offload_params_to_cpu() - - # FIXME(sgm): the best practice is to delete the cuda tensor - # rebind the model weights, can be any cpu tensor - if self.origin_params is not None: - for name, param in self.origin_params.items(): - self.params[name] = param - - # self.inference_engine.sync_model_weights(params) - self.inference_engine.offload_model_weights() - - self.module.train() - - # add empty cache after each compute - torch.cuda.empty_cache() - - def preprocess_data(self, data: DataProto) -> DataProto: - # prompts are identical for each training tp. We select for each inference tp - micro_dp_size = get_micro_data_parallel_world_size() - micro_dp_rank = get_micro_data_parallel_rank() - - # broadcast from tp=0 to other tp ranks - broadcast_dict_tensor(data.batch, - src=mpu.get_tensor_model_parallel_src_rank(), - group=mpu.get_tensor_model_parallel_group()) - - if micro_dp_size > 1: - local_prompts = data.chunk(chunks=micro_dp_size) - data = local_prompts[micro_dp_rank] - - return data - - def postprocess_data(self, data: DataProto) -> DataProto: - meta_info = data.meta_info - # all gather batch among micro-dp groups - micro_dp_size = get_micro_data_parallel_world_size() - if micro_dp_size > 1: - data.batch = allgather_dict_tensors(data.batch.contiguous(), - size=get_micro_data_parallel_world_size(), - group=get_micro_data_parallel_group(), - dim=0) - - # all gather batch among pp group - if meta_info.get('allgather_pp_output', True): - data.batch = allgather_dict_tensors(data.batch.contiguous(), - size=mpu.get_pipeline_model_parallel_world_size(), - group=mpu.get_pipeline_model_parallel_group(), - dim=0) - return data - - -""" -Micro Data parallel group -""" - - -def get_micro_data_parallel_group(): - assert _MICRO_DATA_PARALLEL_GROUP is not None - return _MICRO_DATA_PARALLEL_GROUP - - -def get_micro_data_parallel_world_size(): - return torch.distributed.get_world_size(group=get_micro_data_parallel_group()) - - -def get_micro_data_parallel_rank(): - return torch.distributed.get_rank(group=get_micro_data_parallel_group())